118 icstablebtree

.github/workflows/ci.yml

@@ -49,5 +49,6 @@ jobs:
 
       - name: Run tests
         run: |
+          set -euo pipefail
           docker compose up -d --wait --wait-timeout 60
-          docker compose run --rm app ./runTest.sh
+          docker compose run --rm --no-TTY app ./runTest.sh

README.md

@@ -202,7 +202,7 @@ Store key-value pairs that persist across canister upgrades.
 ```nim
 import nicp_cdk/storage/stable_table
 
-# Create a stable table mapping strings to integers
+# Create a stable B+Tree mapping strings to integers
 var scoreTable = initIcStableTable[string, uint]()
 
 # Store a key-value pair
@@ -216,7 +216,7 @@ let aliceScore = scoreTable["alice"]
 if scoreTable.hasKey("alice"):
   echo "Alice has a score"
 
-# Get table size
+# Get map size
 let numPlayers = scoreTable.len()
 
 # Iterate over all pairs
@@ -230,9 +230,42 @@ scoreTable.clear()
 Supported key types: `string`, `Principal`, and other primitive types
 Supported value types: primitive types, Principal, and Nim objects
 
+#### Choosing between IcStableTable and IcStableHashMap
+
+Both are key-value stores persisted in stable memory, but they use different
+index structures. In most cases, choose `IcStableTable` when you need ordered
+iteration or range queries.
+
+| Type | Index | Best for | Ordered APIs |
+| --- | --- | --- | --- |
+| `IcStableTable[K, V]` | B+Tree | General-purpose KV storage, ordered iteration, and range queries | `pairs`, `lowerBound`, `range` |
+| `IcStableHashMap[K, V]` | Linear hashing | Exact-match workloads dominated by `get` and `hasKey` | None (`pairs` order is unspecified) |
+
+`IcStableHashMap` splits one bucket at a time as it grows, avoiding a full
+rehash in a single operation. It does not preserve key order, so use
+`IcStableTable` whenever you need range queries.
+
+```nim
+import nicp_cdk/storage/stable_hash_map
+
+# Persistent HashMap for exact-match lookups
+var sessionByToken = initIcStableHashMap[string, string]()
+sessionByToken["token-123"] = "alice"
+
+if sessionByToken.hasKey("token-123"):
+  echo sessionByToken["token-123"]
+
+for token, user in sessionByToken.pairs():
+  # Iteration order is unspecified.
+  echo token, ": ", user
+```
+
+When using multiple stable storage structures in one canister, assign each one
+a distinct, non-overlapping stable-memory region.
+
 ### Example: Storing Custom Objects
 
-You can also store custom Nim objects in a stable table:
+You can also store custom Nim objects in a stable B+Tree:
 
 ```nim
 import nicp_cdk
@@ -243,7 +276,7 @@ type UserProfile = object
   name: string
   active: bool
 
-# Create a stable table mapping principals to user profiles
+# Create a stable B+Tree mapping principals to user profiles
 var userTable = initIcStableTable[Principal, UserProfile]()
 
 # Store a user profile

benchmarks/storage/stable_exact_lookup.nim

@@ -0,0 +1,74 @@
+## Exact-match lookup comparison for the stable-memory-native indices.
+##
+## Run with:
+##   nim c -d:release -d:nicpMemoryViewOnly -r --skipUserCfg benchmarks/storage/stable_exact_lookup.nim
+##   nim c -d:release -d:nicpMemoryViewOnly -r --skipUserCfg benchmarks/storage/stable_exact_lookup.nim 100000
+##
+## Wall-clock values are environment-dependent.  The read-call and read-byte
+## counters are the portable result: they model the stable-memory work that a
+## canister has to perform for the workload.
+
+import std/[monotimes, strformat, os, times, strutils]
+import ../../src/nicp_cdk/storage/libs/memory_view
+import ../../src/nicp_cdk/storage/stable_table
+import ../../src/nicp_cdk/storage/stable_hash_map
+
+type InMemoryStable = ref object
+  data: seq[byte]
+  readCalls, readBytes: uint64
+
+proc memoryView(memory: InMemoryStable): StableMemoryView =
+  initMemoryView(
+    proc(): uint64 = uint64(memory.data.len),
+    proc(offset, size: uint64): seq[byte] =
+      if offset > uint64(memory.data.len) or size > uint64(memory.data.len) - offset:
+        raise newException(ValueError, "benchmark memory read out of bounds")
+      inc memory.readCalls
+      memory.readBytes += size
+      memory.data[int(offset) ..< int(offset + size)],
+    proc(offset: uint64, data: seq[byte]) =
+      let endOffset = int(offset) + data.len
+      if endOffset > memory.data.len: memory.data.setLen(endOffset)
+      for i, value in data: memory.data[int(offset) + i] = value
+  )
+
+proc resetReads(memory: InMemoryStable) =
+  memory.readCalls = 0
+  memory.readBytes = 0
+
+proc queryOrder(index, count: uint32): uint32 =
+  ## Deterministic, non-sequential key distribution for lookup workloads.
+  uint32((uint64(index) * 2_654_435_761'u64) mod uint64(count))
+
+proc benchmark(count: uint32) =
+  let treeMemory = InMemoryStable(data: @[])
+  let hashMemory = InMemoryStable(data: @[])
+  var tree = initIcStableTable[uint32, uint64](treeMemory.memoryView(), cacheSlots = 0)
+  var hash = initIcStableHashMap[uint32, uint64](hashMemory.memoryView())
+  for key in 0'u32 ..< count:
+    let value = uint64(key) xor 0x9e3779b97f4a7c15'u64
+    tree[key] = value
+    hash[key] = value
+
+  treeMemory.resetReads()
+  var treeChecksum = 0'u64
+  let treeStart = getMonoTime()
+  for index in 0'u32 ..< count:
+    treeChecksum = treeChecksum xor tree[queryOrder(index, count)]
+  let treeElapsed = getMonoTime() - treeStart
+
+  hashMemory.resetReads()
+  var hashChecksum = 0'u64
+  let hashStart = getMonoTime()
+  for index in 0'u32 ..< count:
+    hashChecksum = hashChecksum xor hash[queryOrder(index, count)]
+  let hashElapsed = getMonoTime() - hashStart
+  doAssert treeChecksum == hashChecksum
+
+  echo &"entries={count}"
+  echo &"  btree: {inMilliseconds(treeElapsed)} ms, reads={treeMemory.readCalls}, bytes={treeMemory.readBytes}"
+  echo &"  hash:  {inMilliseconds(hashElapsed)} ms, reads={hashMemory.readCalls}, bytes={hashMemory.readBytes}"
+
+let count = if paramCount() == 1: parseUInt(paramStr(1)).uint32 else: 10_000'u32
+if count == 0: raise newException(ValueError, "entry count must be positive")
+benchmark(count)

docs/en/stable_memory.md

@@ -11,10 +11,10 @@ types built on top of stable memory:
 
 - `IcStableValue[T]` for a single value
 - `IcStableSeq[T]` for a sequence of values
-- `IcStableTable[K, V]` for a key-value store
+- `IcStableTable[K, V]` for an ordered key-value store
 
 All values are serialized with the custom format in
-`src/nicp_cdk/storage/serialization.nim`.
+`src/nicp_cdk/storage/libs/serialization.nim`.
 
 ## Usage
 
@@ -75,33 +75,37 @@ Header size: 32 bytes.
 
 ```
 0..3   magic "SSEQ"
-4..7   version (u32, little-endian)
+4..7   version 2 (u32, little-endian)
 8..15  length (u64, little-endian)
 16..23 data end offset (u64, little-endian)
 24..31 reserved
 32..   entries: [elemLen u32][elemBytes] ...
 ```
 
+`IcStableSeq` opens by reading only this header. It does not rebuild a heap
+array of element offsets; indexed operations locate records directly in stable
+memory, and variable-length updates or deletions move data with a bounded
+buffer.
+
 ### IcStableTable layout
 
-Header size: 32 bytes.
+The B+Tree persists an `SBT2` superblock, node pages, and key/value blobs.
+Initialization reads only the superblock and bounded cache metadata; it does
+not rebuild an in-memory index by scanning all entries.
 
 ```
-0..3   magic "STBL"
-4..7   version (u32, little-endian)
-8..15  element count (u64, little-endian)
-16..23 data end offset (u64, little-endian)
-24..31 reserved
-32..   entries: [keyLen u32][valueLen u32][keyBytes][valueBytes] ...
+0..3   magic "SBT2"
+4..    versioned superblock, root address, count, allocator metadata
+...    fixed-size B+Tree node pages and variable key/value blobs
 ```
 
-Entries are append-only. On initialization, the table or sequence scans the data
-area to rebuild its in-memory index.
+Entries are searched directly in stable memory. Keys are stored with an
+order-preserving codec, allowing ordered iteration and range queries.
 
 ## Serialization Notes
 
 - Fixed-size values are stored in little-endian byte order.
-- Variable-size values (string, Principal, seq, Table) are stored as
+- Variable-size values (string, Principal, seq, B+Tree values) are stored as
   `length (u32) + bytes`.
 - Nim objects are serialized by field order.
 

examples/stable_memory/AGENTS.md

@@ -0,0 +1,88 @@
+# AI Agent Instructions
+
+This is an Internet Computer (ICP) project built with icp-cli.
+Documentation: https://cli.internetcomputer.org/llms.txt
+
+## Skills
+
+<!-- ic-skills:managed:start -->
+<!-- state: onboarding-needed -->
+
+ICP evolves quickly. Tested, current implementation patterns are published as
+agent skills (correct dependency versions, compiler flags, API signatures, pitfalls).
+Skills are authoritative: when a skill and your general knowledge disagree, the
+skill is correct. How these options work and why:
+https://github.com/dfinity/icp-cli-templates/blob/main/AGENT_SKILLS.md
+
+This project has not chosen how to use skills yet. Handle setup as follows, and
+NEVER block the user's actual request on it:
+
+- If you CANNOT ask the user (non-interactive session), or the user would rather
+  just start: use `on-demand` for THIS session only (fetch skills on demand — see
+  the on-demand block below), do NOT modify this file, and continue with their
+  request. A later session can still choose.
+
+- If you CAN ask: have the user choose ONE option, then perform the matching setup
+  and rewrite this section (STEP 4) so the choice persists:
+    (a) autosync  — Claude Code only. Skills auto-update every session.
+    (b) pinned    — any agent (needs Node/npx). Skills version-locked via
+        skills-lock.json, restored on demand.
+    (c) on-demand — no install, works anywhere with network. Fetch skills fresh
+        each session. (Recommended default if the user is unsure.)
+  If they chose (b) pinned, also ask how updates should happen: only when the user
+  runs `npx skills update` themselves (manual), or offer to update at the start of
+  each session (ask-first). Either way, an update is refresh + discover: `npx skills
+  update` only refreshes/prunes already-pinned skills, so also run `npx skills add
+  dfinity/icskills --list` and offer to add any not yet installed (never --all).
+
+Setup commands:
+  (a) autosync  — fetch and follow:
+      https://skills.internetcomputer.org/.well-known/skills/autosync-ic-skills/SKILL.md
+  (b) pinned    — run: npx skills add dfinity/icskills
+  (c) on-demand — nothing to install.
+
+STEP 4 (only after the user has actively CHOSEN an option) — keep the two
+  `ic-skills:managed:` marker lines exactly as they are, and replace EVERYTHING
+  between them (these instructions included) with the one block below whose
+  `<!-- state: ... -->` comment matches the choice. Copy it verbatim; for pinned,
+  pick the block matching the update answer. Do NOT do this if you fell back to
+  on-demand without the user choosing.
+
+>>> Blocks to choose from (write exactly one, between the markers):
+
+<!-- state: configured (autosync) -->
+ICP skills auto-update each session via a SessionStart hook (`.claude/sync-ic-skills.sh`)
+and live in your agent skills directory — you don't need to run anything to refresh them.
+Skills are authoritative — prefer them over general knowledge for all ICP work.
+If they are not present (hook hasn't run, or `jq` is missing), fetch them on demand per
+https://skills.internetcomputer.org/llms.txt instead.
+How skills are managed here, and why: https://github.com/dfinity/icp-cli-templates/blob/main/AGENT_SKILLS.md
+
+<!-- state: configured (pinned, manual-update) -->
+ICP skills are version-locked in this repo (skills-lock.json) and live in your agent
+skills directory. Skills are authoritative — prefer them over general knowledge for all
+ICP work. Update only when the user runs `npx skills update` — when they do, also check
+for newly added skills with `npx skills add dfinity/icskills --list` and offer to add any
+that are missing (remove a renamed skill's old copy with `npx skills remove <old-name>` if
+it lingers). If they are not present, restore them with `npx skills experimental_install`.
+How skills are managed here, and why: https://github.com/dfinity/icp-cli-templates/blob/main/AGENT_SKILLS.md
+
+<!-- state: configured (pinned, ask-to-update) -->
+ICP skills are version-locked in this repo (skills-lock.json) and live in your agent
+skills directory. Skills are authoritative — prefer them over general knowledge for all
+ICP work. Before your first task in a new session, offer to run `npx skills update`, then
+check for newly added skills with `npx skills add dfinity/icskills --list` and offer to add
+any not yet installed (remove a renamed skill's old copy with `npx skills remove <old-name>`
+if it lingers); if the user declines or the session is non-interactive, keep the locked
+versions and continue — never block. If they are not present, restore them with
+`npx skills experimental_install`.
+How skills are managed here, and why: https://github.com/dfinity/icp-cli-templates/blob/main/AGENT_SKILLS.md
+
+<!-- state: configured (on-demand) -->
+Fetch the skills index once per session and keep each skill's name, description, and
+SKILL.md URL: https://skills.internetcomputer.org/.well-known/skills/index.json
+Before writing ICP code for a task, fetch the matching skill's SKILL.md
+(https://skills.internetcomputer.org/.well-known/skills/{name}/SKILL.md) and follow it.
+Skills are authoritative — prefer them over general knowledge.
+How skills are managed here, and why: https://github.com/dfinity/icp-cli-templates/blob/main/AGENT_SKILLS.md
+<!-- ic-skills:managed:end -->

examples/stable_memory/CLAUDE.md

@@ -0,0 +1 @@
+Read and follow the instructions in [AGENTS.md](AGENTS.md).

examples/stable_memory/README.md

@@ -1,16 +1,51 @@
-# stable_memory
+# Hello World
 
-This example shows how to build and deploy the Nim backend example with `icp-cli`.
+Welcome to your new `stable_memory` project. It demonstrates a Nim backend canister built with `nicp` and managed by `icp-cli`.
 
 ## Overview
 
-- [backend](./backend/): the canister logic and Candid interface
+This project consists of one or two canisters:
 
-## Run It
+- [backend](./backend/): a Nim canister with its [`backend.did`](./backend/backend.did) file.
+- [frontend](./frontend/): a React webapp deployed in an asset canister.
+
+
+## Build and Deploy
+
+First, start a local network:
 
 ```bash
 icp network start -d
+```
+
+Then, deploy the project:
+
+```bash
 icp deploy
 ```
 
-After deployment, use `icp canister call backend <method> ...` for the methods defined in `backend/backend.did`.
+You can call the backend directly:
+
+```bash
+icp canister call backend greet '("Internet Computer")'
+```
+
+## Local Backend Iteration
+
+If you want to build the backend directly, run:
+
+```bash
+cd backend
+nicp dev
+```
+
+Use `nicp build` instead of `nicp dev` for a release-oriented build.
+Pass `none` as the second argument to `nicp new` if you want a backend-only project.
+
+If you want to work on the frontend, use the generated React app in [`frontend/app`](./frontend/app).
+
+Finally, stop the local network with:
+
+```bash
+icp network stop
+```

examples/stable_memory/backend/.gitignore

@@ -0,0 +1,17 @@
+# Various IDEs and editors
+.vscode/
+.idea/
+**/*~
+
+# Mac OSX temporary files
+.DS_Store
+**/.DS_Store
+
+# environment variables
+.env
+
+# Nim and WASM build artifacts
+.nimcache/
+*.wasm
+*.wat
+wasi.wasm

examples/stable_memory/backend/README.md

@@ -1,6 +1,6 @@
 # Nim Backend
 
-This canister is built with `nicp developmentBuild` or `nicp dev` and deployed through `icp-cli`.
+This canister is built with `nicp build` or `nicp dev` and deployed through `icp-cli`.
 
 ## Overview