From cd9ebeae9de0edca1542cccfa779f27f5e567d07 Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:12:56 -0700 Subject: [PATCH] Prepare pinned offline Qwen embedding and sqlite-vec compatibility experiment --- .../semantic-runtime-preparation.yml | 658 ++++++++++++++++++ .../2026-10-08-semantic-memory-experiment.md | 182 +++++ ...2026-10-08-semantic-runtime-preparation.md | 75 ++ 3 files changed, 915 insertions(+) create mode 100644 .github/workflows/semantic-runtime-preparation.yml create mode 100644 docs/plans/2026-10-08-semantic-memory-experiment.md create mode 100644 docs/plans/2026-10-08-semantic-runtime-preparation.md diff --git a/.github/workflows/semantic-runtime-preparation.yml b/.github/workflows/semantic-runtime-preparation.yml new file mode 100644 index 000000000..938a69683 --- /dev/null +++ b/.github/workflows/semantic-runtime-preparation.yml @@ -0,0 +1,658 @@ +name: Semantic runtime preparation +'on': + pull_request: + paths: + - .github/workflows/semantic-runtime-preparation.yml + - docs/plans/2026-10-08-semantic-runtime-preparation.md + workflow_dispatch: +permissions: + contents: read +concurrency: + group: semantic-runtime-preparation-${{ github.run_id }}-${{ github.run_attempt }} + cancel-in-progress: false +jobs: + preparation: + name: semantic-runtime-preparation + runs-on: ubuntu-24.04 + timeout-minutes: 50 + env: + OLLAMA_IMAGE: ollama/ollama:0.35.1@sha256:17e945f1aa6e505707eeb621e3f36847d0a4a063d4778f7007645acccfb8bd4f + PYTHON_IMAGE: python:3.11.14-slim-bookworm@sha256:83f339c1be6340ae1096010fdccf6552ac932d8f410d45d206014916bdf37e48 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Admit actual capacity and create bounded disposable filesystems + run: | + set -euo pipefail + publish="$RUNNER_TEMP/semantic-publish/payload" + mkdir -p "$publish" + chmod 0700 "$publish" + git rev-parse HEAD > "$publish/checked-commit.txt" + printf '%s\n' "$GITHUB_REPOSITORY" > "$publish/repository.txt" + printf '%s\n' "$GITHUB_RUN_ID" > "$publish/run-id.txt" + printf '%s\n' "$GITHUB_RUN_ATTEMPT" > "$publish/run-attempt.txt" + ruby -rpsych -e 'Psych.safe_load(File.read(ARGV.fetch(0)), aliases: false)' .github/workflows/semantic-runtime-preparation.yml + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + python3 -I -B -S - <<'PY' + import json, os, pathlib, shutil + memory = dict(line.split(':', 1) for line in pathlib.Path('/proc/meminfo').read_text().splitlines()) + ram = int(memory['MemTotal'].split()[0]) * 1024 + free = shutil.disk_usage(os.environ['RUNNER_TEMP']).free + observed = {'schema_version': 1, 'ram_bytes': ram, 'free_disk_bytes': free, + 'minimum_ram_bytes': 14 * 2**30, 'minimum_free_disk_bytes': 32 * 2**30, + 'scientific_effect': 'NONE', 'scientific_status_authority': False} + path = pathlib.Path(os.environ['RUNNER_TEMP']) / 'semantic-publish/payload/capacity.json' + path.write_text(json.dumps(observed, sort_keys=True, indent=2) + '\n') + if ram < observed['minimum_ram_bytes'] or free < observed['minimum_free_disk_bytes']: + raise ValueError('actual hosted capacity is below preparation admission') + PY + # Evidence is bounded too; preserve the capacity/native records across its mount. + mv "$publish" "$RUNNER_TEMP/semantic-initial-evidence" + for specification in models:10240 work:512 control:64 publish:64; do + label="${specification%:*}" + size="${specification#*:}" + directory="$RUNNER_TEMP/semantic-$label" + mkdir -p "$directory" + truncate -s "${size}M" "$RUNNER_TEMP/semantic-$label.img" + mkfs.ext4 -F "$RUNNER_TEMP/semantic-$label.img" > "$RUNNER_TEMP/semantic-initial-evidence/filesystem-$label.log" 2>&1 + options=loop,nodev,nosuid,noexec + if [ "$label" = control ]; then options=loop,nodev,nosuid; fi + sudo mount -o "$options" "$RUNNER_TEMP/semantic-$label.img" "$directory" + # Keep ext4 root metadata and lost+found untouched; only this new payload belongs to the runner. + sudo mkdir "$directory/payload" + sudo chmod 0755 "$directory/payload" + sudo chown "$(id -u):$(id -g)" "$directory/payload" + done + cp "$RUNNER_TEMP/semantic-initial-evidence/"* "$publish/" + chmod 0700 "$publish" "$RUNNER_TEMP/semantic-work/payload" + sudo chown 65532:65532 "$RUNNER_TEMP/semantic-work/payload" + mkdir -p "$RUNNER_TEMP/semantic-models/payload/blobs" "$RUNNER_TEMP/semantic-models/payload/manifests/registry.ollama.ai/library/qwen3" "$RUNNER_TEMP/semantic-control/payload/downloads" + cp .github/workflows/semantic-runtime-preparation.yml "$publish/checked-workflow.yml" + cp docs/plans/2026-10-08-semantic-runtime-preparation.md "$publish/checked-plan.md" + - name: Acquire only fixed public inputs and verify every byte pin + run: | + set -euo pipefail + publish="$RUNNER_TEMP/semantic-publish/payload" + trap 'status=$?; printf "%s\n" "$status" > "$publish/provision-exit-status.txt"' EXIT + models="$RUNNER_TEMP/semantic-models/payload" + control="$RUNNER_TEMP/semantic-control/payload" + fetch() { + # Registry-origin HTTPS delivery redirects are permitted; bytes are never trusted by URL alone. + curl --fail --silent --show-error --location --max-redirs 3 --proto '=https' --proto-redir '=https' \ + --tlsv1.2 --connect-timeout 10 --max-time "$4" --max-filesize "$3" "$1" -o "$2" + } + fetch https://registry.ollama.ai/v2/library/qwen3/manifests/8b "$models/manifests/registry.ollama.ai/library/qwen3/8b" 16384 60 + test "$(sha256sum "$models/manifests/registry.ollama.ai/library/qwen3/8b" | cut -d ' ' -f 1)" = 500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41 + for pin in \ + 05a61d37b08453e59290add468e3bb2f688e23a01e967fecb0e2fa41218cea76:487 \ + a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f:5225374496 \ + ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4:1723 \ + d18a5cc71b84bc4af394a31116bd3932b42241de70c77d2b76d69a314ec8aa12:11338 \ + cff3f395ef3756ab63e58b0ad1b32bb6f802905cae1472e6a12034e4246fbbdb:120; do + digest="${pin%:*}" + size="${pin#*:}" + duration=60 + if [ "$size" = 5225374496 ]; then duration=900; fi + fetch "https://registry.ollama.ai/v2/library/qwen3/blobs/sha256:$digest" "$models/blobs/sha256-$digest" "$size" "$duration" + done + fetch https://files.pythonhosted.org/packages/6f/ad/6afd073b0f817b3e03f9e37ad626ae341805891f23c74b5292818f49ac63/sqlite_vec-0.1.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64.whl "$control/downloads/sqlite-vec-0.1.9-linux-x64.whl" 163388 60 + python3 -I -B -S - <<'PY' + import email.parser, hashlib, json, os, pathlib, stat, zipfile + base = pathlib.Path(os.environ['RUNNER_TEMP']) + models, control, publish = (base / ('semantic-' + name) / 'payload' for name in ('models', 'control', 'publish')) + def require(condition, message): + if not condition: + raise ValueError(message) + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate primary manifest key') + result[key] = value + return result + def identity(path, expected_size, expected_hash): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= expected_size, 'unsafe public input') + digest, length = hashlib.sha256(), 0 + with path.open('rb') as stream: + while part := stream.read(1048576): + length += len(part) + require(length <= expected_size, 'public input grew') + digest.update(part) + row = {'path': path.relative_to(models).as_posix(), 'bytes': length, 'sha256': digest.hexdigest(), + 'expected_bytes': expected_size, 'expected_sha256': expected_hash, + 'archive_policy': 'ARCHIVE_ORIGINAL_SMALL_INPUT' if expected_size < 1048576 else 'NOT_ARCHIVED_FULL_WEIGHTS'} + require(length == expected_size and row['sha256'] == expected_hash, 'fixed public input pin mismatch') + return row + manifest_path = models / 'manifests/registry.ollama.ai/library/qwen3/8b' + raw = manifest_path.read_bytes() + require(hashlib.sha256(raw).hexdigest() == '500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41', 'model manifest pin mismatch') + manifest = json.loads(raw, object_pairs_hook=unique) + pins = [('config', '05a61d37b08453e59290add468e3bb2f688e23a01e967fecb0e2fa41218cea76', 487), + ('model', 'a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f', 5225374496), + ('template', 'ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4', 1723), + ('license', 'd18a5cc71b84bc4af394a31116bd3932b42241de70c77d2b76d69a314ec8aa12', 11338), + ('params', 'cff3f395ef3756ab63e58b0ad1b32bb6f802905cae1472e6a12034e4246fbbdb', 120)] + require(set(manifest) == {'schemaVersion', 'mediaType', 'config', 'layers'} and manifest['schemaVersion'] == 2 + and len(manifest['layers']) == 4, 'unexpected fixed model manifest topology') + actual = [manifest['config'], *manifest['layers']] + rows = [{'path': manifest_path.relative_to(models).as_posix(), 'bytes': len(raw), + 'sha256': hashlib.sha256(raw).hexdigest(), 'expected_bytes': len(raw), + 'expected_sha256': hashlib.sha256(raw).hexdigest(), 'archive_policy': 'ARCHIVE_ORIGINAL_SMALL_INPUT'}] + for item, (kind, digest, size) in zip(actual, pins): + require(item['digest'] == 'sha256:' + digest and type(item['size']) is int and item['size'] == size, 'model layer identity drift') + row = identity(models / ('blobs/sha256-' + digest), size, digest) + row['kind'] = kind + rows.append(row) + (publish / 'model-binding-before.json').write_text(json.dumps(rows, sort_keys=True, indent=2) + '\n') + wheel = control / 'downloads/sqlite-vec-0.1.9-linux-x64.whl' + wheel_raw = wheel.read_bytes() + require(len(wheel_raw) == 163388 and hashlib.sha256(wheel_raw).hexdigest() == '1515727990b49e79bcaf75fdee2ffc7d461f8b66905013231251f1c8938e7786', 'Linux wheel pin mismatch') + with zipfile.ZipFile(wheel) as archive: + expected = {'sqlite_vec/__init__.py', 'sqlite_vec/vec0.so', 'sqlite_vec-0.1.9.dist-info/METADATA', + 'sqlite_vec-0.1.9.dist-info/WHEEL', 'sqlite_vec-0.1.9.dist-info/top_level.txt', 'sqlite_vec-0.1.9.dist-info/RECORD'} + require(len(archive.infolist()) == 6 and set(archive.namelist()) == expected + and sum(item.file_size for item in archive.infolist()) <= 1048576, 'unexpected pinned wheel membership') + library = archive.read('sqlite_vec/vec0.so') + require(hashlib.sha256(library).hexdigest() == '5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8', 'Linux extension binary pin mismatch') + metadata_raw = archive.read('sqlite_vec-0.1.9.dist-info/METADATA') + metadata = email.parser.BytesParser().parsebytes(metadata_raw) + require(metadata['Name'] == 'sqlite-vec' and metadata['Version'] == '0.1.9', 'wheel metadata identity mismatch') + (control / 'vec0.so').write_bytes(library) + (control / 'sqlite-vec-METADATA.txt').write_bytes(metadata_raw) + declared = {'schema_version': 1, 'model_declared_license': 'Apache-2.0', 'sqlite_vec_declared_license': metadata['License'], + 'wheel_license_file': 'ABSENT', 'legal_approval': 'NOT_ASSESSED', 'scientific_effect': 'NONE', + 'ollama_image_component_license_inventory': 'NOT_ASSESSED', 'python_image_component_license_inventory': 'NOT_ASSESSED', + 'scientific_status_authority': False, 'wheel_bytes': len(wheel_raw), 'wheel_sha256': hashlib.sha256(wheel_raw).hexdigest()} + (publish / 'declared-licenses.json').write_text(json.dumps(declared, sort_keys=True, indent=2) + '\n') + PY + for pair in "$OLLAMA_IMAGE:ollama" "$PYTHON_IMAGE:python"; do + image="${pair%:*}" + label="${pair##*:}" + (ulimit -f 2048; timeout --signal=TERM --kill-after=10s 600s docker pull --platform linux/amd64 "$image") > "$publish/$label-image-pull.log" 2>&1 + docker image inspect "$image" > "$publish/$label-image.json" + done + chmod -R a-w "$models" + - name: Write fixed public compatibility controls + run: | + set -euo pipefail + cat > "$RUNNER_TEMP/semantic-control/payload/compatibility.py" <<'PY' + import hashlib, json, math, os, pathlib, platform, sqlite3, struct, sys, time, urllib.error, urllib.request + root = pathlib.Path('/work/evidence') + root.mkdir() + def require(condition, message): + if not condition: + raise ValueError(message) + def save(name, value): + raw = (json.dumps(value, ensure_ascii=True, allow_nan=False, sort_keys=True, indent=2) + '\n').encode() + require(len(raw) <= 16777216, 'compatibility evidence byte limit') + with (root / name).open('xb') as output: + output.write(raw) + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate native response key') + result[key] = value + return result + def invalid_number(value): + raise ValueError('nonfinite native JSON number') + require(sys.version_info[:3] == (3, 11, 14) and platform.system() == 'Linux' and platform.machine() == 'x86_64', 'actual client runtime drift') + extension = pathlib.Path('/control/vec0.so') + require(hashlib.sha256(extension.read_bytes()).hexdigest() == '5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8', 'loaded Linux extension pin mismatch') + database = root / 'compatibility.sqlite' + def connect(): + db = sqlite3.connect(database) + db.enable_load_extension(True) + try: + db.load_extension(str(extension)) + finally: + db.enable_load_extension(False) + require(db.execute('select vec_version()').fetchone()[0] in ('v0.1.9', '0.1.9'), 'actual loaded extension version drift') + db.execute('pragma journal_mode=DELETE') + db.execute('pragma synchronous=FULL') + return db + query = struct.pack('<3f', 1.0, 0.0, 0.0) + with connect() as db: + version = db.execute('select vec_version()').fetchone()[0] + compile_options = [row[0] for row in db.execute('pragma compile_options')] + db.execute('create virtual table tiny using vec0(embedding float[3] distance_metric=cosine)') + db.executemany('insert into tiny(rowid, embedding) values (?, ?)', [(1, query), (2, struct.pack('<3f', 0.0, 1.0, 0.0))]) + db.close() + with connect() as db: + tiny_rows = db.execute('select rowid, distance from tiny where embedding match ? and k=2 order by distance', (query,)).fetchall() + tiny_rows.sort(key=lambda item: (item[1], item[0])) + require(len(tiny_rows) == 2 and [row[0] for row in tiny_rows] == [1, 2] + and abs(tiny_rows[0][1]) <= 1e-6 and abs(tiny_rows[1][1] - 1) <= 1e-6, 'reopened tiny cosine query mismatch') + db.close() + save('extension-result.json', {'schema_version': 1, 'python': platform.python_version(), 'platform': platform.platform(), + 'sqlite_version': sqlite3.sqlite_version, 'sqlite_compile_options': compile_options, 'vec_version': version, + 'tiny_cosine_rows': tiny_rows, 'actual_loaded_extension': True, 'reopened': True, + 'scientific_effect': 'NONE', 'scientific_status_authority': False}) + # No proxy handler, redirect handler or endpoint input is admitted to this loopback-only client. + class DenyRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, fp, code, message, headers, url): + raise ValueError('loopback API redirect refused') + client = urllib.request.build_opener(urllib.request.ProxyHandler({}), DenyRedirect()) + def api(path, label, payload=None, timeout=30): + require(path in ('/api/version', '/api/tags', '/api/show', '/api/embed', '/api/ps'), 'unadmitted API path') + data = None if payload is None else json.dumps(payload, allow_nan=False, separators=(',', ':')).encode() + if data is not None: + save(label + '-request.json', payload) + request = urllib.request.Request('http://127.0.0.1:11434' + path, data=data, headers={'Content-Type': 'application/json'}) + try: + response = client.open(request, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + with response: + raw = response.read(16777217) + require(len(raw) <= 16777216, 'native API response byte limit') + with (root / (label + '-response.json')).open('xb') as output: + output.write(raw) + save(label + '-http.json', {'status': response.status, 'response_bytes': len(raw), 'path': path}) + require(response.status == 200, 'native API rejected compatibility request') + return json.loads(raw, object_pairs_hook=unique, parse_constant=invalid_number) + # Readiness waits precede the one embedding attempt; they cannot substitute a different model or endpoint. + readiness = [] + for attempt in range(30): + try: + version_raw = client.open('http://127.0.0.1:11434/api/version', timeout=2) + with version_raw: + raw = version_raw.read(16385) + require(version_raw.status == 200 and len(raw) <= 16384, 'invalid readiness response') + ready_version = json.loads(raw, object_pairs_hook=unique, parse_constant=invalid_number) + require(ready_version == {'version': '0.35.1'}, 'actual Ollama version drift') + readiness.append({'attempt': attempt + 1, 'outcome': 'ready'}) + break + except urllib.error.URLError as error: + readiness.append({'attempt': attempt + 1, 'outcome': type(error).__name__}) + time.sleep(1) + else: + save('readiness.json', readiness) + raise ValueError('bounded Ollama startup did not become ready') + save('readiness.json', readiness) + model = 'qwen3:8b' + manifest_digest = '500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41' + def model_identity(label): + require(api('/api/version', label + '-version') == {'version': '0.35.1'}, 'native version drift') + tags = api('/api/tags', label + '-tags') + require(type(tags) is dict and type(tags.get('models')) is list and len(tags['models']) == 1 + and tags['models'][0]['name'] == model and tags['models'][0]['digest'] == manifest_digest, 'actual served model manifest drift') + show = api('/api/show', label + '-show', {'model': model, 'verbose': True}) + require(type(show) is dict and type(show.get('model_info')) is dict and type(show.get('template')) is str, 'native show identity unavailable') + require(hashlib.sha256(show['template'].encode()).hexdigest() == 'ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4', 'actual template byte drift') + require(show['model_info'].get('general.architecture') == 'qwen3' + and type(show['model_info'].get('qwen3.embedding_length')) is int and show['model_info']['qwen3.embedding_length'] == 4096, 'model metadata identity drift') + return show + before = model_identity('before') + try: + request = {'model': model, 'input': ['TEST preparation: water boils.', 'TEST preparation: a square has four sides.'], + 'truncate': False, 'keep_alive': '60s', 'options': {'num_ctx': 4096, 'num_thread': 4, 'num_gpu': 0}} + embeddings = api('/api/embed', 'embedding', request, timeout=600) + require(type(embeddings) is dict and embeddings.get('model') == model + and type(embeddings.get('embeddings')) is list and len(embeddings['embeddings']) == 2, 'actual embedding cardinality/model mismatch') + vectors = embeddings['embeddings'] + norms = [] + for vector in vectors: + require(type(vector) is list and len(vector) == 4096 + and all(type(value) in (int, float) and math.isfinite(value) for value in vector), 'invalid actual embedding dimensions or values') + norm = math.sqrt(math.fsum(value * value for value in vector)) + require(math.isfinite(norm) and norm > 0 and abs(norm - 1) <= 1e-3, 'actual embedding norm mismatch') + norms.append(norm) + for name in ('total_duration', 'load_duration', 'prompt_eval_count'): + require(type(embeddings.get(name)) is int and embeddings[name] >= 0, 'missing actual embedding statistics') + require(embeddings['prompt_eval_count'] > 0, 'actual embedding input count absent') + blobs = [struct.pack('<4096f', *vector) for vector in vectors] + rounded = [struct.unpack('<4096f', blob) for blob in blobs] + with connect() as db: + db.execute('create virtual table native using vec0(embedding float[4096] distance_metric=cosine)') + db.executemany('insert into native(rowid, embedding) values (?, ?)', list(enumerate(blobs, start=1))) + db.close() + with connect() as db: + rows = db.execute('select rowid,distance from native where embedding match ? and k=2 order by distance', (blobs[0],)).fetchall() + rows.sort(key=lambda item: (item[1], item[0])) + db.close() + norm_query = math.sqrt(math.fsum(value * value for value in rounded[0])) + oracle = [] + for identifier, vector in enumerate(rounded, start=1): + norm = math.sqrt(math.fsum(value * value for value in vector)) + distance = 1 - math.fsum(a * b for a, b in zip(rounded[0], vector)) / (norm_query * norm) + oracle.append((identifier, distance)) + oracle.sort(key=lambda item: (item[1], item[0])) + require(len(rows) == 2 and [row[0] for row in rows] == [row[0] for row in oracle] + and all(abs(actual[1] - expected[1]) <= 1e-5 for actual, expected in zip(rows, oracle)), 'native float32 cosine/reopen mismatch') + save('embedding-result.json', {'schema_version': 1, 'embedding_requests': 1, 'public_input_count': 2, + 'dimensions': 4096, 'norms': norms, 'float32_cosine_rows': rows, 'exact_cosine_oracle': oracle, + 'truncate': False, 'native_statistics': {name: embeddings[name] for name in ('total_duration', 'load_duration', 'prompt_eval_count')}, + 'reopened': True, 'scientific_effect': 'NONE', 'scientific_status_authority': False}) + finally: + after = model_identity('after') + require(before == after, 'native model/tokenizer/template changed during compatibility attempt') + api('/api/ps', 'after-ps') + save('compatibility-result.json', {'schema_version': 1, 'scope': 'tiny-public-runtime-compatibility', + 'actual_loaded_extension': True, 'actual_embedding_vectors': 2, 'native_identity_unchanged': True, + 'full_prompt_tokenizer_parity': 'NOT_RUN', 'native_selected_model_info_parity': 'NOT_RUN', + 'approximately_500_token_chunking': 'NOT_RUN', 'retrieval_quality': 'NOT_RUN', 'generation_quality': 'NOT_RUN', + 'installed_entrypoints': 'NOT_TOUCHED', 'default_retrieval_switch': False, + 'scientific_effect': 'NONE', 'scientific_status_authority': False}) + PY + chmod -R a-w "$RUNNER_TEMP/semantic-control/payload" + - name: Exercise the fixed runtime with external networking denied + run: | + set -euo pipefail + publish="$RUNNER_TEMP/semantic-publish/payload" + # Native HOME is not assigned or repurposed. Only the fixed image's disposable /root or / case is supported. + docker create --name semantic-server --log-driver none --platform linux/amd64 --network none \ + --read-only --user 65532:65532 --cpus 3.5 --memory 12g --pids-limit 192 --cap-drop ALL --security-opt no-new-privileges \ + --ulimit fsize=16777216:16777216 --tmpfs /tmp:rw,noexec,nosuid,nodev,size=1g,uid=65532,gid=65532 \ + --tmpfs /root:rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700 \ + --tmpfs /.ollama:rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700 \ + --env OLLAMA_HOST=127.0.0.1:11434 --env OLLAMA_MODELS=/models --env OLLAMA_NO_CLOUD=1 --env OLLAMA_NOPRUNE=1 \ + --env OLLAMA_NUM_PARALLEL=1 --env OLLAMA_MAX_LOADED_MODELS=1 --env OLLAMA_MAX_QUEUE=1 \ + --env OLLAMA_CONTEXT_LENGTH=4096 --env OLLAMA_LOAD_TIMEOUT=300s --env OLLAMA_VULKAN=false \ + --env OLLAMA_DEBUG=false --env OLLAMA_DEBUG_LOG_REQUESTS=false \ + --mount type=bind,source="$RUNNER_TEMP/semantic-models/payload",target=/models,readonly \ + --entrypoint /bin/sh "$OLLAMA_IMAGE" -c 'case "${HOME-}" in /root|/) exec /bin/ollama serve ;; *) exit 78 ;; esac' > "$publish/server-container-id.txt" + docker inspect semantic-server > "$publish/server-container-before.json" + set +e + # Preserve the original startup console, including any newly generated public key printed upstream. + # Private key files and ephemeral home files are outside the closed collection payload. + (ulimit -f 2048; timeout --signal=TERM --kill-after=10s 900s docker start --attach semantic-server) > "$publish/server-console.log" 2>&1 & + server_process=$! + set -e + # Wait only for the already-created sandbox to run; no inference request is retried. + for attempt in $(seq 1 20); do + docker inspect --format '{{.State.Status}}' semantic-server > "$publish/server-start-state.txt" + state="$(cat "$publish/server-start-state.txt")" + if [ "$state" = running ]; then break; fi + if [ "$state" = exited ] || [ "$state" = dead ]; then exit 1; fi + sleep 1 + done + test "$state" = running + docker create --name semantic-client --log-driver none --platform linux/amd64 --network container:semantic-server \ + --read-only --user 65532:65532 --cpus 0.5 --memory 512m --pids-limit 64 --cap-drop ALL --security-opt no-new-privileges \ + --ulimit fsize=16777216:16777216 --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m \ + --mount type=bind,source="$RUNNER_TEMP/semantic-control/payload",target=/control,readonly \ + --mount type=bind,source="$RUNNER_TEMP/semantic-work/payload",target=/work \ + --entrypoint python3 "$PYTHON_IMAGE" -I -B -S /control/compatibility.py > "$publish/client-container-id.txt" + docker inspect semantic-client > "$publish/client-container-before.json" + set +e + (ulimit -f 2048; timeout --signal=TERM --kill-after=10s 800s docker start --attach semantic-client) > "$publish/client-console.log" 2>&1 + status=$? + printf '%s\n' "$status" > "$publish/client-attach-exit-status.txt" + set -e + docker inspect semantic-client > "$publish/client-container-after.json" + if [ "$status" = 0 ]; then + test "$(docker inspect --format '{{.State.Status}}' semantic-client)" = exited + status="$(docker inspect --format '{{.State.ExitCode}}' semantic-client)" + fi + printf '%s\n' "$status" > "$publish/compatibility-exit-status.txt" + docker inspect semantic-server > "$publish/server-container-after.json" + docker stop --time 10 semantic-server > "$publish/server-stop.txt" + docker inspect semantic-server > "$publish/server-container-stopped.json" + set +e + wait "$server_process" + server_status=$? + printf '%s\n' "$server_status" > "$publish/server-attach-exit-status.txt" + set -e + exit "$status" + - name: Dispose both sandboxes before collecting their outputs + if: always() + run: | + set -euo pipefail + for name in semantic-client semantic-server; do + present="$(docker ps -aq --filter "name=^/$name$")" + if [ -n "$present" ]; then docker rm --force "$name"; fi + remaining="$(docker ps -aq --filter "name=^/$name$")" + test -z "$remaining" + done + - name: Collect only bounded regular original evidence and rehash the model cache + if: always() + run: | + set -euo pipefail + for name in semantic-client semantic-server; do + remaining="$(docker ps -aq --filter "name=^/$name$")" + test -z "$remaining" + done + # An interrupted setup can leave admission records staged before this payload is created. + sudo mkdir -p "$RUNNER_TEMP/semantic-publish/payload" + sudo chmod 0700 "$RUNNER_TEMP/semantic-publish/payload" + sudo chown "$(id -u):$(id -g)" "$RUNNER_TEMP/semantic-publish/payload" + sudo python3 -I -B -S - "$RUNNER_TEMP" "$(id -u)" "$(id -g)" <<'PY' + import hashlib, json, os, pathlib, stat, sys + base = pathlib.Path(sys.argv[1]) + destination = base / 'semantic-publish/payload' + host_uid, host_gid = map(int, sys.argv[2:]) + total, missing, refused = 0, [], [] + def require(condition, message): + if not condition: + raise ValueError(message) + require(0 <= host_uid < 2147483648 and 0 <= host_gid < 2147483648, 'invalid trusted collector identity') + def anchored(root, relative): + descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + parts = relative.split('/') + require(all(part not in ('', '.', '..') for part in parts), 'unsafe collector path') + for part in parts[:-1]: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + return os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=descriptor) + finally: + os.close(descriptor) + def write(name, raw): + require('/' not in name and name not in ('', '.', '..'), 'unsafe retained destination') + directory = os.open(destination, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + file = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=directory) + try: + os.fchown(file, host_uid, host_gid) + with os.fdopen(file, 'wb', closefd=False) as stream: + stream.write(raw) + finally: + os.close(file) + finally: + os.close(directory) + def retain(root, source, name, limit): + global total + file = anchored(root, source) + with os.fdopen(file, 'rb') as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= limit, 'unsafe retained original') + raw = stream.read(limit + 1) + require(len(raw) <= limit, 'retained original grew') + total += len(raw) + require(total <= 48 * 2**20, 'retained payload total limit') + write(name, raw) + # An interrupted mount setup still retains the original admission/native records. + initial = base / 'semantic-initial-evidence' + for name in ('checked-commit.txt', 'repository.txt', 'run-id.txt', 'run-attempt.txt', 'capacity.json', + 'filesystem-models.log', 'filesystem-work.log', 'filesystem-control.log', 'filesystem-publish.log'): + if not (destination / name).exists(): + try: + retain(initial, name, name, 65536) + except FileNotFoundError: + missing.append(name) + except (OSError, ValueError): + refused.append(name) + control = base / 'semantic-control/payload' + models = base / 'semantic-models/payload' + members = [(control, 'downloads/sqlite-vec-0.1.9-linux-x64.whl', 'sqlite-vec-0.1.9-linux-x64.whl', 163388), + (control, 'vec0.so', 'vec0.so', 1048576), (control, 'sqlite-vec-METADATA.txt', 'sqlite-vec-METADATA.txt', 1048576), + (control, 'compatibility.py', 'compatibility-control.py', 65536), + (models, 'manifests/registry.ollama.ai/library/qwen3/8b', 'original-model-manifest.json', 16384)] + for name in ('config', 'template', 'license', 'params'): + digest = {'config': '05a61d37b08453e59290add468e3bb2f688e23a01e967fecb0e2fa41218cea76', + 'template': 'ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4', + 'license': 'd18a5cc71b84bc4af394a31116bd3932b42241de70c77d2b76d69a314ec8aa12', + 'params': 'cff3f395ef3756ab63e58b0ad1b32bb6f802905cae1472e6a12034e4246fbbdb'}[name] + members.append((models, 'blobs/sha256-' + digest, 'original-model-' + name + '.txt', 16384)) + fixed = ['extension-result.json', 'readiness.json', 'compatibility.sqlite', 'embedding-result.json', 'compatibility-result.json'] + for prefix, paths in (('before', ('version', 'tags', 'show')), ('after', ('version', 'tags', 'show', 'ps')), ('embedding', ('',))): + for path in paths: + label = prefix + ('-' + path if path else '') + fixed.extend([label + '-response.json', label + '-http.json']) + if path == 'show' or prefix == 'embedding': + fixed.append(label + '-request.json') + for name in fixed: + members.append((base / 'semantic-work/payload', 'evidence/' + name, name, 16777216 if '-show-response' in name else 4194304)) + for root, source, name, limit in members: + try: + retain(root, source, name, limit) + except FileNotFoundError: + missing.append(name) + except (OSError, ValueError): + refused.append(name) + # The large immutable model is streamed only for an identity record; its bytes are not copied into evidence. + try: + before = json.loads((destination / 'model-binding-before.json').read_bytes()) + after = [] + for row in before: + file = anchored(models, row['path']) + digest, length = hashlib.sha256(), 0 + with os.fdopen(file, 'rb') as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= row['expected_bytes'], 'unsafe final model input') + while part := stream.read(1048576): + length += len(part) + require(length <= row['expected_bytes'], 'model input grew') + digest.update(part) + final = dict(row, bytes=length, sha256=digest.hexdigest()) + after.append(final) + require(final == row, 'model cache changed during compatibility attempt') + write('model-binding-after.json', (json.dumps(after, sort_keys=True, indent=2) + '\n').encode()) + except FileNotFoundError: + missing.append('model-binding-after.json') + except (OSError, ValueError, TypeError, KeyError): + refused.append('model-binding-after.json') + write('collection.json', (json.dumps({'schema_version': 1, 'missing': sorted(set(missing)), 'refused': sorted(set(refused)), + 'retained_bytes': total, 'full_model_weights': 'NOT_ARCHIVED', 'full_image_layers': 'NOT_ARCHIVED', + 'generated_private_key_files': 'NOT_ARCHIVED', 'ephemeral_home_files': 'NOT_ARCHIVED', + 'generated_public_key_in_original_console': 'MAY_BE_PRESENT', + 'scientific_effect': 'NONE', 'scientific_status_authority': False}, sort_keys=True, indent=2) + '\n').encode()) + if refused: + raise ValueError('bounded original evidence collection refused members') + PY + - name: Verify original identities and emit a compatibility-only receipt + run: | + set -euo pipefail + python3 -I -B -S - <<'PY' + import hashlib, json, os, pathlib + root = pathlib.Path(os.environ['RUNNER_TEMP']) / 'semantic-publish/payload' + def require(condition, message): + if not condition: + raise ValueError(message) + require((root / 'provision-exit-status.txt').read_text() == '0\n' + and (root / 'compatibility-exit-status.txt').read_text() == '0\n' + and (root / 'client-attach-exit-status.txt').read_text() == '0\n' + and (root / 'server-attach-exit-status.txt').read_text() == '0\n', 'actual preparation or attachment attempt failed') + collection = json.loads((root / 'collection.json').read_bytes()) + require(collection['missing'] == [] and collection['refused'] == [], 'original preparation evidence incomplete') + for name, variable in (('checked-commit', 'GITHUB_SHA'), ('repository', 'GITHUB_REPOSITORY'), + ('run-id', 'GITHUB_RUN_ID'), ('run-attempt', 'GITHUB_RUN_ATTEMPT')): + require((root / (name + '.txt')).read_text() == os.environ[variable] + '\n', 'native preparation identity mismatch') + images, image_refs = {}, {} + for label, image, version, digest in ( + ('ollama', 'ollama/ollama', '0.35.1', '17e945f1aa6e505707eeb621e3f36847d0a4a063d4778f7007645acccfb8bd4f'), + ('python', 'python', '3.11.14-slim-bookworm', '83f339c1be6340ae1096010fdccf6552ac932d8f410d45d206014916bdf37e48')): + image_refs[label] = image + ':' + version + '@sha256:' + digest + observed = json.loads((root / (label + '-image.json')).read_bytes()) + require(type(observed) is list and len(observed) == 1 and observed[0]['Os'] == 'linux' + and observed[0]['Architecture'] == 'amd64' and image + '@sha256:' + digest in observed[0]['RepoDigests'], 'actual image identity drift') + require(os.environ[label.upper() + '_IMAGE'] == image_refs[label], 'fixed checked image reference drift') + images[label] = observed[0] + server = json.loads((root / 'server-container-before.json').read_bytes())[0] + client = json.loads((root / 'client-container-before.json').read_bytes())[0] + base = pathlib.Path(os.environ['RUNNER_TEMP']) + bind_sets = { + 'ollama': {(str(base / 'semantic-models/payload'), '/models', False)}, + 'python': {(str(base / 'semantic-control/payload'), '/control', False), + (str(base / 'semantic-work/payload'), '/work', True)}} + tmpfs_sets = { + 'ollama': {'/tmp': 'rw,noexec,nosuid,nodev,size=1g,uid=65532,gid=65532', + '/root': 'rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700', + '/.ollama': 'rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700'}, + 'python': {'/tmp': 'rw,noexec,nosuid,nodev,size=64m'}} + def validate_container(container, label, cpu, memory, pids): + config = container['HostConfig'] + require(container['Image'] == images[label]['Id'] and container['Config']['Image'] == image_refs[label], + 'actual container did not use the corresponding inspected fixed image') + require(container['Config']['User'] == '65532:65532' and config['ReadonlyRootfs'] is True + and config['NanoCpus'] == cpu and config['Memory'] == memory and config['PidsLimit'] == pids + and config['CapDrop'] == ['ALL'] and 'no-new-privileges' in config['SecurityOpt'] + and config['LogConfig']['Type'] == 'none' and config['PortBindings'] in (None, {}), 'actual sandbox restrictions drift') + require(config['Tmpfs'] == tmpfs_sets[label] and config['Binds'] in (None, []) + and config['VolumesFrom'] in (None, []), 'actual temporary or inherited mount set drift') + mounts = container['Mounts'] + require(type(mounts) is list and len(mounts) == len(bind_sets[label]) + and all(item['Type'] == 'bind' and type(item['RW']) is bool for item in mounts) + and {(item['Source'], item['Destination'], item['RW']) for item in mounts} == bind_sets[label], + 'actual bind source/destination/permission set drift') + configured = config['Mounts'] + require(type(configured) is list and len(configured) == len(bind_sets[label]) + and all(item['Type'] == 'bind' and type(item.get('ReadOnly', False)) is bool for item in configured) + and {(item['Source'], item['Target'], not item.get('ReadOnly', False)) for item in configured} == bind_sets[label], + 'actual configured mount set drift') + validate_container(server, 'ollama', 3500000000, 12 * 2**30, 192) + validate_container(client, 'python', 500000000, 512 * 2**20, 64) + require(server['HostConfig']['NetworkMode'] == 'none' + and client['HostConfig']['NetworkMode'] in ('container:' + server['Id'], 'container:semantic-server'), 'actual offline namespace drift') + final_client = json.loads((root / 'client-container-after.json').read_bytes())[0] + final_server = json.loads((root / 'server-container-stopped.json').read_bytes())[0] + validate_container(final_client, 'python', 500000000, 512 * 2**20, 64) + validate_container(final_server, 'ollama', 3500000000, 12 * 2**30, 192) + require(final_client['HostConfig']['NetworkMode'] == client['HostConfig']['NetworkMode'] + and final_server['HostConfig']['NetworkMode'] == 'none', 'final offline namespace drift') + require(final_client['Id'] == client['Id'] and final_client['State']['Status'] == 'exited' + and final_client['State']['ExitCode'] == 0 and final_client['State']['OOMKilled'] is False, + 'actual compatibility process did not finish successfully') + require(final_server['Id'] == server['Id'] and final_server['State']['Status'] == 'exited' + and final_server['State']['ExitCode'] == 0 and final_server['State']['OOMKilled'] is False, + 'actual server did not stop cleanly') + before = json.loads((root / 'model-binding-before.json').read_bytes()) + require(before == json.loads((root / 'model-binding-after.json').read_bytes()), 'original model cache identity changed') + result = json.loads((root / 'compatibility-result.json').read_bytes()) + require(result['actual_loaded_extension'] is True and result['actual_embedding_vectors'] == 2 + and result['native_identity_unchanged'] is True and result['default_retrieval_switch'] is False, 'tiny compatibility result incomplete') + require(hashlib.sha256((root / 'original-model-manifest.json').read_bytes()).hexdigest() + == '500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41', 'retained native manifest drift') + inventory = [] + for path in sorted(root.iterdir()): + require(path.is_file() and not path.is_symlink(), 'unexpected trusted publication member') + raw = path.read_bytes() + inventory.append({'path': path.name, 'bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()}) + receipt = {'schema_version': 1, 'scope': 'semantic-runtime-tiny-public-compatibility', + 'repository': os.environ['GITHUB_REPOSITORY'], 'checked_commit': os.environ['GITHUB_SHA'], + 'run_id': os.environ['GITHUB_RUN_ID'], 'run_attempt': os.environ['GITHUB_RUN_ATTEMPT'], + 'actual_private_inputs': False, 'installed_entrypoints': 'NOT_TOUCHED', 'full_weights': 'NOT_ARCHIVED', 'full_image_layers': 'NOT_ARCHIVED', + 'runtime_images': {label: {'reference': image_refs[label], 'image_id': images[label]['Id']} for label in sorted(images)}, + 'feature_red_green': 'NOT_RUN', 'tokenizer_parity': 'NOT_RUN', 'retrieval_quality': 'NOT_RUN', + 'scientific_effect': 'NONE', 'scientific_status_authority': False, 'custody': 'unknown', + 'final_disposition': 'REQUIRES_NATIVE_JOB_SUCCESS_AND_ORIGINAL_ARCHIVE_READBACK', 'members': inventory} + (root / 'preparation-receipt.json').write_text(json.dumps(receipt, sort_keys=True, indent=2) + '\n') + PY + - name: Hash retained original members even after a failed attempt + if: always() + run: | + set -euo pipefail + cd "$RUNNER_TEMP/semantic-publish/payload" + find . -type f ! -name ARTIFACT_SHA256SUMS -print0 | sort -z | xargs -0 -r sha256sum > ARTIFACT_SHA256SUMS + - name: Retain original compatibility inputs and failure evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: semantic-runtime-preparation-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/semantic-publish/payload/ + if-no-files-found: error + retention-days: 90 + - name: Release every mounted bounded filesystem strictly + if: always() + run: | + set -euo pipefail + status=0 + for label in models work control publish; do + directory="$RUNNER_TEMP/semantic-$label" + if mountpoint -q "$directory"; then + if ! sudo umount "$directory"; then status=1; fi + fi + done + exit "$status" diff --git a/docs/plans/2026-10-08-semantic-memory-experiment.md b/docs/plans/2026-10-08-semantic-memory-experiment.md new file mode 100644 index 000000000..8c5be5fe9 --- /dev/null +++ b/docs/plans/2026-10-08-semantic-memory-experiment.md @@ -0,0 +1,182 @@ +# Semantic Memory Experiment Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox syntax. This document authorizes no implementation in the present plan-only task. + +**Goal:** Measure whether a pinned, approximately 500-token sqlite-vec sidecar improves source retrieval without weakening freshness, citation, privacy, prompt admission, or the two existing local inference contracts. + +**Architecture:** First run a disposable hosted experiment against frozen public/synthetic text, comparing the existing excerpt mechanism, lexical retrieval, and semantic retrieval. A deterministic coordinator admits sources and prompts; models cannot select tools, sources, destinations, or policy. Only measured compatibility and two distinct actual nonlocal reviews of the exact final candidate can support a later reversible integration into both native entrypoints. + +**Tech stack:** Pinned Linux Python/SQLite/sqlite-vec and tokenizer runtimes, a separately pinned hosted Ollama instance using the acquired Qwen3:8b identity, synthetic UTF-8 fixtures, stdlib exact cosine oracle, and disposable GitHub-hosted execution. No hosted model fallback is part of the candidate. + +**Spec:** [Research architecture design, semantic local retrieval](../superpowers/specs/2026-10-08-research-architecture-design.md), with this plan specifying the bounded experiment for Task 5 of [the architecture plan](../superpowers/plans/2026-10-08-research-architecture-plan.md). + +## Global constraints and observed starting point + +- Root owns this plan under claim amendment6070898124. Current owner authorization already exists. Current task changes this plan only: no code/tests, installed state, inference/embedding calls, local product execution, private corpus, DB/config/credential/seal reads, or worker interruptions. +- Drive/R17 remains the governing record. Retrieval rank, vector distance, summaries, review receipts and CI success have scientific effect NONE and scientific-status authority false. Preserve original failures and corrections through the existing records rather than creating another acceptance register. +- `deployment/local_agents.py`, in the coordinator checkout, was source-read and rehashed as `1aff2aef9faa8191c07e84f9a2f842e8a49d4d196980fd2d83039c1385f7cd43`: loopback Ollama11434, `qwen3:8b`, model digest `500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41`, context4096, output768, tools unavailable, source text currently excerpted at12000 characters. It exposes `native_infer(role, text, guidance=None)` and retains role/guidance/proposal and cache contracts. +- `/Users/dylanroy/Library/Application Support/OpenClaw-Custodian/custodian/inference.py` was separately source-read and rehashed as `e201b1cb05d33cd8d4eff3b5361e70715cc93a7afa2fee44a2d9ea92d6727b8d`. Its `summarize(evidence, runtime, model)` path enforces the sole loopback route,16384 context,1024 output, no fallbacks, all tools denied, a12000-byte task-message cap, actual input<=12288 tokens, validated evidence projection/source IDs, and before/after model/runtime checks. This is not the4096 worker contract. This plan does not read its config or sealed guidance. +- Parent-verified canonical selected native `model_info` SHA is `471b2f8be50438f109229559fe1d959636725eda569dccd3e59d81c55d7adda0`; template SHA is `ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4`. Exact tokenizer/native prompt-count parity is NOT_RUN. The4096 embedding_length field is metadata, not proof that this model implements `/api/embed`. +- Parent-verified Darwin sqlite-vec0.1.9 binary SHA is `193e480c50b59a55977d166f4aaf0e1bc8832d6963516e5950f39e4d2ce0b793`. Its existence does not establish an actual loaded extension, working queries, or Linux ABI compatibility. Linux preparation must resolve and pin its own compatible binary; do not reuse the Darwin binary or infer success from an installed package name. +- Preserve the previous v1 six-call result: baseline0/2 and candidate0/2, with cold-load confounding. The473-byte smoke/register mechanism did not measure true500-token retrieval, usable embeddings, or quality improvement. Do not pool it into this experiment or relabel it positive. +- The last reported installed custody cycle `5c140e44aa3f49f2981df716c5914901`, at22:47, failed StorageAdmissionError:15.78GiB available, below the20GiB reserve. The last reported source index had518 nodes and4 references at09:51 and is stale. These are parent-retained observations, not a fresh installed-state read by this plan author. Do not lower the reserve or retry the failed cycle. Separate healthy-worker observations from this custody failure. +- Root's fresh read-only GET `/api/ps` at23:25:18UTC returned `models:[]` with connect timeout2 seconds/total timeout5 seconds and no model load/inference. Treat the installed model as currently cold at that observation; do not infer worker inactivity, a warm cache, contention-free capacity or successful embeddings. The recorded storage guard remains unresolved, so this plan does not load the roughly5GB installed model or allocate a local index. + +## Review focus + +1. Native token counts can disagree with a superficially matching tokenizer/template. Task2 must prove complete uncached prompt-count parity, including special tokens, both entrypoint profiles and non-ASCII text; a character ratio cannot substitute. +2. Similarity can return a stale, excluded, superseded or inaccessible source. Task4 must enforce admission before scoring and again immediately before prompt publication, including a withdrawal racing a selected result. +3. A rebuild can publish mixed generations or a partially persisted vector table. Task4 must preserve the prior complete generation on every staged failure and prove reopen/restart consistency. +4. A relevant sentence can lie after the current12000-character excerpt or in a confusable older version. Tasks1/5 must freeze deep and version-specific answer spans, preserve original byte offsets, and score source identity separately from lexical relevance. +5. Warm cache speed can conceal model loading, embedding/build costs or peer starvation. Tasks5/6 must separate cache states, charge every attempt, measure both roles under load and prohibit installation from warm speed alone. + +## File and ownership map for later implementation + +No file below is created by this task except this plan. Each implementation unit needs an active owner, source review and actual hosted RED before product code. + +| Proposed file | Responsibility | +| --- | --- | +| `experiments/semantic_memory/runtime-lock.json` | Exact Linux/container/Ollama/model/tokenizer/SQLite/extension identities, licenses and measured resource profile; preparation output first, frozen input thereafter. | +| `experiments/semantic_memory/fixtures/corpus-manifest.json`, `corpus/*.txt`, `development-cases.json`, `heldout-cases.json`, `heldout-oracle.json` | Public/synthetic bytes, exact source/revision/extraction identities, eligibility events and frozen queries/gold byte spans. | +| `experiments/semantic_memory/tokenization.py`, `chunking.py` | Pinned token IDs/pieces/counts, exact UTF-8 byte spans and versioned splitter. | +| `experiments/semantic_memory/embedding.py`, `index.py` | Validated real embedding calls, float32 persistence, exact cosine comparison, immutable complete generations. | +| `experiments/semantic_memory/admission.py`, `prompt.py` | Source eligibility, exact lookup/literal fallback, citations, coverage labels, complete prompt/output admission and cache identity. | +| `experiments/semantic_memory/benchmark.py`, `tools/semantic_memory_experiment.py` | Fixed baselines, randomized paired measurement, reports, bounded experiment CLI. | +| `tests/test_semantic_memory_{runtime,chunking,index,admission,prompt,benchmark}.py` | External controls; no import of installed worker code or real private fixture. | +| `.github/workflows/semantic-memory-experiment.yml` | Root-owned future preparation/execution/retention lane; this plan does not amend existing required checks. | +| Coordinator-owned future `deployment/semantic_memory.py` and `deployment/local_agents.py`; custodian-owned future `custodian/retrieval.py` and `custodian/inference.py` | Separate reversible native integrations after experiment and exact-final review, in the owning source repositories rather than by editing installed copies. | + +Future CLI modes are `prepare-report`, `chunk`, `build`, `retrieve`, `benchmark`, and `verify-report`. All consume explicit pinned fixture/runtime manifests; installed/private paths are outside their scope. Metadata inputs are strict UTF-8 JSON, <=4MiB/document and64 container levels, with duplicate keys, nonfinite numbers, unknown wrapper fields and booleans used as integers refused. Corpus access is anchored to the allowlisted immutable fixture directory and rejects symlinks/special files/traversal; queries and corpus annotations cannot select imports, SQL identifiers or destinations. Preparation resolves identities, then freezes them before the capability/quality experiment. A mutable upstream tag is never an experiment identity. Existing source/extraction/export ABI must be reconciled by the local coordinator before an installed adapter is implemented; no arbitrary private DB schema is invented here. + +## Frozen data and interface contracts + +`SourceRecord` carries `{source_id, revision_id, source_sha256, extraction_sha256, extractor_version, text_sha256, text_bytes, content_ref, observed_at, policy_epoch, access_state, exclusion_state, supersession_state}`. In the hosted experiment `content_ref` is a manifest-allowlisted relative fixture file, never a URL to fetch. Source and extraction hashes can differ; offsets always address the exact extracted UTF-8 text bytes named by text_sha256. A source update cannot retain an old revision identity with new bytes. + +`ChunkRecord` carries source identity plus `{chunk_id, start_byte, end_byte, text_sha256, chunk_sha256, token_count, overlap_tokens, tokenizer_identity, splitter_version, embedding_identity}`. Spans are half-open byte intervals; original text slices must round-trip exactly. chunk_id is the canonical hash of these immutable identity fields. Citation includes source/revision/text hash and byte span, not just a title or chunk ordinal. Do not silently normalize whitespace, Unicode, line endings or mathematical notation. + +`GenerationManifest` binds sorted source and chunk inventories, policy epoch, extraction/tokenizer/splitter/embedding/extension identities, embedding dimensions and representation, table count, DB SHA, build statistics and source-admission snapshot. A DB contains one generation. The complete manifest has no model-generated provenance fields. A generation cache key also binds query bytes, query-embedding identity, role, guidance digest, prompt/template version, context/output/byte limits, selection policy and selected chunk manifest. + +`RetrievalRequest` is `{query, exact_source_id, exact_ref, role, task_scope, top_k, expected_generation, admission_epoch, freshness_now}`. Query is bounded to2048 UTF-8 bytes and128 pinned tokens; top_k<=4. Exact ID/ref lookup is first; a valid exact revision request never falls through to a different revision by similarity. `task_scope` is `navigation`, `focused_excerpt`, or `whole_document`; similarity alone cannot satisfy whole-document coverage. + +`RetrievalResult` is `{generation, policy_epoch, selection_mode, chunks, citations, coverage, unavailable_reasons, scientific_effect:"NONE",scientific_status_authority:false}`. Modes are `exact`, `literal`, `semantic`, or `no_admitted_source`. Source loss removes support and invalidates cached results; no stale text is returned merely because its vector exists. Unknown freshness/access yields an unresolved result, not an affirmative admission. + +`PromptAdmission` binds the exact final serialized request bytes and template rendering, selected evidence/citation manifest, tokenizer count, output reserve and profile identity. It either produces a fully admitted request or a typed refusal. It must not truncate bytes after counting or subtract counts of independently tokenized pieces as though tokenization were additive. + +## Task1: Freeze the corpus, adversarial events and measurement protocol + +**Files:** Fixture files and `tests/test_semantic_memory_benchmark.py`; no installed source changes. + +**Interfaces:** Produce one canonical corpus/case manifest with32 documents, <=384KiB total extracted text, <=128 final chunks and no private source. Produce8 development queries and40 held-out queries:8 each for paraphrase, evidence beyond character12000, confusable revisions, exact ID/ref, and no-answer. Preselect20 generation cases,4 per stratum, for each of the two prompt profiles. Gold span/answer records remain outside indexed text and model requests. + +- [ ] Author synthetic documents and frozen expected source/revision/hash/byte spans before observing semantic rankings. Include math/code, Unicode, tables, negations, same titles, similar statements with different constants and superseded versus current versions. Deep cases place all decisive evidence after the original12000-character boundary; also record its UTF-8 byte position. +- [ ] Add test methods `test_frozen_case_inventory_and_hashes`, `test_gold_never_enters_index_or_prompt`, and `test_deep_and_confusable_spans_are_real`. Require40 unique held-out IDs,8 per stratum,20 preselected generation IDs, exact gold slice hashes and explicit absent-answer gold records. No skip or zero discovery can count as success. +- [ ] Publish tests/fixtures first; execute them in the restricted hosted lane and retain actual RED for the absent harness/validator. Implement only that fixture/inventory validator, rerun exact candidate normal and optimized controls, then source-review its data boundary. +- [ ] Freeze all fixture hashes, evaluation seed `20261008`, query order schedule, baseline definitions and gates. Tune any similarity threshold or abstention rule using the8 development queries only; commit its exact value before held-out execution. Exposure is recorded; this is held-out evaluation, not a claim of blinded review. + +## Task2: Pin hosted runtime and prove the tokenizer contract + +**Files:** `runtime-lock.json`, `tokenization.py`, `tests/test_semantic_memory_runtime.py`; root owns future workflow preparation. + +**Interfaces:** `tokenize_text(raw_utf8, tokenizer_identity) -> TokenizedText` with exact IDs and byte pieces; `count_native_prompt(request_bytes, profile, runtime_lock) -> TokenCountReceipt`. ProfileW is4096/768; profileC is16384/1024, additionally retaining the installed custodian's12000-byte message and12288-input-token admission caps. + +- [ ] Prepare only public model/runtime dependencies in a separate trusted hosted stage. Record image digest, CPU/architecture/RAM/disk, Python and SQLite versions/compile options, Ollama binary/version/manifest/layer hashes, the GGUF SHA, all tokenizer type/vocab/merge/added-token/BOS/EOS/pretokenizer fields, template bytes, and a separately pinned tokenizer implementation. Record licenses and exact Linux sqlite-vec wheel/library SHA and ABI. Do not assume the current Darwin version has a compatible Linux build. +- [ ] Verify that the acquired model manifest matches digest500a1f... and the relevant selected model_info/template identities above. A different manifest, tokenizer, quantization or template starts a distinct compatibility candidate, not a silent substitution. A dependency unavailable at the frozen identity ends this candidate as WATCH. +- [ ] Author external tests `test_actual_token_piece_reconstruction`, `test_full_native_prompt_count_parity`, `test_special_tokens_are_data_in_source_text`, and `test_each_entrypoint_budget_is_distinct`; observe actual hosted RED before implementing the adapter. A pinned GGUF/llama.cpp tokenization path is a candidate implementation; it is not assumed identical to Ollama merely because both support Qwen. +- [ ] In the hosted instance only, calibrate20 complete requests spanning plain text, CJK/combining characters, math/code, whitespace, literal special-token strings, citations, guidance and both profiles. Use one generated token, thinking false and uncached context for count-only probes. Compare the local full rendered prompt count exactly to the native processed input count; if cache accounting or rendering is ambiguous, retain the raw response and refuse parity. Never use character estimates or a constant offset learned from one prompt. +- [ ] Require exact byte-piece reconstruction and all20 count comparisons. Full native token IDs are not advertised as independently verified unless the pinned native runtime actually exposes them; processed token-count parity is the required admission evidence. Resolve native Go-template rendering/special-token behavior in source and controls; a different template engine is not sufficient proof. + +## Task3: True approximately500-token chunks, real embeddings and exact vector queries + +**Files:** `chunking.py`, `embedding.py`, `index.py`, chunk/runtime/index tests. + +**Interfaces:** `split_text(source, tokenized, splitter_identity) -> list[ChunkRecord]`; `embed_texts(texts, embedding_identity, truncate=False) -> EmbeddingReceipt`; `rank_exact_cosine(query_vector, stored_vectors) -> ranked_chunk_ids`. + +- [ ] Author controls `test_chunk_spans_round_trip_and_cover_text`, `test_500_token_and_overlap_bounds`, `test_unicode_token_boundaries_do_not_corrupt_spans`, and `test_model_or_splitter_change_invalidates_chunks`; obtain hosted RED before implementation. Target500 independently counted tokens per chunk, nonfinal chunks480–512 and final chunks1–512; target64 overlap tokens, permitted56–72 after UTF-8 boundary adjustment. A short whole document is one final chunk. Preserve gap-free union coverage, strictly advancing starts and exact source bytes; record actual counts, never rename500 characters or whitespace words as tokens. +- [ ] Implement windowing from native-compatible token pieces and valid UTF-8 boundaries, then retokenize each exact slice independently. If a boundary adjustment cannot satisfy the frozen bounds, refuse/retain the case rather than silently using a different splitter. The complete rendered prompt is counted separately later. +- [ ] Probe the actual pinned Qwen `/api/embed` capability on a hosted loopback instance using one and multiple public strings with `truncate:false`. Require returned model identity, one vector per input, common positive dimension, finite values and finite nonzero norm. Record native processed counts/timings and before/after manifest identity. Test over-context refusal with truncate:false, cardinality/dimension drift, NaN/Infinity/zero vectors and unavailable capability. Metadata embedding_length4096 is not the expected vector dimension; the observed dimension becomes a frozen candidate field. +- [ ] If Qwen embedding is unsupported, retain that negative and stop this candidate. A separately selected, licensed embedding model requires its own explicit pinned candidate and repeated cost/quality experiment; no automatic download, substitution or paid fallback occurs. +- [ ] Normalize finite nonzero vectors using a declared rule, persist float32 bytes, and compute the independent exact cosine-distance oracle `1-dot(a,b)/(norm(a)*norm(b))` from those same stored float32 values. Explicitly select and test cosine distance in the frozen extension/query implementation; do not compare default L2 distances to cosine scores. Load the exact Linux extension, record `vec_version()`/actual library identity, execute actual insert/query/delete transactions, close/reopen and verify the same vectors/counts. Package presence does not pass this step. +- [ ] Add `test_sqlite_vec_matches_exact_cosine_after_reopen`, `test_vector_dimension_and_nonfinite_refuse`, and `test_ties_sort_by_chunk_identity`. Require top4 exact set/order with deterministic chunk-ID ties and distance error<=1e-5 on all fixtures, including near-ties. If the extension cannot honor tie ordering, obtain all tied candidates and order them explicitly; an approximate or lost result fails this candidate. + +## Task4: Generation promotion, freshness and loss-only support withdrawal + +**Files:** `index.py`, `admission.py`, index/admission tests. + +**Interfaces:** `build_generation(source_manifest, runtime_lock) -> completed_generation`; `publish_generation(completed_generation, expected_current) -> generation_id`; `admit_source(source_record, live_admission, now) -> decision`; `retrieve(request, admitted_generation) -> RetrievalResult`. + +- [ ] Write and observe hosted RED for `test_failed_rebuild_preserves_previous_generation`, `test_restart_never_observes_mixed_manifest_and_db`, `test_compare_and_swap_generation_conflict`, `test_stale_excluded_access_lost_sources_are_never_selected`, and `test_withdrawal_between_selection_and_prompt_refuses`. +- [ ] Build in a private staged generation directory inside the hosted fixture sandbox. Transactionally finish tables; checkpoint/close any WAL, record DB/hash/inventory and fsync completed files/directories before publication. Use one authoritative generation_head row in a separate small coordinator SQLite state DB; `BEGIN IMMEDIATE`, compare expected_current and commit the new generation/manifest/DB identity together. Do not implement compare-and-swap as an unlocked file read followed by replace. Readers read one committed pointer then hold that immutable generation for a request. Before pointer commit, embedding/insert/manifest/fsync failures preserve the old head; interrupted commit admits only the old or new complete generation. A missing acknowledgment is retained as unknown delivery and resolved by head readback, never by pretending a committed generation rolled back. +- [ ] For the hosted fixture policy, freshness TTL is300 seconds with an injected monotonic clock; clock rollback, missing observation, changed source/extraction/revision bytes, exclusion, supersession or access loss refuse admission. Enforce eligibility before exact/literal/semantic selection and again at prompt admission using an unchanged policy epoch. Vector/lexical SQL uses bound query values and an explicit admitted identity set; neither an unsupported vec0 filter nor filtering only after top4 selection can substitute. Native installed policy will remain authoritative later; this experiment's300-second TTL is not a replacement for it. +- [ ] Apply each event individually and in combinations; require zero withdrawn/stale chunk bytes in returned evidence, cached outputs and prompt requests. Invalidate query/result caches by policy epoch and generation. If all sources withdraw, return no_admitted_source. Do not retain stale support through fallback or treat an empty vector lookup as an invitation to fetch a URL. +- [ ] Rebuild after a revision update; measure full and incremental work separately. Verify the old revision remains retained as history but cannot satisfy a current exact reference. A failed rebuild cannot rehabilitate sources withdrawn from the old generation. + +## Task5: Complete prompt admission and fair held-out quality benchmark + +**Files:** `prompt.py`, `benchmark.py`, prompt/benchmark tests. + +**Interfaces:** `admit_prompt(retrieval, task, guidance_identity, profile) -> PromptAdmission`; `run_benchmark(frozen_cases, runtime_lock, schedule) -> BenchmarkReport`. + +- [ ] Write/observe hosted RED for `test_full_prompt_plus_output_reserve`, `test_whole_document_requires_coverage`, `test_exact_ref_wins_and_literal_fallback_remains`, `test_cache_identity_binds_every_consumed_input`, and `test_source_instructions_remain_inert`. Include a source telling the worker to fetch a URL, expose a secret, claim acceptance or overwrite another file; require zero tools/fallback/action and preserve the text as quoted data only. +- [ ] Count the final serialized request including system/role, verified guidance representation, task/query, source delimiters, citations, JSON escaping, template and special tokens. Require W prompt+768<=4096; C prompt+1024<=16384 and C input<=12288 plus its12000-byte task-message cap. Refuse an oversize request, or drop complete low-ranked chunks and recount with an explicit excerpt/coverage label. Never truncate already-cited spans or increase native limits to make retrieval fit. +- [ ] Define three retrieval arms using the same admitted source set and task: E uses the existing12000-character source excerpt, then the same token admission (oversize is a measured refusal); L uses deterministic literal/SQLite FTS5 retrieval with exact-ref-first behavior; S uses exact-ref-first, literal fallback and semantic top4. E's failure to expose deep evidence is measured, not hidden by granting it a longer excerpt. Freeze development-selected thresholds before held-out ranking; no answer oracle enters model requests. +- [ ] Run all40 held-out retrieval cases for every arm; record rank/recall, exact source/revision/span fidelity and no-answer support separately. Run the preselected20 generation cases for every arm under both profiles:120 generation calls maximum, same model/digest, system/task/output schema, temperature0, thinking false and output reserves. Use identical request/guidance bytes apart from the selected evidence. Randomize paired arm order with the frozen seed, alternate profile order, and retain every raw response/refusal. This generic synthetic answer/citation harness measures retrieval-assisted quality; it does not claim to be an installed worker integration. +- [ ] Validate cited source IDs/revisions/hashes/spans exactly and score answers against frozen facts/abstentions. Unsupported claims, wrong-version facts and confident no-answer outputs fail safety. Report failures/refusals in denominators; no successful-only average. Provide per-stratum results and paired differences, not only one aggregate number. +- [ ] Report exact retrieval bytes/tokens, complete prompt tokens, reserved/actual output, cited-support precision, answer score, recall@4, abstention precision, unauthorized request/tool attempts, all call counts and chargeable resource/cost units. Verify caches change on query/chunk/source/generation/model/tokenizer/guidance/profile/role/policy changes. + +## Task6: Cold/warm/build costs, peer fairness and bounded hosted execution + +**Files:** benchmark tests/CLI; root-owned future workflow and retained reports. + +**Interfaces:** Produce original outputs and a canonical report binding the exact code/test/fixture/runtime identities and run/attempt/native checked commit. The report states observed capability and disposition per phase; an absent/failed phase cannot be called PASS. + +- [ ] Prepare on a runner whose observed resources can admit the frozen profile: at least14GiB physical RAM and32GiB free disk before dependency preparation. Record the actual runner, CPU count/flags, architecture, kernel and SQLite/runtime identities. Bound experimental compute to4 CPUs,12GiB RAM,256 PIDs and1GiB temporary storage. Model/dependency cache is a separately bounded10GiB read-only prepared mount; generation output is bounded512MiB and published evidence64MiB. Record both process RSS and actual cgroup peak usage; shared model mappings do not justify hiding a memory-limit failure. Run server and clients in the same isolated network namespace with loopback only, no external interface/route, nonroot user, dropped capabilities/no-new-privileges, read-only prepared source/models and bounded disposable outputs. Trusted dependency preparation is the separate network-enabled stage. No installed credentials, private mounts or persistent local worker access enter the container. +- [ ] Use three separately bounded hosted phases with timeout caps60/80/60 minutes for preparation+capability, build+retrieval, and generation+measurement. Total cap200 minutes; maximum640 input vectors/400 embedding requests and270 generation requests:20 one-token parity,120 quality,120 paired fairness/baseline,6 cold-state calibration, with4 requests reserved for explicit negative controls. Corpus plus one full rebuild consumes at most256 vectors; held-out/warm/quality/fairness queries consume at most260, leaving124 for capability/development/fault controls. Every embedding input/request and generated attempt counts, including failures. No automatic retries after timeout/unknown delivery. Sequential admission stops at the cap and retains WATCH/incomplete evidence rather than silently narrowing the frozen suite. If actual preparation needs more capacity, return measured WATCH and author a distinct revised resource candidate. +- [ ] Measure model-cold, process/index-cold with model warm, and fully warm conditions separately. Model-cold requires an actually unloaded model and no reused prompt/KV cache on the disposable host; index-cold uses a fresh process/generation open and declares whether OS page-cache coldness was achieved. Do not interrupt installed workers or claim OS-cold from a process restart alone. +- [ ] For warm measurements, run three frozen paired schedules of the40 retrieval queries. Query/result caches are disabled for the primary comparison; models/index pages can stay warm. Charge actual query embedding time to S and report separately any additional cache-enabled workload. Baselines receive the same warm state and source-admission checks. Retrieval-only replay may reuse validated corpus vectors, never claim it measures re-embedding cost. +- [ ] Record p50/p95/max wall latency, model load/prompt-eval/generation time, CPU-seconds, peak RSS, model and SQLite residency, DB/model/text/evidence disk bytes, build/rebuild time, embedding tokens/vectors/requests, failure/unknown calls, and runner billing units. Compute break-even query count from measured construction/rebuild cost and per-query gain; if S is slower, no positive break-even is invented. +- [ ] Run a frozen two-role contention schedule on the disposable instance:20 alternating W/C requests plus20 paired arrivals of one W and one C request,60 requests total, concurrency at most2. Use preselected short cases admitted by both profiles and both arms; repeats are deliberate contention measurements, not additional held-out quality cases. Repeat the identical60-request schedule for the non-retrieval baseline with equivalent source/task/output work. Report per-role queue wait, completion latency, refused work and resource peaks; relative to that baseline require no p95 increase>10% and no starvation/timeouts. This is fairness evidence for a candidate scheduler, not proof about live workers. +- [ ] Retain normal/optimized test inventories with exact method IDs, actual native calls, raw fixture outputs, runtime lock, DB/generation hashes and the report in allowlisted regular-file artifacts. Confirm process/container disposal before staging, record manifest hashes, download the original archive and independently verify raw bytes/native identities. Keep all negative attempts. No missing/skipped discovery or count-only acceptance. + +## Measured disposition gates + +The numerical criteria are frozen experiment gates, not universal claims about retrieval or science. Do not change them after seeing held-out results. + +| Disposition | Required evidence and permitted consequence | +| --- | --- | +| REJECT candidate | Any private/unauthorized disclosure or tool/fallback action; any stale/excluded/access-lost support; corrupted/mixed generation; false exact revision/citation; unsupported no-answer generation; nonfinite/dimension mismatch silently admitted; or tokenizer/native prompt parity failure. Preserve artifact and cause; no installation. | +| WATCH | Embedding capability, Linux runtime/extension load, full token parity, complete held-out suite, resource capacity, freshness export ABI or exact-final reviews remain missing/ambiguous; candidate stays uninstalled. Budget exhaustion is incomplete evidence, not a narrowed PASS. | +| ADAPT, opt-in only | All safety/custody/identity controls pass; exact ID/ref8/8; citations100% byte/source/revision valid; generated no-answer subset4/4 abstains under both profiles; other quality gates below are noninferior, but improvement, amortization or fairness is inadequate. Retain literal/default excerpt paths; installation still needs actual technical reviews and native integration evidence. | +| ADOPT narrowly | All prior safety controls; answerable retrieval recall>=29/32; hard paraphrase+deep recall>=14/16 and at least2 more successes than the better E/L arm; no confusable-version or exact-ref regression; generation score>=18/20 under each profile and no loss against the better baseline; zero unsupported answers and complete valid citations; warm complete-task S p95<=1.25x L including query embedding/admission/generation, break-even<=500 queries when a latency gain exists, peak experimental RSS<=12GiB, and fairness gate passes. Raw retrieval latency is also reported without hiding embedding cost. If quality improves but no cost amortization exists, classify ADAPT rather than invent savings. Install only for the measured task/source/profile scope after Task7. | + +For whole-document tasks, the sidecar can provide navigation or explicitly labeled excerpts. ADOPT does not license a partial top4 selection to stand in for complete document review. A default-on decision requires a separately frozen representative workload and ongoing freshness/fallback evidence beyond this small synthetic experiment. + +## Task7: Later actual two-entrypoint integration and installation gate + +**Files:** Coordinator/custodian source repositories and their external tests; this main plan does not edit installed source/config/state. + +**Interfaces:** W keeps `native_infer(role, text, guidance=None)` and its original proposal contract; C keeps `summarize(evidence, runtime, model)` and its original projected-evidence/source-ID contract. A deterministic adapter produces admitted source excerpts/citations before either model call. The model never queries SQLite, selects a path, runs tools or controls source eligibility. + +- [ ] Obtain the owning coordinator's current source/export/admission contract and exact deployment candidate. Recheck source identities only within the active handoff, without reading unrelated DB/config/credentials/seals. Confirm the storage admission issue has been resolved through an independent later cycle with the original20GiB reserve intact; do not retry this plan's recorded failure or label the09:51 index fresh. +- [ ] Author external integration controls before changing either native source: both actual source/extraction revisions and freshness/export identities must bind the retrieval generation; W role/guidance/cache/proposal/4096/768 behavior remains enforced; C privacy projection, source-ID validation,12000-byte/12288-input/16384/1024 limits, model checks, zero-tool accounting and no-fallback route remain enforced. Test concurrent profile use against C's actual post-run requirement of one loaded model with context16384; context switching or multiple runtime instances must not be masked by relaxing that check. Unsupported whole-document, access loss and source instructions must refuse or retain explicit excerpt limits. +- [ ] Observe genuine isolated hosted RED against exact source copies, implement the smallest adapters in the owning source trees, then run exact-candidate normal/optimized and controlled public-source calls through both actual entrypoints. An experimental harness alone is not this evidence. Preserve original request/runtime/response/source/generation receipts and compare legacy literal/excerpt behavior; neither entrypoint may silently gain a hosted route or larger budget. +- [ ] Obtain two distinct actual nonlocal evidence reviews of the exact final code/tests/runtime/fixture/benchmark/integration bytes, including at least one nonauthor reviewer. Retain raw completed reports, provider/model/run identities, authorship/exposure and exact verdicts. Duplicated models/runs, missing traces or source-only intentions do not satisfy the rule; same-provider technical reviews receive no organizational-independence claim. If operational guidance/rules change, satisfy their separate tested rule quorum before deployment; do not edit seals to bypass it. +- [ ] After the installation gate is satisfied, install a reversible, explicitly scoped opt-in sidecar via the coordinator's native deployment mechanism. Verify actual compatible extension loading on Darwin separately, not from the Linux benchmark; run selected public sources first, then only separately admitted selected acquired sources under existing private policy. Preserve no-tools/zero-hosted-fallback and peer fairness. A private live source test remains private; only policy-permitted derived evidence is published. +- [ ] Read back the actual installed source/runtime/generation/request receipts for both entrypoints, test live freshness/access-loss withdrawal and rollback, and append measured scope/disposition to existing Work Events with original evidence hashes. Only this evidence can close the installed semantic-retrieval requirement. Otherwise retain ADAPT/WATCH and the uninstalled/residual scope explicitly. + +## Outstanding determinations and stop conditions + +1. The exact Linux image/Ollama/tokenizer/sqlite-vec identities, model acquisition license and actual usable hosted capacity are unresolved until Task2 preparation. Do not fill them from the installed Darwin package or a mutable upstream default. +2. Qwen `/api/embed` capability, vector dimensions/normalization, prompt-token parity and persistence/query behavior remain UNKNOWN/NOT_RUN. A rejected capability needs a distinct candidate; it cannot be patched over by interpreting4096 metadata as vectors. +3. The native selected-source/extraction/freshness/export ABI and current installed storage admission are separate coordinator-owned evidence. Public fixtures cannot establish private-corpus coverage, availability or current deployment readiness. +4. The original native W/C message limits and validation schemas differ; actual integration must preserve both. If current native templates/guidance add unaccounted tokens, admission stops until exact parity and full budgets are tested. +5. Actual two-distinct-nonlocal exact-final review and any separate rule quorum are future technical gates. Standing owner authorization removes no substantive verification requirement, and this plan introduces no renewed owner-permission queue. + +## Primary mechanism references and limits + +- [Ollama embedding API](https://docs.ollama.com/api/embed) documents batched inputs and truncate:false refusal. This supports the probe shape, not the availability of embeddings for the installed Qwen model. +- [Ollama generation API](https://docs.ollama.com/api/generate) documents template/raw behavior and native token/timing accounting. Actual pinned-runtime response semantics must be checked rather than assumed from current documentation. +- [llama.cpp server tokenizer API](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#post-tokenize-tokenize-a-given-text) documents token IDs/byte pieces and special-token flags. The live master page is a design reference; preparation pins exact executable source/model bytes and validates parity with native Ollama. +- [sqlite-vec API](https://alexgarcia.xyz/sqlite-vec/api-reference.html) and [vec0 tables](https://alexgarcia.xyz/sqlite-vec/features/vec0.html) describe vector-query primitives. Current documentation may describe a different release; actual frozen0.1.x Linux compatibility, extension loading and exact query/persistence controls are required. +- [SQLite atomic commit](https://www.sqlite.org/atomiccommit.html) describes transaction durability assumptions. The generation pointer/manifest protocol additionally needs tested filesystem fsync/atomic replacement behavior; a SQLite transaction alone does not atomically publish separate files. + +This plan is a mechanism-to-compatibility-to-controlled-test-to-benchmark-to-disposition sequence. Its present status is PLAN_ONLY, experiment NOT_RUN and installed integration NOT_DEPLOYED. All eight parent architecture goals remain active; no measurement or installation is asserted by this document. diff --git a/docs/plans/2026-10-08-semantic-runtime-preparation.md b/docs/plans/2026-10-08-semantic-runtime-preparation.md new file mode 100644 index 000000000..1dd0bf252 --- /dev/null +++ b/docs/plans/2026-10-08-semantic-runtime-preparation.md @@ -0,0 +1,75 @@ +# Semantic Runtime Preparation: Tiny Public Compatibility + +Status: SOURCE_PREPARED. Publication, workflow parsing, dependency acquisition, Docker execution, model loading, native embeddings, extension loading and original hosted artifact readback are NOT_RUN for this unit. Source inspection and primary metadata verification do not establish runtime compatibility. + +Root owns these two paths under claim 6070431864 and preparation amendment 6071136915. This unit implements only the public hosted preparation boundary for the accepted [semantic memory experiment](2026-10-08-semantic-memory-experiment.md), whose frozen source SHA is `a9b90c1cc0b8df014d85c93d79e1357e1953a159bdc373affd034bd438fd7446`. That experiment remains the authority for tokenizer parity, chunks of approximately 500 tokens, source admission, complete generations, retrieval/generation quality, cold/build costs, fairness and any later integration. No installed worker entrypoint, source index, guidance, state, configuration, credential, seal or local model is read or modified by this unit. + +The corresponding workflow is [.github/workflows/semantic-runtime-preparation.yml](../../.github/workflows/semantic-runtime-preparation.yml). It can run from a pull request touching these two paths or an explicit workflow dispatch. Root controls publication, review, execution and the integration queue. A fresh distinct nonauthor source review of the final source is required before publication; existing owner authorization already covers the work and does not substitute for that review or actual tests. + +## Frozen identities and observations + +| Input | Frozen candidate identity | Preparation disposition | +| --- | --- | --- | +| Ollama server image | `ollama/ollama:0.35.1@sha256:17e945f1aa6e505707eeb621e3f36847d0a4a063d4778f7007645acccfb8bd4f`, Linux amd64 | Root verified the raw OCI manifest hash using primary public registry data. No image pull or execution occurred during source preparation. The four compressed layers total approximately3.78GB; the large layer is3638058746 bytes. | +| Python client image | `python:3.11.14-slim-bookworm@sha256:83f339c1be6340ae1096010fdccf6552ac932d8f410d45d206014916bdf37e48`, Linux amd64 | Previously exercised by the project's restricted hosted Python controls; this workflow must read back its own actual image identity. | +| Qwen manifest | `500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41` | Root's distinct canonical tag GET returned original bytes with this SHA, equal to the previously observed installed identity. A digest-path GET returned404; that failure remains an unsupported retrieval path, with no retry or alias substitution. | +| Qwen model layer | 5225374496 bytes; SHA `a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f` | NOT_DOWNLOADED during source preparation. Hosted preparation verifies the complete layer before and after the compatibility attempt. Full weight bytes are NOT_ARCHIVED in the evidence artifact. | +| Config layer | 487 bytes; SHA `05a61d37b08453e59290add468e3bb2f688e23a01e967fecb0e2fa41218cea76` | Hosted acquisition and archival of original bytes required. | +| Template layer | 1723 bytes; SHA `ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4` | Root downloaded and verified the original public bytes separately. This workflow must acquire and verify its own copy and compare the served template bytes. | +| License layer | 11338 bytes; SHA `d18a5cc71b84bc4af394a31116bd3932b42241de70c77d2b76d69a314ec8aa12` | Root verified the original public Apache2.0 text separately. Legal approval is NOT_ASSESSED. | +| Params layer | 120 bytes; SHA `cff3f395ef3756ab63e58b0ad1b32bb6f802905cae1472e6a12034e4246fbbdb` | Hosted acquisition and archival of original bytes required. | +| Linux sqlite-vec wheel | Version0.1.9;163388 bytes; SHA `1515727990b49e79bcaf75fdee2ffc7d461f8b66905013231251f1c8938e7786` | Root verified primary PyPI metadata and the downloaded original wheel. Only six fixed members occur in the pinned wheel. Its metadata declares MIT plus Apache2.0; no LICENSE member occurs; legal approval is NOT_ASSESSED. No package installation or import of its Python wrapper is necessary. | +| Linux extension member | `sqlite_vec/vec0.so`; SHA `5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8` | Root read and hashed the member as data. Loading and SQL execution remain NOT_RUN until the hosted compatibility attempt. The Darwin extension is not reused. | + +The runtime source keeps the exact fixed registry and PyPI URLs, sizes and digests. The initial mutable Qwen tag is a delivery path: its raw SHA must match the frozen manifest before any layer is acquired. The manifest's exact config/layer identities must also match the five independently fixed pins. No generic `ollama pull`, latest dependency resolution, alternate model, paid endpoint or retry option is admitted. Curl may follow at most three registry-origin HTTPS delivery redirects; each final input remains subject to the frozen byte length and SHA. This is an input allowlist and content-integrity boundary, not a claim of a network firewall restricting the trusted provisioning host's egress. Runtime image component license inventories remain NOT_ASSESSED and are explicitly retained as pending; this preparation cannot establish legal approval for installation. + +Primary references are [the official Ollama v0.35.1 Dockerfile](https://github.com/ollama/ollama/blob/v0.35.1/Dockerfile), [its environment configuration](https://github.com/ollama/ollama/blob/v0.35.1/envconfig/config.go), [the official Ollama API schema](https://github.com/ollama/ollama/blob/v0.35.1/docs/openapi.yaml), [sqlite-vec's Python loading guide](https://alexgarcia.xyz/sqlite-vec/python.html), [its vec0 query documentation](https://alexgarcia.xyz/sqlite-vec/features/vec0.html), and [the exact PyPI0.1.9 release metadata](https://pypi.org/pypi/sqlite-vec/0.1.9/json). API documentation describes what to test; actual endpoint responses determine whether this pinned model implements it. + +## Boundary and resource profile + +Preparation begins with actual `/proc/meminfo` RAM and actual free bytes on the disposable hosted runner. Require RAM>=14GiB and free disk>=32GiB before model acquisition, image pulls or filesystem allocation. Retain the observed capacity record even when this admission refuses the attempt. The installed custodian's reported storage rejection is not retried and its20GiB reserve is not lowered; no local model probe or index allocation is part of preparation. + +Four separate ext4 loopback filesystems bound model storage10GiB, scratch/output512MiB, prepared control/extension64MiB, and published evidence64MiB. Each mounted root keeps its original root-owned metadata and `lost+found`; a fresh `payload` subdirectory alone is created, assigned to the appropriate runner/sandbox identity and used for staging, container binds, recursive permission changes, hashes, collection and upload. Strict publication checks still refuse directories, symlinks and nonregular members; filesystem metadata is outside the publication payload. Cleanup unmounts the original four roots. Models, work and evidence use nodev/nosuid/noexec. The control filesystem permits executable mappings so Linux can actually load the pinned shared library; its payload uses nodev/nosuid and is mounted read-only inside the client. Neither model weights nor sandbox output are written into the checked repository. The trusted host provisions only fixed public data and fixed digest images. No dependency or model code executes during the networked acquisition stage. + +The offline server has UID/GID65532, read-only root, read-only prepared model cache, Docker networknone, no published host port, capabilities dropped, no-new-privileges,3.5CPU,12GiB RAM and192 PIDs. Its temporary filesystem is1GiB with noexec/nosuid/nodev. Explicit Ollama settings bind127.0.0.1:11434, disable cloud and pruning, use one loaded model, one parallel request, queue1, context4096,300-second load-stall bound, and disabled Vulkan/request-body diagnostics. No GPU/device is admitted. Source intent is not actual proof of these restrictions: the receipt checks native Docker observations. + +The client has UID/GID65532, read-only root, read-only prepared control/extension,0.5CPU,512MiB RAM and64 PIDs. It joins only the server's isolated network namespace, receives no proxy environment, has no published port and executes isolated Python with `-I -B -S`. It receives only the fixed compatibility control and its bounded disposable work mount. It receives no checkout, Git credentials, API account keys, private capture or installed path. Its temporary filesystem is64MiB with noexec/nosuid/nodev. The pair's CPU quota totals4 and PID bounds total256; client RAM and temporary space are explicitly additional to the12GiB/1GiB server limits. + +The [official v0.35.1 server entrypoint](https://github.com/ollama/ollama/blob/v0.35.1/cmd/cmd.go#L1968) initializes a local key before serving. This unit never assigns or repurposes HOME. It admits only the fixed image's native `/root` or `/` home case and supplies separate1MiB disposable tmpfs locations at `/root` and `/.ollama`. Any unexpected native home, key initialization failure, read-only model-cache incompatibility or server startup failure ends the attempt. Generated private-key files and all ephemeral home files stay outside collection and are NOT_ARCHIVED. Upstream prints the generated public key in its startup console; that original console is preserved and may contain the public key. The disposable key files are not installed user/provider credentials, and cloud access stays disabled. + +The job has a50-minute wall limit. Fixed small downloads have60-second bounds; the large model layer has900 seconds; each digest image pull has600 seconds. Server attachment has900 seconds and client attachment800 seconds, with10-second termination escalation. Each of the four pull/attachment logs has a2MiB file limit, leaving publication space for the48MiB payload ceiling and trusted metadata; Docker uses log-drivernone. Attachment failure, including a log limit termination, refuses a positive receipt. No download, embedding or failed model load is retried. Bounded startup observations wait for this already-created sandbox; they do not repeat an embedding attempt or change the model. + +## Actual compatibility controls required + +1. Verify Python3.11.14/Linux/x86_64 and the exact loaded extension SHA. Enable SQLite extension loading only around the one absolute pinned extension path and disable it immediately afterward. Require actual `vec_version()`0.1.9, retain SQLite version and compile options, create a3-dimensional cosine vec0 table with vectors `[1,0,0]` and `[0,1,0]`, close/reopen the DB, and require row IDs1/2 with distances0/1 within1e-6. The fixed v0.1.9 KNN query accepts only `ORDER BY distance`: fetch the complete `k=2` result and apply `(distance,rowid)` tie ordering in Python, after SQL execution. No string-only version report counts as an actual loaded extension or working query. [Pinned upstream planner](https://github.com/asg017/sqlite-vec/blob/v0.1.9/sqlite-vec.c#L5205) +2. Admit only loopback API paths `/api/version`, `/api/tags`, `/api/show`, `/api/embed`, and `/api/ps`; refuse redirects and ignore proxies. Require actual server version0.35.1, exactly one served model named `qwen3:8b`, and native tag digest equal to the fixed manifest. Obtain original verbose show responses before and after the attempt, verify the exact template bytes, and require unchanged complete show records. The model's `qwen3.embedding_length`4096 is checked as identity metadata and does not substitute for real embedding outputs. +3. Make exactly one embedding request with the two public strings `TEST preparation: water boils.` and `TEST preparation: a square has four sides.`, `truncate:false`, context4096, CPU execution and no endpoint/model fallback. Require two returned vectors, each exactly4096 finite numeric components, nonzero norm and unit norm within1e-3; require actual total/load/prompt statistics and positive prompt_eval_count. Store original request, bounded response and HTTP status bytes before semantic response validation. An unsupported or failed endpoint/model response remains a genuine failed attempt. +4. Persist the two actual vectors as little-endian float32 into a distinct cosine vec0 table, close/reopen the DB, fetch the complete `k=2` result with the same single `ORDER BY distance`, sort in Python by `(distance,rowid)`, and compare IDs and distances against a stdlib cosine oracle over the exact persisted float32 values within1e-5. This checks two-vector interoperability only. These two public sentences and a query equal to the first vector cannot establish retrieval relevance, chunk quality, held-out performance or model selection. +5. Record native server/client container IDs, image inspections, configured restrictions and client terminal exit state. Stop the server after the attempt; require successful actual client execution and clean server shutdown for a positive receipt. Dispose both sandboxes and confirm their absence before any untrusted output collection. Preserve unsupported responses, partial extension success, original error logs and nonzero exits rather than rewriting them into success. + +## Custody, collection and interpretation + +The collector runs only after both sandboxes are absent. It uses isolated trusted Python, anchored O_DIRECTORY/O_NOFOLLOW parent traversal, O_NOFOLLOW/O_NONBLOCK file opens, regular-file and single-link checks, exact fixed member names, per-file limits and a48MiB payload ceiling. Output files are freshly created O_EXCL and owned by the runner. Untrusted sandbox paths cannot select source or destination names. Verbose show responses have16MiB limits; other API records and the tiny DB have4MiB limits. The64MiB publication filesystem bounds the entire artifact, including trusted logs and manifests. Interrupted setup retains the original capacity/native records and whichever bounded originals actually exist. + +Retain the original manifest, config/template/license/params layers, Linux wheel, extracted extension and metadata, exact compatibility control, checked workflow/plan bytes, API originals, the tiny disposable DB, Docker/capacity observations, exits and collection missing/refused inventory. Stream all six prepared model files again after disposal and require their complete length/SHA records to match the before-attempt records. The5GB weight layer and image layers remain explicitly NOT_ARCHIVED; the artifact does not imply full raw-byte custody of those runtime dependencies. A partial preparation never produces a success receipt merely because a few small files were retained. + +The preparation receipt binds repository, checked commit, run ID and attempt to original member hashes. It verifies the exact Linux amd64 image repo digests and additionally joins every initial/final container `Image` to the corresponding inspected image `Id`, requiring its exact pinned `Config.Image` reference. Both native runtime and configured bind inventories must equal the admitted source/destination/read-write tuples, with no extra bind or inherited volume; each complete tmpfs map must equal the admitted options, including its size limits. The admitted sources are only the prepared model/control/work `payload` directories. Existing resource restrictions, offline namespace, ID and successful terminal checks remain required. It also verifies the original model manifest, unchanged cache identities, loaded extension and two actual embedding vectors. It carries custodyunknown, scientific effectNONE, scientific-status authorityfalse and disposition `REQUIRES_NATIVE_JOB_SUCCESS_AND_ORIGINAL_ARCHIVE_READBACK`. A hash manifest and90-day original artifact are uploaded from the publication `payload` even after failed attempts. Mounted filesystem roots are released strictly after upload; an unmount failure keeps the native job failed. A receipt or upload cannot override native job failure. + +For positive delivery, root must independently read the native job conclusion and its actual checked commit/tree/parents, fetch the original hosted artifact, compare its raw ZIP hash and every retained member, read back the native repository/run/attempt identities, and preserve exact errors or acquisition gaps. Local source checks, reconstructed ZIPs, a rerun from another revision, an artifact ID alone or a Docker image name cannot substitute for that evidence. The source author is not the independent source reviewer or custodian. + +## Explicit remaining scope + +Even a successful native compatibility attempt leaves full prompt-tokenizer parity, the selected native model_info parity hash, approximately500-token chunk construction, corpus/revision identity, admission and withdrawal races, complete atomic generations, exact lookup/literal fallback, held-out retrieval/generation acceptance, cold/load/build cost, p95/fairness and representative-workload evidence NOT_RUN. It makes no change to Housekeeper/UIUX defaults and no scientific promotion. The accepted experiment requires later external tests and genuine hosted RED before product implementation, actual measured gates and two distinct actual nonlocal reviews of the exact final installed candidate, including a nonauthor. This preparation unit cannot satisfy that installation quorum or bypass the coordinator's native contracts. + +## Review correction record + +The prior source freeze, workflow SHA `c4ff430594479f090f5e1c4bcae336849b6988eb2f0f66b382fc6995d81580a4` and plan SHA `6f7bc6ca3cadf8492774189caa0acd56d4e2e8215ab8a61d78d3a3b58022a15f`, received REQUEST_CHANGES from the distinct nonauthor operations-contract reviewer. The four findings were source findings; they were not an executed RED run or evidence of successful runtime compatibility. Root read and accepted that review. This source revision records the corrections without discarding the rejection: + +| Finding | Correction now in source | Verification still required | +| --- | --- | --- | +| P1: ext4 mount roots expose root-owned `lost+found` to recursive permission changes and strict publication inventory. | Use fresh payload subdirectories under all four mounted roots; leave root metadata untouched; bind, hash, collect and upload only payloads; unmount original roots. | Actual hosted setup, permissions, collection and strict unmount. | +| P1: sqlite-vec0.1.9 rejects KNN queries with two ORDER BY terms. | Both queries use only `ORDER BY distance`; sort the complete two-row result in Python by `(distance,rowid)` before comparison. | Actual extension load, both queries and DB reopen. | +| P2: a valid image-inspection record did not prove that the tested containers used it; mount checks admitted unspecified extras. | Join initial/final native container image IDs to inspected image IDs, require exact pinned configured references, exact source/destination/permission bind sets and complete tmpfs maps including limits. | Actual native Docker records and receipt enforcement. | +| P2: saying every generated home key was unarchived conflicted with original startup output containing the public key. | Keep original console unchanged; explicitly exclude private-key/home files while allowing the printed public key to appear in that console. | Original bounded console and closed artifact membership readback. | + +Current disposition remains SOURCE_PREPARED / NOT_RUN and requires a fresh distinct nonauthor review of this revised freeze before publication. All source-review identities here have organizational independence0; distinct agent/nonauthor identity is not independent organizational evidence. YAML parsing, execution, acquisition, model/extension capability and hosted custody remain NOT_RUN. Unsupported capability or insufficient capacity yields retained negative evidence or WATCH, not silent substitution.