diff --git a/.github/workflows/semantic-runtime-preparation.yml b/.github/workflows/semantic-runtime-preparation.yml new file mode 100644 index 000000000..00224d23a --- /dev/null +++ b/.github/workflows/semantic-runtime-preparation.yml @@ -0,0 +1,768 @@ +name: Semantic runtime preparation +'on': + pull_request: + paths: + - .github/workflows/semantic-runtime-preparation.yml + - docs/plans/2026-10-08-semantic-runtime-preparation.md + - docs/plans/2026-10-08-semantic-memory-experiment.md + workflow_dispatch: +permissions: + contents: read +concurrency: + group: semantic-runtime-preparation-${{ github.run_id }}-${{ github.run_attempt }} + cancel-in-progress: false +jobs: + preparation: + name: semantic-runtime-preparation + runs-on: ubuntu-24.04 + timeout-minutes: 50 + env: + OLLAMA_IMAGE: ollama/ollama:0.35.1@sha256:17e945f1aa6e505707eeb621e3f36847d0a4a063d4778f7007645acccfb8bd4f + PYTHON_IMAGE: python:3.11.14-slim-bookworm@sha256:83f339c1be6340ae1096010fdccf6552ac932d8f410d45d206014916bdf37e48 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Admit actual capacity and create bounded disposable filesystems + run: | + set -euo pipefail + publish="$RUNNER_TEMP/semantic-publish/payload" + mkdir -p "$publish" + chmod 0700 "$publish" + git rev-parse HEAD > "$publish/checked-commit.txt" + printf '%s\n' "$GITHUB_REPOSITORY" > "$publish/repository.txt" + printf '%s\n' "$GITHUB_RUN_ID" > "$publish/run-id.txt" + printf '%s\n' "$GITHUB_RUN_ATTEMPT" > "$publish/run-attempt.txt" + ruby -rpsych -e 'Psych.safe_load(File.read(ARGV.fetch(0)), aliases: false)' .github/workflows/semantic-runtime-preparation.yml + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + python3 -I -B -S - <<'PY' + import json, os, pathlib, shutil + memory = dict(line.split(':', 1) for line in pathlib.Path('/proc/meminfo').read_text().splitlines()) + ram = int(memory['MemTotal'].split()[0]) * 1024 + free = shutil.disk_usage(os.environ['RUNNER_TEMP']).free + observed = {'schema_version': 1, 'ram_bytes': ram, 'free_disk_bytes': free, + 'minimum_ram_bytes': 14 * 2**30, 'minimum_free_disk_bytes': 32 * 2**30, + 'scientific_effect': 'NONE', 'scientific_status_authority': False} + path = pathlib.Path(os.environ['RUNNER_TEMP']) / 'semantic-publish/payload/capacity.json' + path.write_text(json.dumps(observed, sort_keys=True, indent=2) + '\n') + if ram < observed['minimum_ram_bytes'] or free < observed['minimum_free_disk_bytes']: + raise ValueError('actual hosted capacity is below preparation admission') + PY + # Evidence is bounded too; preserve the capacity/native records across its mount. + mv "$publish" "$RUNNER_TEMP/semantic-initial-evidence" + for specification in models:10240 work:512 control:64 publish:64; do + label="${specification%:*}" + size="${specification#*:}" + directory="$RUNNER_TEMP/semantic-$label" + mkdir -p "$directory" + truncate -s "${size}M" "$RUNNER_TEMP/semantic-$label.img" + mkfs.ext4 -F "$RUNNER_TEMP/semantic-$label.img" > "$RUNNER_TEMP/semantic-initial-evidence/filesystem-$label.log" 2>&1 + options=loop,nodev,nosuid,noexec + if [ "$label" = control ]; then options=loop,nodev,nosuid; fi + sudo mount -o "$options" "$RUNNER_TEMP/semantic-$label.img" "$directory" + # Keep ext4 root metadata and lost+found untouched; only this new payload belongs to the runner. + sudo mkdir "$directory/payload" + sudo chmod 0755 "$directory/payload" + sudo chown "$(id -u):$(id -g)" "$directory/payload" + done + cp "$RUNNER_TEMP/semantic-initial-evidence/"* "$publish/" + chmod 0700 "$publish" "$RUNNER_TEMP/semantic-work/payload" + sudo chown 65532:65532 "$RUNNER_TEMP/semantic-work/payload" + mkdir -p "$RUNNER_TEMP/semantic-models/payload/blobs" "$RUNNER_TEMP/semantic-models/payload/manifests/registry.ollama.ai/library/qwen3-embedding" "$RUNNER_TEMP/semantic-control/payload/downloads" + cp .github/workflows/semantic-runtime-preparation.yml "$publish/checked-workflow.yml" + cp docs/plans/2026-10-08-semantic-runtime-preparation.md "$publish/checked-plan.md" + cp docs/plans/2026-10-08-semantic-memory-experiment.md "$publish/checked-experiment.md" + - name: Acquire only fixed public inputs and verify every byte pin + run: | + set -euo pipefail + publish="$RUNNER_TEMP/semantic-publish/payload" + trap 'status=$?; printf "%s\n" "$status" > "$publish/provision-exit-status.txt"' EXIT + models="$RUNNER_TEMP/semantic-models/payload" + control="$RUNNER_TEMP/semantic-control/payload" + fetch() { + # Fixed public-input HTTPS delivery redirects are permitted; bytes are never trusted by URL alone. + curl --fail --silent --show-error --location --max-redirs 3 --proto '=https' --proto-redir '=https' \ + --tlsv1.2 --connect-timeout 10 --max-time "$4" --max-filesize "$3" "$1" -o "$2" + } + fetch https://registry.ollama.ai/v2/library/qwen3-embedding/manifests/8b "$models/manifests/registry.ollama.ai/library/qwen3-embedding/8b" 531 60 + test "$(sha256sum "$models/manifests/registry.ollama.ai/library/qwen3-embedding/8b" | cut -d ' ' -f 1)" = 64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b + for pin in \ + 0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89:265 \ + 3fcd3febec8b3fd64435204db75bf0dd73b91e8d0661e0331acfe7e7c3120b85:4676804928; do + digest="${pin%:*}" + size="${pin#*:}" + duration=60 + if [ "$size" = 4676804928 ]; then duration=900; fi + fetch "https://registry.ollama.ai/v2/library/qwen3-embedding/blobs/sha256:$digest" "$models/blobs/sha256-$digest" "$size" "$duration" + done + # This immutable upstream license is separate public source metadata, not an encoder registry layer. + fetch https://huggingface.co/Qwen/Qwen3-Embedding-8B/raw/1d8ad4ca9b3dd8059ad90a75d4983776a23d44af/LICENSE "$control/downloads/upstream-encoder-LICENSE.txt" 11343 60 + fetch https://files.pythonhosted.org/packages/6f/ad/6afd073b0f817b3e03f9e37ad626ae341805891f23c74b5292818f49ac63/sqlite_vec-0.1.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64.whl "$control/downloads/sqlite-vec-0.1.9-linux-x64.whl" 163388 60 + python3 -I -B -S - <<'PY' + import email.parser, hashlib, json, os, pathlib, stat, zipfile + base = pathlib.Path(os.environ['RUNNER_TEMP']) + models, control, publish = (base / ('semantic-' + name) / 'payload' for name in ('models', 'control', 'publish')) + def require(condition, message): + if not condition: + raise ValueError(message) + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate primary manifest key') + result[key] = value + return result + def identity(path, expected_size, expected_hash): + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= expected_size, 'unsafe public input') + digest, length = hashlib.sha256(), 0 + with path.open('rb') as stream: + while part := stream.read(1048576): + length += len(part) + require(length <= expected_size, 'public input grew') + digest.update(part) + row = {'path': path.relative_to(models).as_posix(), 'bytes': length, 'sha256': digest.hexdigest(), + 'expected_bytes': expected_size, 'expected_sha256': expected_hash, + 'archive_policy': 'ARCHIVE_ORIGINAL_SMALL_INPUT' if expected_size < 1048576 else 'NOT_ARCHIVED_FULL_WEIGHTS'} + require(length == expected_size and row['sha256'] == expected_hash, 'fixed public input pin mismatch') + return row + manifest_path = models / 'manifests/registry.ollama.ai/library/qwen3-embedding/8b' + raw = manifest_path.read_bytes() + require(len(raw) == 531 and hashlib.sha256(raw).hexdigest() == '64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b', 'encoder manifest pin mismatch') + manifest = json.loads(raw, object_pairs_hook=unique) + pins = [('config', '0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89', 265), + ('model', '3fcd3febec8b3fd64435204db75bf0dd73b91e8d0661e0331acfe7e7c3120b85', 4676804928)] + require(set(manifest) == {'schemaVersion', 'mediaType', 'config', 'layers'} and manifest['schemaVersion'] == 2 + and manifest['mediaType'] == 'application/vnd.docker.distribution.manifest.v2+json' + and type(manifest['layers']) is list and len(manifest['layers']) == 1, 'unexpected fixed encoder manifest topology') + actual = [manifest['config'], *manifest['layers']] + rows = [{'path': manifest_path.relative_to(models).as_posix(), 'bytes': len(raw), + 'sha256': hashlib.sha256(raw).hexdigest(), 'expected_bytes': 531, + 'expected_sha256': '64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b', 'archive_policy': 'ARCHIVE_ORIGINAL_SMALL_INPUT'}] + for item, (kind, digest, size) in zip(actual, pins): + require(item['digest'] == 'sha256:' + digest and type(item['size']) is int and item['size'] == size, 'model layer identity drift') + require(item['mediaType'] == ('application/vnd.docker.container.image.v1+json' if kind == 'config' + else 'application/vnd.ollama.image.model'), 'encoder layer type drift') + row = identity(models / ('blobs/sha256-' + digest), size, digest) + row['kind'] = kind + rows.append(row) + require(len(rows) == 3, 'encoder cache must contain manifest, config and model identities only') + config = json.loads((models / ('blobs/sha256-' + pins[0][1])).read_bytes(), object_pairs_hook=unique) + require(config == {'model_format': 'gguf', 'model_family': 'qwen3', 'model_families': ['qwen3'], 'model_type': '7.6B', + 'file_type': 'Q4_K_M', 'architecture': 'amd64', 'os': 'linux', + 'rootfs': {'type': 'layers', 'diff_ids': ['sha256:' + pins[1][1]]}}, 'encoder config drift') + (publish / 'model-binding-before.json').write_text(json.dumps(rows, sort_keys=True, indent=2) + '\n') + wheel = control / 'downloads/sqlite-vec-0.1.9-linux-x64.whl' + wheel_raw = wheel.read_bytes() + require(len(wheel_raw) == 163388 and hashlib.sha256(wheel_raw).hexdigest() == '1515727990b49e79bcaf75fdee2ffc7d461f8b66905013231251f1c8938e7786', 'Linux wheel pin mismatch') + with zipfile.ZipFile(wheel) as archive: + expected = {'sqlite_vec/__init__.py', 'sqlite_vec/vec0.so', 'sqlite_vec-0.1.9.dist-info/METADATA', + 'sqlite_vec-0.1.9.dist-info/WHEEL', 'sqlite_vec-0.1.9.dist-info/top_level.txt', 'sqlite_vec-0.1.9.dist-info/RECORD'} + require(len(archive.infolist()) == 6 and set(archive.namelist()) == expected + and sum(item.file_size for item in archive.infolist()) <= 1048576, 'unexpected pinned wheel membership') + library = archive.read('sqlite_vec/vec0.so') + require(hashlib.sha256(library).hexdigest() == '5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8', 'Linux extension binary pin mismatch') + metadata_raw = archive.read('sqlite_vec-0.1.9.dist-info/METADATA') + metadata = email.parser.BytesParser().parsebytes(metadata_raw) + require(metadata['Name'] == 'sqlite-vec' and metadata['Version'] == '0.1.9', 'wheel metadata identity mismatch') + (control / 'vec0.so').write_bytes(library) + (control / 'sqlite-vec-METADATA.txt').write_bytes(metadata_raw) + license_raw = (control / 'downloads/upstream-encoder-LICENSE.txt').read_bytes() + require(len(license_raw) == 11343 and hashlib.sha256(license_raw).hexdigest() == '832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e', 'immutable upstream encoder license drift') + declared = {'schema_version': 1, 'model_declared_license': 'Apache-2.0', + 'model_upstream_source': 'https://huggingface.co/Qwen/Qwen3-Embedding-8B/tree/1d8ad4ca9b3dd8059ad90a75d4983776a23d44af', + 'model_upstream_license_url': 'https://huggingface.co/Qwen/Qwen3-Embedding-8B/raw/1d8ad4ca9b3dd8059ad90a75d4983776a23d44af/LICENSE', + 'model_upstream_license_bytes': len(license_raw), 'model_upstream_license_sha256': hashlib.sha256(license_raw).hexdigest(), + 'model_registry_license_layer': 'ABSENT', 'model_registry_template_layer': 'ABSENT', 'model_registry_params_layer': 'ABSENT', + 'gguf_conversion_lineage': 'NOT_INDEPENDENTLY_VERIFIED', 'sqlite_vec_declared_license': metadata['License'], + 'wheel_license_file': 'ABSENT', 'legal_approval': 'NOT_ASSESSED', 'scientific_effect': 'NONE', + 'ollama_image_component_license_inventory': 'NOT_ASSESSED', 'python_image_component_license_inventory': 'NOT_ASSESSED', + 'scientific_status_authority': False, 'wheel_bytes': len(wheel_raw), 'wheel_sha256': hashlib.sha256(wheel_raw).hexdigest()} + (publish / 'declared-licenses.json').write_text(json.dumps(declared, sort_keys=True, indent=2) + '\n') + PY + for pair in "$OLLAMA_IMAGE:ollama" "$PYTHON_IMAGE:python"; do + image="${pair%:*}" + label="${pair##*:}" + (ulimit -f 2048; timeout --signal=TERM --kill-after=10s 600s docker pull --platform linux/amd64 "$image") > "$publish/$label-image-pull.log" 2>&1 + docker image inspect "$image" > "$publish/$label-image.json" + done + chmod -R a-w "$models" + - name: Write fixed public compatibility controls + run: | + set -euo pipefail + cat > "$RUNNER_TEMP/semantic-control/payload/compatibility.py" <<'PY' + import hashlib, json, math, os, pathlib, platform, sqlite3, stat, struct, sys, time, urllib.error, urllib.request + root = pathlib.Path('/work/evidence') + root.mkdir() + def require(condition, message): + if not condition: + raise ValueError(message) + def save(name, value): + raw = (json.dumps(value, ensure_ascii=True, allow_nan=False, sort_keys=True, indent=2) + '\n').encode() + require(len(raw) <= 16777216, 'compatibility evidence byte limit') + with (root / name).open('xb') as output: + output.write(raw) + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate native response key') + result[key] = value + return result + def invalid_number(value): + raise ValueError('nonfinite native JSON number') + require(sys.version_info[:3] == (3, 11, 14) and platform.system() == 'Linux' and platform.machine() == 'x86_64', 'actual client runtime drift') + extension = pathlib.Path('/control/vec0.so') + require(hashlib.sha256(extension.read_bytes()).hexdigest() == '5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8', 'loaded Linux extension pin mismatch') + database = root / 'compatibility.sqlite' + def connect(): + db = sqlite3.connect(database) + db.enable_load_extension(True) + try: + db.load_extension(str(extension)) + finally: + db.enable_load_extension(False) + require(db.execute('select vec_version()').fetchone()[0] in ('v0.1.9', '0.1.9'), 'actual loaded extension version drift') + db.execute('pragma journal_mode=DELETE') + db.execute('pragma synchronous=FULL') + return db + query = struct.pack('<3f', 1.0, 0.0, 0.0) + with connect() as db: + version = db.execute('select vec_version()').fetchone()[0] + compile_options = [row[0] for row in db.execute('pragma compile_options')] + db.execute('create virtual table tiny using vec0(embedding float[3] distance_metric=cosine)') + db.executemany('insert into tiny(rowid, embedding) values (?, ?)', [(1, query), (2, struct.pack('<3f', 0.0, 1.0, 0.0))]) + db.close() + with connect() as db: + tiny_rows = db.execute('select rowid, distance from tiny where embedding match ? and k=2 order by distance', (query,)).fetchall() + tiny_rows.sort(key=lambda item: (item[1], item[0])) + require(len(tiny_rows) == 2 and [row[0] for row in tiny_rows] == [1, 2] + and abs(tiny_rows[0][1]) <= 1e-6 and abs(tiny_rows[1][1] - 1) <= 1e-6, 'reopened tiny cosine query mismatch') + db.close() + save('extension-result.json', {'schema_version': 1, 'python': platform.python_version(), 'platform': platform.platform(), + 'sqlite_version': sqlite3.sqlite_version, 'sqlite_compile_options': compile_options, 'vec_version': version, + 'tiny_cosine_rows': tiny_rows, 'actual_loaded_extension': True, 'reopened': True, + 'scientific_effect': 'NONE', 'scientific_status_authority': False}) + # No proxy handler, redirect handler or endpoint input is admitted to this loopback-only client. + class DenyRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, fp, code, message, headers, url): + raise ValueError('loopback API redirect refused') + client = urllib.request.build_opener(urllib.request.ProxyHandler({}), DenyRedirect()) + def api(path, label, payload=None, timeout=30): + require(path in ('/api/version', '/api/tags', '/api/show', '/api/embed', '/api/ps'), 'unadmitted API path') + data = None if payload is None else json.dumps(payload, allow_nan=False, separators=(',', ':')).encode() + if data is not None: + save(label + '-request.json', payload) + request = urllib.request.Request('http://127.0.0.1:11434' + path, data=data, headers={'Content-Type': 'application/json'}) + try: + response = client.open(request, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + with response: + raw = response.read(16777217) + require(len(raw) <= 16777216, 'native API response byte limit') + with (root / (label + '-response.json')).open('xb') as output: + output.write(raw) + save(label + '-http.json', {'status': response.status, 'response_bytes': len(raw), 'path': path}) + require(response.status == 200, 'native API rejected compatibility request') + return json.loads(raw, object_pairs_hook=unique, parse_constant=invalid_number) + # Readiness waits precede the one embedding attempt; they cannot substitute a different model or endpoint. + readiness = [] + for attempt in range(30): + try: + version_raw = client.open('http://127.0.0.1:11434/api/version', timeout=2) + with version_raw: + raw = version_raw.read(16385) + require(version_raw.status == 200 and len(raw) <= 16384, 'invalid readiness response') + ready_version = json.loads(raw, object_pairs_hook=unique, parse_constant=invalid_number) + require(ready_version == {'version': '0.35.1'}, 'actual Ollama version drift') + readiness.append({'attempt': attempt + 1, 'outcome': 'ready'}) + break + except urllib.error.URLError as error: + readiness.append({'attempt': attempt + 1, 'outcome': type(error).__name__}) + time.sleep(1) + else: + save('readiness.json', readiness) + raise ValueError('bounded Ollama startup did not become ready') + save('readiness.json', readiness) + # A distinct encoder candidate follows the retained generator HTTP501; this is not a retry of that candidate. + model = 'qwen3-embedding:8b' + manifest_digest = '64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b' + def model_identity(label): + require(api('/api/version', label + '-version') == {'version': '0.35.1'}, 'native version drift') + tags = api('/api/tags', label + '-tags') + require(type(tags) is dict and type(tags.get('models')) is list and len(tags['models']) == 1 + and tags['models'][0]['name'] == model and tags['models'][0]['digest'] == manifest_digest, 'actual served model manifest drift') + show = api('/api/show', label + '-show', {'model': model, 'verbose': True}) + require(type(show) is dict and type(show.get('model_info')) is dict and type(show.get('template')) is str, 'native show identity unavailable') + require(type(show.get('details')) is dict and show['details'].get('format') == 'gguf' + and show['details'].get('family') == 'qwen3' and show['details'].get('quantization_level') == 'Q4_K_M', 'encoder details drift') + require(type(show.get('capabilities')) is list and 'embedding' in show['capabilities'], 'encoder embedding capability metadata absent') + info = show['model_info'] + require(info.get('general.architecture') == 'qwen3', 'encoder architecture drift') + for key, expected in (('qwen3.embedding_length', 4096), ('qwen3.context_length', 40960), + ('tokenizer.ggml.eos_token_id', 151643), ('tokenizer.ggml.eot_token_id', 151645)): + require(type(info.get(key)) is int and info[key] == expected, 'encoder metadata integer drift: ' + key) + require(info.get('tokenizer.ggml.add_bos_token') is False and info.get('tokenizer.ggml.add_eos_token') is True, + 'encoder special-token flag drift') + require(type(info.get('tokenizer.ggml.tokens')) is list and len(info['tokenizer.ggml.tokens']) == 151665, + 'encoder vocabulary cardinality drift') + # No generator-template hash or unverified numeric pooling enum is applied to this encoder. + # The complete original show/template/pooling fields are retained and compared before/after. + return show + before = model_identity('before') + try: + request = {'model': model, 'input': ['TEST preparation: water boils.', 'TEST preparation: a square has four sides.'], + 'truncate': False, 'keep_alive': '60s', 'options': {'num_ctx': 4096, 'num_thread': 4, 'num_gpu': 0}} + embeddings = api('/api/embed', 'embedding', request, timeout=600) + require(type(embeddings) is dict and embeddings.get('model') == model + and type(embeddings.get('embeddings')) is list and len(embeddings['embeddings']) == 2, 'actual embedding cardinality/model mismatch') + vectors = embeddings['embeddings'] + norms = [] + for vector in vectors: + require(type(vector) is list and len(vector) == 4096 + and all(type(value) in (int, float) and math.isfinite(value) for value in vector), 'invalid actual embedding dimensions or values') + norm = math.sqrt(math.fsum(value * value for value in vector)) + require(math.isfinite(norm) and norm > 0 and abs(norm - 1) <= 1e-3, 'actual embedding norm mismatch') + norms.append(norm) + for name in ('total_duration', 'load_duration', 'prompt_eval_count'): + require(type(embeddings.get(name)) is int and embeddings[name] >= 0, 'missing actual embedding statistics') + require(embeddings['prompt_eval_count'] > 0, 'actual embedding input count absent') + blobs = [struct.pack('<4096f', *vector) for vector in vectors] + rounded = [struct.unpack('<4096f', blob) for blob in blobs] + native_schema_sql = 'CREATE VIRTUAL TABLE native USING vec0(embedding float[4096] distance_metric=cosine, chunk_size=16)' + def stored_native_identity(db): + schema = db.execute("select sql from sqlite_schema where type='table' and name='native'").fetchall() + require(schema == [(native_schema_sql,)], 'stored native vector schema drift') + stored = db.execute('select rowid, embedding from native order by rowid').fetchall() + require(len(stored) == 2 and all(type(row[0]) is int for row in stored) + and [row[0] for row in stored] == [1, 2] + and all(type(row[1]) is bytes and row[1] == expected for row, expected in zip(stored, blobs)), + 'stored native float32 vector identity drift') + return {'schema_sql': schema[0][0], 'stored_vectors': [ + {'rowid': row[0], 'bytes': len(row[1]), 'sha256': hashlib.sha256(row[1]).hexdigest()} for row in stored]} + def database_identity(): + descriptor = os.open(database, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, 'rb') as stream: + before = os.fstat(stream.fileno()) + require(stat.S_ISREG(before.st_mode) and before.st_nlink == 1 + and 0 < before.st_size <= 4194304, 'native database is not a bounded regular file') + digest, length = hashlib.sha256(), 0 + while part := stream.read(65536): + length += len(part) + require(length <= 4194304, 'native database grew beyond retained-member limit') + digest.update(part) + after = os.fstat(stream.fileno()) + require(stat.S_ISREG(after.st_mode) and after.st_nlink == 1 + and (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) == + (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) + and length == before.st_size, 'native database changed while hashing') + return {'bytes': length, 'sha256': digest.hexdigest()} + with connect() as db: + db.execute(native_schema_sql) + db.executemany('insert into native(rowid, embedding) values (?, ?)', list(enumerate(blobs, start=1))) + committed_native = stored_native_identity(db) + db.close() + committed_database = database_identity() + with connect() as db: + reopened_native = stored_native_identity(db) + require(reopened_native == committed_native, 'native vector identities changed after reopen') + rows = db.execute('select rowid,distance from native where embedding match ? and k=2 order by distance', (blobs[0],)).fetchall() + rows.sort(key=lambda item: (item[1], item[0])) + db.close() + reopened_database = database_identity() + require(reopened_database == committed_database, 'native database bytes changed after reopen/query') + norm_query = math.sqrt(math.fsum(value * value for value in rounded[0])) + oracle = [] + for identifier, vector in enumerate(rounded, start=1): + norm = math.sqrt(math.fsum(value * value for value in vector)) + distance = 1 - math.fsum(a * b for a, b in zip(rounded[0], vector)) / (norm_query * norm) + oracle.append((identifier, distance)) + oracle.sort(key=lambda item: (item[1], item[0])) + require(len(rows) == 2 and [row[0] for row in rows] == [row[0] for row in oracle] + and all(abs(actual[1] - expected[1]) <= 1e-5 for actual, expected in zip(rows, oracle)), 'native float32 cosine/reopen mismatch') + save('embedding-result.json', {'schema_version': 1, 'embedding_requests': 1, 'public_input_count': 2, + 'model': model, 'manifest_sha256': manifest_digest, 'candidate': 'ADAPT_DEDICATED_ENCODER', + 'dimensions': 4096, 'norms': norms, 'float32_cosine_rows': rows, 'exact_cosine_oracle': oracle, + 'native_storage': {'chunk_size': 16, 'database_limit_bytes': 4194304, + 'committed_table': committed_native, 'reopened_table': reopened_native, + 'committed_database': committed_database, 'reopened_database': reopened_database}, + 'truncate': False, 'native_statistics': {name: embeddings[name] for name in ('total_duration', 'load_duration', 'prompt_eval_count')}, + 'reopened': True, 'scientific_effect': 'NONE', 'scientific_status_authority': False}) + finally: + after = model_identity('after') + require(before == after, 'native model/tokenizer/template changed during compatibility attempt') + api('/api/ps', 'after-ps') + save('compatibility-result.json', {'schema_version': 1, 'scope': 'tiny-public-runtime-compatibility', + 'candidate': 'ADAPT_DEDICATED_ENCODER', 'encoder_model': model, 'encoder_manifest_sha256': manifest_digest, + 'actual_loaded_extension': True, 'actual_embedding_vectors': 2, 'native_identity_unchanged': True, + 'encoder_tokenizer_parity': 'NOT_RUN', 'encoder_pooling_numeric_enum_parity': 'NOT_RUN', + 'full_prompt_tokenizer_parity': 'NOT_RUN', 'native_selected_model_info_parity': 'NOT_RUN', + 'approximately_500_token_chunking': 'NOT_RUN', 'retrieval_quality': 'NOT_RUN', 'generation_quality': 'NOT_RUN', + 'installed_entrypoints': 'NOT_TOUCHED', 'default_retrieval_switch': False, + 'scientific_effect': 'NONE', 'scientific_status_authority': False}) + PY + chmod -R a-w "$RUNNER_TEMP/semantic-control/payload" + - name: Exercise the fixed runtime with external networking denied + run: | + set -euo pipefail + publish="$RUNNER_TEMP/semantic-publish/payload" + # Native HOME is not assigned or repurposed. Only the fixed image's disposable /root or / case is supported. + docker create --name semantic-server --log-driver none --platform linux/amd64 --network none \ + --read-only --user 65532:65532 --cpus 3.5 --memory 12g --pids-limit 192 --cap-drop ALL --security-opt no-new-privileges \ + --ulimit fsize=16777216:16777216 --tmpfs /tmp:rw,noexec,nosuid,nodev,size=1g,uid=65532,gid=65532 \ + --tmpfs /root:rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700 \ + --tmpfs /.ollama:rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700 \ + --env OLLAMA_HOST=127.0.0.1:11434 --env OLLAMA_MODELS=/models --env OLLAMA_NO_CLOUD=1 --env OLLAMA_NOPRUNE=1 \ + --env OLLAMA_NUM_PARALLEL=1 --env OLLAMA_MAX_LOADED_MODELS=1 --env OLLAMA_MAX_QUEUE=1 \ + --env OLLAMA_CONTEXT_LENGTH=4096 --env OLLAMA_LOAD_TIMEOUT=300s --env OLLAMA_VULKAN=false \ + --env OLLAMA_DEBUG=false --env OLLAMA_DEBUG_LOG_REQUESTS=false \ + --mount type=bind,source="$RUNNER_TEMP/semantic-models/payload",target=/models,readonly \ + --entrypoint /bin/sh "$OLLAMA_IMAGE" -c 'case "${HOME-}" in /root|/) exec /bin/ollama serve ;; *) exit 78 ;; esac' > "$publish/server-container-id.txt" + docker inspect semantic-server > "$publish/server-container-before.json" + set +e + # Preserve the original startup console, including any newly generated public key printed upstream. + # Private key files and ephemeral home files are outside the closed collection payload. + (ulimit -f 2048; timeout --signal=TERM --kill-after=10s 900s docker start --attach semantic-server) > "$publish/server-console.log" 2>&1 & + server_process=$! + set -e + # Wait only for the already-created sandbox to run; no inference request is retried. + for attempt in $(seq 1 20); do + docker inspect --format '{{.State.Status}}' semantic-server > "$publish/server-start-state.txt" + state="$(cat "$publish/server-start-state.txt")" + if [ "$state" = running ]; then break; fi + if [ "$state" = exited ] || [ "$state" = dead ]; then exit 1; fi + sleep 1 + done + test "$state" = running + docker create --name semantic-client --log-driver none --platform linux/amd64 --network container:semantic-server \ + --read-only --user 65532:65532 --cpus 0.5 --memory 512m --pids-limit 64 --cap-drop ALL --security-opt no-new-privileges \ + --ulimit fsize=16777216:16777216 --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m \ + --mount type=bind,source="$RUNNER_TEMP/semantic-control/payload",target=/control,readonly \ + --mount type=bind,source="$RUNNER_TEMP/semantic-work/payload",target=/work \ + --entrypoint python3 "$PYTHON_IMAGE" -I -B -S /control/compatibility.py > "$publish/client-container-id.txt" + docker inspect semantic-client > "$publish/client-container-before.json" + set +e + (ulimit -f 2048; timeout --signal=TERM --kill-after=10s 800s docker start --attach semantic-client) > "$publish/client-console.log" 2>&1 + status=$? + printf '%s\n' "$status" > "$publish/client-attach-exit-status.txt" + set -e + docker inspect semantic-client > "$publish/client-container-after.json" + if [ "$status" = 0 ]; then + test "$(docker inspect --format '{{.State.Status}}' semantic-client)" = exited + status="$(docker inspect --format '{{.State.ExitCode}}' semantic-client)" + fi + printf '%s\n' "$status" > "$publish/compatibility-exit-status.txt" + docker inspect semantic-server > "$publish/server-container-after.json" + docker stop --time 10 semantic-server > "$publish/server-stop.txt" + docker inspect semantic-server > "$publish/server-container-stopped.json" + set +e + wait "$server_process" + server_status=$? + printf '%s\n' "$server_status" > "$publish/server-attach-exit-status.txt" + set -e + exit "$status" + - name: Dispose both sandboxes before collecting their outputs + if: always() + run: | + set -euo pipefail + for name in semantic-client semantic-server; do + present="$(docker ps -aq --filter "name=^/$name$")" + if [ -n "$present" ]; then docker rm --force "$name"; fi + remaining="$(docker ps -aq --filter "name=^/$name$")" + test -z "$remaining" + done + - name: Collect only bounded regular original evidence and rehash the model cache + if: always() + run: | + set -euo pipefail + for name in semantic-client semantic-server; do + remaining="$(docker ps -aq --filter "name=^/$name$")" + test -z "$remaining" + done + # An interrupted setup can leave admission records staged before this payload is created. + sudo mkdir -p "$RUNNER_TEMP/semantic-publish/payload" + sudo chmod 0700 "$RUNNER_TEMP/semantic-publish/payload" + sudo chown "$(id -u):$(id -g)" "$RUNNER_TEMP/semantic-publish/payload" + sudo python3 -I -B -S - "$RUNNER_TEMP" "$(id -u)" "$(id -g)" <<'PY' + import hashlib, json, os, pathlib, stat, sys + base = pathlib.Path(sys.argv[1]) + destination = base / 'semantic-publish/payload' + host_uid, host_gid = map(int, sys.argv[2:]) + total, missing, refused = 0, [], [] + def require(condition, message): + if not condition: + raise ValueError(message) + require(0 <= host_uid < 2147483648 and 0 <= host_gid < 2147483648, 'invalid trusted collector identity') + def anchored(root, relative): + descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + parts = relative.split('/') + require(all(part not in ('', '.', '..') for part in parts), 'unsafe collector path') + for part in parts[:-1]: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + return os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=descriptor) + finally: + os.close(descriptor) + def write(name, raw): + require('/' not in name and name not in ('', '.', '..'), 'unsafe retained destination') + directory = os.open(destination, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + file = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=directory) + try: + os.fchown(file, host_uid, host_gid) + with os.fdopen(file, 'wb', closefd=False) as stream: + stream.write(raw) + finally: + os.close(file) + finally: + os.close(directory) + def retain(root, source, name, limit): + global total + file = anchored(root, source) + with os.fdopen(file, 'rb') as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= limit, 'unsafe retained original') + raw = stream.read(limit + 1) + after = os.fstat(stream.fileno()) + require(stat.S_ISREG(after.st_mode) and after.st_nlink == 1 + and (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) == + (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns), 'retained original changed') + require(len(raw) <= limit, 'retained original grew') + total += len(raw) + require(total <= 48 * 2**20, 'collector copied-original subtotal limit') + write(name, raw) + # An interrupted mount setup still retains the original admission/native records. + initial = base / 'semantic-initial-evidence' + for name in ('checked-commit.txt', 'repository.txt', 'run-id.txt', 'run-attempt.txt', 'capacity.json', + 'filesystem-models.log', 'filesystem-work.log', 'filesystem-control.log', 'filesystem-publish.log'): + if not (destination / name).exists(): + try: + retain(initial, name, name, 65536) + except FileNotFoundError: + missing.append(name) + except (OSError, ValueError): + refused.append(name) + control = base / 'semantic-control/payload' + models = base / 'semantic-models/payload' + members = [(control, 'downloads/sqlite-vec-0.1.9-linux-x64.whl', 'sqlite-vec-0.1.9-linux-x64.whl', 163388), + (control, 'vec0.so', 'vec0.so', 1048576), (control, 'sqlite-vec-METADATA.txt', 'sqlite-vec-METADATA.txt', 1048576), + (control, 'compatibility.py', 'compatibility-control.py', 65536), + (control, 'downloads/upstream-encoder-LICENSE.txt', 'upstream-encoder-LICENSE.txt', 11343), + (models, 'manifests/registry.ollama.ai/library/qwen3-embedding/8b', 'original-model-manifest.json', 531), + (models, 'blobs/sha256-0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89', 'original-model-config.json', 265)] + fixed = ['extension-result.json', 'readiness.json', 'compatibility.sqlite', 'embedding-result.json', 'compatibility-result.json'] + for prefix, paths in (('before', ('version', 'tags', 'show')), ('after', ('version', 'tags', 'show', 'ps')), ('embedding', ('',))): + for path in paths: + label = prefix + ('-' + path if path else '') + fixed.extend([label + '-response.json', label + '-http.json']) + if path == 'show' or prefix == 'embedding': + fixed.append(label + '-request.json') + for name in fixed: + members.append((base / 'semantic-work/payload', 'evidence/' + name, name, 16777216 if '-show-response' in name else 4194304)) + for root, source, name, limit in members: + try: + retain(root, source, name, limit) + except FileNotFoundError: + missing.append(name) + except (OSError, ValueError): + refused.append(name) + # The large immutable model is streamed only for an identity record; its bytes are not copied into evidence. + try: + before = json.loads((destination / 'model-binding-before.json').read_bytes()) + require(type(before) is list and len(before) == 3 + and [row['path'] for row in before] == ['manifests/registry.ollama.ai/library/qwen3-embedding/8b', + 'blobs/sha256-0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89', + 'blobs/sha256-3fcd3febec8b3fd64435204db75bf0dd73b91e8d0661e0331acfe7e7c3120b85'], 'encoder cache inventory drift') + after = [] + for row in before: + file = anchored(models, row['path']) + digest, length = hashlib.sha256(), 0 + with os.fdopen(file, 'rb') as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_size <= row['expected_bytes'], 'unsafe final model input') + while part := stream.read(1048576): + length += len(part) + require(length <= row['expected_bytes'], 'model input grew') + digest.update(part) + observed = os.fstat(stream.fileno()) + require(stat.S_ISREG(observed.st_mode) and observed.st_nlink == 1 + and (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) == + (observed.st_dev, observed.st_ino, observed.st_size, observed.st_mtime_ns, observed.st_ctime_ns), + 'final model input changed while hashing') + final = dict(row, bytes=length, sha256=digest.hexdigest()) + after.append(final) + require(final == row, 'model cache changed during compatibility attempt') + write('model-binding-after.json', (json.dumps(after, sort_keys=True, indent=2) + '\n').encode()) + except FileNotFoundError: + missing.append('model-binding-after.json') + except (OSError, ValueError, TypeError, KeyError): + refused.append('model-binding-after.json') + publication_root = base / 'semantic-publish' + publication_mounted = os.path.ismount(publication_root) + publication_identity = None + if publication_mounted: + mount_info = os.stat(publication_root, follow_symlinks=False) + capacity = os.statvfs(publication_root) + require(stat.S_ISDIR(mount_info.st_mode), 'publication mount root is not a directory') + publication_identity = {'host_device': mount_info.st_dev, + 'reported_total_bytes': capacity.f_frsize * capacity.f_blocks} + write('collection.json', (json.dumps({'schema_version': 1, 'missing': sorted(set(missing)), 'refused': sorted(set(refused)), + 'collector_counted_copy_input_bytes': total, 'collector_copy_limit_bytes': 48 * 2**20, + 'collector_counter_may_include_refused_copy_attempts': True, + 'configured_publication_filesystem_capacity_bytes': 64 * 2**20, + 'publication_mount_observed': publication_mounted, 'actual_publication_filesystem': publication_identity, + 'collector_subtotal_excludes_prior_originals_and_its_metadata': True, + 'full_model_weights': 'NOT_ARCHIVED', 'full_image_layers': 'NOT_ARCHIVED', + 'generated_private_key_files': 'NOT_ARCHIVED', 'ephemeral_home_files': 'NOT_ARCHIVED', + 'generated_public_key_in_original_console': 'MAY_BE_PRESENT', + 'scientific_effect': 'NONE', 'scientific_status_authority': False}, sort_keys=True, indent=2) + '\n').encode()) + if refused: + raise ValueError('bounded original evidence collection refused members') + PY + - name: Verify original identities and emit a compatibility-only receipt + run: | + set -euo pipefail + python3 -I -B -S - <<'PY' + import hashlib, json, os, pathlib + root = pathlib.Path(os.environ['RUNNER_TEMP']) / 'semantic-publish/payload' + def require(condition, message): + if not condition: + raise ValueError(message) + require((root / 'provision-exit-status.txt').read_text() == '0\n' + and (root / 'compatibility-exit-status.txt').read_text() == '0\n' + and (root / 'client-attach-exit-status.txt').read_text() == '0\n' + and (root / 'server-attach-exit-status.txt').read_text() == '0\n', 'actual preparation or attachment attempt failed') + collection = json.loads((root / 'collection.json').read_bytes()) + require(collection['missing'] == [] and collection['refused'] == [], 'original preparation evidence incomplete') + require(collection['publication_mount_observed'] is True + and collection['configured_publication_filesystem_capacity_bytes'] == 64 * 2**20 + and type(collection['actual_publication_filesystem']) is dict + and type(collection['actual_publication_filesystem']['host_device']) is int + and type(collection['actual_publication_filesystem']['reported_total_bytes']) is int + and 0 < collection['actual_publication_filesystem']['reported_total_bytes'] <= 64 * 2**20, + 'publication filesystem capacity was not actually observed') + for name, variable in (('checked-commit', 'GITHUB_SHA'), ('repository', 'GITHUB_REPOSITORY'), + ('run-id', 'GITHUB_RUN_ID'), ('run-attempt', 'GITHUB_RUN_ATTEMPT')): + require((root / (name + '.txt')).read_text() == os.environ[variable] + '\n', 'native preparation identity mismatch') + images, image_refs = {}, {} + for label, image, version, digest in ( + ('ollama', 'ollama/ollama', '0.35.1', '17e945f1aa6e505707eeb621e3f36847d0a4a063d4778f7007645acccfb8bd4f'), + ('python', 'python', '3.11.14-slim-bookworm', '83f339c1be6340ae1096010fdccf6552ac932d8f410d45d206014916bdf37e48')): + image_refs[label] = image + ':' + version + '@sha256:' + digest + observed = json.loads((root / (label + '-image.json')).read_bytes()) + require(type(observed) is list and len(observed) == 1 and observed[0]['Os'] == 'linux' + and observed[0]['Architecture'] == 'amd64' and image + '@sha256:' + digest in observed[0]['RepoDigests'], 'actual image identity drift') + require(os.environ[label.upper() + '_IMAGE'] == image_refs[label], 'fixed checked image reference drift') + images[label] = observed[0] + server = json.loads((root / 'server-container-before.json').read_bytes())[0] + client = json.loads((root / 'client-container-before.json').read_bytes())[0] + base = pathlib.Path(os.environ['RUNNER_TEMP']) + bind_sets = { + 'ollama': {(str(base / 'semantic-models/payload'), '/models', False)}, + 'python': {(str(base / 'semantic-control/payload'), '/control', False), + (str(base / 'semantic-work/payload'), '/work', True)}} + tmpfs_sets = { + 'ollama': {'/tmp': 'rw,noexec,nosuid,nodev,size=1g,uid=65532,gid=65532', + '/root': 'rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700', + '/.ollama': 'rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700'}, + 'python': {'/tmp': 'rw,noexec,nosuid,nodev,size=64m'}} + def validate_container(container, label, cpu, memory, pids): + config = container['HostConfig'] + require(container['Image'] == images[label]['Id'] and container['Config']['Image'] == image_refs[label], + 'actual container did not use the corresponding inspected fixed image') + require(container['Config']['User'] == '65532:65532' and config['ReadonlyRootfs'] is True + and config['NanoCpus'] == cpu and config['Memory'] == memory and config['PidsLimit'] == pids + and config['CapDrop'] == ['ALL'] and 'no-new-privileges' in config['SecurityOpt'] + and config['LogConfig']['Type'] == 'none' and config['PortBindings'] in (None, {}), 'actual sandbox restrictions drift') + require(config['Tmpfs'] == tmpfs_sets[label] and config['Binds'] in (None, []) + and config['VolumesFrom'] in (None, []), 'actual temporary or inherited mount set drift') + mounts = container['Mounts'] + require(type(mounts) is list and len(mounts) == len(bind_sets[label]) + and all(item['Type'] == 'bind' and type(item['RW']) is bool for item in mounts) + and {(item['Source'], item['Destination'], item['RW']) for item in mounts} == bind_sets[label], + 'actual bind source/destination/permission set drift') + configured = config['Mounts'] + require(type(configured) is list and len(configured) == len(bind_sets[label]) + and all(item['Type'] == 'bind' and type(item.get('ReadOnly', False)) is bool for item in configured) + and {(item['Source'], item['Target'], not item.get('ReadOnly', False)) for item in configured} == bind_sets[label], + 'actual configured mount set drift') + validate_container(server, 'ollama', 3500000000, 12 * 2**30, 192) + validate_container(client, 'python', 500000000, 512 * 2**20, 64) + require(server['HostConfig']['NetworkMode'] == 'none' + and client['HostConfig']['NetworkMode'] in ('container:' + server['Id'], 'container:semantic-server'), 'actual offline namespace drift') + final_client = json.loads((root / 'client-container-after.json').read_bytes())[0] + final_server = json.loads((root / 'server-container-stopped.json').read_bytes())[0] + validate_container(final_client, 'python', 500000000, 512 * 2**20, 64) + validate_container(final_server, 'ollama', 3500000000, 12 * 2**30, 192) + require(final_client['HostConfig']['NetworkMode'] == client['HostConfig']['NetworkMode'] + and final_server['HostConfig']['NetworkMode'] == 'none', 'final offline namespace drift') + require(final_client['Id'] == client['Id'] and final_client['State']['Status'] == 'exited' + and final_client['State']['ExitCode'] == 0 and final_client['State']['OOMKilled'] is False, + 'actual compatibility process did not finish successfully') + require(final_server['Id'] == server['Id'] and final_server['State']['Status'] == 'exited' + and final_server['State']['ExitCode'] == 0 and final_server['State']['OOMKilled'] is False, + 'actual server did not stop cleanly') + before = json.loads((root / 'model-binding-before.json').read_bytes()) + require(before == json.loads((root / 'model-binding-after.json').read_bytes()), 'original model cache identity changed') + result = json.loads((root / 'compatibility-result.json').read_bytes()) + require(result['actual_loaded_extension'] is True and result['actual_embedding_vectors'] == 2 + and result['native_identity_unchanged'] is True and result['default_retrieval_switch'] is False + and result['candidate'] == 'ADAPT_DEDICATED_ENCODER' and result['encoder_model'] == 'qwen3-embedding:8b' + and result['encoder_manifest_sha256'] == '64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b', 'tiny encoder compatibility result incomplete') + require(hashlib.sha256((root / 'original-model-manifest.json').read_bytes()).hexdigest() + == '64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b', 'retained encoder manifest drift') + require(len((root / 'original-model-config.json').read_bytes()) == 265 + and hashlib.sha256((root / 'original-model-config.json').read_bytes()).hexdigest() + == '0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89', 'retained encoder config drift') + require(len((root / 'upstream-encoder-LICENSE.txt').read_bytes()) == 11343 + and hashlib.sha256((root / 'upstream-encoder-LICENSE.txt').read_bytes()).hexdigest() + == '832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e', 'retained upstream encoder license drift') + inventory = [] + for path in sorted(root.iterdir()): + require(path.is_file() and not path.is_symlink(), 'unexpected trusted publication member') + raw = path.read_bytes() + inventory.append({'path': path.name, 'bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()}) + receipt = {'schema_version': 1, 'scope': 'semantic-runtime-tiny-public-compatibility', + 'candidate': 'ADAPT_DEDICATED_ENCODER', 'encoder_model': 'qwen3-embedding:8b', + 'encoder_manifest_sha256': '64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b', + 'generator_installed_model': 'NOT_TOUCHED', 'encoder_tokenizer_parity': 'NOT_RUN', + 'collector_copy_limit_bytes': 48 * 2**20, 'configured_publication_filesystem_capacity_bytes': 64 * 2**20, + 'publication_mount_observed': collection['publication_mount_observed'], + 'actual_publication_filesystem': collection['actual_publication_filesystem'], + 'collector_subtotal_is_not_whole_publication_bytes': True, + 'repository': os.environ['GITHUB_REPOSITORY'], 'checked_commit': os.environ['GITHUB_SHA'], + 'run_id': os.environ['GITHUB_RUN_ID'], 'run_attempt': os.environ['GITHUB_RUN_ATTEMPT'], + 'actual_private_inputs': False, 'installed_entrypoints': 'NOT_TOUCHED', 'full_weights': 'NOT_ARCHIVED', 'full_image_layers': 'NOT_ARCHIVED', + 'runtime_images': {label: {'reference': image_refs[label], 'image_id': images[label]['Id']} for label in sorted(images)}, + 'feature_red_green': 'NOT_RUN', 'tokenizer_parity': 'NOT_RUN', 'retrieval_quality': 'NOT_RUN', + 'scientific_effect': 'NONE', 'scientific_status_authority': False, 'custody': 'unknown', + 'final_disposition': 'REQUIRES_NATIVE_JOB_SUCCESS_AND_ORIGINAL_ARCHIVE_READBACK', 'members': inventory} + (root / 'preparation-receipt.json').write_text(json.dumps(receipt, sort_keys=True, indent=2) + '\n') + PY + - name: Hash retained original members even after a failed attempt + if: always() + run: | + set -euo pipefail + cd "$RUNNER_TEMP/semantic-publish/payload" + find . -type f ! -name ARTIFACT_SHA256SUMS -print0 | sort -z | xargs -0 -r sha256sum > ARTIFACT_SHA256SUMS + - name: Retain original compatibility inputs and failure evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: semantic-runtime-preparation-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/semantic-publish/payload/ + if-no-files-found: error + retention-days: 90 + - name: Release every mounted bounded filesystem strictly + if: always() + run: | + set -euo pipefail + status=0 + for label in models work control publish; do + directory="$RUNNER_TEMP/semantic-$label" + if mountpoint -q "$directory"; then + if ! sudo umount "$directory"; then status=1; fi + fi + done + exit "$status" diff --git a/docs/plans/2026-10-08-semantic-memory-experiment.md b/docs/plans/2026-10-08-semantic-memory-experiment.md new file mode 100644 index 000000000..4186f40d1 --- /dev/null +++ b/docs/plans/2026-10-08-semantic-memory-experiment.md @@ -0,0 +1,195 @@ +# Semantic Memory Experiment Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans for later experiment implementation task-by-task. Steps use checkbox syntax. The current root-assigned three-source amendment implements only the distinct hosted encoder preparation workflow and updates these two plans; it does not implement the sidecar, benchmark or installed integrations. + +**Goal:** Measure whether a pinned, approximately 500-token sqlite-vec sidecar improves source retrieval without weakening freshness, citation, privacy, prompt admission, or the two existing local inference contracts. + +**Architecture:** First run a disposable hosted experiment against frozen public/synthetic text, comparing the existing excerpt mechanism, lexical retrieval, and semantic retrieval. A deterministic coordinator admits sources and prompts; models cannot select tools, sources, destinations, or policy. Only measured compatibility and two distinct actual nonlocal reviews of the exact final candidate can support a later reversible integration into both native entrypoints. + +**Tech stack:** Pinned Linux Python/SQLite/sqlite-vec and separate encoder/generator tokenizer runtimes, disposable hosted Ollama, Qwen3-Embedding:8b for the distinct embedding candidate and the unchanged Qwen3:8b identity for later generation, synthetic UTF-8 fixtures and a stdlib exact cosine oracle. The current preparation acquires only the encoder in an ephemeral hosted cache. No fallback or installed model replacement is part of the candidate. + +**Spec:** [Research architecture design, semantic local retrieval](../superpowers/specs/2026-10-08-research-architecture-design.md), with this plan specifying the bounded experiment for Task 5 of [the architecture plan](../superpowers/plans/2026-10-08-research-architecture-plan.md). + +## Global constraints and observed starting point + +- Root's earlier plan claim6070898124 remains history. The current claimed ADAPT amendment6071864716, renewed6085859861 after owner token restoration6085820307, assigns operations_contract_review exactly the existing preparation workflow and two semantic plans. Standing owner authorization covers these source edits; fresh nonauthor review precedes publication. No sidecar/tests, installed state, local model/product execution, private corpus, DB/config/credential/seal access or worker interruption belongs to this amendment. +- Drive/R17 remains the governing record. Retrieval rank, vector distance, summaries, review receipts and CI success have scientific effect NONE and scientific-status authority false. Preserve original failures and corrections through the existing records rather than creating another acceptance register. +- `deployment/local_agents.py`, in the coordinator checkout, was source-read and rehashed as `1aff2aef9faa8191c07e84f9a2f842e8a49d4d196980fd2d83039c1385f7cd43`: loopback Ollama11434, `qwen3:8b`, model digest `500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41`, context4096, output768, tools unavailable, source text currently excerpted at12000 characters. It exposes `native_infer(role, text, guidance=None)` and retains role/guidance/proposal and cache contracts. +- `/Users/dylanroy/Library/Application Support/OpenClaw-Custodian/custodian/inference.py` was separately source-read and rehashed as `e201b1cb05d33cd8d4eff3b5361e70715cc93a7afa2fee44a2d9ea92d6727b8d`. Its `summarize(evidence, runtime, model)` path enforces the sole loopback route,16384 context,1024 output, no fallbacks, all tools denied, a12000-byte task-message cap, actual input<=12288 tokens, validated evidence projection/source IDs, and before/after model/runtime checks. This is not the4096 worker contract. This plan does not read its config or sealed guidance. +- Parent-verified canonical selected native `model_info` SHA is `471b2f8be50438f109229559fe1d959636725eda569dccd3e59d81c55d7adda0`; template SHA is `ae370d884f108d16e7cc8fd5259ebc5773a0afa6e078b11f4ed7e39a27e0dfc4`. Exact tokenizer/native prompt-count parity is NOT_RUN. The4096 embedding_length field is metadata, not proof that this model implements `/api/embed`. +- Parent-verified Darwin sqlite-vec0.1.9 binary SHA is `193e480c50b59a55977d166f4aaf0e1bc8832d6963516e5950f39e4d2ce0b793`. Its existence does not establish an actual loaded extension, working queries, or Linux ABI compatibility. Linux preparation must resolve and pin its own compatible binary; do not reuse the Darwin binary or infer success from an installed package name. +- Preserve the previous v1 six-call result: baseline0/2 and candidate0/2, with cold-load confounding. The473-byte smoke/register mechanism did not measure true500-token retrieval, usable embeddings, or quality improvement. Do not pool it into this experiment or relabel it positive. +- The last reported installed custody cycle `5c140e44aa3f49f2981df716c5914901`, at22:47, failed StorageAdmissionError:15.78GiB available, below the20GiB reserve. The last reported source index had518 nodes and4 references at09:51 and is stale. These are parent-retained observations, not a fresh installed-state read by this plan author. Do not lower the reserve or retry the failed cycle. Separate healthy-worker observations from this custody failure. +- Root's fresh read-only GET `/api/ps` at23:25:18UTC returned `models:[]` with connect timeout2 seconds/total timeout5 seconds and no model load/inference. Treat the installed model as currently cold at that observation; do not infer worker inactivity, a warm cache, contention-free capacity or successful embeddings. The recorded storage guard remains unresolved, so this plan does not load the roughly5GB installed model or allocate a local index. +- The first actual hosted preparation of the generator candidate failed: native run37863672155, checked commit prefix6081e3, PR333 head prefixcd9, original artifact11587377480, raw archive SHA `b9a3f24e3714ef2a6f3be87666f2621a21a486eb93ce012346543ac862b8d331`. Root verified all65 original payload hashes. One embed request containing two public strings returned HTTP501 with an81-byte unsupported-embeddings error; no vectors were obtained. Actual Linux sqlite-vec v0.1.9 loading and tiny3D cosine query/reopen succeeded. This is scoped extension evidence and a failed generator embedding candidate, not approximately500-token chunking, RAG, quality, native integration or scientific acceptance. +- Root selected the separate ADAPT encoder candidate Qwen3-Embedding:8b: manifest531B SHA `64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b`; config265B SHA `0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89`; model4676804928B SHA `3fcd3febec8b3fd64435204db75bf0dd73b91e8d0661e0331acfe7e7c3120b85`. Only manifest/config/model enter its hosted model cache. The small public metadata was verified as data; full weights were not downloaded locally. No encoder registry template/license/params layers exist. A separate immutable upstream Apache2.0 LICENSE is pinned by [the preparation plan](2026-10-08-semantic-runtime-preparation.md); legal approval and GGUF conversion lineage remain unassessed/unverified. + +### Distinct tokenizer and capability domains + +The [encoder GGUF profile](https://ollama.com/library/qwen3-embedding:8b/blobs/3fcd3febec8b) has width4096, context metadata40960, Q4_K_M, Last pooling, vocabulary axis151665, addBOSfalse, addEOStrue, EOS151643 and EOT151645. The [generator GGUF profile](https://ollama.com/library/qwen3:8b/blobs/a3de86cd1c13) has vocabulary151936 and EOS151645; its addEOS/EOT/pooling fields are not listed and remain unspecified here. Do not infer them as false/zero, assert an unverified numeric pooling enum, or treat a shared qwen3 architecture as tokenizer equivalence. The immutable upstream encoder config's EOS151645 and model-card32k wording are separate provenance from the converted GGUF. Actual usable runtime limits, token pieces and parity require hosted controls. + +Generator counts govern approximately500-token chunk targets, overlap and final W/C prompt admission. Encoder counts independently govern each exact chunk/query input, including its own special tokens and native embedding limit. This preparation uses num_ctx4096 with truncate:false, irrespective of context metadata40960. Two-vector success would not prove either tokenizer's parity. Later runtime locks, chunk/generation/cache identities must bind both domains; generator prompt/template/output contracts remain W4096/768 and C16384/1024, C input12288 and message12000bytes. No encoder count is reused as a generator prompt count, and no generator count admits an encoder input. + +## Review focus + +1. Native token counts can disagree with a superficially matching tokenizer/template. Task2 must prove complete uncached prompt-count parity, including special tokens, both entrypoint profiles and non-ASCII text; a character ratio cannot substitute. +2. Similarity can return a stale, excluded, superseded or inaccessible source. Task4 must enforce admission before scoring and again immediately before prompt publication, including a withdrawal racing a selected result. +3. A rebuild can publish mixed generations or a partially persisted vector table. Task4 must preserve the prior complete generation on every staged failure and prove reopen/restart consistency. +4. A relevant sentence can lie after the current12000-character excerpt or in a confusable older version. Tasks1/5 must freeze deep and version-specific answer spans, preserve original byte offsets, and score source identity separately from lexical relevance. +5. Warm cache speed can conceal model loading, embedding/build costs or peer starvation. Tasks5/6 must separate cache states, charge every attempt, measure both roles under load and prohibit installation from warm speed alone. + +## File and ownership map for later implementation + +No future product file below is created by the current three-source preparation amendment. Each later implementation unit needs active ownership, source review and actual hosted RED before product code. + +| Proposed file | Responsibility | +| --- | --- | +| `experiments/semantic_memory/runtime-lock.json` | Exact Linux/container/Ollama/model/tokenizer/SQLite/extension identities, licenses and measured resource profile; preparation output first, frozen input thereafter. | +| `experiments/semantic_memory/fixtures/corpus-manifest.json`, `corpus/*.txt`, `development-cases.json`, `heldout-cases.json`, `heldout-oracle.json` | Public/synthetic bytes, exact source/revision/extraction identities, eligibility events and frozen queries/gold byte spans. | +| `experiments/semantic_memory/tokenization.py`, `chunking.py` | Pinned token IDs/pieces/counts, exact UTF-8 byte spans and versioned splitter. | +| `experiments/semantic_memory/embedding.py`, `index.py` | Validated real embedding calls, float32 persistence, exact cosine comparison, immutable complete generations. | +| `experiments/semantic_memory/admission.py`, `prompt.py` | Source eligibility, exact lookup/literal fallback, citations, coverage labels, complete prompt/output admission and cache identity. | +| `experiments/semantic_memory/benchmark.py`, `tools/semantic_memory_experiment.py` | Fixed baselines, randomized paired measurement, reports, bounded experiment CLI. | +| `tests/test_semantic_memory_{runtime,chunking,index,admission,prompt,benchmark}.py` | External controls; no import of installed worker code or real private fixture. | +| `.github/workflows/semantic-memory-experiment.yml` | Root-owned future preparation/execution/retention lane; this plan does not amend existing required checks. | +| Coordinator-owned future `deployment/semantic_memory.py` and `deployment/local_agents.py`; custodian-owned future `custodian/retrieval.py` and `custodian/inference.py` | Separate reversible native integrations after experiment and exact-final review, in the owning source repositories rather than by editing installed copies. | + +Future CLI modes are `prepare-report`, `chunk`, `build`, `retrieve`, `benchmark`, and `verify-report`. All consume explicit pinned fixture/runtime manifests; installed/private paths are outside their scope. Metadata inputs are strict UTF-8 JSON, <=4MiB/document and64 container levels, with duplicate keys, nonfinite numbers, unknown wrapper fields and booleans used as integers refused. Corpus access is anchored to the allowlisted immutable fixture directory and rejects symlinks/special files/traversal; queries and corpus annotations cannot select imports, SQL identifiers or destinations. Preparation resolves identities, then freezes them before the capability/quality experiment. A mutable upstream tag is never an experiment identity. Existing source/extraction/export ABI must be reconciled by the local coordinator before an installed adapter is implemented; no arbitrary private DB schema is invented here. + +## Frozen data and interface contracts + +`SourceRecord` carries `{source_id, revision_id, source_sha256, extraction_sha256, extractor_version, text_sha256, text_bytes, content_ref, observed_at, policy_epoch, access_state, exclusion_state, supersession_state}`. In the hosted experiment `content_ref` is a manifest-allowlisted relative fixture file, never a URL to fetch. Source and extraction hashes can differ; offsets always address the exact extracted UTF-8 text bytes named by text_sha256. A source update cannot retain an old revision identity with new bytes. + +`ChunkRecord` carries source identity plus `{chunk_id, start_byte, end_byte, text_sha256, chunk_sha256, token_count, overlap_tokens, tokenizer_identity, encoder_input_token_count, encoder_tokenizer_identity, splitter_version, embedding_identity}`. Here token_count/overlap_tokens/tokenizer_identity refer to the generator domain; encoder_input_token_count includes the separately verified encoder special-token policy. Both tokenizer identities bind their domain, exact model/manifest/GGUF and tokenizer implementation/settings. Spans are half-open original UTF-8 byte intervals and must round-trip exactly. chunk_id hashes these immutable fields. Citation includes source/revision/text hash and byte span. No silent normalization of whitespace, Unicode, line endings or mathematical notation is admitted. + +`GenerationManifest` binds sorted source/chunk inventories, policy epoch, extraction, both tokenizer domains, splitter/encoder/extension identities, embedding dimensions/representation, table count, DB SHA, build statistics and source-admission snapshot. A DB contains one generation. No model-generated provenance fields are admitted. Generation/query caches also bind query bytes, encoder query identity, generator model/template, both tokenizer profiles, role, guidance, context/output/byte limits, selection policy and selected chunk manifest. Encoder or generator identity drift invalidates the relevant complete generation/cache; no old vector is relabelled under a changed encoder. + +`RetrievalRequest` is `{query, exact_source_id, exact_ref, role, task_scope, top_k, expected_generation, admission_epoch, freshness_now}`. Query is bounded to2048 UTF-8 bytes and128 independently counted generator tokens; an actual semantic request additionally needs independent encoder token admission under its frozen native context/special-token profile. top_k<=4. Exact ID/ref lookup is first; a valid exact revision request never falls through to another revision by similarity. task_scope is navigation, focused_excerpt or whole_document; similarity alone cannot establish whole-document coverage. + +`RetrievalResult` is `{generation, policy_epoch, selection_mode, chunks, citations, coverage, unavailable_reasons, scientific_effect:"NONE",scientific_status_authority:false}`. Modes are `exact`, `literal`, `semantic`, or `no_admitted_source`. Source loss removes support and invalidates cached results; no stale text is returned merely because its vector exists. Unknown freshness/access yields an unresolved result, not an affirmative admission. + +`PromptAdmission` binds the exact final serialized request bytes and template rendering, selected evidence/citation manifest, tokenizer count, output reserve and profile identity. It either produces a fully admitted request or a typed refusal. It must not truncate bytes after counting or subtract counts of independently tokenized pieces as though tokenization were additive. + +## Task1: Freeze the corpus, adversarial events and measurement protocol + +**Files:** Fixture files and `tests/test_semantic_memory_benchmark.py`; no installed source changes. + +**Interfaces:** Produce one canonical corpus/case manifest with32 documents, <=384KiB total extracted text, <=128 final chunks and no private source. Produce8 development queries and40 held-out queries:8 each for paraphrase, evidence beyond character12000, confusable revisions, exact ID/ref, and no-answer. Preselect20 generation cases,4 per stratum, for each of the two prompt profiles. Gold span/answer records remain outside indexed text and model requests. + +- [ ] Author synthetic documents and frozen expected source/revision/hash/byte spans before observing semantic rankings. Include math/code, Unicode, tables, negations, same titles, similar statements with different constants and superseded versus current versions. Deep cases place all decisive evidence after the original12000-character boundary; also record its UTF-8 byte position. +- [ ] Add test methods `test_frozen_case_inventory_and_hashes`, `test_gold_never_enters_index_or_prompt`, and `test_deep_and_confusable_spans_are_real`. Require40 unique held-out IDs,8 per stratum,20 preselected generation IDs, exact gold slice hashes and explicit absent-answer gold records. No skip or zero discovery can count as success. +- [ ] Publish tests/fixtures first; execute them in the restricted hosted lane and retain actual RED for the absent harness/validator. Implement only that fixture/inventory validator, rerun exact candidate normal and optimized controls, then source-review its data boundary. +- [ ] Freeze all fixture hashes, evaluation seed `20261008`, query order schedule, baseline definitions and gates. Tune any similarity threshold or abstention rule using the8 development queries only; commit its exact value before held-out execution. Exposure is recorded; this is held-out evaluation, not a claim of blinded review. + +## Task2: Pin hosted runtime and prove the tokenizer contract + +**Files:** `runtime-lock.json`, `tokenization.py`, `tests/test_semantic_memory_runtime.py`; root owns future workflow preparation. + +**Interfaces:** `tokenize_text(raw_utf8, tokenizer_identity) -> TokenizedText` binds an explicit encoder or generator domain with exact IDs/byte pieces; `count_native_prompt(request_bytes, profile, runtime_lock) -> TokenCountReceipt` uses the generator domain only; `count_encoder_input(raw_utf8, encoder_profile, runtime_lock) -> EncoderInputReceipt` admits the exact independently rendered encoder input/special tokens. ProfileW stays4096/768; profileC stays16384/1024 with12000-byte message and12288-input-token caps. + +- [ ] Prepare only public dependencies in a separate trusted hosted stage. Record image/runtime/architecture/RAM/disk, Python/SQLite/compile options, Ollama binary/version, both model manifest/layer/GGUF identities, both complete tokenizer type/vocab/merge/added-token/BOS/EOS/EOT/pretokenizer profiles, actual template bytes and separately pinned tokenizer implementations. Distinguish an absent metadata key from false/zero. Record licenses and exact Linux extension SHA/ABI. The current encoder preparation does not acquire or run the generator or establish the later dual-profile runtime lock. +- [ ] Require encoder manifest64b933... and config/model pins above for embedding; require original generator manifest500a1f... and selected generator model_info/template identities for later generation. Keep separate tokenizer/profile bindings. A changed manifest/tokenizer/quantization/template is another candidate. This explicit ADAPT does not retry the failed generator capability or override installed models; unavailable fixed dependencies yield WATCH. +- [ ] Author external tests `test_actual_token_piece_reconstruction`, `test_full_native_prompt_count_parity`, `test_special_tokens_are_data_in_source_text`, and `test_each_entrypoint_budget_is_distinct`; observe actual hosted RED before implementing the adapter. A pinned GGUF/llama.cpp tokenization path is a candidate implementation; it is not assumed identical to Ollama merely because both support Qwen. +- [ ] In the hosted instance only, calibrate20 complete requests spanning plain text, CJK/combining characters, math/code, whitespace, literal special-token strings, citations, guidance and both profiles. Use one generated token, thinking false and uncached context for count-only probes. Compare the local full rendered prompt count exactly to the native processed input count; if cache accounting or rendering is ambiguous, retain the raw response and refuse parity. Never use character estimates or a constant offset learned from one prompt. +- [ ] Require exact byte-piece reconstruction and all20 count comparisons. Full native token IDs are not advertised as independently verified unless the pinned native runtime actually exposes them; processed token-count parity is the required admission evidence. Resolve native Go-template rendering/special-token behavior in source and controls; a different template engine is not sufficient proof. +- [ ] Author separate encoder controls for byte-piece reconstruction, encoder-specific EOS/EOT/BOS handling, per-input native processed-count parity and truncate:false over-context refusal. Count each calibration input/request within the existing640-vector/400-request cap; no extra generation budget is created. Aggregate prompt_eval_count from the current two-input preparation is recorded statistics, not individual count parity. Unknown tokenizer rendering or any mismatch retains WATCH/refusal before chunk embedding. + +## Task3: True approximately500-token chunks, real embeddings and exact vector queries + +**Files:** `chunking.py`, `embedding.py`, `index.py`, chunk/runtime/index tests. + +**Interfaces:** `split_text(source, tokenized, splitter_identity) -> list[ChunkRecord]`; `embed_texts(texts, embedding_identity, truncate=False) -> EmbeddingReceipt`; `rank_exact_cosine(query_vector, stored_vectors) -> ranked_chunk_ids`. + +- [ ] Author controls `test_chunk_spans_round_trip_and_cover_text`, `test_500_token_and_overlap_bounds`, `test_unicode_token_boundaries_do_not_corrupt_spans`, and `test_model_or_splitter_change_invalidates_chunks`; obtain hosted RED before implementation. Target500 independently counted generator tokens per chunk, nonfinal480–512 and final1–512; target64 generator overlap tokens, permitted56–72 after UTF-8 adjustment. Independently count each exact encoder input and special tokens; refuse oversize rather than silently truncating or changing the500-token definition. Preserve gap-free coverage, advancing starts and original bytes; never rename characters/words as tokens. +- [ ] Implement windowing from native-compatible token pieces and valid UTF-8 boundaries, then retokenize each exact slice independently. If a boundary adjustment cannot satisfy the frozen bounds, refuse/retain the case rather than silently using a different splitter. The complete rendered prompt is counted separately later. +- [ ] Probe the separately pinned Qwen3-Embedding:8b encoder on hosted loopback with truncate:false and one/multiple public strings. Require its exact returned model/manifest identity, one vector per input, frozen candidate width4096, finite values and nonzero norms; validate actual dimensions rather than passing metadata alone. Record native counts/timings and before/after identities. Test over-context refusal, cardinality/dimension drift, NaN/Infinity/zero and unavailable capability. The present preparation makes exactly one two-input request; broader controls remain future work. +- [ ] Preserve the generator candidate's HTTP501 as failed capability. This declared ADAPT encoder requires its own actual hosted success, independent token admission and repeated complete cost/quality experiment. If the encoder is unsupported, stop it too; no automatic alternate download, substitute, retry or paid fallback occurs. +- [ ] Normalize finite nonzero vectors using a declared rule, persist float32 bytes, and compute the independent exact cosine-distance oracle `1-dot(a,b)/(norm(a)*norm(b))` from those same stored float32 values. Explicitly select and test cosine distance in the frozen extension/query implementation; do not compare default L2 distances to cosine scores. Load the exact Linux extension, record `vec_version()`/actual library identity, execute actual insert/query/delete transactions, close/reopen and verify the same vectors/counts. Package presence does not pass this step. +- [ ] Add `test_sqlite_vec_matches_exact_cosine_after_reopen`, `test_vector_dimension_and_nonfinite_refuse`, and `test_ties_sort_by_chunk_identity`. Require top4 exact set/order with deterministic chunk-ID ties and distance error<=1e-5 on all fixtures, including near-ties. If the extension cannot honor tie ordering, obtain all tied candidates and order them explicitly; an approximate or lost result fails this candidate. + +## Task4: Generation promotion, freshness and loss-only support withdrawal + +**Files:** `index.py`, `admission.py`, index/admission tests. + +**Interfaces:** `build_generation(source_manifest, runtime_lock) -> completed_generation`; `publish_generation(completed_generation, expected_current) -> generation_id`; `admit_source(source_record, live_admission, now) -> decision`; `retrieve(request, admitted_generation) -> RetrievalResult`. + +- [ ] Write and observe hosted RED for `test_failed_rebuild_preserves_previous_generation`, `test_restart_never_observes_mixed_manifest_and_db`, `test_compare_and_swap_generation_conflict`, `test_stale_excluded_access_lost_sources_are_never_selected`, and `test_withdrawal_between_selection_and_prompt_refuses`. +- [ ] Build in a private staged generation directory inside the hosted fixture sandbox. Transactionally finish tables; checkpoint/close any WAL, record DB/hash/inventory and fsync completed files/directories before publication. Use one authoritative generation_head row in a separate small coordinator SQLite state DB; `BEGIN IMMEDIATE`, compare expected_current and commit the new generation/manifest/DB identity together. Do not implement compare-and-swap as an unlocked file read followed by replace. Readers read one committed pointer then hold that immutable generation for a request. Before pointer commit, embedding/insert/manifest/fsync failures preserve the old head; interrupted commit admits only the old or new complete generation. A missing acknowledgment is retained as unknown delivery and resolved by head readback, never by pretending a committed generation rolled back. +- [ ] For the hosted fixture policy, freshness TTL is300 seconds with an injected monotonic clock; clock rollback, missing observation, changed source/extraction/revision bytes, exclusion, supersession or access loss refuse admission. Enforce eligibility before exact/literal/semantic selection and again at prompt admission using an unchanged policy epoch. Vector/lexical SQL uses bound query values and an explicit admitted identity set; neither an unsupported vec0 filter nor filtering only after top4 selection can substitute. Native installed policy will remain authoritative later; this experiment's300-second TTL is not a replacement for it. +- [ ] Apply each event individually and in combinations; require zero withdrawn/stale chunk bytes in returned evidence, cached outputs and prompt requests. Invalidate query/result caches by policy epoch and generation. If all sources withdraw, return no_admitted_source. Do not retain stale support through fallback or treat an empty vector lookup as an invitation to fetch a URL. +- [ ] Rebuild after a revision update; measure full and incremental work separately. Verify the old revision remains retained as history but cannot satisfy a current exact reference. A failed rebuild cannot rehabilitate sources withdrawn from the old generation. + +## Task5: Complete prompt admission and fair held-out quality benchmark + +**Files:** `prompt.py`, `benchmark.py`, prompt/benchmark tests. + +**Interfaces:** `admit_prompt(retrieval, task, guidance_identity, profile) -> PromptAdmission`; `run_benchmark(frozen_cases, runtime_lock, schedule) -> BenchmarkReport`. + +- [ ] Write/observe hosted RED for `test_full_prompt_plus_output_reserve`, `test_whole_document_requires_coverage`, `test_exact_ref_wins_and_literal_fallback_remains`, `test_cache_identity_binds_every_consumed_input`, and `test_source_instructions_remain_inert`. Include a source telling the worker to fetch a URL, expose a secret, claim acceptance or overwrite another file; require zero tools/fallback/action and preserve the text as quoted data only. +- [ ] Count the final serialized request including system/role, verified guidance representation, task/query, source delimiters, citations, JSON escaping, template and special tokens. Require W prompt+768<=4096; C prompt+1024<=16384 and C input<=12288 plus its12000-byte task-message cap. Refuse an oversize request, or drop complete low-ranked chunks and recount with an explicit excerpt/coverage label. Never truncate already-cited spans or increase native limits to make retrieval fit. +- [ ] Define three retrieval arms using the same admitted source set and task: E uses the existing12000-character source excerpt, then the same token admission (oversize is a measured refusal); L uses deterministic literal/SQLite FTS5 retrieval with exact-ref-first behavior; S uses exact-ref-first, literal fallback and semantic top4. E's failure to expose deep evidence is measured, not hidden by granting it a longer excerpt. Freeze development-selected thresholds before held-out ranking; no answer oracle enters model requests. +- [ ] Run all40 held-out retrieval cases for every arm; record rank/recall, exact source/revision/span fidelity and no-answer support separately. Run the preselected20 generation cases for every arm under both profiles:120 generation calls maximum, same model/digest, system/task/output schema, temperature0, thinking false and output reserves. Use identical request/guidance bytes apart from the selected evidence. Randomize paired arm order with the frozen seed, alternate profile order, and retain every raw response/refusal. This generic synthetic answer/citation harness measures retrieval-assisted quality; it does not claim to be an installed worker integration. +- [ ] Validate cited source IDs/revisions/hashes/spans exactly and score answers against frozen facts/abstentions. Unsupported claims, wrong-version facts and confident no-answer outputs fail safety. Report failures/refusals in denominators; no successful-only average. Provide per-stratum results and paired differences, not only one aggregate number. +- [ ] Report exact retrieval bytes/tokens, complete prompt tokens, reserved/actual output, cited-support precision, answer score, recall@4, abstention precision, unauthorized request/tool attempts, all call counts and chargeable resource/cost units. Verify caches change on query/chunk/source/generation/model/tokenizer/guidance/profile/role/policy changes. + +## Task6: Cold/warm/build costs, peer fairness and bounded hosted execution + +**Files:** benchmark tests/CLI; root-owned future workflow and retained reports. + +**Interfaces:** Produce original outputs and a canonical report binding the exact code/test/fixture/runtime identities and run/attempt/native checked commit. The report states observed capability and disposition per phase; an absent/failed phase cannot be called PASS. + +- [ ] Prepare on a runner whose observed resources can admit the frozen profile: at least14GiB physical RAM and32GiB free disk before dependency preparation. Record the actual runner, CPU count/flags, architecture, kernel and SQLite/runtime identities. Bound experimental compute to4 CPUs,12GiB RAM,256 PIDs and1GiB temporary storage. Model/dependency cache is a separately bounded10GiB read-only prepared mount; generation output is bounded512MiB and published evidence64MiB. Record both process RSS and actual cgroup peak usage; shared model mappings do not justify hiding a memory-limit failure. Run server and clients in the same isolated network namespace with loopback only, no external interface/route, nonroot user, dropped capabilities/no-new-privileges, read-only prepared source/models and bounded disposable outputs. Trusted dependency preparation is the separate network-enabled stage. No installed credentials, private mounts or persistent local worker access enter the container. +- [ ] Use three separately bounded hosted phases with timeout caps60/80/60 minutes for preparation+capability, build+retrieval, and generation+measurement. Total cap200 minutes; maximum640 input vectors/400 embedding requests and270 generation requests:20 one-token parity,120 quality,120 paired fairness/baseline,6 cold-state calibration, with4 requests reserved for explicit negative controls. Corpus plus one full rebuild consumes at most256 vectors; held-out/warm/quality/fairness queries consume at most260, leaving124 for capability/development/fault controls. Every embedding input/request and generated attempt counts, including failures. No automatic retries after timeout/unknown delivery. Sequential admission stops at the cap and retains WATCH/incomplete evidence rather than silently narrowing the frozen suite. If actual preparation needs more capacity, return measured WATCH and author a distinct revised resource candidate. +- [ ] Measure encoder-cold and generator-cold independently, process/index-cold with declared model state, and fully warm conditions. Charge encoder-to-generator model switching/load costs and observe the existing one-loaded-model limit; no second simultaneously loaded model is silently admitted. A dual-model prepared cache needs measured space admission under the existing10GiB bound. Model-cold requires actual unload/no reused prompt/KV cache; index-cold declares OS page-cache state. No installed worker interruption or OS-cold inference from process restart is admitted. +- [ ] For warm measurements, run three frozen paired schedules of the40 retrieval queries. Query/result caches are disabled for the primary comparison; models/index pages can stay warm. Charge actual query embedding time to S and report separately any additional cache-enabled workload. Baselines receive the same warm state and source-admission checks. Retrieval-only replay may reuse validated corpus vectors, never claim it measures re-embedding cost. +- [ ] Record p50/p95/max wall latency, model load/prompt-eval/generation time, CPU-seconds, peak RSS, model and SQLite residency, DB/model/text/evidence disk bytes, build/rebuild time, embedding tokens/vectors/requests, failure/unknown calls, and runner billing units. Compute break-even query count from measured construction/rebuild cost and per-query gain; if S is slower, no positive break-even is invented. +- [ ] Run a frozen two-role contention schedule on the disposable instance:20 alternating W/C requests plus20 paired arrivals of one W and one C request,60 requests total, concurrency at most2. Use preselected short cases admitted by both profiles and both arms; repeats are deliberate contention measurements, not additional held-out quality cases. Repeat the identical60-request schedule for the non-retrieval baseline with equivalent source/task/output work. Report per-role queue wait, completion latency, refused work and resource peaks; relative to that baseline require no p95 increase>10% and no starvation/timeouts. This is fairness evidence for a candidate scheduler, not proof about live workers. +- [ ] Retain normal/optimized test inventories with exact method IDs, actual native calls, raw fixture outputs, runtime lock, DB/generation hashes and the report in allowlisted regular-file artifacts. Confirm process/container disposal before staging, record manifest hashes, download the original archive and independently verify raw bytes/native identities. Keep all negative attempts. No missing/skipped discovery or count-only acceptance. + +## Measured disposition gates + +The numerical criteria are frozen experiment gates, not universal claims about retrieval or science. Do not change them after seeing held-out results. + +| Disposition | Required evidence and permitted consequence | +| --- | --- | +| REJECT candidate | Any private/unauthorized disclosure or tool/fallback action; any stale/excluded/access-lost support; corrupted/mixed generation; false exact revision/citation; unsupported no-answer generation; nonfinite/dimension mismatch silently admitted; or tokenizer/native prompt parity failure. Preserve artifact and cause; no installation. | +| WATCH | Embedding capability, Linux runtime/extension load, full token parity, complete held-out suite, resource capacity, freshness export ABI or exact-final reviews remain missing/ambiguous; candidate stays uninstalled. Budget exhaustion is incomplete evidence, not a narrowed PASS. | +| ADAPT, opt-in only | All safety/custody/identity controls pass; exact ID/ref8/8; citations100% byte/source/revision valid; generated no-answer subset4/4 abstains under both profiles; other quality gates below are noninferior, but improvement, amortization or fairness is inadequate. Retain literal/default excerpt paths; installation still needs actual technical reviews and native integration evidence. | +| ADOPT narrowly | All prior safety controls; answerable retrieval recall>=29/32; hard paraphrase+deep recall>=14/16 and at least2 more successes than the better E/L arm; no confusable-version or exact-ref regression; generation score>=18/20 under each profile and no loss against the better baseline; zero unsupported answers and complete valid citations; warm complete-task S p95<=1.25x L including query embedding/admission/generation, break-even<=500 queries when a latency gain exists, peak experimental RSS<=12GiB, and fairness gate passes. Raw retrieval latency is also reported without hiding embedding cost. If quality improves but no cost amortization exists, classify ADAPT rather than invent savings. Install only for the measured task/source/profile scope after Task7. | + +For whole-document tasks, the sidecar can provide navigation or explicitly labeled excerpts. ADOPT does not license a partial top4 selection to stand in for complete document review. A default-on decision requires a separately frozen representative workload and ongoing freshness/fallback evidence beyond this small synthetic experiment. + +## Task7: Later actual two-entrypoint integration and installation gate + +**Files:** Coordinator/custodian source repositories and their external tests; this main plan does not edit installed source/config/state. + +**Interfaces:** W keeps `native_infer(role, text, guidance=None)` and its original proposal contract; C keeps `summarize(evidence, runtime, model)` and its original projected-evidence/source-ID contract. A deterministic adapter produces admitted source excerpts/citations before either model call. The model never queries SQLite, selects a path, runs tools or controls source eligibility. + +- [ ] Obtain the owning coordinator's current source/export/admission contract and exact deployment candidate. Recheck source identities only within the active handoff, without reading unrelated DB/config/credentials/seals. Confirm the storage admission issue has been resolved through an independent later cycle with the original20GiB reserve intact; do not retry this plan's recorded failure or label the09:51 index fresh. +- [ ] Author external integration controls before changing either native source: both actual source/extraction revisions and freshness/export identities must bind the retrieval generation; W role/guidance/cache/proposal/4096/768 behavior remains enforced; C privacy projection, source-ID validation,12000-byte/12288-input/16384/1024 limits, model checks, zero-tool accounting and no-fallback route remain enforced. Test concurrent profile use against C's actual post-run requirement of one loaded model with context16384; context switching or multiple runtime instances must not be masked by relaxing that check. Unsupported whole-document, access loss and source instructions must refuse or retain explicit excerpt limits. +- [ ] Observe genuine isolated hosted RED against exact source copies, implement the smallest adapters in the owning source trees, then run exact-candidate normal/optimized and controlled public-source calls through both actual entrypoints. An experimental harness alone is not this evidence. Preserve original request/runtime/response/source/generation receipts and compare legacy literal/excerpt behavior; neither entrypoint may silently gain a hosted route or larger budget. +- [ ] Obtain two distinct actual nonlocal evidence reviews of the exact final code/tests/runtime/fixture/benchmark/integration bytes, including at least one nonauthor reviewer. Retain raw completed reports, provider/model/run identities, authorship/exposure and exact verdicts. Duplicated models/runs, missing traces or source-only intentions do not satisfy the rule; same-provider technical reviews receive no organizational-independence claim. If operational guidance/rules change, satisfy their separate tested rule quorum before deployment; do not edit seals to bypass it. +- [ ] After the installation gate is satisfied, install a reversible, explicitly scoped opt-in sidecar via the coordinator's native deployment mechanism. Verify actual compatible extension loading on Darwin separately, not from the Linux benchmark; run selected public sources first, then only separately admitted selected acquired sources under existing private policy. Preserve no-tools/zero-hosted-fallback and peer fairness. A private live source test remains private; only policy-permitted derived evidence is published. +- [ ] Read back the actual installed source/runtime/generation/request receipts for both entrypoints, test live freshness/access-loss withdrawal and rollback, and append measured scope/disposition to existing Work Events with original evidence hashes. Only this evidence can close the installed semantic-retrieval requirement. Otherwise retain ADAPT/WATCH and the uninstalled/residual scope explicitly. + +## Outstanding determinations and stop conditions + +1. Image/Ollama/Linux extension pins and original tiny3D runtime evidence exist, but this new encoder's loading/usable capacity, complete dual-tokenizer profiles and later dual-model resource admission remain NOT_RUN. Upstream license provenance is separate from legal approval/conversion lineage. No installed Darwin package or mutable default fills these gaps. +2. The original generator embedding capability is a retained HTTP501 failure. The dedicated encoder's actual vectors, normalization, encoder count parity, generator prompt parity and4096D persistence/query behavior remain NOT_RUN for the successor. Metadata or successful tiny3D vectors cannot satisfy them. +3. The native selected-source/extraction/freshness/export ABI and current installed storage admission are separate coordinator-owned evidence. Public fixtures cannot establish private-corpus coverage, availability or current deployment readiness. +4. The original native W/C message limits and validation schemas differ; actual integration must preserve both. If current native templates/guidance add unaccounted tokens, admission stops until exact parity and full budgets are tested. +5. Actual two-distinct-nonlocal exact-final review and any separate rule quorum are future technical gates. Standing owner authorization removes no substantive verification requirement, and this plan introduces no renewed owner-permission queue. + +## Primary mechanism references and limits + +- [Ollama embedding API](https://docs.ollama.com/api/embed) documents batched inputs and truncate:false refusal. This supports the probe shape, not the availability of embeddings for the installed Qwen model. +- [Ollama generation API](https://docs.ollama.com/api/generate) documents template/raw behavior and native token/timing accounting. Actual pinned-runtime response semantics must be checked rather than assumed from current documentation. +- [llama.cpp server tokenizer API](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md#post-tokenize-tokenize-a-given-text) documents token IDs/byte pieces and special-token flags. The live master page is a design reference; preparation pins exact executable source/model bytes and validates parity with native Ollama. +- [sqlite-vec API](https://alexgarcia.xyz/sqlite-vec/api-reference.html) and [vec0 tables](https://alexgarcia.xyz/sqlite-vec/features/vec0.html) describe vector-query primitives. Current documentation may describe a different release; actual frozen0.1.x Linux compatibility, extension loading and exact query/persistence controls are required. +- [SQLite atomic commit](https://www.sqlite.org/atomiccommit.html) describes transaction durability assumptions. The generation pointer/manifest protocol additionally needs tested filesystem fsync/atomic replacement behavior; a SQLite transaction alone does not atomically publish separate files. + +This plan remains a mechanism-to-compatibility-to-controlled-test-to-benchmark-to-disposition sequence. The original preparation's scoped extension positive and unsupported generator embedding result are retained observations; the complete approximately500-token/quality experiment remains NOT_RUN and installed integration NOT_DEPLOYED. The encoder successor is SOURCE_PREPARED/ADAPT for fresh exact three-source nonauthor review. ADAPT here names the changed compatibility candidate, not satisfaction of the later opt-in disposition gates. All eight parent architecture goals remain active; no successful embedding, quality improvement, installation or scientific promotion is asserted. + +## Retained experiment freeze and amendment boundary + +The original accepted experiment source SHA `a9b90c1cc0b8df014d85c93d79e1357e1953a159bdc373affd034bd438fd7446` remains history. The original preparation's corrected workflow/plan freeze, distinct source acceptance, native failed run and original archive are preserved in the preparation plan. This amendment preserves corpus inventories, held-out gates, generation-call budgets, W/C contracts and later actual review/installation requirements while making encoder/generator identities and token admission distinct. It changes only the claimed three sources and does not provide a hosted RED for future experiment product code. The64MiB preparation publication filesystem remains the whole physical bound after successful mount setup; its48MiB collector counter covers attempted original-copy input bytes and can include refused copy attempts, as labelled in the preparation evidence. It excludes prior originals/new metadata and is not a whole-artifact byte claim. diff --git a/docs/plans/2026-10-08-semantic-runtime-preparation.md b/docs/plans/2026-10-08-semantic-runtime-preparation.md new file mode 100644 index 000000000..0c10ce486 --- /dev/null +++ b/docs/plans/2026-10-08-semantic-runtime-preparation.md @@ -0,0 +1,101 @@ +# Semantic Runtime Preparation: Tiny Public Compatibility + +Status: ADAPT candidate / SOURCE_PREPARED. The original generative-model preparation failed its embedding request. The first separately pinned encoder preparation returned two actual4096D vectors, then failed its SQLite persistence transaction. Both original failed attempts remain retained below. This storage-corrected successor has not been published or executed; its amended schema, stored-vector identity checks, bounded database checks, persistence/reopen/query and original hosted readback remain NOT_RUN. The earlier vectors and tiny3D extension positive do not establish successful4096D persistence or complete compatibility. + +Root assigned the original encoder adaptation's workflow, this plan and the [semantic memory experiment](2026-10-08-semantic-memory-experiment.md) to operations_contract_review as source author under adaptation claim 6071864716, renewed by root 6085859861 after owner token restoration 6085820307. Earlier source ownership remains history under 6070431864/6071136915. The experiment's previous accepted freeze was `a9b90c1cc0b8df014d85c93d79e1357e1953a159bdc373affd034bd438fd7446`; those three amended sources required a new exact freeze and distinct nonauthor review. The current storage correction recorded below assigns only the workflow and this plan to graph_tests_review under the existing claim; the experiment stays unchanged. The experiment remains the authority for separate encoder/generator tokenizer parity, approximately 500-token chunks, source admission, complete generations, quality, cold/build costs, fairness and later integration. This preparation acquires only a dedicated encoder in a disposable hosted cache; installed generative Qwen3:8b, worker entrypoints, source index, guidance, state, configuration, credentials and seals are untouched. + +The corresponding workflow is [.github/workflows/semantic-runtime-preparation.yml](../../.github/workflows/semantic-runtime-preparation.yml). It can run from a pull request touching these three paths or an explicit workflow dispatch. Root controls publication, review, execution and the integration queue. A fresh distinct nonauthor source review of all three final sources is required before publication; existing owner authorization covers the work and does not substitute for that review or actual tests. + +## Frozen identities and observations + +| Input | Frozen candidate identity | Preparation disposition | +| --- | --- | --- | +| Ollama server image | `ollama/ollama:0.35.1@sha256:17e945f1aa6e505707eeb621e3f36847d0a4a063d4778f7007645acccfb8bd4f`, Linux amd64 | Root verified the raw OCI manifest hash using primary public registry data. No image pull or execution occurred during source preparation. The four compressed layers total approximately3.78GB; the large layer is3638058746 bytes. | +| Python client image | `python:3.11.14-slim-bookworm@sha256:83f339c1be6340ae1096010fdccf6552ac932d8f410d45d206014916bdf37e48`, Linux amd64 | Previously exercised by the project's restricted hosted Python controls; this workflow must read back its own actual image identity. | +| Encoder manifest | Qwen3-Embedding:8b, 531 bytes; SHA `64b933495768fbd3b87c20583d379728a07471e0c66733a9df87cd1901b3c44b` | Root captured exact public registry bytes in `work/captures/qwen3-embedding-8b-registry-manifest-20261009T0033Z.json`. The tag is only a delivery path; hosted acquisition must match the frozen SHA and length. | +| Encoder model layer | 4676804928 bytes; SHA `3fcd3febec8b3fd64435204db75bf0dd73b91e8d0661e0331acfe7e7c3120b85` | NOT_DOWNLOADED locally during this amendment. Hosted preparation verifies the entire layer before and after the one compatibility attempt. Full weight bytes are NOT_ARCHIVED. | +| Encoder config layer | 265 bytes; SHA `0a3d61b01340ca9dceb4a661e21b3dfe1418ed5206d91291ab933a3762f8bb89` | Root captured original public bytes in `work/captures/qwen3-embedding-8b-config-20261009.json`; exact config content and rootfs model digest must match. | +| Separate immutable upstream license | Qwen/Qwen3-Embedding-8B revision `1d8ad4ca9b3dd8059ad90a75d4983776a23d44af`; LICENSE 11343 bytes; SHA `832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e` | Small original Apache2.0 source metadata independently fetched and rehashed as data. Hosted preparation acquires and retains its own fixed copy. This is not a registry layer; GGUF conversion lineage is NOT_INDEPENDENTLY_VERIFIED and legal approval is NOT_ASSESSED. | +| Linux sqlite-vec wheel | Version0.1.9;163388 bytes; SHA `1515727990b49e79bcaf75fdee2ffc7d461f8b66905013231251f1c8938e7786` | Root verified primary PyPI metadata and the downloaded original wheel. Only six fixed members occur in the pinned wheel. Its metadata declares MIT plus Apache2.0; no LICENSE member occurs; legal approval is NOT_ASSESSED. No package installation or import of its Python wrapper is necessary. | +| Linux extension member | `sqlite_vec/vec0.so`; SHA `5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8` | Root read and hashed the member as data. The original hosted preparation loaded it and passed the tiny3D reopen/query control; loading and SQL execution for this encoder successor remain NOT_RUN. The Darwin extension is not reused. | + +The encoder manifest contains exactly one config descriptor and one model layer. Exactly three cache identities are bound before and after: manifest, config and weights. There are no encoder registry template, license or params layers; the generator's six-file cache topology and template hash are not reused. The runtime keeps fixed registry, immutable upstream LICENSE and PyPI URLs, lengths and hashes. The mutable encoder tag must match the frozen manifest before layer acquisition; any mismatch stops this candidate. No generic ollama pull, latest resolution, model fallback, paid endpoint or retry is admitted. Curl follows at most three HTTPS delivery redirects with every input still bound by its full byte pin. This is a trusted-host input allowlist, not an egress firewall. Image license inventories and legal approval remain NOT_ASSESSED. + +The [official encoder GGUF metadata](https://ollama.com/library/qwen3-embedding:8b/blobs/3fcd3febec8b) declares Q4_K_M, architecture qwen3, vector width4096, context metadata40960, pooling Last, addBOSfalse, addEOStrue, EOS151643, EOT151645 and token-embedding vocabulary axis151665. These fields differ from the [generator metadata](https://ollama.com/library/qwen3:8b/blobs/a3de86cd1c13): vocabulary151936 and EOS151645; addEOS/EOT/pooling are not listed there and are not inferred. The preparation stays at num_ctx4096, regardless of the larger context metadata. It checks actual verbose show fields and retains the complete unchanged before/after template and pooling observations, without asserting an unverified numeric pooling enum or any tokenizer parity. The [immutable upstream source](https://huggingface.co/Qwen/Qwen3-Embedding-8B/tree/1d8ad4ca9b3dd8059ad90a75d4983776a23d44af) is separate provenance: its config uses EOS151645 and model card advertises32k, so those values are not substituted for the converted GGUF profile. + +Primary references are [the official Ollama v0.35.1 Dockerfile](https://github.com/ollama/ollama/blob/v0.35.1/Dockerfile), [its environment configuration](https://github.com/ollama/ollama/blob/v0.35.1/envconfig/config.go), [the official Ollama API schema](https://github.com/ollama/ollama/blob/v0.35.1/docs/openapi.yaml), [sqlite-vec's Python loading guide](https://alexgarcia.xyz/sqlite-vec/python.html), [its vec0 query documentation](https://alexgarcia.xyz/sqlite-vec/features/vec0.html), and [the exact PyPI0.1.9 release metadata](https://pypi.org/pypi/sqlite-vec/0.1.9/json). API documentation describes what to test; actual endpoint responses determine whether this pinned model implements it. + +## Boundary and resource profile + +Preparation begins with actual `/proc/meminfo` RAM and actual free bytes on the disposable hosted runner. Require RAM>=14GiB and free disk>=32GiB before model acquisition, image pulls or filesystem allocation. Retain the observed capacity record even when this admission refuses the attempt. The installed custodian's reported storage rejection is not retried and its20GiB reserve is not lowered; no local model probe or index allocation is part of preparation. + +Four separate ext4 loopback filesystems bound model storage10GiB, scratch/output512MiB, prepared control/extension64MiB, and the entire mounted publication filesystem64MiB. Each mounted root keeps its original root-owned metadata and lost+found; a fresh payload subdirectory alone is assigned and used for staging, binds, permission changes, hashes, collection and upload. Filesystem metadata remains outside the publication payload. Cleanup unmounts the original roots. Models, work and evidence use nodev/nosuid/noexec; control permits executable mappings for the pinned extension and is read-only inside the client. No weights or sandbox outputs enter the checkout; only fixed public data and digest images are provisioned. The collector's48MiB copy admission counter excludes originals already written by provisioning/execution, its collection JSON, the later receipt and hash manifest. It is not a48MiB whole-artifact guarantee. The successfully mounted64MiB filesystem supplies the separate whole-publication physical bound, including logs/metadata; exhaustion is retained failure, not permission to discard evidence or raise limits. Early setup refusal can retain bounded admission records in the original runner staging directory before that filesystem exists; such a failure archive does not prove a mounted publication bound or successful preparation. + +The offline server has UID/GID65532, read-only root, read-only prepared model cache, Docker networknone, no published host port, capabilities dropped, no-new-privileges,3.5CPU,12GiB RAM and192 PIDs. Its temporary filesystem is1GiB with noexec/nosuid/nodev. Explicit Ollama settings bind127.0.0.1:11434, disable cloud and pruning, use one loaded model, one parallel request, queue1, context4096,300-second load-stall bound, and disabled Vulkan/request-body diagnostics. No GPU/device is admitted. Source intent is not actual proof of these restrictions: the receipt checks native Docker observations. + +The client has UID/GID65532, read-only root, read-only prepared control/extension,0.5CPU,512MiB RAM and64 PIDs. It joins only the server's isolated network namespace, receives no proxy environment, has no published port and executes isolated Python with `-I -B -S`. It receives only the fixed compatibility control and its bounded disposable work mount. It receives no checkout, Git credentials, API account keys, private capture or installed path. Its temporary filesystem is64MiB with noexec/nosuid/nodev. The pair's CPU quota totals4 and PID bounds total256; client RAM and temporary space are explicitly additional to the12GiB/1GiB server limits. + +The [official v0.35.1 server entrypoint](https://github.com/ollama/ollama/blob/v0.35.1/cmd/cmd.go#L1968) initializes a local key before serving. This unit never assigns or repurposes HOME. It admits only the fixed image's native `/root` or `/` home case and supplies separate1MiB disposable tmpfs locations at `/root` and `/.ollama`. Any unexpected native home, key initialization failure, read-only model-cache incompatibility or server startup failure ends the attempt. Generated private-key files and all ephemeral home files stay outside collection and are NOT_ARCHIVED. Upstream prints the generated public key in its startup console; that original console is preserved and may contain the public key. The disposable key files are not installed user/provider credentials, and cloud access stays disabled. + +The job has a50-minute wall limit. Fixed small downloads have60-second bounds; the model layer900 seconds; each digest image pull600 seconds. Server/client attachments are bounded900/800 seconds with10-second termination escalation. Each of four pull/attachment logs has a2MiB file limit, separately consuming the64MiB publication filesystem alongside the collector's48MiB copy admission subtotal and other originals/metadata. No whole-artifact48MiB guarantee is asserted. Docker logging is disabled; attachment/log-limit failure refuses a positive receipt. No download, embedding or failed load is retried. Bounded startup observations wait for the created sandbox and do not repeat an embedding attempt or change its model. + +## Actual compatibility controls required + +1. Verify Python3.11.14/Linux/x86_64 and the exact loaded extension SHA. Enable SQLite extension loading only around the one absolute pinned extension path and disable it immediately afterward. Require actual `vec_version()`0.1.9, retain SQLite version and compile options, create a3-dimensional cosine vec0 table with vectors `[1,0,0]` and `[0,1,0]`, close/reopen the DB, and require row IDs1/2 with distances0/1 within1e-6. The fixed v0.1.9 KNN query accepts only `ORDER BY distance`: fetch the complete `k=2` result and apply `(distance,rowid)` tie ordering in Python, after SQL execution. No string-only version report counts as an actual loaded extension or working query. [Pinned upstream planner](https://github.com/asg017/sqlite-vec/blob/v0.1.9/sqlite-vec.c#L5205) +2. Admit only loopback API paths `/api/version`, `/api/tags`, `/api/show`, `/api/embed`, and `/api/ps`; refuse redirects and ignore proxies. Require server0.35.1, exactly one served encoder named qwen3-embedding:8b and its fixed manifest digest. Verify actual verbose show architecture, Q4_K_M details, embedding capability metadata, width4096, context40960, vocabulary151665 and the encoder-specific BOS/EOS/EOT fields above. Require unchanged complete show/template records before/after, with no generator template hash or numeric pooling enum substituted. Capability metadata is an admission check; actual returned vectors are required for success. +3. Make exactly one embedding request with the two public strings `TEST preparation: water boils.` and `TEST preparation: a square has four sides.`, `truncate:false`, context4096, CPU execution and no endpoint/model fallback. Require two returned vectors, each exactly4096 finite numeric components, nonzero norm and unit norm within1e-3; require actual total/load/prompt statistics and positive prompt_eval_count. Store original request, bounded response and HTTP status bytes before semantic response validation. An unsupported or failed endpoint/model response remains a genuine failed attempt. +4. Persist the two actual vectors as little-endian float32 into the fixed `CREATE VIRTUAL TABLE native USING vec0(embedding float[4096] distance_metric=cosine, chunk_size=16)` schema. After commit and again after close/reopen, require exact sqlite_schema.sql equality and exactly row IDs1/2 with the original16,384-byte float32 blobs; retain their lengths/hashes. After each close, use a no-follow/nonblocking regular-file/single-link read with pre/post identity/growth checks to require positive DB size<=4MiB, retain its actual size/SHA, and require unchanged DB bytes across reopen/query. Fetch the complete `k=2` result with the same single `ORDER BY distance`, sort in Python by `(distance,rowid)`, and compare IDs and distances against the unchanged stdlib cosine oracle over the exact persisted float32 values within1e-5. Retain committed/reopened schema, vector and DB identities alongside the original query/oracle fields. This checks two-vector interoperability only. These two public sentences and a query equal to the first vector cannot establish retrieval relevance, chunk quality, held-out performance or model selection. +5. Record native server/client container IDs, image inspections, configured restrictions and client terminal exit state. Stop the server after the attempt; require successful actual client execution and clean server shutdown for a positive receipt. Dispose both sandboxes and confirm their absence before any untrusted output collection. Preserve unsupported responses, partial extension success, original error logs and nonzero exits rather than rewriting them into success. + +## Custody, collection and interpretation + +The collector runs only after both sandboxes are absent. It uses isolated trusted Python, anchored O_DIRECTORY/O_NOFOLLOW traversal, O_NOFOLLOW/O_NONBLOCK opens, pre/post regular-file/single-link/identity checks, fixed member names and per-file limits. Its48MiB copy admission counter excludes prior trusted originals and collection/receipt/hash metadata; it counts input bytes before writing and can include refused or failed copy attempts in negative evidence. collection.json labels those semantics; the receipt explicitly distinguishes this subtotal from whole-publication bytes. Output files are freshly created O_EXCL and runner-owned; sandbox paths cannot select names. Verbose show responses have16MiB limits, other API records and the tiny DB4MiB. The configured64MiB publication filesystem supplies the whole-publication physical bound only after actual mount establishment. The trusted collector records configured_publication_filesystem_capacity_bytes separately from host os.path.ismount observation and actual statvfs reported_total_bytes/device; actual_publication_filesystem is null when the root is not mounted. A positive receipt requires observed mounting and typed actual capacity greater than zero and no more than64MiB, and retains those observations. Interrupted setup preserves original capacity/native records and whichever bounded originals exist without claiming that the configured publication bound was established. + +Retain the encoder manifest and config, the separately sourced upstream LICENSE, Linux wheel/extension/metadata, exact control, all three checked source files, API originals, tiny DB, Docker/capacity observations, exits and missing/refused inventory. After disposal, stream exactly the three prepared encoder files again and match their complete lengths/hashes to the before-attempt records. The4.676GB weights and image layers remain NOT_ARCHIVED; the small artifact is not full dependency raw-byte custody. No missing template/license/params layers are invented or requested. Partial retention cannot produce a success receipt. + +The preparation receipt binds repository, checked commit, run ID and attempt to original member hashes. It verifies the exact Linux amd64 image repo digests and additionally joins every initial/final container `Image` to the corresponding inspected image `Id`, requiring its exact pinned `Config.Image` reference. Both native runtime and configured bind inventories must equal the admitted source/destination/read-write tuples, with no extra bind or inherited volume; each complete tmpfs map must equal the admitted options, including its size limits. The admitted sources are only the prepared model/control/work `payload` directories. Existing resource restrictions, offline namespace, ID and successful terminal checks remain required. It also verifies the original model manifest, unchanged cache identities, loaded extension and two actual embedding vectors. It carries custodyunknown, scientific effectNONE, scientific-status authorityfalse and disposition `REQUIRES_NATIVE_JOB_SUCCESS_AND_ORIGINAL_ARCHIVE_READBACK`. A hash manifest and90-day original artifact are uploaded from the publication `payload` even after failed attempts. Mounted filesystem roots are released strictly after upload; an unmount failure keeps the native job failed. A receipt or upload cannot override native job failure. + +For positive delivery, root must independently read the native job conclusion and its actual checked commit/tree/parents, fetch the original hosted artifact, compare its raw ZIP hash and every retained member, read back the native repository/run/attempt identities, and preserve exact errors or acquisition gaps. Local source checks, reconstructed ZIPs, a rerun from another revision, an artifact ID alone or a Docker image name cannot substitute for that evidence. The source author is not the independent source reviewer or custodian. + +## Explicit remaining scope + +Even a successful native compatibility attempt leaves full prompt-tokenizer parity, the selected native model_info parity hash, approximately500-token chunk construction, corpus/revision identity, admission and withdrawal races, complete atomic generations, exact lookup/literal fallback, held-out retrieval/generation acceptance, cold/load/build cost, p95/fairness and representative-workload evidence NOT_RUN. It makes no change to Housekeeper/UIUX defaults and no scientific promotion. The accepted experiment requires later external tests and genuine hosted RED before product implementation, actual measured gates and two distinct actual nonlocal reviews of the exact final installed candidate, including a nonauthor. This preparation unit cannot satisfy that installation quorum or bypass the coordinator's native contracts. + +## Review correction record + +The prior source freeze, workflow SHA `c4ff430594479f090f5e1c4bcae336849b6988eb2f0f66b382fc6995d81580a4` and plan SHA `6f7bc6ca3cadf8492774189caa0acd56d4e2e8215ab8a61d78d3a3b58022a15f`, received REQUEST_CHANGES from the distinct nonauthor operations-contract reviewer. The four findings were source findings; they were not an executed RED run or evidence of successful runtime compatibility. Root read and accepted that review. This source revision records the corrections without discarding the rejection: + +| Finding | Correction now in source | Verification still required | +| --- | --- | --- | +| P1: ext4 mount roots expose root-owned `lost+found` to recursive permission changes and strict publication inventory. | Use fresh payload subdirectories under all four mounted roots; leave root metadata untouched; bind, hash, collect and upload only payloads; unmount original roots. | Actual hosted setup, permissions, collection and strict unmount. | +| P1: sqlite-vec0.1.9 rejects KNN queries with two ORDER BY terms. | Both queries use only `ORDER BY distance`; sort the complete two-row result in Python by `(distance,rowid)` before comparison. | Actual extension load, both queries and DB reopen. | +| P2: a valid image-inspection record did not prove that the tested containers used it; mount checks admitted unspecified extras. | Join initial/final native container image IDs to inspected image IDs, require exact pinned configured references, exact source/destination/permission bind sets and complete tmpfs maps including limits. | Actual native Docker records and receipt enforcement. | +| P2: saying every generated home key was unarchived conflicted with original startup output containing the public key. | Keep original console unchanged; explicitly exclude private-key/home files while allowing the printed public key to appear in that console. | Original bounded console and closed artifact membership readback. | + +The corrected original freeze, workflow SHA `9c3a08b25d187d3cd923d8cc9a3729b10ede857ccf4b6f85d73cdf77d93c9b6b` and plan SHA `9e36c1ee1fd772428ae02231cdb4e381eb0e006fc46589341de770f354e584c8`, received distinct nonauthor scoped source acceptance before PR333 publication. That review did not establish runtime capability. Organizational independence was0 and remains0. + +## Preserved first hosted failure and distinct adaptation + +The original PR333 candidate at head prefixcd9 ran as native run37863672155, checked commit prefix6081e3, and produced original artifact11587377480. Root read back the original archive, SHA256 `b9a3f24e3714ef2a6f3be87666f2621a21a486eb93ce012346543ac862b8d331`, and verified all65 retained payload hashes. Exactly one embed API request with two public strings returned HTTP501 and an81-byte error stating that the generative model does not support embeddings. No embedding vectors were obtained. The pinned Linux sqlite-vec v0.1.9 was actually loaded, and the tiny3-dimensional cosine table/query succeeded after DB reopen. That positive is scoped extension evidence; it is not native embedding persistence, approximately500-token chunking, RAG, relevance, generation quality or installation evidence. + +The generator candidate's manifest500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41, GGUF a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f and original template/license/params bytes remain in that archived failure history. Its unsupported digest-path GET404 also remains history. Neither failure is retried or relabeled. Root's explicitly claimed ADAPT chooses a distinct dedicated encoder, with different model/tokenizer identity and its own hosted compatibility attempt. The installed generator is unchanged; this source amendment is not an automatic model override or an inferred runtime success. + +This three-source successor is SOURCE_PREPARED for fresh nonauthor review before publication. Preparation source author is operations_contract_review; the metadata-only subagent encoder_metadata_check verified official profiles/upstream source but did not supply runtime or full-source acceptance. Small public LICENSE bytes were fetched and hashed again without saving a new project file; no local Python, Node, YAML parser, project tests, Docker, full weights, inference, Git publication, paid calls or installed/private configuration access occurred. The successor's actual encoder capability, tokenizer parity, approximately500-token chunks, complete experiment and original hosted custody remain NOT_RUN/WATCH. The first original native failure stays a failed result. + +## Retained early-negative custody finding and correction + +The encoder adaptation freeze, workflow SHA `a2b707da5d172ecaac15d42e5045c08176368449c8b7140a5fdad456ac4aca8a`, preparation plan SHA `318659843abe55001b099f0b4ac88fa9b907e6fb12d3c1532186fe90f4ce51d1` and experiment SHA `e4a8f1ab196b95727f44267ccb608925da2ee7f039232a7621b19c952f61f17b`, received REQUEST_CHANGES from distinct nonauthor graph_tests_review for a static custody claim: the always collector can recreate an ordinary publication payload after failed mount setup, but its64MiB field did not distinguish configured capacity from actual established storage. This was a source finding, not an executed capacity failure, hosted RED or successful encoder runtime result. Root authorized the narrow correction within the existing three-source scope; operations_contract_review remains author. + +The corrected collector records configured capacity, actual host mount observation and observed filesystem device/capacity, with null actual identity when unmounted. The positive receipt requires typed observed capacity greater than zero and at most64MiB. The existing48MiB attempted-copy counter and its exclusions are unchanged. The experiment source is unchanged because its existing physical-bound claim already requires successful mount setup. All original generator failure, extension positive, model/profile pins, resource limits, disposal/strict-unmount rules and review history remain retained. The complete corrected three-source freeze requires fresh independent source review before root publication; no source execution, YAML parsing, local Python/Node/DB/Docker/model execution, network request or Git mutation occurred for this correction. Corrected hosted runtime remains NOT_RUN and organizational independence0. + +## Preserved first encoder persistence failure and bounded storage correction + +The configured-versus-observed mount correction received distinct nonauthor graph_tests_review ACCEPT_SCOPED_SOURCE at workflow SHA `d1cf5c13cbc2b10460974ab571131dd857658dd00126cd26d81976aac939812d`, preparation plan SHA `8b9e62949101c054fc3d126354bc571439965e824f06638e993c60c4a94f2a23` and unchanged experiment SHA `e4a8f1ab196b95727f44267ccb608925da2ee7f039232a7621b19c952f61f17b`. That acceptance was source-only. The earlier NOT_RUN statements in that freeze and the retained experiment describe their pre-attempt observations; this appended native failure record supplies the subsequent bounded observations without relabeling either failure. + +The first dedicated encoder attempt, run37968610841/job113949184791, failed. Root retained original artifact11634631531,3,115,550 raw ZIP bytes/SHA `9e1ff96193a0cc3364782efcc659f9a67e8253ecaabc30b47cbae6fff6d70074`, as `work/evidence/semantic-runtime-encoder-failure-37968610841.zip` and verified all65 original payload hashes. Its one two-string embedding request returned HTTP200 with104,421 original response bytes, two4096D finite/nonzero vectors and prompt_eval_count17. The control passed its vector/statistics admission, then raised sqlite3.OperationalError `disk I/O error` at compatibility-control.py139, transaction exit after native table creation140 and the two inserts141. The original DB is77,824 bytes. Native client observations retain file-size soft/hard16,777,216 bytes,512MiB memory, exit1 and OOMKilledfalse; the earlier tiny3D extension/reopen control passed. No reopened4096D query, embedding-result or compatibility-result was produced. The original model before/after and failed-attempt evidence remain retained; no positive preparation receipt overrides the failed job. + +The proximate operating-system errno and extended SQLite I/O error were NOT_RETAINED. Source-only analysis of [exact sqlite-vecv0.1.9](https://raw.githubusercontent.com/asg017/sqlite-vec/v0.1.9/sqlite-vec.c) establishes a necessary storage incompatibility: the default chunk_size1024 preallocates4096*4*1024=16,777,216 vector bytes alone, plus row IDs/validity and SQLite pages/table overhead, so the whole DB cannot fit the16MiB per-file limit. This strongly explains the transaction-exit failure but is an allocation inference, not a measured EFBIG attribution; other generic filesystem/journal I/O errors were not independently excluded. The constructor admits positive multiples of8 up to4096. The narrow amendment uses chunk_size16 for this two-vector fixture, reducing its vector chunk payload to262,144 bytes, with no limit increase. + +Root authorized graph_tests_review as this correction's source author within the existing PR333 workflow/preparation-plan claim; operations_contract_review remains author of the prior encoder adaptation and separate adapter tests. The correction changes only native table allocation and explicit committed/reopened schema, stored-float32, regular DB-size/hash controls and result fields. It preserves4096 dimensions, the same two public strings/one API request, model/dependency pins, oracle/query,16MiB file limit,512MiB work filesystem,512MiB client RAM, all other CPU/PID/time/API/publication limits, native joins, disposal and strict unmount. The experiment stays byte-identical at `e4a8f1ab196b95727f44267ccb608925da2ee7f039232a7621b19c952f61f17b`; its future storage/resource strategy remains subject to separate measured admission. + +This two-source amendment is SOURCE_PREPARED for a distinct complete nonauthor source review before root publication and a new hosted attempt. The correction author does not award its own nonauthor acceptance. No local Python/Node/extension/DB/model/project/test/YAML execution, Git mutation or private/installed access occurred. Corrected hosted persistence/reopen/query, complete compatibility and original custody remain NOT_RUN; tokenizer parity, approximately500-token chunks, experiment quality/cost/fairness and installation remain NOT_RUN/WATCH. Both original generator HTTP501 and encoder I/O failures remain failed historical results. Source/test exposure and organizational independence0 remain explicit; scientific effectNONE, scientific-status authorityfalse and custodyunknown are unchanged.