diff --git a/__tests__/e2e/device/visionMmprojMultiModel.e2e.sh b/__tests__/e2e/device/visionMmprojMultiModel.e2e.sh new file mode 100644 index 00000000..35b1d8eb --- /dev/null +++ b/__tests__/e2e/device/visionMmprojMultiModel.e2e.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# On-device e2e: multi-model mmproj invariants (Android=adb, iOS=devicectl log pull). +# +# Two properties the A1 fix guarantees, proven on a REAL device via disk state + the app's debug log +# (un-fakeable — no reliance on UI assertions): +# +# A) QUANT DEDUP — downloading a 2nd quantization of the SAME model must NOT re-download the mmproj. +# One projector serves every quant (mmProjLocalName is quant-independent), and checkMmProjExists +# skips the sidecar when it is already on disk. ASSERT: after the 2nd quant, the model still has +# exactly ONE mmproj file on disk, and the log shows the mmproj was reused, not re-fetched. +# +# B) FAMILY SEPARATION — two different models of the same family (Gemma 4 E2B vs E4B) must each keep +# their OWN mmproj and never mispair. ASSERT: two DISTINCT mmproj files on disk, and each model +# loads with `[WIRE-VISION] … initialized:true` naming ITS OWN mmproj, with no +# "Multimodal support not enabled" anywhere. +# +# PLATFORM: pass PLATFORM=android (adb) or PLATFORM=ios (devicectl). The DOWNLOADS + model LOADS are +# driven by the operator/harness (coordinate taps drift — screenshot-verify); this script owns the +# deterministic ASSERTIONS. Run each phase with PHASE=baseline|dedup|family. +# +# PLATFORM=android bash visionMmprojMultiModel.e2e.sh +set -uo pipefail +PLATFORM="${PLATFORM:-android}" +IOS_UDID="${IOS_UDID:-00008150-000225103CD8C01C}" +PKG=ai.offgridmobile.dev + +models_ls() { + if [ "$PLATFORM" = ios ]; then + xcrun devicectl device info files --device "$IOS_UDID" --domain-type appDataContainer \ + --domain-identifier "$PKG" --subdirectory Documents/models 2>/dev/null | grep -oE '[^ /]+\.gguf' || true + else + export PATH="$PATH:$HOME/Library/Android/sdk/platform-tools" + adb exec-out run-as "$PKG" sh -c 'ls -1 files/models 2>/dev/null' + fi +} + +read_log() { + if [ "$PLATFORM" = ios ]; then + xcrun devicectl device copy from --device "$IOS_UDID" --domain-type appDataContainer \ + --domain-identifier "$PKG" --source Documents/offgrid-debug.log --destination /tmp/mm-e2e-ios.log >/dev/null 2>&1 + cat /tmp/mm-e2e-ios.log 2>/dev/null + else + export PATH="$PATH:$HOME/Library/Android/sdk/platform-tools" + adb exec-out run-as "$PKG" sh -c 'cat files/offgrid-debug.log 2>/dev/null' + fi +} + +# count a model-family's mmproj files on disk (e.g. STEM=gemma-4-e2b) +mmproj_count() { models_ls | grep -i mmproj | grep -ic "$1" || true; } + +case "${1:?phase: baseline|dedup|family}" in + baseline) + echo "[$PLATFORM] gemma-4-e2b mmproj files: $(mmproj_count gemma-4-e2b)" + echo "[$PLATFORM] gemma-4-e4b mmproj files: $(mmproj_count gemma-4-e4b)" + models_ls | tr '\n' ' '; echo ;; + + dedup) # run AFTER downloading a 2nd quant of gemma-4-E2B + N=$(mmproj_count gemma-4-e2b) + WEIGHTS=$(models_ls | grep -vi mmproj | grep -ic gemma-4-e2b) + echo "gemma-4-e2b: weights=$WEIGHTS mmproj=$N" + # the 2nd quant's finalization must report the mmproj already present (not re-downloaded) + REUSE=$(read_log | grep -aiE "mmproj already on disk|mmProjFileExists\":true.*gemma-4-e2b|mmproj.*skip" | tail -1) + if [ "$N" -eq 1 ] && [ "$WEIGHTS" -ge 2 ]; then + echo "PASS(A): 2 gemma-4-E2B quants share ONE mmproj (no re-download). reuse-log: ${REUSE:0:80}" + else + echo "FAIL(A): expected 1 mmproj + >=2 weights, got mmproj=$N weights=$WEIGHTS"; exit 1 + fi ;; + + family) # run AFTER downloading gemma-4-E4B alongside E2B + E2B=$(mmproj_count gemma-4-e2b); E4B=$(mmproj_count gemma-4-e4b) + echo "mmproj files: e2b=$E2B e4b=$E4B" + DUMP=$(read_log) + ERR=$(echo "$DUMP" | grep -aic "multimodal support not enabled") + + # PAIRING proof — the un-fakeable signal is the app's own pairing decision, logged two ways: + # 1. [linkOrphanMmProj] (fires at boot / finalization) prints " — linking ". + # 2. [WIRE-VISION] (fires at model LOAD) prints the model + mmproj it initialized with. + # Family separation holds iff E2B weights pair the e2b mmproj AND E4B weights pair the e4b mmproj + # (NEVER the other family's). A load-triggered WIRE-VISION is nice-to-have but needs a manual + # message-send; the boot-time linkOrphanMmProj line is deterministic, so accept EITHER. + pair_ok() { # $1=weights-family stem $2=required mmproj stem + echo "$DUMP" | grep -aiE "linkOrphanMmProj|WIRE-VISION" | grep -i "$1" | grep -i "$2" | tail -1 + } + mispair() { # $1=weights-family stem $2=FORBIDDEN mmproj stem (the other family's) + echo "$DUMP" | grep -aiE "linkOrphanMmProj|WIRE-VISION" | grep -i "$1" | grep -ic "$2" + } + P_E2B=$(pair_ok "gemma-4-e2b-it-q" "gemma-4-e2b-it-mmproj") + P_E4B=$(pair_ok "gemma-4-e4b-it-q" "gemma-4-e4b-it-mmproj") + X_E2B=$(mispair "gemma-4-e2b-it-q" "gemma-4-e4b-it-mmproj") # E2B must NOT pair e4b mmproj + X_E4B=$(mispair "gemma-4-e4b-it-q" "gemma-4-e2b-it-mmproj") # E4B must NOT pair e2b mmproj + echo "pair e2b->e2b: ${P_E2B:+yes} | pair e4b->e4b: ${P_E4B:+yes} | mispairs: e2b->e4b=$X_E2B e4b->e2b=$X_E4B" + if [ "$E2B" -ge 1 ] && [ "$E4B" -ge 1 ] && [ -n "$P_E2B" ] && [ -n "$P_E4B" ] \ + && [ "$X_E2B" -eq 0 ] && [ "$X_E4B" -eq 0 ] && [ "$ERR" -eq 0 ]; then + echo "PASS(B): distinct mmproj on disk; E2B pairs its OWN mmproj and E4B pairs its OWN mmproj; no cross-pairing, no error" + else + echo "FAIL(B): e2b_files=$E2B e4b_files=$E4B e2b_paired=${P_E2B:+1} e4b_paired=${P_E4B:+1} mispair_e2b=$X_E2B mispair_e4b=$X_E4B errors=$ERR"; exit 1 + fi ;; +esac diff --git a/__tests__/e2e/device/visionModelOnDevice.android.e2e.sh b/__tests__/e2e/device/visionModelOnDevice.android.e2e.sh new file mode 100755 index 00000000..2d602e55 --- /dev/null +++ b/__tests__/e2e/device/visionModelOnDevice.android.e2e.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# On-device e2e (Android, adb): download a vision GGUF, load it, and PROVE multimodal initialised. +# +# This is a REAL device test — no Provit journey engine, just adb driving + the app's own debug log. +# It codifies the flow we hand-drove during the A1 (mmproj) verification so it is repeatable. +# +# Assertion (the mmproj-fix proof): after the model loads, offgrid-debug.log must contain +# [WIRE-VISION] {... "initialized":true, "support":{"vision":true ...}} +# and MUST NOT contain "Multimodal support not enabled". The load-time init is the exact thing the +# fix restores; attaching an image is a bonus, not required to prove the fix. +# +# ⚠️ NAVIGATION RELIABILITY: the taps below are COORDINATE-based (calibrated for 1080x2378). The RN app +# does NOT expose its tab-bar/search/testIDs to `uiautomator`, so text/element-based taps aren't available +# on Android, and blind coordinate sequences DRIFT (a tap can hit a promo link → Chrome, or a permission +# dialog). Treat this script as a RECORDED reference: run it with a human watching (or screenshot-verify +# each step). The ASSERTIONS are hardened (they check the model's own GGUF on disk + a fresh model-named +# [WIRE-VISION] initialized:true), so it can never false-pass even if navigation drifts — it will FAIL +# loudly instead. iOS (WDA) exposes accessibility ids (home-tab/models-tab/…) and is more reliable there. +# +# Prereqs (see rules.md → On-device testing playbook): +# - Debuggable Debug build installed (run-as works → log file readable). +# - Metro running on the branch + `adb reverse tcp:8081 tcp:8081` (Android Debug loads JS from Metro). +# - Device unlocked, Auto-Lock long. Coords below are calibrated for 1080x2378 (OnePlus CPH2707). +# +# Usage: MODEL_QUERY="Qwen3.5-0.8B" bash visionModelOnDevice.android.e2e.sh +set -uo pipefail +export PATH="$PATH:$HOME/Library/Android/sdk/platform-tools" +PKG=ai.offgridmobile.dev +QUERY="${MODEL_QUERY:?set MODEL_QUERY, e.g. Qwen3.5-0.8B or gemma-4-E2B}" +LOG() { adb exec-out run-as "$PKG" cat files/offgrid-debug.log 2>/dev/null; } +TAP() { adb shell input tap "$1" "$2"; sleep "${3:-2}"; } +SHOT() { adb exec-out screencap -p > /tmp/e2e-android.png 2>/dev/null; } + +echo "== reset app to Home ==" +adb shell input keyevent 3; sleep 1 +adb shell monkey -p "$PKG" -c android.intent.category.LAUNCHER 1 >/dev/null 2>&1; sleep 4 +# skip onboarding if present (optional taps) +TAP 949 215 2 # Skip (intro) — no-op if absent +TAP 538 2299 2 # Skip for Now — no-op if absent + +echo "== Models tab -> search '$QUERY' ==" +TAP 757 2245 2 # Models tab (bottom nav, 4th) +TAP 402 740 1 # search field +adb shell input text "$QUERY"; sleep 4 +TAP 538 1000 1 # dismiss keyboard (first tap on result) +TAP 538 1000 3 # open top result detail + +echo "== download the smallest (topmost) file ==" +SHOT +TAP 949 975 4 # download icon on the first (smallest) file row +TAP 538 2217 2 # dismiss any memory-warning "Download Anyway"/OK — no-op if absent + +echo "== wait for download to finalize — assert the model's OWN GGUF lands on disk (not a log grep) ==" +STEM=$(echo "$QUERY" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9') # e.g. qwen3508b +DL_OK=0 +for i in $(seq 1 90); do + ON_DISK=$(adb exec-out run-as "$PKG" sh -c 'ls files/models 2>/dev/null') + # a matching WEIGHTS file (not the mmproj) whose name reduces to our query stem + if echo "$ON_DISK" | grep -vi mmproj | tr -d '[:upper:] .-_' | grep -qi "$STEM"; then + echo "download complete — files: $(echo "$ON_DISK" | tr '\n' ' ')"; DL_OK=1; break + fi + echo " [$i] downloading… (on disk: $(echo "$ON_DISK" | tr '\n' ' '))"; sleep 20 +done +[ "$DL_OK" = 1 ] || { echo "FAIL: $QUERY never landed on disk — navigation/download did not start"; exit 1; } + +echo "== load the model (Home -> Select Model -> first local -> New Chat -> send text) ==" +adb shell input keyevent 3; sleep 1 +adb shell monkey -p "$PKG" -c android.intent.category.LAUNCHER 1 >/dev/null 2>&1; sleep 3 +TAP 750 878 2 # Select Model (Home card) +TAP 538 1374 8 # first local model (the one just downloaded, top of LOCAL MODELS) +TAP 538 743 3 # New Chat +TAP 400 2266 1 # message input +adb shell input text "hello"; sleep 1 +TAP 978 1451 20 # send (keyboard-up position) -> triggers load + [WIRE-VISION] + +echo "== ASSERT: a FRESH [WIRE-VISION] for THIS model, initialised, no error ==" +# Un-fakeable: the WIRE-VISION line must (a) name THIS model's on-disk file, (b) be initialized:true, +# (c) vision:true — a stale line for a different model (e.g. SmolVLM) can never satisfy this. +DUMP=$(LOG) +VISION=$(echo "$DUMP" | grep -aE "WIRE-VISION" | grep -i "$STEM" | tail -1) +echo " $VISION" +if [ -n "$VISION" ] && echo "$VISION" | grep -q '"initialized":true' && echo "$VISION" | grep -q '"vision":true'; then + echo "PASS: $QUERY vision initialised on Android"; exit 0 +else + echo "FAIL: no fresh initialised [WIRE-VISION] for $QUERY (model may not have loaded / wrong model loaded)"; exit 1 +fi