From 788634def1513e8bc2e7d6acd48228da1b5fcb67 Mon Sep 17 00:00:00 2001 From: charan Date: Fri, 21 Aug 2026 03:43:07 +0530 Subject: [PATCH 1/2] Harden the installer: GPU packages, pinned model checksum, preflight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The installer had two real defects. It minted the API key with `openssl rand`, which Termux does not ship, so a fresh setup failed at the key step; it now uses python's secrets module, which is always present. And it installed only the CPU build, so a fresh phone silently ran everything on the CPU at a quarter of the speed; it now pulls the Vulkan backend and Mesa's turnip driver that put prompt processing on the Adreno GPU. It also installs the RAG pipeline (ragcore plus the three entry points) and the python runtime, runs a preflight (Termux present, aarch64, enough free space) that fails early with a clear message, prints a GPU-detected line when Vulkan comes up, and is idempotent — an existing key is never overwritten. fetch-model.sh now pins and verifies the model's SHA256: it skips the download when the file is already present and correct, and fails loudly on a mismatch instead of leaving a corrupt model in place. Verified against the model on the device — the pinned hash matches and a re-run exits without re-downloading. --- bin/fetch-model.sh | 34 +++++++++++++++++-- install.sh | 82 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 15 deletions(-) diff --git a/bin/fetch-model.sh b/bin/fetch-model.sh index 597d58c..4d44657 100755 --- a/bin/fetch-model.sh +++ b/bin/fetch-model.sh @@ -1,15 +1,43 @@ #!/data/data/com.termux/files/usr/bin/bash -# Download the GGUF model. Resumable — safe to re-run after a dropped connection. -# Runs on the DEVICE (inside Termux). +# Download and verify the chat model (GGUF). Runs on the DEVICE (inside Termux). +# Resumable — safe to re-run after a dropped connection. set -euo pipefail +# Pin the exact build so a rebuild is reproducible. Override URL/DEST/SHA to use +# a different quant or model. URL="${LLM_MODEL_URL:-https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_M.gguf}" DEST="${LLM_MODEL:-$HOME/models/qwen3-4b.gguf}" +SHA256="${LLM_MODEL_SHA256:-3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597}" mkdir -p "$(dirname "$DEST")" termux-wake-lock -# -C - resumes a partial file; --retry-all-errors survives flaky campus Wi-Fi. +verify() { + [ -r "$DEST" ] || return 1 + [ -n "$SHA256" ] || return 0 # nothing to check against + printf '%s %s\n' "$SHA256" "$DEST" | sha256sum -c --status +} + +if verify; then + echo "model already present and verified at $DEST" + exit 0 +fi + +# -C - resumes a partial file; --retry-all-errors survives a flaky connection. +echo "downloading $(basename "$URL") ..." curl -4 -L -C - --retry 999 --retry-delay 5 --retry-all-errors -o "$DEST" "$URL" +if [ -n "$SHA256" ]; then + echo "verifying checksum ..." + if verify; then + echo "checksum OK" + else + echo "CHECKSUM MISMATCH for $DEST" >&2 + echo " expected: $SHA256" >&2 + echo " got: $(sha256sum "$DEST" | cut -d' ' -f1)" >&2 + echo "delete the file and re-run, or set LLM_MODEL_SHA256 if you changed the model." >&2 + exit 1 + fi +fi + echo "model at $DEST ($(du -h "$DEST" | cut -f1))" diff --git a/install.sh b/install.sh index c556416..6ba6eb3 100755 --- a/install.sh +++ b/install.sh @@ -1,28 +1,86 @@ #!/data/data/com.termux/files/usr/bin/bash # One-shot setup. Runs on the DEVICE (inside Termux) on a fresh install. +# Idempotent: safe to re-run — it only fills in what is missing. set -euo pipefail -echo "==> installing packages" +say() { printf '==> %s\n' "$*"; } +die() { printf 'error: %s\n' "$*" >&2; exit 1; } + +# --- preflight ------------------------------------------------------------- +# Fail early with a clear message rather than halfway through a package install. +[ -n "${PREFIX:-}" ] && [ -d "$PREFIX" ] || die "this must run inside Termux (\$PREFIX not set)" +case "$(uname -m)" in + aarch64|arm64) : ;; + *) die "built for aarch64 phones; this device is $(uname -m)" ;; +esac +# The model alone is 2.3 GB; ask for a little headroom on top. +avail_kb=$(df -Pk "$HOME" | awk 'NR==2{print $4}') +[ "${avail_kb:-0}" -ge 3500000 ] || die "need ~3.5 GB free under \$HOME, have $((avail_kb/1024)) MB" + +# --- packages -------------------------------------------------------------- +# llama-cpp is the server. The Vulkan backend plus Mesa's turnip driver are what +# let prompt processing run on the Adreno GPU (4x faster than CPU); without them +# everything silently falls back to CPU. python runs the RAG pipeline and mints +# the API key. See bench/RESULTS.md for why the GPU packages matter. +say "installing packages (this pulls the Vulkan GPU backend)" pkg update -y -pkg install -y llama-cpp openssh tmux termux-api +pkg install -y \ + llama-cpp \ + llama-cpp-backend-vulkan \ + mesa-vulkan-icd-freedreno \ + python \ + openssh \ + tmux \ + termux-api -echo "==> creating directories" -mkdir -p "$HOME/bin" "$HOME/models" "$HOME/.config" "$HOME/.termux/boot" +# --- directories ----------------------------------------------------------- +say "creating directories" +mkdir -p "$HOME/bin" "$HOME/models" "$HOME/.config" "$HOME/.termux/boot" \ + "$HOME/rag/bin" "$HOME/rag/corpus" -echo "==> installing scripts" -install -m 700 bin/llm-server.sh "$HOME/bin/llm-server.sh" -install -m 700 bin/fetch-model.sh "$HOME/bin/fetch-model.sh" -install -m 755 boot/start-lab.sh "$HOME/.termux/boot/start-lab.sh" +# --- scripts --------------------------------------------------------------- +say "installing scripts" +install -m 700 bin/llm-server.sh "$HOME/bin/llm-server.sh" +install -m 700 bin/fetch-model.sh "$HOME/bin/fetch-model.sh" +install -m 755 boot/start-lab.sh "$HOME/.termux/boot/start-lab.sh" +# RAG pipeline: shared core plus the three entry points. +install -m 644 rag/bin/ragcore.py "$HOME/rag/bin/ragcore.py" +install -m 700 rag/bin/rag-embed-server.sh "$HOME/rag/bin/rag-embed-server.sh" +install -m 755 rag/bin/rag-index.py "$HOME/rag/bin/rag-index.py" +install -m 755 rag/bin/rag-ask.py "$HOME/rag/bin/rag-ask.py" +install -m 755 rag/bin/rag-web.py "$HOME/rag/bin/rag-web.py" +# --- API key --------------------------------------------------------------- +# Termux ships no openssl, so the key is minted with python's secrets module, +# which is always present. Never overwrite an existing key on a re-run. KEYFILE="$HOME/.config/llm-api-key" if [ ! -r "$KEYFILE" ]; then - echo "==> generating API key" - openssl rand -hex 24 > "$KEYFILE" + say "generating API key" + python3 -c "import secrets; print(secrets.token_hex(24))" > "$KEYFILE" chmod 600 "$KEYFILE" fi +# --- GPU sanity check ------------------------------------------------------ +# Not fatal: the server still runs on CPU if the GPU is unavailable, just slower. +if command -v vulkaninfo >/dev/null 2>&1; then + if vulkaninfo --summary 2>/dev/null | grep -qi adreno; then + say "GPU detected: $(vulkaninfo --summary 2>/dev/null | grep -i adreno | head -1 | sed 's/^[[:space:]]*//')" + fi +fi + echo -echo "setup done. API key:" +say "setup done. API key (also at $KEYFILE):" cat "$KEYFILE" echo -echo "next: ~/bin/fetch-model.sh then ~/bin/llm-server.sh" +cat <<'NEXT' +next steps: + ~/bin/fetch-model.sh download + verify the chat model (~2.3 GB, resumable) + ~/bin/llm-server.sh start the chat server on :8081 + +for the notes assistant (RAG), additionally: + - put an embedding model at ~/models/nomic-embed.gguf (see rag/README.md) + - put your notes (markdown) under ~/rag/corpus/ + - ~/rag/bin/rag-embed-server.sh then python3 ~/rag/bin/rag-index.py + +everything autostarts on reboot via ~/.termux/boot/start-lab.sh +NEXT From 01a72517122a4a25dec09d2fd3d0050a39f79bcd Mon Sep 17 00:00:00 2001 From: charan Date: Fri, 21 Aug 2026 03:43:39 +0530 Subject: [PATCH 2/2] Add CHANGELOG for the v0.1.0 release --- CHANGELOG.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a94fb13 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,54 @@ +# Changelog + +All notable changes to this project are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project aims to +follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] — 2026-08-21 + +First tagged release: an offline LLM server on an unrooted Android phone, plus a +retrieval-augmented notes assistant, both reachable from a laptop or other +devices on the same network. + +### Added +- **Chat server** — `llama-server` serving an OpenAI-compatible API on `:8081`, + API-key protected, autostarting on boot via Termux:Boot and tmux. +- **Notes assistant (RAG)** — a pipeline that embeds a corpus of markdown notes, + retrieves the most relevant chunks for a question by cosine similarity (pure + standard library, no numpy), and answers from them with cited sources. +- **Browser UI + OpenAI-compatible RAG endpoint** (`rag/bin/rag-web.py`, `:8083`) + — a chat page and a `/v1/chat/completions` endpoint that apply retrieval before + answering and add a `sources` field. Binds LAN-wide behind the bearer token, or + loopback-only with `RAG_WEB_HOST=127.0.0.1`. +- **Shared core** (`rag/bin/ragcore.py`) — chunking, scoring, prompt assembly used + by the indexer, the CLI, and the web server so they cannot drift apart. Vectors + held as pre-normalized float32 arrays (~4.8 MB vs ~38 MB of Python floats). +- **Test suite** (`tests/`) — 60 tests covering the whole path against stand-in + model servers, so they run on a laptop and in CI with no phone and no model + weights. +- **CI** (`.github/workflows/ci.yml`) — pytest on Python 3.11–3.13, ruff, and + shellcheck. +- **Benchmark harness** (`bench/`) — `probe.py` reports live prompt/generation + throughput; `RESULTS.md` records the full matrix and the GPU story. + +### Performance +- **GPU offload** — prompt processing runs on the Adreno GPU through Mesa's turnip + Vulkan driver, taking prompt-eval from **18 to 70 tokens/sec**. Generation runs + on the six pinned performance cores at **12 tokens/sec**. A 605-token retrieval + prompt starts answering in **8.9 s instead of 36 s**. +- Retrieved context is bounded by a character budget rather than a fixed chunk + count, since time-to-first-token is prompt-eval bound. + +### Security +- API key minted with python's `secrets` module (Termux ships no openssl) and + stored `chmod 600`; never committed. +- The notes corpus and built index are gitignored — the index contains full chunk + text, so publishing it would publish the notes. + +### Setup +- `install.sh` is idempotent, runs a preflight (Termux, aarch64, free space), + installs the Vulkan GPU packages, and generates the key if absent. +- `bin/fetch-model.sh` pins and verifies the model's SHA256, skipping the + download when the file is already present and correct. + +[0.1.0]: https://github.com/ankamteja/android-llm-server/releases/tag/v0.1.0