From 809740a671fb553a67d735c5bba08bb3aa0eaa91 Mon Sep 17 00:00:00 2001 From: charan Date: Fri, 21 Aug 2026 02:37:39 +0530 Subject: [PATCH 1/4] Add browser UI and OpenAI-compatible endpoint for the notes assistant The chat server on :8081 answers from model weights alone; retrieval only happened inside rag-ask.py, so anything else pointed at the phone -- curl, a browser, any OpenAI client -- silently got answers that had never seen the notes. This moves retrieval behind an HTTP surface of its own. rag-web.py serves a chat page and an OpenAI-compatible /v1/chat/completions on :8083, both applying retrieval first, and holds the index in memory instead of re-reading it per question. It binds loopback by default and is reached over adb forward, so it is never exposed to the campus network; a bearer token is required if it is ever bound to a routable address. ragcore.py now holds the chunking, scoring and prompt assembly that rag-index, rag-ask and rag-web all share, so the three cannot drift apart. Vectors are kept as pre-normalized float32 arrays, which cuts the resident index from about 38 MB to 4.8 MB -- worth doing on a phone already near its RAM ceiling -- and turns cosine similarity into a plain dot product. Retrieved context is now bounded by a character budget rather than a fixed chunk count. Time to first token is prompt-eval bound at roughly 20 tokens per second, so prompt length is the wait; a fixed top_k let that wait vary with whatever the chunker happened to produce. Tests cover the whole path against stand-in embedding and chat servers, so they run on a laptop and in CI with no phone and no model weights. --- .github/workflows/ci.yml | 53 ++++++ .gitignore | 5 + pyproject.toml | 21 +++ rag/bin/rag-ask.py | 112 ++++++------- rag/bin/rag-index.py | 131 ++++++--------- rag/bin/rag-web.py | 343 +++++++++++++++++++++++++++++++++++++++ rag/bin/ragcore.py | 339 ++++++++++++++++++++++++++++++++++++++ tests/conftest.py | 73 +++++++++ tests/fakes.py | 113 +++++++++++++ tests/test_ragcore.py | 254 +++++++++++++++++++++++++++++ tests/test_web.py | 246 ++++++++++++++++++++++++++++ 11 files changed, 1542 insertions(+), 148 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 pyproject.toml create mode 100644 rag/bin/rag-web.py create mode 100644 rag/bin/ragcore.py create mode 100644 tests/conftest.py create mode 100644 tests/fakes.py create mode 100644 tests/test_ragcore.py create mode 100644 tests/test_web.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a633f3f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + tests: + # The device-side code is plain standard library, so the whole RAG path is + # exercised here against stand-in model servers — no phone, no GGUF weights. + name: tests (python ${{ matrix.python }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: python -m pip install --upgrade pip pytest + - run: python -m pytest tests/ -q + + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip ruff + - name: ruff + run: ruff check . + + shell: + name: shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: shellcheck + run: | + sudo apt-get update -qq && sudo apt-get install -y shellcheck + # Termux scripts use a Termux shebang that shellcheck cannot resolve, + # so the shell dialect is named explicitly. + find . -name '*.sh' -not -path './.git/*' -print0 \ + | xargs -0 -r shellcheck --shell=bash --external-sources diff --git a/.gitignore b/.gitignore index bb70bef..c42c497 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,8 @@ bench/raw/ rag/corpus/ rag/index/ *.jsonl + +# Local Claude Code worktrees +.claude/worktrees/ +__pycache__/ +*.pyc diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..632160f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +# Tooling config only — nothing here is installed on the phone, where the RAG +# scripts run against Termux's system Python with no third-party packages. + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "B", "UP"] +ignore = [ + # ragcore/rag-web must import after sys.path is extended, since the device + # has no package install step. + "E402", +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["B011"] diff --git a/rag/bin/rag-ask.py b/rag/bin/rag-ask.py index 2520c9f..da452d3 100644 --- a/rag/bin/rag-ask.py +++ b/rag/bin/rag-ask.py @@ -6,71 +6,57 @@ Embeds the question, finds the most similar note chunks by cosine similarity (pure Python, no numpy), then asks the chat model to answer using only that retrieved context. Prints the answer followed by the sources it drew from. -""" -import json, math, os, sys, urllib.request - -INDEX = os.path.expanduser(os.environ.get("RAG_INDEX", "~/rag/index.jsonl")) -EMBED_URL = os.environ.get("RAG_EMBED_URL", "http://127.0.0.1:8082/v1/embeddings") -CHAT_URL = os.environ.get("RAG_CHAT_URL", "http://127.0.0.1:8081/v1/chat/completions") -KEYFILE = os.path.expanduser(os.environ.get("LLM_KEYFILE", "~/.config/llm-api-key")) -TOP_K = int(os.environ.get("RAG_TOP_K", "5")) - -SYSTEM = ( - "You are a penetration-testing study assistant for the HTB CPTS exam. " - "Answer using ONLY the provided notes context. Give exact commands and flags. " - "Cite the source path in brackets after each step. " - "If the context does not cover the question, say so plainly instead of guessing." -) - -def embed(text): - req = urllib.request.Request( - EMBED_URL, data=json.dumps({"input": text, "model": "nomic"}).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=60) as r: - return json.load(r)["data"][0]["embedding"] - -def cosine(a, b): - dot = sum(x*y for x, y in zip(a, b)) - na = math.sqrt(sum(x*x for x in a)); nb = math.sqrt(sum(y*y for y in b)) - return dot / (na*nb + 1e-9) -def main(): - if len(sys.argv) < 2: - print('usage: rag-ask.py "your question"', file=sys.stderr); sys.exit(1) - question = " ".join(sys.argv[1:]) - - if not os.path.exists(INDEX): - print(f"no index at {INDEX} — run rag-index.py first", file=sys.stderr); sys.exit(1) - docs = [json.loads(l) for l in open(INDEX)] - - qv = embed(f"search_query: {question}") - ranked = sorted(docs, key=lambda d: cosine(qv, d["vector"]), reverse=True)[:TOP_K] - - context = "\n\n".join( - f"[{d['source']} — {d['heading']}]\n{d['text']}" for d in ranked) - key = open(KEYFILE).read().strip() - payload = { - "messages": [ - {"role": "system", "content": SYSTEM}, - {"role": "user", - "content": f"Notes context:\n\n{context}\n\n---\nQuestion: {question}"}, - ], - "stream": False, "temperature": 0.2, "max_tokens": 768, - } - req = urllib.request.Request( - CHAT_URL, data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json", - "Authorization": f"Bearer {key}"}) - with urllib.request.urlopen(req, timeout=300) as r: - ans = json.load(r)["choices"][0]["message"]["content"] +For a browser window or an OpenAI-compatible endpoint over the same retrieval, +run rag-web.py instead — it keeps the index in memory between questions. +""" +import argparse +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import ragcore # noqa: E402 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("question", nargs="+") + ap.add_argument("--top-k", type=int, default=ragcore.TOP_K) + ap.add_argument("--index", default=None) + ap.add_argument("--no-stream", action="store_true", + help="wait for the whole answer instead of printing tokens") + ap.add_argument("--show-scores", action="store_true", + help="print the similarity score next to each source") + args = ap.parse_args(argv) + question = " ".join(args.question) + + try: + index = ragcore.Index.load(args.index) + hits = ragcore.retrieve(index, question, top_k=args.top_k) + messages, hits = ragcore.build_messages(question, hits) + if args.no_stream: + print(ragcore.chat(messages)) + else: + for token in ragcore.chat(messages, stream=True): + sys.stdout.write(token) + sys.stdout.flush() + print() + except ragcore.RagError as exc: + print(f"rag-ask: {exc}", file=sys.stderr) + return 1 - print(ans.strip()) print("\n--- sources ---") - seen = set() - for d in ranked: - if d["source"] not in seen: - print(f" {d['source']}") - seen.add(d["source"]) + if args.show_scores: + seen = set() + for score, doc in hits: + if doc["source"] not in seen: + seen.add(doc["source"]) + print(f" {score:.3f} {doc['source']}") + else: + for src in ragcore.sources(hits): + print(f" {src}") + return 0 + if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/rag/bin/rag-index.py b/rag/bin/rag-index.py index 5c95e26..7c92987 100644 --- a/rag/bin/rag-index.py +++ b/rag/bin/rag-index.py @@ -9,103 +9,64 @@ Defaults: ~/rag/corpus -> ~/rag/index.jsonl Talks to the embedding server on http://127.0.0.1:8082 (see rag-embed-server.sh). +Chunking and embedding live in ragcore.py, shared with rag-ask.py / rag-web.py. """ -import json, os, re, sys, urllib.request, time +import argparse +import json +import os +import sys +import time -CORPUS = os.path.expanduser(sys.argv[1] if len(sys.argv) > 1 else "~/rag/corpus") -INDEX = os.path.expanduser(sys.argv[2] if len(sys.argv) > 2 else "~/rag/index.jsonl") -EMBED_URL = os.environ.get("RAG_EMBED_URL", "http://127.0.0.1:8082/v1/embeddings") +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import ragcore # noqa: E402 -MAX_WORDS = 350 # target chunk size; larger sections are split at blank lines -MIN_WORDS = 20 # sections smaller than this are merged forward -def breadcrumb(path): - """exploitation/password-attacks/john-the-ripper/README.md - -> 'exploitation > password-attacks > john-the-ripper'""" - rel = os.path.relpath(path, CORPUS) - parts = rel.replace(".md", "").split(os.sep) - if parts and parts[-1].lower() in ("readme", "index"): - parts = parts[:-1] - return " > ".join(p.replace("-", " ") for p in parts) - -def split_sections(text): - """Yield (heading, body) splitting on ### / #### headings and *** rules, - keeping fenced ``` code blocks intact.""" - lines = text.splitlines() - sections, cur_head, cur, in_code = [], "", [], False - def flush(h, buf): - body = "\n".join(buf).strip() - if body: - sections.append((h, body)) - for ln in lines: - if ln.strip().startswith("```"): - in_code = not in_code - cur.append(ln); continue - if not in_code and (re.match(r"^#{1,4}\s+", ln) or ln.strip() == "***"): - flush(cur_head, cur) - cur = [] - cur_head = re.sub(r"^#{1,4}\s+", "", ln).strip() if ln.strip() != "***" else cur_head - continue - cur.append(ln) - flush(cur_head, cur) - return sections +def collect(corpus): + files = [] + for root, _, names in os.walk(corpus): + for name in names: + if name.endswith(".md"): + files.append(os.path.join(root, name)) + return sorted(files) -def pack(sections): - """Merge tiny sections forward, split oversized ones at blank lines.""" - out, buf_h, buf = [], "", [] - def wc(s): return len(s.split()) - for head, body in sections: - if wc(body) < MIN_WORDS and buf: - buf.append(body); continue - if buf: - out.append((buf_h, "\n\n".join(buf))); buf = [] - if wc(body) > MAX_WORDS: - para, acc = body.split("\n\n"), [] - for p in para: - if sum(wc(x) for x in acc) + wc(p) > MAX_WORDS and acc: - out.append((head, "\n\n".join(acc))); acc = [] - acc.append(p) - if acc: out.append((head, "\n\n".join(acc))) - else: - buf_h, buf = head, [body] - if buf: out.append((buf_h, "\n\n".join(buf))) - return out -def embed(text): - text = text[:6000] # ~1800 tokens, under the embed ctx/batch ceiling - req = urllib.request.Request( - EMBED_URL, - data=json.dumps({"input": text, "model": "nomic"}).encode(), - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=120) as r: - return json.load(r)["data"][0]["embedding"] +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("corpus", nargs="?", default="~/rag/corpus") + ap.add_argument("index", nargs="?", default="~/rag/index.jsonl") + args = ap.parse_args(argv) + corpus = os.path.expanduser(args.corpus) + index = os.path.expanduser(args.index) -def main(): - files = [] - for root, _, names in os.walk(CORPUS): - for n in names: - if n.endswith(".md"): - files.append(os.path.join(root, n)) - files.sort() + files = collect(corpus) print(f"corpus: {len(files)} files", flush=True) + if not files: + print(f"no .md files under {corpus}", file=sys.stderr) + return 1 chunks = [] - for f in files: - crumb = breadcrumb(f) - for head, body in pack(split_sections(open(f, encoding="utf-8", errors="ignore").read())): - chunks.append({"source": os.path.relpath(f, CORPUS), - "breadcrumb": crumb, "heading": head, "text": body}) + for path in files: + chunks.extend(ragcore.chunk_file(path, corpus)) print(f"chunks: {len(chunks)}", flush=True) t0 = time.time() - with open(INDEX, "w") as out: - for i, c in enumerate(chunks): - doc = f"search_document: {c['breadcrumb']} > {c['heading']}\n{c['text']}" - c["vector"] = embed(doc) - out.write(json.dumps(c) + "\n") - if (i + 1) % 25 == 0 or i + 1 == len(chunks): - print(f" embedded {i+1}/{len(chunks)}", flush=True) - print(f"done: {len(chunks)} chunks -> {INDEX} in {time.time()-t0:.0f}s", flush=True) + tmp = index + ".partial" + try: + with open(tmp, "w") as out: + for i, chunk in enumerate(chunks): + chunk["vector"] = ragcore.embed(ragcore.document_text(chunk)) + out.write(json.dumps(chunk) + "\n") + if (i + 1) % 25 == 0 or i + 1 == len(chunks): + print(f" embedded {i+1}/{len(chunks)}", flush=True) + except ragcore.RagError as exc: + print(f"rag-index: {exc}", file=sys.stderr) + print(f"partial index left at {tmp}", file=sys.stderr) + return 1 + # Only replace a working index once the whole run succeeded. + os.replace(tmp, index) + print(f"done: {len(chunks)} chunks -> {index} in {time.time()-t0:.0f}s", flush=True) + return 0 + if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/rag/bin/rag-web.py b/rag/bin/rag-web.py new file mode 100644 index 0000000..acb36c6 --- /dev/null +++ b/rag/bin/rag-web.py @@ -0,0 +1,343 @@ +#!/data/data/com.termux/files/usr/bin/env python3 +"""Browser front end and OpenAI-compatible endpoint for the notes assistant. + +Runs on the DEVICE, default 127.0.0.1:8083. The chat server on :8081 answers +from model weights alone; everything here goes through retrieval first, so: + + :8081 raw model, no notes + :8083 same model, your notes retrieved and pasted in first + + GET / the chat page + GET /health {"status":"ok","chunks":N} + POST /ask {"question": "..."} -> SSE token stream + POST /v1/chat/completions OpenAI-compatible, RAG applied automatically + +Reach it from the laptop with: adb forward tcp:8083 tcp:8083 + +The index is loaded once at startup (a few seconds) instead of per question, +which is most of why this answers faster than the CLI. +""" +import argparse +import json +import os +import sys +import time +import traceback +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import ragcore # noqa: E402 + +MAX_BODY = 256 * 1024 + +PAGE = """ + + +CPTS notes assistant + +

CPTS notes assistant

+

Answers from your note chunks — retrieval first, +then the local model.

+
+
+
+
+ +""" + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "rag-web" + + # -- plumbing ---------------------------------------------------------- + def log_message(self, fmt, *args): + sys.stderr.write(f"{self.log_date_time_string()} {fmt % args}\n") + + def _send(self, code, body, ctype="application/json", extra=None): + if isinstance(body, (dict, list)): + body = json.dumps(body) + raw = body.encode() if isinstance(body, str) else body + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(raw))) + for k, v in (extra or {}).items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write(raw) + + def _read_json(self): + length = int(self.headers.get("Content-Length") or 0) + if length <= 0: + raise ValueError("empty request body") + if length > MAX_BODY: + raise ValueError(f"request body over {MAX_BODY} bytes") + return json.loads(self.rfile.read(length).decode("utf-8")) + + def _authorized(self): + """No key needed when bound to loopback; required otherwise.""" + if not self.server.require_key: + return True + sent = self.headers.get("Authorization", "") + return sent.startswith("Bearer ") and sent[7:].strip() == self.server.key + + # -- routes ------------------------------------------------------------ + def do_GET(self): + path = self.path.split("?")[0].rstrip("/") or "/" + if path == "/": + self._send(200, PAGE, "text/html; charset=utf-8") + elif path == "/health": + self._send(200, {"status": "ok", "chunks": len(self.server.index)}) + elif path == "/v1/models": + self._send(200, {"object": "list", "data": [ + {"id": "cpts-notes-rag", "object": "model", "owned_by": "local"}]}) + else: + self._send(404, {"error": "not found"}) + + def do_POST(self): + path = self.path.split("?")[0].rstrip("/") or "/" + if path not in ("/ask", "/v1/chat/completions"): + self._send(404, {"error": "not found"}) + return + if not self._authorized(): + self._send(401, {"error": "missing or bad bearer token"}) + return + try: + payload = self._read_json() + except ValueError as exc: + self._send(400, {"error": str(exc)}) + return + try: + if path == "/ask": + self._handle_ask(payload) + else: + self._handle_openai(payload) + except ragcore.RagError as exc: + self._send(503, {"error": str(exc)}) + except Exception: + traceback.print_exc() + self._send(500, {"error": "internal error, see server log"}) + + # -- browser stream ---------------------------------------------------- + def _handle_ask(self, payload): + question = (payload.get("question") or "").strip() + if not question: + self._send(400, {"error": "no question"}) + return + top_k = int(payload.get("top_k") or self.server.top_k) + + t0 = time.time() + hits = ragcore.retrieve(self.server.index, question, top_k=top_k) + retrieve_ms = int((time.time() - t0) * 1000) + messages, hits = ragcore.build_messages(question, hits) + prompt_chars = len(messages[1]["content"]) + + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-store") + self.end_headers() + + def event(obj): + self.wfile.write(f"data: {json.dumps(obj)}\n\n".encode()) + self.wfile.flush() + + ttft_ms, tokens = None, 0 + try: + for token in ragcore.chat(messages, stream=True, key=self.server.key): + if ttft_ms is None: + ttft_ms = int((time.time() - t0) * 1000) + tokens += 1 + event({"token": token}) + except ragcore.RagError as exc: + event({"error": str(exc)}) + except (BrokenPipeError, ConnectionResetError): + return # browser navigated away mid-answer + total = time.time() - t0 + gen_s = total - (ttft_ms or 0) / 1000 + event({"sources": ragcore.sources(hits), "chunks": len(self.server.index), + "retrieve_ms": retrieve_ms, "ttft_ms": ttft_ms, + "prompt_chars": prompt_chars, "tokens": tokens, + "tok_per_s": round(tokens / gen_s, 1) if gen_s > 0 else None, + "scores": [round(s, 3) for s, _ in hits]}) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + # -- OpenAI-compatible ------------------------------------------------- + def _handle_openai(self, payload): + messages = payload.get("messages") or [] + question = next((m.get("content", "") for m in reversed(messages) + if m.get("role") == "user"), "").strip() + if not question: + self._send(400, {"error": "no user message"}) + return + top_k = int(payload.get("top_k") or self.server.top_k) + hits = ragcore.retrieve(self.server.index, question, top_k=top_k) + rag_messages, hits = ragcore.build_messages(question, hits) + src = ragcore.sources(hits) + + params = {k: payload[k] for k in ("temperature", "max_tokens", "top_p") + if k in payload} + + if not payload.get("stream"): + answer = ragcore.chat(rag_messages, key=self.server.key, **params) + self._send(200, { + "id": f"chatcmpl-rag-{int(time.time()*1000)}", + "object": "chat.completion", "created": int(time.time()), + "model": "cpts-notes-rag", + "choices": [{"index": 0, "finish_reason": "stop", + "message": {"role": "assistant", "content": answer}}], + "sources": src, + }) + return + + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-store") + self.end_headers() + created, cid = int(time.time()), f"chatcmpl-rag-{int(time.time()*1000)}" + + def chunk(delta, finish=None): + body = {"id": cid, "object": "chat.completion.chunk", "created": created, + "model": "cpts-notes-rag", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} + self.wfile.write(f"data: {json.dumps(body)}\n\n".encode()) + self.wfile.flush() + + try: + for token in ragcore.chat(rag_messages, stream=True, key=self.server.key, + **params): + chunk({"content": token}) + except (BrokenPipeError, ConnectionResetError): + return + chunk({}, finish="stop") + self.wfile.write(f"data: {json.dumps({'sources': src})}\n\n".encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + +def build_server(host, port, index, key, top_k): + httpd = ThreadingHTTPServer((host, port), Handler) + httpd.index = index + httpd.key = key + httpd.top_k = top_k + # Loopback is already limited to processes on the phone (and whatever the + # laptop forwards over USB), so no token is asked for there. Any other bind + # address is reachable from the network and must present the API key. + httpd.require_key = host not in ("127.0.0.1", "localhost", "::1") + return httpd + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--host", default=os.environ.get("RAG_WEB_HOST", "127.0.0.1")) + ap.add_argument("--port", type=int, + default=int(os.environ.get("RAG_WEB_PORT", "8083"))) + ap.add_argument("--index", default=None, help="path to index.jsonl") + ap.add_argument("--top-k", type=int, default=ragcore.TOP_K) + args = ap.parse_args(argv) + + t0 = time.time() + try: + index = ragcore.Index.load(args.index) + key = ragcore.api_key() + except ragcore.RagError as exc: + print(f"rag-web: {exc}", file=sys.stderr) + return 1 + print(f"loaded {len(index)} chunks in {time.time()-t0:.1f}s", flush=True) + + httpd = build_server(args.host, args.port, index, key, args.top_k) + if httpd.require_key: + print(f"listening on {args.host}:{args.port} — bearer token REQUIRED " + f"(not loopback)", flush=True) + else: + print(f"listening on http://{args.host}:{args.port} " + f"(loopback only; adb forward tcp:{args.port} tcp:{args.port})", + flush=True) + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rag/bin/ragcore.py b/rag/bin/ragcore.py new file mode 100644 index 0000000..26c8914 --- /dev/null +++ b/rag/bin/ragcore.py @@ -0,0 +1,339 @@ +"""Shared RAG pieces: chunking, index loading, retrieval, prompt assembly. + +Imported by rag-index.py (build), rag-ask.py (CLI) and rag-web.py (browser + +OpenAI-compatible endpoint) so all three agree on how notes are cut up, scored +and handed to the model. Pure standard library — Termux has no numpy. +""" +import json +import math +import os +import re +import urllib.error +import urllib.request +from array import array +from operator import mul + +# --- configuration --------------------------------------------------------- +# Every value is env-overridable so tests can point at a fake server. + +EMBED_URL = os.environ.get("RAG_EMBED_URL", "http://127.0.0.1:8082/v1/embeddings") +CHAT_URL = os.environ.get("RAG_CHAT_URL", "http://127.0.0.1:8081/v1/chat/completions") +INDEX_PATH = os.path.expanduser(os.environ.get("RAG_INDEX", "~/rag/index.jsonl")) +KEYFILE = os.path.expanduser(os.environ.get("LLM_KEYFILE", "~/.config/llm-api-key")) +TOP_K = int(os.environ.get("RAG_TOP_K", "5")) + +MAX_WORDS = 350 # target chunk size; larger sections are split at blank lines +MIN_WORDS = 20 # sections smaller than this are merged forward +EMBED_CHARS = 6000 # ~1800 tokens, under the embed ctx/batch ceiling + +# Time to first token is prompt-eval bound, and on this phone that is the whole +# wait: retrieval takes ~160 ms while prompt eval runs at roughly 20 tokens per +# second. A token is about 4 characters, so every 1000 characters of retrieved +# notes costs ~12 seconds before the answer starts. We therefore pull a generous +# top_k (ranking is cheap) and then spend a fixed character budget on the +# best-scoring chunks, rather than pasting in a fixed number of them and letting +# the prompt size — and the wait — vary with whatever the chunker produced. +# Raise it for more thorough answers, lower it for a faster first token. +CONTEXT_CHARS = int(os.environ.get("RAG_CONTEXT_CHARS", "2000")) +MAX_TOKENS = int(os.environ.get("RAG_MAX_TOKENS", "512")) + +SYSTEM = ( + "You are a penetration-testing study assistant for the HTB CPTS exam. " + "Answer using ONLY the provided notes context. Give exact commands and flags. " + "Cite the source path in brackets after each step. " + "If the context does not cover the question, say so plainly instead of guessing." +) + + +class RagError(RuntimeError): + """Anything the caller should show the user rather than traceback on.""" + + +# --- chunking (used at index time) ----------------------------------------- + +def breadcrumb(path, corpus): + """exploitation/password-attacks/john-the-ripper/README.md + -> 'exploitation > password attacks > john the ripper'""" + rel = os.path.relpath(path, corpus) + parts = rel[:-3].split(os.sep) if rel.endswith(".md") else rel.split(os.sep) + if parts and parts[-1].lower() in ("readme", "index"): + parts = parts[:-1] + return " > ".join(p.replace("-", " ") for p in parts) + + +def split_sections(text): + """Yield (heading, body) splitting on # .. #### headings and *** rules, + keeping fenced ``` code blocks intact.""" + sections, cur_head, cur, in_code = [], "", [], False + + def flush(h, buf): + body = "\n".join(buf).strip() + if body: + sections.append((h, body)) + + for ln in text.splitlines(): + if ln.strip().startswith("```"): + in_code = not in_code + cur.append(ln) + continue + if not in_code and (re.match(r"^#{1,4}\s+", ln) or ln.strip() == "***"): + flush(cur_head, cur) + cur = [] + if ln.strip() != "***": + cur_head = re.sub(r"^#{1,4}\s+", "", ln).strip() + continue + cur.append(ln) + flush(cur_head, cur) + return sections + + +def pack(sections): + """Merge tiny sections forward, split oversized ones at blank lines.""" + out, buf_h, buf = [], "", [] + + def wc(s): + return len(s.split()) + + for head, body in sections: + if wc(body) < MIN_WORDS and buf: + buf.append(body) + continue + if buf: + out.append((buf_h, "\n\n".join(buf))) + buf = [] + if wc(body) > MAX_WORDS: + acc = [] + for p in body.split("\n\n"): + if acc and sum(wc(x) for x in acc) + wc(p) > MAX_WORDS: + out.append((head, "\n\n".join(acc))) + acc = [] + acc.append(p) + if acc: + out.append((head, "\n\n".join(acc))) + else: + buf_h, buf = head, [body] + if buf: + out.append((buf_h, "\n\n".join(buf))) + return out + + +def chunk_file(path, corpus): + """Read one markdown file into index-ready chunk dicts (no vectors yet).""" + with open(path, encoding="utf-8", errors="ignore") as fh: + text = fh.read() + crumb = breadcrumb(path, corpus) + return [ + {"source": os.path.relpath(path, corpus), "breadcrumb": crumb, + "heading": head, "text": body} + for head, body in pack(split_sections(text)) + ] + + +def document_text(chunk): + """The string that gets embedded for a chunk (nomic needs its prefix).""" + return (f"search_document: {chunk['breadcrumb']} > {chunk['heading']}\n" + f"{chunk['text']}")[:EMBED_CHARS] + + +# --- vectors --------------------------------------------------------------- + +def normalize(vec): + """Unit-length float array, so cosine similarity is a plain dot product.""" + norm = math.sqrt(sum(x * x for x in vec)) + if norm == 0: + return array("f", vec) + return array("f", (x / norm for x in vec)) + + +def dot(a, b): + return sum(map(mul, a, b)) + + +def cosine(a, b): + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return sum(map(mul, a, b)) / (na * nb + 1e-9) + + +# --- index ----------------------------------------------------------------- + +class Index: + """The embedded corpus held in memory. + + Vectors are stored pre-normalized as float32 arrays: 1560 chunks x 768 dims + is ~4.8 MB this way versus ~38 MB as lists of Python floats, which matters + on a phone that is already near its RAM ceiling. + """ + + def __init__(self, docs, vectors): + self.docs = docs + self.vectors = vectors + + def __len__(self): + return len(self.docs) + + @classmethod + def load(cls, path=None): + path = os.path.expanduser(path or INDEX_PATH) + if not os.path.exists(path): + raise RagError(f"no index at {path} — run rag-index.py first") + docs, vectors = [], [] + with open(path, encoding="utf-8") as fh: + for lineno, line in enumerate(fh, 1): + line = line.strip() + if not line: + continue + try: + rec = json.loads(line) + vec = rec.pop("vector") + except (ValueError, KeyError) as exc: + raise RagError( + f"{path}:{lineno} is not a valid index record ({exc})") from exc + docs.append(rec) + vectors.append(normalize(vec)) + if not docs: + raise RagError(f"index at {path} is empty") + return cls(docs, vectors) + + def search(self, query_vec, top_k=TOP_K): + """Return [(score, doc)] for the top_k most similar chunks.""" + qv = normalize(query_vec) + scored = ((dot(qv, v), i) for i, v in enumerate(self.vectors)) + best = sorted(scored, reverse=True)[:top_k] + return [(score, self.docs[i]) for score, i in best] + + +# --- talking to the two model servers -------------------------------------- + +def _post(url, payload, headers, timeout): + req = urllib.request.Request( + url, data=json.dumps(payload).encode(), headers=headers) + try: + return urllib.request.urlopen(req, timeout=timeout) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", "replace")[:300] + raise RagError(f"{url} returned {exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise RagError( + f"cannot reach {url}: {exc.reason} — is the server up?") from exc + + +def embed(text, url=None, timeout=120): + """One embedding vector from the local nomic server.""" + resp = _post(url or EMBED_URL, {"input": text[:EMBED_CHARS], "model": "nomic"}, + {"Content-Type": "application/json"}, timeout) + with resp as r: + return json.load(r)["data"][0]["embedding"] + + +def embed_query(question, url=None, timeout=120): + """nomic asymmetric retrieval: questions get a different prefix to notes.""" + return embed(f"search_query: {question}", url=url, timeout=timeout) + + +def api_key(keyfile=None): + path = os.path.expanduser(keyfile or KEYFILE) + try: + with open(path) as fh: + key = fh.read().strip() + except OSError as exc: + raise RagError(f"cannot read API key at {path}: {exc}") from exc + if not key: + raise RagError(f"API key file {path} is empty") + return key + + +def fit_context(hits, budget=None): + """Best-scoring chunks that fit the character budget, best first. + + Returns (kept_hits, context_string). The top hit is always kept even if it + alone exceeds the budget, in which case it is truncated — an answer from a + clipped note beats no answer. + """ + budget = CONTEXT_CHARS if budget is None else budget + kept, blocks, used = [], [], 0 + for score, doc in hits: + block = f"[{doc['source']} — {doc['heading']}]\n{doc['text']}" + if not kept and len(block) > budget: + block = block[:budget] + "\n[...truncated]" + elif used + len(block) > budget: + continue # a later, shorter chunk may still fit + kept.append((score, doc)) + blocks.append(block) + used += len(block) + return kept, "\n\n".join(blocks) + + +def build_context(hits, budget=None): + """The notes block pasted in front of the question.""" + return fit_context(hits, budget)[1] + + +def build_messages(question, hits, system=SYSTEM, budget=None): + """System + user messages for a retrieval-augmented answer. + + Returns (messages, kept_hits) so the caller cites only the notes that + actually made it into the prompt.""" + kept, context = fit_context(hits, budget) + messages = [ + {"role": "system", "content": system}, + {"role": "user", + "content": f"Notes context:\n\n{context}\n\n---\nQuestion: {question}"}, + ] + return messages, kept + + +def sources(hits): + """Unique source paths, best match first.""" + seen, out = set(), [] + for _, doc in hits: + if doc["source"] not in seen: + seen.add(doc["source"]) + out.append(doc["source"]) + return out + + +def retrieve(index, question, top_k=TOP_K, embed_url=None): + """Embed the question and pull the most similar chunks.""" + return index.search(embed_query(question, url=embed_url), top_k=top_k) + + +def chat(messages, stream=False, url=None, key=None, timeout=300, **params): + """Call the chat server. Returns the answer string, or a token iterator + when stream=True.""" + payload = {"messages": messages, "stream": bool(stream), + "temperature": 0.2, "max_tokens": MAX_TOKENS} + payload.update(params) + headers = {"Content-Type": "application/json", + "Authorization": f"Bearer {key or api_key()}"} + resp = _post(url or CHAT_URL, payload, headers, timeout) + if not stream: + with resp as r: + return json.load(r)["choices"][0]["message"]["content"].strip() + return _iter_stream(resp) + + +def _iter_stream(resp): + """Yield content deltas from an OpenAI-style SSE response.""" + with resp as r: + for raw in r: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + return + try: + delta = json.loads(data)["choices"][0].get("delta", {}) + except (ValueError, KeyError, IndexError): + continue + piece = delta.get("content") + if piece: + yield piece + + +def ask(index, question, top_k=TOP_K, stream=False, budget=None, **kw): + """Retrieve then answer. Returns (answer_or_iterator, kept_hits).""" + hits = retrieve(index, question, top_k=top_k) + messages, kept = build_messages(question, hits, budget=budget) + return chat(messages, stream=stream, **kw), kept diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..eefaac1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,73 @@ +"""Shared fixtures. Everything here runs on a laptop or in CI — no phone, no GGUF.""" +import json +import os +import sys + +import pytest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "rag", "bin")) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from fakes import ChatHandler, EmbedHandler, FakeServer, TEST_KEY, fake_vector # noqa: E402 + +import ragcore # noqa: E402 + + +@pytest.fixture +def embed_server(): + with FakeServer(EmbedHandler) as srv: + yield srv + + +@pytest.fixture +def chat_server(): + with FakeServer(ChatHandler) as srv: + yield srv + + +@pytest.fixture +def keyfile(tmp_path): + path = tmp_path / "llm-api-key" + path.write_text(TEST_KEY + "\n") + path.chmod(0o600) + return str(path) + + +CORPUS_CHUNKS = [ + {"source": "recon/smb.md", "breadcrumb": "recon > smb", + "heading": "Enumerate shares", + "text": "smbclient -N -L //TARGET lists shares anonymously."}, + {"source": "recon/snmp.md", "breadcrumb": "recon > snmp", + "heading": "Community strings", + "text": "onesixtyone TARGET wordlist brute forces snmp community strings."}, + {"source": "creds/hashcat.md", "breadcrumb": "creds > hashcat", + "heading": "NTLMv2", + "text": "hashcat -m 5600 hashes rockyou cracks ntlmv2 responses."}, +] + + +@pytest.fixture +def index_file(tmp_path): + """A tiny index.jsonl embedded with the deterministic fake embedder.""" + path = tmp_path / "index.jsonl" + with path.open("w") as fh: + for chunk in CORPUS_CHUNKS: + record = dict(chunk) + record["vector"] = fake_vector(ragcore.document_text(chunk)) + fh.write(json.dumps(record) + "\n") + return str(path) + + +@pytest.fixture +def index(index_file): + return ragcore.Index.load(index_file) + + +@pytest.fixture +def wired(monkeypatch, embed_server, chat_server, keyfile): + """Point ragcore's module-level defaults at the fake servers.""" + monkeypatch.setattr(ragcore, "EMBED_URL", f"{embed_server.url}/v1/embeddings") + monkeypatch.setattr(ragcore, "CHAT_URL", f"{chat_server.url}/v1/chat/completions") + monkeypatch.setattr(ragcore, "KEYFILE", keyfile) + return embed_server, chat_server diff --git a/tests/fakes.py b/tests/fakes.py new file mode 100644 index 0000000..c8995ed --- /dev/null +++ b/tests/fakes.py @@ -0,0 +1,113 @@ +"""Stand-in embedding and chat servers. + +The real ones are llama-server processes holding multi-gigabyte GGUF weights on +a phone. These speak the same HTTP shapes in a few hundred lines so the whole +RAG path can be tested on a laptop, in CI, with no model and no device. +""" +import hashlib +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +DIM = 32 +TEST_KEY = "testkey" + + +def fake_vector(text): + """Deterministic pseudo-embedding: same text always gives the same vector, + and texts sharing words land closer together than unrelated ones.""" + vec = [0.0] * DIM + words = text.lower().replace("\n", " ").split() + for word in words: + digest = hashlib.sha256(word.encode()).digest() + for i in range(DIM): + vec[i] += (digest[i % len(digest)] - 128) / 128.0 + if not words: + vec[0] = 1.0 + return vec + + +class _Base(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def _json_body(self): + length = int(self.headers.get("Content-Length") or 0) + return json.loads(self.rfile.read(length) or b"{}") + + def _send(self, code, obj): + raw = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + +class EmbedHandler(_Base): + def do_POST(self): + body = self._json_body() + self.server.calls.append(body) + self._send(200, {"data": [{"embedding": fake_vector(body.get("input", ""))}]}) + + +class ChatHandler(_Base): + """Echoes back whether it was given retrieved notes, so a test can tell a + RAG answer apart from a bare-model answer.""" + + def do_POST(self): + if self.headers.get("Authorization") != f"Bearer {TEST_KEY}": + self._send(401, {"error": "unauthorized"}) + return + body = self._json_body() + self.server.calls.append(body) + user = body["messages"][-1]["content"] + answer = "SAW_CONTEXT" if "Notes context:" in user else "NO_CONTEXT" + + if not body.get("stream"): + self._send(200, {"choices": [ + {"index": 0, "finish_reason": "stop", + "message": {"role": "assistant", "content": answer}}]}) + return + + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + for piece in (answer[:4], answer[4:]): + chunk = {"choices": [{"delta": {"content": piece}}]} + self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode()) + self.wfile.flush() + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + +class FakeServer: + """Context manager running one handler on an ephemeral port.""" + + def __init__(self, handler): + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.httpd.calls = [] + self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + + @property + def port(self): + return self.httpd.server_address[1] + + @property + def url(self): + return f"http://127.0.0.1:{self.port}" + + @property + def calls(self): + return self.httpd.calls + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *exc): + self.httpd.shutdown() + self.httpd.server_close() + self.thread.join(timeout=5) diff --git a/tests/test_ragcore.py b/tests/test_ragcore.py new file mode 100644 index 0000000..2c3196b --- /dev/null +++ b/tests/test_ragcore.py @@ -0,0 +1,254 @@ +"""Chunking, vector maths, retrieval and prompt assembly.""" +import math +import os + +import pytest + +import ragcore +from fakes import fake_vector + + +# --- chunking -------------------------------------------------------------- + +def test_split_sections_keeps_code_fences_whole(): + text = ( + "# Recon\n" + "Start here.\n" + "```bash\n" + "# this hash is a comment, not a heading\n" + "nmap -sC -sV target\n" + "```\n" + "## Next\n" + "More.\n" + ) + sections = dict(ragcore.split_sections(text)) + assert "nmap -sC -sV target" in sections["Recon"] + # The '#' inside the fence must not have started a new section. + assert "this hash is a comment" in sections["Recon"] + assert sections["Next"].strip() == "More." + + +def test_split_sections_horizontal_rule_keeps_heading(): + text = "## Tools\nfirst\n***\nsecond\n" + sections = ragcore.split_sections(text) + assert [h for h, _ in sections] == ["Tools", "Tools"] + assert [b.strip() for _, b in sections] == ["first", "second"] + + +def test_pack_merges_tiny_sections_forward(): + sections = [("Big", "word " * 30), ("Tiny", "short")] + packed = ragcore.pack(sections) + assert len(packed) == 1 + assert packed[0][0] == "Big" + assert "short" in packed[0][1] + + +def test_pack_splits_oversized_sections(): + para = "word " * 200 + packed = ragcore.pack([("Huge", f"{para}\n\n{para}\n\n{para}")]) + assert len(packed) > 1 + assert all(h == "Huge" for h, _ in packed) + assert all(len(body.split()) <= ragcore.MAX_WORDS * 1.5 for _, body in packed) + + +def test_breadcrumb_drops_readme_and_dashes(): + crumb = ragcore.breadcrumb("/c/exploitation/password-attacks/README.md", "/c") + assert crumb == "exploitation > password attacks" + + +def test_chunk_file_records_relative_source(tmp_path): + sub = tmp_path / "recon" + sub.mkdir() + (sub / "smb.md").write_text("## Shares\nsmbclient -N -L //TARGET\n") + chunks = ragcore.chunk_file(str(sub / "smb.md"), str(tmp_path)) + assert chunks[0]["source"] == os.path.join("recon", "smb.md") + assert chunks[0]["heading"] == "Shares" + assert chunks[0]["breadcrumb"] == "recon > smb" + + +def test_document_text_carries_nomic_prefix_and_is_clipped(): + chunk = {"breadcrumb": "a > b", "heading": "H", "text": "x" * 20000} + doc = ragcore.document_text(chunk) + assert doc.startswith("search_document: a > b > H") + assert len(doc) == ragcore.EMBED_CHARS + + +# --- vectors --------------------------------------------------------------- + +def test_normalize_gives_unit_length(): + unit = ragcore.normalize([3.0, 4.0]) + assert math.isclose(math.sqrt(sum(x * x for x in unit)), 1.0, rel_tol=1e-6) + + +def test_cosine_known_values(): + assert math.isclose(ragcore.cosine([1, 0], [1, 0]), 1.0, rel_tol=1e-6) + assert math.isclose(ragcore.cosine([1, 0], [0, 1]), 0.0, abs_tol=1e-6) + assert math.isclose(ragcore.cosine([1, 0], [-1, 0]), -1.0, rel_tol=1e-6) + assert math.isclose(ragcore.cosine([1, 1], [2, 2]), 1.0, rel_tol=1e-6) + + +def test_dot_of_normalized_matches_cosine(): + a, b = [0.3, -1.2, 4.0, 0.5], [2.0, 1.0, -0.5, 3.0] + assert math.isclose(ragcore.dot(ragcore.normalize(a), ragcore.normalize(b)), + ragcore.cosine(a, b), abs_tol=1e-6) + + +def test_normalize_survives_zero_vector(): + assert list(ragcore.normalize([0.0, 0.0])) == [0.0, 0.0] + + +# --- index ----------------------------------------------------------------- + +def test_index_load_counts_chunks(index): + assert len(index) == 3 + assert {d["source"] for d in index.docs} == { + "recon/smb.md", "recon/snmp.md", "creds/hashcat.md"} + + +def test_index_load_strips_vectors_from_docs(index): + assert all("vector" not in doc for doc in index.docs) + + +def test_index_load_missing_file_raises(tmp_path): + with pytest.raises(ragcore.RagError, match="no index at"): + ragcore.Index.load(str(tmp_path / "nope.jsonl")) + + +def test_index_load_empty_file_raises(tmp_path): + path = tmp_path / "empty.jsonl" + path.write_text("") + with pytest.raises(ragcore.RagError, match="empty"): + ragcore.Index.load(str(path)) + + +def test_index_load_corrupt_line_raises(tmp_path): + path = tmp_path / "bad.jsonl" + path.write_text('{"source":"a","text":"b"}\n') # no vector key + with pytest.raises(ragcore.RagError, match="not a valid index record"): + ragcore.Index.load(str(path)) + + +def test_search_ranks_the_matching_note_first(index): + hits = index.search(fake_vector("search_query: how do I enumerate smb shares"), + top_k=3) + assert hits[0][1]["source"] == "recon/smb.md" + assert [s for s, _ in hits] == sorted((s for s, _ in hits), reverse=True) + + +def test_search_respects_top_k(index): + assert len(index.search(fake_vector("anything"), top_k=2)) == 2 + + +def test_search_scores_are_cosines_in_range(index): + for score, _ in index.search(fake_vector("smb shares"), top_k=3): + assert -1.0001 <= score <= 1.0001 + + +# --- prompt assembly ------------------------------------------------------- + +def test_build_messages_puts_notes_before_question(index): + hits = index.search(fake_vector("smb shares"), top_k=2) + messages, kept = ragcore.build_messages("how do I enumerate smb?", hits) + assert len(kept) == 2 + assert messages[0]["role"] == "system" + assert "ONLY the provided notes context" in messages[0]["content"] + user = messages[1]["content"] + assert user.index("Notes context:") < user.index("Question: ") + assert "[recon/smb.md — Enumerate shares]" in user + + +def test_sources_are_unique_and_ordered(index): + hits = index.search(fake_vector("smb shares"), top_k=3) + duplicated = hits + hits + assert ragcore.sources(duplicated) == ragcore.sources(hits) + assert ragcore.sources(hits)[0] == hits[0][1]["source"] + + +# --- server calls ---------------------------------------------------------- + +def test_embed_query_uses_the_search_query_prefix(wired, index): + embed_server, _ = wired + ragcore.embed_query("how do I enumerate smb?") + assert embed_server.calls[-1]["input"].startswith("search_query: ") + + +def test_api_key_reads_and_strips(keyfile): + assert ragcore.api_key(keyfile) == "testkey" + + +def test_api_key_missing_file_raises(tmp_path): + with pytest.raises(ragcore.RagError, match="cannot read API key"): + ragcore.api_key(str(tmp_path / "absent")) + + +def test_api_key_empty_file_raises(tmp_path): + path = tmp_path / "blank" + path.write_text(" \n") + with pytest.raises(ragcore.RagError, match="is empty"): + ragcore.api_key(str(path)) + + +def test_chat_rejects_a_bad_key(wired): + with pytest.raises(ragcore.RagError, match="401"): + ragcore.chat([{"role": "user", "content": "hi"}], key="wrong") + + +def test_chat_unreachable_server_gives_a_readable_error(): + with pytest.raises(ragcore.RagError, match="cannot reach"): + # Port 1 is never listening, and the message must not be a traceback. + ragcore.chat([{"role": "user", "content": "hi"}], + url="http://127.0.0.1:1/v1/chat/completions", key="k") + + +def test_ask_sends_retrieved_notes_to_the_model(wired, index): + _, chat_server = wired + answer, hits = ragcore.ask(index, "how do I enumerate smb shares?") + assert answer == "SAW_CONTEXT" + assert hits[0][1]["source"] == "recon/smb.md" + sent = chat_server.calls[-1]["messages"][-1]["content"] + assert "smbclient -N -L //TARGET" in sent + + +def test_streaming_yields_the_same_answer(wired, index): + stream, _ = ragcore.ask(index, "smb shares", stream=True) + assert "".join(stream) == "SAW_CONTEXT" + + +# --- context budget -------------------------------------------------------- + +def test_fit_context_stops_at_the_budget(index): + hits = index.search(fake_vector("smb shares"), top_k=3) + kept, context = ragcore.fit_context(hits, budget=90) + assert len(kept) < len(hits) + assert len(context) <= 90 + + +def test_fit_context_always_keeps_the_best_hit_even_if_oversized(index): + hits = index.search(fake_vector("smb shares"), top_k=3) + kept, context = ragcore.fit_context(hits, budget=10) + assert len(kept) == 1 + assert kept[0] == hits[0] + assert context.endswith("[...truncated]") + + +def test_fit_context_keeps_best_first(index): + hits = index.search(fake_vector("smb shares"), top_k=3) + kept, _ = ragcore.fit_context(hits, budget=10_000) + assert [s for s, _ in kept] == sorted((s for s, _ in kept), reverse=True) + + +def test_build_messages_cites_only_what_was_sent(index): + hits = index.search(fake_vector("smb shares"), top_k=3) + messages, kept = ragcore.build_messages("q", hits, budget=90) + user = messages[1]["content"] + for _, doc in kept: + assert doc["source"] in user + dropped = [doc for _, doc in hits if (0, doc) not in [(0, d) for _, d in kept]] + for doc in dropped: + assert f"[{doc['source']} — {doc['heading']}]" not in user + + +def test_max_tokens_comes_from_config(wired, index): + _, chat_server = wired + ragcore.ask(index, "smb shares") + assert chat_server.calls[-1]["max_tokens"] == ragcore.MAX_TOKENS diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 0000000..330872a --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,246 @@ +"""The HTTP surface of rag-web.py: browser page, SSE stream, OpenAI endpoint, auth.""" +import importlib.util +import json +import os +import threading +import urllib.error +import urllib.request + +import pytest + +import ragcore +from fakes import TEST_KEY + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _load_rag_web(): + """rag-web.py has a dash in its name, so it needs a manual import.""" + path = os.path.join(REPO, "rag", "bin", "rag-web.py") + spec = importlib.util.spec_from_file_location("rag_web", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rag_web = _load_rag_web() + + +@pytest.fixture +def web(wired, index): + """rag-web bound to an ephemeral loopback port, wired to the fake models.""" + httpd = rag_web.build_server("127.0.0.1", 0, index, TEST_KEY, ragcore.TOP_K) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + httpd.url = f"http://127.0.0.1:{httpd.server_address[1]}" + try: + yield httpd + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + +def get(url, **kw): + return urllib.request.urlopen(url, timeout=15, **kw) + + +def post(url, obj, headers=None): + req = urllib.request.Request( + url, data=json.dumps(obj).encode(), + headers={"Content-Type": "application/json", **(headers or {})}) + return urllib.request.urlopen(req, timeout=30) + + +def sse_events(resp): + """Parse an SSE body into the list of decoded JSON events.""" + events = [] + for line in resp.read().decode().splitlines(): + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + continue + events.append(json.loads(payload)) + return events + + +# --- static surface -------------------------------------------------------- + +def test_index_page_is_served(web): + resp = get(web.url + "/") + body = resp.read().decode() + assert resp.status == 200 + assert resp.headers["Content-Type"].startswith("text/html") + assert "CPTS notes assistant" in body + + +def test_health_reports_chunk_count(web): + body = json.load(get(web.url + "/health")) + assert body == {"status": "ok", "chunks": 3} + + +def test_models_endpoint_names_the_rag_model(web): + body = json.load(get(web.url + "/v1/models")) + assert body["data"][0]["id"] == "cpts-notes-rag" + + +def test_unknown_path_is_404(web): + with pytest.raises(urllib.error.HTTPError) as exc: + get(web.url + "/nope") + assert exc.value.code == 404 + + +# --- /ask ------------------------------------------------------------------ + +def test_ask_streams_tokens_then_sources(web): + events = sse_events(post(web.url + "/ask", + {"question": "how do I enumerate smb shares?"})) + answer = "".join(e["token"] for e in events if "token" in e) + assert answer == "SAW_CONTEXT" + final = events[-1] + assert final["sources"][0] == "recon/smb.md" + assert final["chunks"] == 3 + + +def test_ask_reports_timing_telemetry(web): + final = sse_events(post(web.url + "/ask", {"question": "smb shares"}))[-1] + for field in ("retrieve_ms", "ttft_ms", "prompt_chars", "tokens", "scores"): + assert field in final, field + assert final["ttft_ms"] >= final["retrieve_ms"] + assert final["prompt_chars"] > 0 + + +def test_ask_actually_retrieves_before_asking(web, wired): + _, chat_server = wired + post(web.url + "/ask", {"question": "how do I enumerate smb shares?"}).read() + sent = chat_server.calls[-1]["messages"][-1]["content"] + assert "Notes context:" in sent + assert "smbclient -N -L //TARGET" in sent + + +def test_ask_rejects_an_empty_question(web): + with pytest.raises(urllib.error.HTTPError) as exc: + post(web.url + "/ask", {"question": " "}) + assert exc.value.code == 400 + + +def test_ask_rejects_malformed_json(web): + req = urllib.request.Request(web.url + "/ask", data=b"{not json", + headers={"Content-Type": "application/json"}) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(req, timeout=15) + assert exc.value.code == 400 + + +def test_ask_honours_top_k(web, wired): + embed_server, chat_server = wired + post(web.url + "/ask", {"question": "smb", "top_k": 1}).read() + sent = chat_server.calls[-1]["messages"][-1]["content"] + assert sent.count("[recon/") + sent.count("[creds/") == 1 + + +# --- OpenAI-compatible endpoint ------------------------------------------- + +def test_openai_endpoint_applies_rag(web, wired): + _, chat_server = wired + body = json.load(post(web.url + "/v1/chat/completions", + {"messages": [{"role": "user", + "content": "how do I enumerate smb shares?"}]})) + assert body["choices"][0]["message"]["content"] == "SAW_CONTEXT" + assert body["model"] == "cpts-notes-rag" + assert body["sources"][0] == "recon/smb.md" + + +def test_openai_endpoint_streams(web): + resp = post(web.url + "/v1/chat/completions", + {"messages": [{"role": "user", "content": "smb shares"}], + "stream": True}) + events = sse_events(resp) + answer = "".join(e["choices"][0]["delta"].get("content", "") + for e in events if e.get("choices")) + assert answer == "SAW_CONTEXT" + assert events[-1]["sources"][0] == "recon/smb.md" + + +def test_openai_endpoint_uses_the_last_user_message(web, wired): + _, chat_server = wired + post(web.url + "/v1/chat/completions", {"messages": [ + {"role": "user", "content": "ignore this one"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "how do I enumerate smb shares?"}, + ]}).read() + assert "Question: how do I enumerate smb shares?" in \ + chat_server.calls[-1]["messages"][-1]["content"] + + +def test_openai_endpoint_forwards_sampling_params(web, wired): + _, chat_server = wired + post(web.url + "/v1/chat/completions", + {"messages": [{"role": "user", "content": "smb"}], + "temperature": 0.9, "max_tokens": 42}).read() + assert chat_server.calls[-1]["temperature"] == 0.9 + assert chat_server.calls[-1]["max_tokens"] == 42 + + +def test_openai_endpoint_rejects_no_user_message(web): + with pytest.raises(urllib.error.HTTPError) as exc: + post(web.url + "/v1/chat/completions", + {"messages": [{"role": "system", "content": "hi"}]}) + assert exc.value.code == 400 + + +# --- auth ------------------------------------------------------------------ + +def test_loopback_bind_needs_no_token(index): + httpd = rag_web.build_server("127.0.0.1", 0, index, TEST_KEY, 5) + try: + assert httpd.require_key is False + finally: + httpd.server_close() + + +def test_non_loopback_bind_requires_a_token(index): + httpd = rag_web.build_server("0.0.0.0", 0, index, TEST_KEY, 5) + try: + assert httpd.require_key is True + finally: + httpd.server_close() + + +def test_token_is_enforced_when_required(web, monkeypatch): + monkeypatch.setattr(web, "require_key", True) + with pytest.raises(urllib.error.HTTPError) as exc: + post(web.url + "/ask", {"question": "smb"}) + assert exc.value.code == 401 + + events = sse_events(post(web.url + "/ask", {"question": "smb"}, + headers={"Authorization": f"Bearer {TEST_KEY}"})) + assert any("token" in e for e in events) + + +def test_health_stays_open_when_a_token_is_required(web, monkeypatch): + monkeypatch.setattr(web, "require_key", True) + assert json.load(get(web.url + "/health"))["status"] == "ok" + + +def test_oversized_body_is_rejected(web): + big = {"question": "x" * (rag_web.MAX_BODY + 10)} + with pytest.raises(urllib.error.HTTPError) as exc: + post(web.url + "/ask", big) + assert exc.value.code == 400 + + +# --- failure handling ------------------------------------------------------ + +def test_chat_server_down_is_reported_in_the_stream(web, monkeypatch): + monkeypatch.setattr(ragcore, "CHAT_URL", "http://127.0.0.1:1/v1/chat/completions") + events = sse_events(post(web.url + "/ask", {"question": "smb"})) + assert any("cannot reach" in e.get("error", "") for e in events) + + +def test_embed_server_down_returns_503(web, monkeypatch): + monkeypatch.setattr(ragcore, "EMBED_URL", "http://127.0.0.1:1/v1/embeddings") + with pytest.raises(urllib.error.HTTPError) as exc: + post(web.url + "/ask", {"question": "smb"}) + assert exc.value.code == 503 From 89467f5c405a9716a29db9b101bf2ff25952a2b9 Mon Sep 17 00:00:00 2001 From: charan Date: Fri, 21 Aug 2026 02:52:02 +0530 Subject: [PATCH 2/4] Put the Adreno GPU and the right CPU cores to work Prompt processing, not generation, is what you wait on before an answer starts, and it was running at 17.9 tokens per second. Two things were leaving the phone idle. The GPU was never used. The handoff recorded "no GPU backend" as a property of the phone, but it is a property of the installed build: the Adreno 830 is reachable from unrooted Termux through Mesa's turnip Vulkan driver, and llama.cpp ships a matching backend as a Termux package. Prompt eval goes to 70.2 tokens per second, so a 605-token RAG prompt starts answering after 8.9 seconds instead of 36. The vendor OpenCL path is a dead end and worth recording as one: the Android linker refuses an absolute path into /vendor/lib64 from an app namespace, and the bare soname resolves to Termux's own ICD loader, so it can never find a platform. Vulkan sidesteps this because turnip runs entirely in userspace. The CPU is 6 performance cores plus 2 prime cores, and spreading 8 threads over all of them is slower than pinning 6 threads to the matched ones -- generation goes 9.1 to 12.2 tokens per second. The threads landing on the prime cores finish early and idle, and the two cores left free absorb the OS and the GPU driver. The chat server now takes cpu0-5 and the embedding server cpu6-7, which also ends the contention of the two asking for 12 threads on an 8-core phone. GGML_BACKEND_PATH is left deliberately unset, with a comment saying why: pointing it at a single .so restricts ggml to that one backend and silently drops the GPU, which cost an hour of chasing during this work. --prio is dropped: raising thread priority needs root and only logged "Operation not permitted" once per thread. bench/RESULTS.md records the full matrix and bench/probe.py reproduces the headline numbers against a running server. --- bench/RESULTS.md | 95 +++++++++++++++++++++++++++++++++++++ bench/probe.py | 34 +++++++++++++ bin/llm-server.sh | 34 ++++++++++++- boot/start-lab.sh | 8 ++++ rag/bin/rag-embed-server.sh | 18 ++++++- 5 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 bench/RESULTS.md create mode 100644 bench/probe.py diff --git a/bench/RESULTS.md b/bench/RESULTS.md new file mode 100644 index 0000000..e75f97a --- /dev/null +++ b/bench/RESULTS.md @@ -0,0 +1,95 @@ +# Measured performance + +Galaxy S25 (`SM-S931B`), Snapdragon 8 Elite, 12 GB RAM, unrooted Termux. +Model: Qwen3-4B-Instruct-2507, Q4_K_M, 2.32 GiB. +llama.cpp build b10516 (Termux `llama-cpp` package). +Numbers from `llama-bench -p 256 -n 32`, and from `bench/probe.py` against the +live server. `pp` is prompt processing, `tg` is token generation. + +## Summary + +| | prompt eval | generation | +|---|---:|---:| +| before (CPU only, 8 threads) | 17.9 tok/s | 10.4 tok/s | +| after (GPU + 6 pinned cores) | **70.2 tok/s** | **12.0 tok/s** | + +Prompt processing is what you wait on before an answer starts. On a real RAG +question (605-token prompt) that is **36 s → 8.9 s**. + +## Where the time goes + +Retrieval is not the bottleneck and never was: embedding the question, scoring +all 1560 chunks in pure Python and picking the top 5 takes **161 ms**. The rest +is llama-server. + +## The CPU is heterogeneous + +`/sys/devices/system/cpu/cpu*/cpufreq/cpuinfo_max_freq`: + +| cores | clock | role | +|---|---|---| +| cpu0–5 | 3.53 GHz | performance | +| cpu6–7 | 4.47 GHz | prime | + +Splitting work evenly across all 8 is *slower* than using the 6 matched cores. +Threads on the prime cores finish their share early and idle, and leaving those +two cores free lets the OS and the GPU driver run without preempting a worker. + +| config | pp256 | tg32 | +|---|---:|---:| +| `-t 8` unpinned | 75.12 | 9.14 | +| `-t 8 -C 0xff --cpu-strict 1` | 75.16 | 7.80 | +| **`-t 6 -C 0x3f --cpu-strict 1`** | **75.35** | **12.16** | +| `-t 2 -C 0xc0 --cpu-strict 1` | 74.94 | 8.00 | + +So the chat server takes cpu0–5 and the embedding server takes cpu6–7. Before +this split the two asked for 12 threads between them on an 8-core phone. + +## The GPU was never being used + +The Adreno 830 is reachable from unrooted Termux, but not the way it first +appears: + +- The vendor OpenCL driver cannot be loaded. `/vendor/lib64/libOpenCL.so` is + listed in `/vendor/etc/public.libraries.txt`, but the Android linker refuses + an absolute path into `/vendor/lib64` from an app namespace, and the bare + soname resolves to Termux's own ICD loader instead. `ggml_opencl: platform + IDs not available` is that dead end. +- Vulkan works. `pkg install llama-cpp-backend-vulkan mesa-vulkan-icd-freedreno` + gives Mesa's **turnip** driver, entirely in userspace, and `vulkaninfo` + then reports `Adreno (TM) 830`. + +CPU-only prompt eval, for comparison — note it scales with thread count, so the +CPU is genuinely compute-starved here in a way the GPU fixes: + +| threads | flash-attn | pp256 | tg32 | +|---|---|---:|---:| +| 4 | off | 11.89 | 11.04 | +| 6 | off | 15.78 | 12.92 | +| 8 | off | 18.66 | 12.82 | +| 4 | on | 12.10 | 11.52 | +| 6 | on | 15.99 | 13.43 | +| 8 | on | 17.57 | 14.34 | + +Flash-attention makes no meaningful difference to prompt eval on this CPU. + +### Do not set `GGML_BACKEND_PATH` + +ggml finds its backend libraries by looking next to the `llama-server` binary. +Setting `GGML_BACKEND_PATH` to a single `.so` **restricts** it to that one +backend — pointing it at `libggml-cpu.so` silently disables the GPU and costs +4x on prompt eval. Setting it to a directory, or to a colon-separated list, +fails outright. Leave it unset and start the server from a Termux shell (which +is what `boot/start-lab.sh` does via tmux); started from a bare `ssh` command +the binary path resolves to `/apex/...` and no backend is found at all. + +## Reproducing + +```bash +# On the phone +llama-bench -m ~/models/qwen3-4b.gguf -p 256 -n 32 -t 6 -C 0x3f --cpu-strict 1 + +# From the laptop, against the running server +adb forward tcp:8081 tcp:8081 +python3 bench/probe.py +``` diff --git a/bench/probe.py b/bench/probe.py new file mode 100644 index 0000000..37f5416 --- /dev/null +++ b/bench/probe.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Report prompt-eval and generation speed straight from llama-server's timings. + + python3 bench/probe.py [URL] + +Run it against the chat server (default http://localhost:8081) after an +`adb forward tcp:8081 tcp:8081`. Prompt-eval speed is the number that decides +how long you wait before an answer starts, and it is the one that changes when +the GPU backend is or is not in play. +""" +import json +import os +import sys +import urllib.request + +URL = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8081/v1/chat/completions" +KEYFILE = os.path.expanduser(os.environ.get("LLM_KEYFILE", "~/.config/llm-api-key")) +KEY = os.environ.get("LLM_API_KEY") or ( + open(KEYFILE).read().strip() if os.path.exists(KEYFILE) else "") + +payload = { + # Long enough that prompt processing dominates and is measured accurately. + "messages": [{"role": "user", "content": "Explain SMB enumeration. " * 60}], + "max_tokens": 16, + "temperature": 0, +} +req = urllib.request.Request( + URL, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {KEY}"}) +with urllib.request.urlopen(req, timeout=600) as resp: + t = json.load(resp)["timings"] + +print(f"prompt: {t['prompt_n']:>5} tok at {t['prompt_per_second']:>6.1f} tok/s") +print(f"gen: {t['predicted_n']:>5} tok at {t['predicted_per_second']:>6.1f} tok/s") diff --git a/bin/llm-server.sh b/bin/llm-server.sh index 0495b7c..89dc0b7 100755 --- a/bin/llm-server.sh +++ b/bin/llm-server.sh @@ -6,16 +6,42 @@ set -euo pipefail MODEL="${LLM_MODEL:-$HOME/models/qwen3-4b.gguf}" KEYFILE="${LLM_KEYFILE:-$HOME/.config/llm-api-key}" PORT="${LLM_PORT:-8081}" -THREADS="${LLM_THREADS:-8}" CTX="${LLM_CTX:-8192}" +# Snapdragon 8 Elite is 6 performance cores (cpu0-5, 3.53 GHz) plus 2 prime +# cores (cpu6-7, 4.47 GHz). Splitting work evenly across all 8 is slower than +# using the 6 matched cores: the threads on the prime cores finish their share +# early and then idle, and the two cores left free absorb the OS and the GPU +# driver. Measured on this device, generation goes 9.1 -> 12.2 tokens/sec by +# dropping from 8 unpinned threads to 6 pinned ones. +THREADS="${LLM_THREADS:-6}" +CPU_MASK="${LLM_CPU_MASK:-0x3f}" # cpu0-5 + +# The GPU (Adreno 830, reached through Mesa's turnip Vulkan driver) is about +# 4x faster than the CPU at prompt processing -- 75 vs 19 tokens/sec -- and +# prompt processing is what decides how long you wait before an answer starts. +# It is slower at generating tokens, so the split below is deliberate: GPU for +# the prompt, the pinned CPU cores for generation. +# +# ggml discovers its backend libraries by looking next to the llama-server +# binary, which only resolves correctly when the server is started from a +# Termux shell (as boot/start-lab.sh does via tmux). Do NOT set +# GGML_BACKEND_PATH here: pointing it at a single .so restricts ggml to that +# one backend, which silently drops the GPU and costs 4x on prompt eval. +NGL="${LLM_NGL:-99}" + [ -r "$MODEL" ] || { echo "no model at $MODEL — run bin/fetch-model.sh first" >&2; exit 1; } [ -r "$KEYFILE" ] || { echo "no API key at $KEYFILE — run install.sh first" >&2; exit 1; } # Android suspends the CPU when idle; without this the server stalls mid-request. termux-wake-lock -# Flags tuned for a phone CPU: +# Flags tuned for this phone: +# -ngl 99 offload to the Adreno GPU for prompt processing +# --cpu-strict keep the worker threads on the cores chosen above +# --poll 100 spin rather than sleep between batches +# (--prio is deliberately absent: raising thread priority needs root, and an +# unrooted Termux just logs "Operation not permitted" once per thread.) # --flash-attn on faster attention, lower memory # --cache-type-k/v q8_0 quantized KV cache — fits an 8192 context in RAM # --host 0.0.0.0 LAN-reachable, which is why --api-key is mandatory @@ -25,7 +51,11 @@ exec llama-server \ --port "$PORT" \ --api-key "$(cat "$KEYFILE")" \ --ctx-size "$CTX" \ + --n-gpu-layers "$NGL" \ --threads "$THREADS" \ + --cpu-mask "$CPU_MASK" \ + --cpu-strict 1 \ + --poll 100 \ --flash-attn on \ --cache-type-k q8_0 \ --cache-type-v q8_0 \ diff --git a/boot/start-lab.sh b/boot/start-lab.sh index ee1ced8..542f79d 100755 --- a/boot/start-lab.sh +++ b/boot/start-lab.sh @@ -22,3 +22,11 @@ if [ -r "$HOME/models/nomic-embed.gguf" ] && [ -x "$HOME/rag/bin/rag-embed-serve tmux has-session -t embsrv 2>/dev/null || \ tmux new-session -d -s embsrv "$HOME/rag/bin/rag-embed-server.sh" fi + +# Browser front end for the notes assistant on :8083 (localhost only). Open it +# from the laptop with: adb forward tcp:8083 tcp:8083 && xdg-open http://localhost:8083 +# Unlike :8081 this applies retrieval before answering, so it needs the index. +if [ -r "$HOME/rag/index.jsonl" ] && [ -r "$HOME/rag/bin/rag-web.py" ]; then + tmux has-session -t ragweb 2>/dev/null || \ + tmux new-session -d -s ragweb "python3 $HOME/rag/bin/rag-web.py" +fi diff --git a/rag/bin/rag-embed-server.sh b/rag/bin/rag-embed-server.sh index 491d786..8d08daf 100644 --- a/rag/bin/rag-embed-server.sh +++ b/rag/bin/rag-embed-server.sh @@ -4,6 +4,19 @@ set -euo pipefail MODEL="${RAG_EMBED_MODEL:-$HOME/models/nomic-embed.gguf}" PORT="${RAG_EMBED_PORT:-8082}" + +# The chat server takes the six performance cores (cpu0-5); this one gets the +# two prime cores (cpu6-7), so the two never fight for the same core. Before +# this split they asked for 12 threads between them on an 8-core phone. +THREADS="${RAG_EMBED_THREADS:-2}" +CPU_MASK="${RAG_EMBED_CPU_MASK:-0xc0}" # cpu6-7 + +# Embedding is pure prompt processing, which is exactly what the Adreno GPU is +# good at, so this is where offload pays off most — it is the difference +# between a fast and a slow rag-index.py run over the whole corpus. As in +# bin/llm-server.sh, GGML_BACKEND_PATH is deliberately left unset so that ggml +# discovers every backend rather than just one. + [ -r "$MODEL" ] || { echo "no embed model at $MODEL" >&2; exit 1; } termux-wake-lock exec llama-server \ @@ -15,4 +28,7 @@ exec llama-server \ --ctx-size 2048 \ --batch-size 2048 \ --ubatch-size 2048 \ - --threads 4 + --n-gpu-layers 99 \ + --threads "$THREADS" \ + --cpu-mask "$CPU_MASK" \ + --cpu-strict 1 From 74ae8b8b649c57ef76a7452b45725a26d872e53c Mon Sep 17 00:00:00 2001 From: charan Date: Fri, 21 Aug 2026 02:53:17 +0530 Subject: [PATCH 3/4] Document the browser UI and replace the estimated benchmarks with measured ones The README claimed 15 tokens/sec generation and 33 prompt, both estimates that turned out to be wrong in opposite directions once measured. It now carries the numbers from bench/RESULTS.md and says which half runs on the GPU and why. rag/README gains the part that was easiest to get wrong: :8081 answers from the model's weights alone and :8083 is the one that applies retrieval, so anything pointed at the chat server directly silently gets answers that never saw the notes. --- README.md | 17 +++++++++++----- rag/README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 823ae55..d16b789 100644 --- a/README.md +++ b/README.md @@ -73,18 +73,25 @@ over Wi-Fi otherwise. ## Measured performance -On the Galaxy S25 (Snapdragon 8 Elite, 6 of 8 cores, Qwen3-4B-Instruct Q4_K_M): +On the Galaxy S25 (Snapdragon 8 Elite, Qwen3-4B-Instruct Q4_K_M). Prompt +processing runs on the Adreno GPU, generation on six pinned CPU cores: | Metric | Value | |---|---| -| Generation | ~15 tokens/sec (flash-attn, 8 threads) | -| Prompt processing | ~33 tokens/sec | +| Prompt processing | 70 tokens/sec (Adreno 830, Vulkan) | +| Generation | 12 tokens/sec (6 pinned CPU cores) | | Model load time | ~2.6 s | | Idle RAM headroom | ~4 GB free with model resident | | Context window | 8192 tokens (q8_0 KV cache) | -| First-token latency | sub-second over USB | -Fast enough to read along with. Not instant, but usable for real work. +Prompt processing is what you wait on before an answer starts, so it is the +number that matters: a 605-token retrieval prompt begins answering after 8.9 +seconds rather than 36. Using the GPU is worth 4x there, and it is *slower* at +generating tokens, which is why the two halves run on different hardware. + +[bench/RESULTS.md](bench/RESULTS.md) has the full matrix, the core-pinning +measurements, and the OpenCL dead end. `bench/probe.py` reproduces the headline +numbers against a running server. ## Endpoint authentication diff --git a/rag/README.md b/rag/README.md index 635e0e2..8f9533d 100644 --- a/rag/README.md +++ b/rag/README.md @@ -43,9 +43,11 @@ advantage of RAG over fine-tuning for a knowledge base that keeps growing. | Path | Runs on | Purpose | |---|---|---| +| `bin/ragcore.py` | phone | Chunking, scoring, prompt assembly — shared by the three below | | `bin/rag-embed-server.sh` | phone | Serves nomic-embed on :8082 (embedding mode) | | `bin/rag-index.py` | phone | Chunks `corpus/`, embeds each chunk → `index.jsonl` | | `bin/rag-ask.py` | phone | Embeds a query, cosine top-k, prompts the chat model | +| `bin/rag-web.py` | phone | Browser page + OpenAI-compatible endpoint on :8083 | ## Usage (on the phone) @@ -56,8 +58,57 @@ python3 ~/rag/bin/rag-index.py # build ~/rag/index.jsonl python3 ~/rag/bin/rag-ask.py "how do I crack an NTLMv2 hash with hashcat?" ``` -From the laptop, run the same `rag-ask.py` over SSH, or call the chat server directly -with your own retrieval client. +## In a browser + +`rag-web.py` serves a chat page on :8083 and keeps the index in memory, so it does +not re-read it per question. From the laptop: + +```sh +adb forward tcp:8083 tcp:8083 +xdg-open http://localhost:8083 +``` + +It binds `127.0.0.1` on the phone, so it is reachable only through that forward and +never sits on the campus network. If you do bind it to a routable address +(`--host 0.0.0.0`), it requires the same bearer token as :8081. + +## Which port applies retrieval + +This is the distinction that matters, and it is easy to get wrong: + +| Port | Retrieval | What it answers from | +|---|---|---| +| `:8081` | **no** | the model's weights alone | +| `:8083` | **yes** | your notes, retrieved and pasted into the prompt | + +`llama-server` on :8081 knows nothing about `corpus/`. Anything pointed straight at +it — curl, a browser, any OpenAI client — gets an answer that never saw your notes. +Point clients at :8083 instead; it speaks the same `/v1/chat/completions` shape and +adds a `sources` field to the response. + +```sh +curl http://localhost:8083/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"how do I enumerate SMB shares?"}]}' +``` + +## Speed + +Retrieval is not the slow part: embedding the question, scoring all 1560 chunks in +pure Python and taking the top 5 costs **161 ms**. The wait is prompt processing on +the chat model, which is why the retrieved context is capped by a character budget +(`RAG_CONTEXT_CHARS`, default 2000) rather than a fixed chunk count — prompt length +is the wait. See [../bench/RESULTS.md](../bench/RESULTS.md). + +## Tests + +`tests/` covers chunking, scoring, retrieval ranking, prompt assembly and the whole +HTTP surface against stand-in model servers, so it runs on a laptop and in CI with +no phone and no model weights: + +```sh +python3 -m pytest tests/ -q +``` ## Chunking From c54884e5d4aa52ef97f7c02dbf8642d5361b01e8 Mon Sep 17 00:00:00 2001 From: charan Date: Fri, 21 Aug 2026 03:13:20 +0530 Subject: [PATCH 4/4] Serve the notes assistant on the LAN, and make the docs read for a general audience Two changes that landed together. Serving: the browser UI and OpenAI-compatible endpoint were loopback-only, reached over adb forward. They now bind LAN-wide at boot so other devices on the same network can use them without a cable, which means every request except /health must carry the bearer token. The page reads whether auth is required from /health, prompts for the token when it is, remembers it per browser, sends it on every request, and re-prompts on a 401. This is a deliberate exposure: the endpoint reads out of a private notes corpus and the token travels in clear text over HTTP, so RAG_WEB_HOST=127.0.0.1 returns it to loopback-only on an untrusted network. Docs: the public documentation no longer addresses a specific person or narrates their plans, and no longer prints a concrete LAN address. Replaced the real IP with a placeholder, generalised the network context, corrected the key-generation command to one that exists on Termux, and removed the claim that a subnet size implies who can reach the service. Deployment-specific and planning notes are kept out of the repository entirely. --- .gitignore | 3 +++ GUIDE.md | 52 +++++++++++++++++++++++++++++-------------- README.md | 36 +++++++++++++++++++++--------- boot/start-lab.sh | 15 +++++++++---- docs/ARCHITECTURE.md | 36 +++++++++++++++++++++--------- docs/NETWORKING.md | 4 ++-- rag/README.md | 6 ++--- rag/bin/rag-web.py | 53 ++++++++++++++++++++++++++++++++++++++------ tests/test_web.py | 17 +++++++++++++- 9 files changed, 167 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index c42c497..c2f5187 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ rag/index/ .claude/worktrees/ __pycache__/ *.pyc + +# Deployment-specific / explanatory docs — never publish (contains local network detail) +private/ diff --git a/GUIDE.md b/GUIDE.md index 517bf96..6d29dcd 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -43,15 +43,17 @@ You talk to the phone from your laptop. First connect (pick ONE): adb forward tcp:8081 tcp:8081 ``` -**By Wi-Fi (both on campus Wi-Fi, no cable):** +**By Wi-Fi (both on the same network, no cable):** ```bash # nothing to set up — just use the phone's address in the commands below -# phone address today: 10.12.219.205 (this can change — see "IP changed?" below) +# phone address today: (this can change — see "IP changed?" below) ``` -Then set your password once per terminal: +Then set your password once per terminal. Read it from the phone rather than +pasting it into a file — a key that lives in a document ends up in a commit: ```bash -KEY=542d409f821fb25b7f291b35ce0af676a60820c04ae2af81 +adb forward tcp:8022 tcp:8022 +KEY=$(ssh -p 8022 localhost 'cat ~/.config/llm-api-key') ``` ### Ask the plain model a question @@ -60,15 +62,30 @@ curl -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ http://localhost:8081/v1/chat/completions \ -d '{"messages":[{"role":"user","content":"explain SMB null sessions"}]}' ``` -(Over Wi-Fi, replace `localhost` with `10.12.219.205`.) +(Over Wi-Fi, replace `localhost` with ``.) ### Ask the CPTS assistant (answers from YOUR notes) -Log into the phone and run: + +**Easiest — in a browser** (from any device on the same Wi-Fi): +``` +http://:8083 +``` +It opens a chat page. Paste the API key once when it asks (the browser remembers it), +type a question, and the answer streams in with the note files it used listed under it. +This is the assistant that reads *your notes* — port 8081 is the plain model that does +not. Any OpenAI-compatible app pointed at `http://:8083/v1` gets the same +notes-aware answers. + +> The key travels in clear text over the local Wi-Fi here, and this page can read the +> private notes. Fine for your own use on a trusted network; see `docs/SECURITY.md` +> before exposing it wider. + +**Or on the command line** (log into the phone): ```bash -ssh -p 8022 10.12.219.205 # or: adb forward tcp:8022 tcp:8022 && ssh -p 8022 localhost +ssh -p 8022 # or: adb forward tcp:8022 tcp:8022 && ssh -p 8022 localhost python3 ~/rag/bin/rag-ask.py "how do I enumerate SNMP?" ``` -It prints an answer, then the note files it used. +It streams an answer, then the note files it used. --- @@ -78,9 +95,10 @@ It prints an answer, then the note files it used. commands and methodology, quick "how do I..." questions. Great as a study aid. **Not good at:** hard multi-step reasoning, anything needing current/internet info, -and it is **slower than ChatGPT** — about 15 words a second. That's the price of running -on a phone CPU with no graphics card. For heavy work you said you'll use the laptop's -RTX 4060 later; this is the always-on study buddy. +and it is **slower than ChatGPT** — about 12 words a second once it starts, after a few +seconds of reading your question on the phone's GPU. That's the price of running on a +phone. It is built to be the always-on, offline study box; heavier work belongs on a +desktop GPU. Think of it as a sharp intern that never sleeps and never phones home — not a genius. @@ -91,7 +109,7 @@ Think of it as a sharp intern that never sleeps and never phones home — not a **"Connection refused" / no answer** The server probably isn't running. Restart it: ```bash -ssh -p 8022 10.12.219.205 +ssh -p 8022 ~/bin/llm-server.sh & # the chat model ~/rag/bin/rag-embed-server.sh & # the notes-search helper ``` @@ -106,7 +124,7 @@ The phone's Wi-Fi address isn't fixed. Get the new one over USB: ```bash adb shell ip -4 addr show wlan0 | grep inet ``` -Use that new address instead of `10.12.219.205`. +Use that new address instead of ``. **I added new notes — how does it learn them?** Put the new `.md` files in `~/rag/corpus/` on the phone, then: @@ -117,13 +135,13 @@ It re-reads everything. No "training" needed — it just re-indexes. --- -## Can I use it away from campus (from anywhere)? +## Can I use it from anywhere (off the local network)? -Not yet. The phone is stuck behind the campus network, which blocks incoming +Not yet. The phone is behind a NAT'd network that blocks incoming connections from the outside world — and the usual tools that get around that -(Tailscale, etc.) are blocked on campus Wi-Fi specifically. The fix is a cheap/free +(Tailscale, etc.) may be blocked on such networks. The fix is a cheap/free cloud server acting as a middleman; it's designed but not built. Details in -`docs/NETWORKING.md`. For now: works on campus Wi-Fi and over USB. +`docs/NETWORKING.md`. For now: works on the local Wi-Fi and over USB. --- diff --git a/README.md b/README.md index d16b789..f8d1bc3 100644 --- a/README.md +++ b/README.md @@ -48,27 +48,39 @@ echo "" > ~/.config/s25-llm-key The client auto-selects USB when a device is attached and falls back to `$LLM_HOST` over Wi-Fi otherwise. +**In a browser** — the notes assistant serves a chat page on :8083 that applies +retrieval before answering (unlike :8081, which is the raw model): + +```sh +adb forward tcp:8083 tcp:8083 && xdg-open http://localhost:8083 +``` + ## What's in here | Path | Runs on | Purpose | |---|---|---| | `install.sh` | phone | One-shot setup: packages, dirs, API key | | `bin/fetch-model.sh` | phone | Resumable GGUF download | -| `bin/llm-server.sh` | phone | Launches `llama-server` with a wake lock | -| `boot/start-lab.sh` | phone | Termux:Boot autostart — sshd, tmux, LLM | +| `bin/llm-server.sh` | phone | Launches `llama-server` (GPU prompt eval, pinned CPU cores) | +| `rag/bin/rag-web.py` | phone | Browser UI + OpenAI-compatible RAG endpoint on :8083 | +| `boot/start-lab.sh` | phone | Termux:Boot autostart — sshd, tmux, LLM, embed, RAG web | | `client/llm` | laptop | CLI client, USB-or-Wi-Fi transport selection | +| `tests/` | laptop / CI | Full test suite against stand-in model servers | ## Documentation - **[GUIDE.md](GUIDE.md)** — start here. Plain-English: what it is, how to use it, how to fix it. -- **[rag/](rag/README.md)** — the CPTS study assistant (RAG over your own notes). +- **[rag/](rag/README.md)** — the CPTS study assistant (RAG over your own notes), and + the browser UI. - **[Architecture](docs/ARCHITECTURE.md)** — how every layer works, from the Android sandbox up through quantization and the request path. Written to be readable with no prior systems background. - **[Setup log](docs/SETUP.md)** — the actual build, in order, including what broke. - **[Networking](docs/NETWORKING.md)** — why remote access is the hard part: NAT, - private addressing, and a DPI-filtered campus network. + private addressing, and a DPI-filtered network. +- **[Benchmarks](bench/RESULTS.md)** — measured throughput, the GPU story, and the + core-pinning numbers. ## Measured performance @@ -110,12 +122,16 @@ without the key. ## Security `llama-server` binds `0.0.0.0`, so it is reachable by anything that can route to the -phone. On the network this was built on, client isolation is **off** — any device on -the same `/20` can reach it. So the API key is mandatory, not decorative: - -- key generated with `openssl rand -hex 24`, stored `chmod 600` at `~/.config/llm-api-key` -- `.gitignore` excludes the key and all `*.gguf` weights -- the key never appears in this repository +phone. Where the local network has client isolation off, other devices on it can reach +the port, so the API key is mandatory, not decorative: + +- key generated with `python3 -c "import secrets; print(secrets.token_hex(24))"`, + stored `chmod 600` at `~/.config/llm-api-key` +- `.gitignore` excludes the key, the notes corpus, the built index, and all `*.gguf` + weights +- the bearer token rides plaintext HTTP, so on an untrusted network it is protection + against casual use, not against someone able to watch the traffic — prefer loopback + plus `adb forward`, or a WireGuard tunnel, there ## Status diff --git a/boot/start-lab.sh b/boot/start-lab.sh index 542f79d..3ec40c0 100755 --- a/boot/start-lab.sh +++ b/boot/start-lab.sh @@ -23,10 +23,17 @@ if [ -r "$HOME/models/nomic-embed.gguf" ] && [ -x "$HOME/rag/bin/rag-embed-serve tmux new-session -d -s embsrv "$HOME/rag/bin/rag-embed-server.sh" fi -# Browser front end for the notes assistant on :8083 (localhost only). Open it -# from the laptop with: adb forward tcp:8083 tcp:8083 && xdg-open http://localhost:8083 -# Unlike :8081 this applies retrieval before answering, so it needs the index. +# Browser front end for the notes assistant on :8083. Unlike :8081 this applies +# retrieval before answering, so it needs the index. +# +# It is bound LAN-wide so other devices on the Wi-Fi can use it, which means +# every request except /health must carry the bearer token. Note what that +# exposes: this endpoint reads out of the private notes corpus and the token is +# sent in clear text over HTTP. Set RAG_WEB_HOST=127.0.0.1 to go back to +# loopback-only (reachable via adb forward tcp:8083 tcp:8083), which is the +# safer default on a network you do not trust. if [ -r "$HOME/rag/index.jsonl" ] && [ -r "$HOME/rag/bin/rag-web.py" ]; then tmux has-session -t ragweb 2>/dev/null || \ - tmux new-session -d -s ragweb "python3 $HOME/rag/bin/rag-web.py" + tmux new-session -d -s ragweb \ + "RAG_WEB_HOST=${RAG_WEB_HOST:-0.0.0.0} python3 $HOME/rag/bin/rag-web.py" fi diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 86dd96a..98b42bb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -105,7 +105,7 @@ Worth understanding properly, because it is the one unsolved piece. Addresses in `10.0.0.0/8`, `192.168.0.0/16` and `172.16.0.0/12` are **private**. They are not globally unique — millions of networks reuse the same numbers — so the public -internet refuses to route them. A phone at `10.12.219.205` is meaningful only inside +internet refuses to route them. A phone at `` is meaningful only inside its own network. ``` @@ -153,25 +153,39 @@ is roughly 8 GB — more than the ~3.4 GB of RAM actually available on this devi `Q4_K_M` stores most weights in about 4 bits, shrinking the file to **2.32 GB** for a modest quality loss. `.gguf` is llama.cpp's container format for such models. -**llama.cpp runs it on CPU.** Most ML tooling assumes an NVIDIA GPU. llama.cpp is -portable C++ that runs on ordinary processors including ARM, using NEON SIMD -instructions. `llama-server` wraps it in an HTTP API that mimics OpenAI's, so existing -clients work unchanged against it. +**llama.cpp runs it, split across the GPU and CPU.** Most ML tooling assumes an +NVIDIA GPU; llama.cpp is portable C++ that runs on ordinary processors including ARM. +On this phone it uses two backends at once: -Launch flags and why each is there: +- **Prompt processing on the Adreno 830 GPU**, through Mesa's open-source *turnip* + Vulkan driver. This is the matmul-heavy work of reading your input, and the GPU is + about 4x faster at it than the CPU (70 vs 18 tokens/sec). It is what decides how + long you wait before an answer starts. +- **Token generation on the CPU**, pinned to the six 3.53 GHz performance cores. The + GPU is actually slower here — generation is memory-bandwidth bound, not compute + bound — so the two halves deliberately run on different hardware. + +`llama-server` wraps all this in an HTTP API that mimics OpenAI's, so existing clients +work unchanged. Launch flags and why each is there: ``` --host 0.0.0.0 listen on all interfaces, not just loopback --port 8081 must be >1024 — non-root cannot bind lower --api-key mandatory: 0.0.0.0 on a LAN without client isolation ---ctx-size 4096 tokens of context retained per conversation ---threads 6 6 of 8 cores; 2 left for Android itself +--ctx-size 8192 tokens of context retained per conversation +--n-gpu-layers 99 offload the whole model to the Adreno GPU for prompt eval +--threads 6 6 worker threads... +--cpu-mask 0x3f ...pinned to cpu0-5, the matched performance cores +--cpu-strict 1 keep them there instead of letting Android migrate them --cont-batching keep the pipeline fed across overlapping requests ---mlock pin weights in RAM so Android cannot swap them out ``` -`--mlock` matters more than it looks: without it, Android's memory manager will page -the model out under pressure and the first token after an idle period takes seconds. +Two flags that are deliberately *absent*: `--mlock` (needs root — `RLIMIT_MEMLOCK` +fails unrooted) and `--prio` (raising thread priority also needs root). And one +environment variable that must stay unset: `GGML_BACKEND_PATH` pointing at a single +backend library restricts ggml to it and silently drops the GPU. See +[../bench/RESULTS.md](../bench/RESULTS.md) for the measurements behind every one of +these choices. --- diff --git a/docs/NETWORKING.md b/docs/NETWORKING.md index 58aee66..163f92b 100644 --- a/docs/NETWORKING.md +++ b/docs/NETWORKING.md @@ -22,7 +22,7 @@ filtered on this network. ## Measurement 1 — the phone's address is private ``` -wlan0: inet 10.12.219.205/20 +wlan0: inet /20 ``` `10.0.0.0/8` is RFC 1918 private space. Not globally routable, not unique, shared by a @@ -96,7 +96,7 @@ non-standard UDP port, presents no hostname to inspect and no known address to m The laptop reaches the phone directly across the wireless network: ``` -$ ssh -p 8022 10.12.219.205 +$ ssh -p 8022 OK from localhost — up 6 days ``` diff --git a/rag/README.md b/rag/README.md index 8f9533d..f85e117 100644 --- a/rag/README.md +++ b/rag/README.md @@ -6,8 +6,8 @@ paths cited — no hallucinated commands, no internet. Built for **HTB CPTS** prep. The corpus here is a private tree of HTB Academy / CPTS module notes (information gathering, exploitation, password attacks, privilege -escalation, post-exploitation). Heavy CTF work is intended to move to a laptop RTX 4060 -later; this runs the always-on study assistant. +escalation, post-exploitation). It runs as an always-on, offline study assistant; +heavier work belongs on a desktop GPU. ## How it works @@ -69,7 +69,7 @@ xdg-open http://localhost:8083 ``` It binds `127.0.0.1` on the phone, so it is reachable only through that forward and -never sits on the campus network. If you do bind it to a routable address +never sits on the wider network. If you do bind it to a routable address (`--host 0.0.0.0`), it requires the same bearer token as :8081. ## Which port applies retrieval diff --git a/rag/bin/rag-web.py b/rag/bin/rag-web.py index acb36c6..8635fde 100644 --- a/rag/bin/rag-web.py +++ b/rag/bin/rag-web.py @@ -1,18 +1,28 @@ #!/data/data/com.termux/files/usr/bin/env python3 """Browser front end and OpenAI-compatible endpoint for the notes assistant. -Runs on the DEVICE, default 127.0.0.1:8083. The chat server on :8081 answers -from model weights alone; everything here goes through retrieval first, so: +Runs on the DEVICE. The chat server on :8081 answers from model weights alone; +everything here goes through retrieval first, so: :8081 raw model, no notes :8083 same model, your notes retrieved and pasted in first GET / the chat page - GET /health {"status":"ok","chunks":N} + GET /health {"status","chunks","auth_required"} — always open POST /ask {"question": "..."} -> SSE token stream POST /v1/chat/completions OpenAI-compatible, RAG applied automatically -Reach it from the laptop with: adb forward tcp:8083 tcp:8083 +Binding decides whether a token is demanded. On loopback there is nothing to +protect against -- only processes on the phone, and whatever the laptop +forwards over USB, can reach it -- so no key is asked for: + + adb forward tcp:8083 tcp:8083 # then http://localhost:8083 + +Bound to a routable address (--host 0.0.0.0) every request except /health must +carry the bearer token from ~/.config/llm-api-key. Be clear-eyed about what +that is worth: this endpoint answers *out of* the private notes corpus, and the +token travels in clear text over HTTP, so anyone able to watch traffic on the +same network can lift it and read the notes through it. See docs/SECURITY.md. The index is loaded once at startup (a few seconds) instead of per question, which is most of why this answers faster than the CLI. @@ -64,19 +74,42 @@ font:inherit;font-weight:600;cursor:pointer} button:disabled{opacity:.5;cursor:default} .err{color:#c0392b} +#keybox{display:flex;gap:8px;align-items:center;margin-bottom:14px;padding:10px 12px; +border:1px solid var(--line);border-radius:8px;background:var(--card);font-size:13px} +#keybox label{color:var(--mut);white-space:nowrap} +#keybox input{flex:1;padding:6px 8px;border:1px solid var(--line);border-radius:6px; +background:var(--bg);color:var(--fg);font:inherit;font-family:ui-monospace,monospace} +#ks{color:var(--mut);white-space:nowrap}

CPTS notes assistant

Answers from your note chunks — retrieval first, then the local model.

+