From 706ef6901ab41a4a16636e1d003757664e5cd091 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:15:42 -0400
Subject: [PATCH 01/11] =?UTF-8?q?=E2=9C=A8=20feat(testing):=20pin=20the=20?=
 =?UTF-8?q?official=20LangChain=20tools=20and=20check=20prompt=20push=20wi?=
 =?UTF-8?q?th=20langsmith?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#747 746.2-oracles Install official LangChain tools through mise to check langstar's results

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 docs/dev/testing/README.md               |  1 +
 docs/dev/testing/official-tool-checks.md | 43 ++++++++++++++++++++++
 mise-tasks/oracle/prompt-roundtrip       | 45 ++++++++++++++++++++++++
 mise.toml                                | 11 ++++++
 4 files changed, 100 insertions(+)
 create mode 100644 docs/dev/testing/official-tool-checks.md
 create mode 100755 mise-tasks/oracle/prompt-roundtrip

diff --git a/docs/dev/testing/README.md b/docs/dev/testing/README.md
index 07d307ec..e8244246 100644
--- a/docs/dev/testing/README.md
+++ b/docs/dev/testing/README.md
@@ -6,6 +6,7 @@ Load this TOC first, then 1-3 docs relevant to your task. **Do not load all docs
 |----------|--------------|
 | `HIGH_LEVEL_TESTING_GUIDELINES.md` | Always - PR requirements, andon cord |
 | `crud-lifecycle-pattern.md` | CRUD features - CLI & SDK verification |
+| `official-tool-checks.md` | Checking a langstar result with the official `langsmith` / `langgraph` tools |
 | `sdk-integration-tests.md` / `cli-integration-tests.md` | SDK or CLI integration tests |
 | `devcontainer-feature-tests.md` | Infrastructure/devcontainer testing |
 | `test-fixtures.md` | Test deployment & fixture management |
diff --git a/docs/dev/testing/official-tool-checks.md b/docs/dev/testing/official-tool-checks.md
new file mode 100644
index 00000000..4f1ad2a1
--- /dev/null
+++ b/docs/dev/testing/official-tool-checks.md
@@ -0,0 +1,43 @@
+# Checking langstar with the official LangChain tools
+
+A langstar command can be wrong in a way its own SDK cannot see, because the CLI and SDK share the same request code. The official tools are a second implementation: create with langstar, read back with the official tool, and compare. This extends [crud-lifecycle-pattern.md](crud-lifecycle-pattern.md).
+
+## The tools
+
+`mise.toml` pins them; `mise install` installs them.
+
+| Tool | Provides |
+|------|----------|
+| [langsmith-cli](https://github.com/langchain-ai/langsmith-cli) | `langsmith` |
+| [langgraph-cli](https://pypi.org/project/langgraph-cli/) | `langgraph` |
+| [langgraph-sdk](https://pypi.org/project/langgraph-sdk/) | Python library; run it with `uv run --with "langgraph-sdk==$LANGGRAPH_SDK_VERSION"` |
+
+Bump a pin by hand and check that the release is older than five days: `minimum_release_age` does not apply to exact pins.
+
+## Which tool checks which command
+
+| langstar command group | Official tool that can read it back |
+|------------------------|-------------------------------------|
+| `prompt` | `langsmith prompt` |
+| `dataset` | `langsmith dataset`, `langsmith example` |
+| `project` | `langsmith project` |
+| `runs` | `langsmith run`, `langsmith trace` |
+| `eval` | `langsmith evaluator`, `langsmith experiment` |
+| `queue` | none |
+| `deployment`, `assistant`, `graph` | `langgraph` (local server and build), `langgraph-sdk` (assistants, threads, runs on a deployment) |
+| `secrets`, `model-config`, `config` | none |
+
+The table says what a tool can reach, not what a check covers today. Run `langsmith <group> --help` to confirm a subcommand exists before writing a check.
+
+## Checks
+
+Each check is a mise task under `mise-tasks/oracle/`. It gets its credentials from `fnox exec` and cleans up what it creates.
+
+| Task | Checks |
+|------|--------|
+| `mise run oracle:prompt-roundtrip` | `langstar prompt push` against `langsmith prompt pull` |
+
+## Findings while writing the first check
+
+- `langstar prompt push --owner -` for a repo that does not exist yet fails: the create-on-push step sends `-/<repo>` as the handle and the API answers 400. The task creates the repo with `langsmith prompt create` first.
+- A push without `--schema` to an empty repo got HTTP 400 "Manifest must have an id field". The task pushes with `--schema`, the path the integration tests use.
diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
new file mode 100755
index 00000000..3abded25
--- /dev/null
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#MISE description="Push a private prompt with langstar, read it back with the official langsmith CLI, then delete it"
+set -euo pipefail
+
+# Outside check for `langstar prompt push`: the official CLI reads the prompt
+# back, so a bug shared by langstar's push and pull cannot hide. The LangSmith
+# credentials come from fnox, so the check re-runs itself under `fnox exec`.
+# What each official tool can check: docs/dev/testing/official-tool-checks.md
+if [ -z "${LANGSMITH_API_KEY:-}" ]; then
+  exec fnox exec -- "$0" "$@"
+fi
+
+repo="oracle-roundtrip-$(date +%s)-$$"
+template="Answer this question: {question}"
+# `-` is the owner handle of a private prompt in the workspace.
+handle="-/${repo}"
+
+cleanup() {
+  langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1 || echo "could not delete ${handle}; delete it by hand" >&2
+}
+trap cleanup EXIT
+
+# The official CLI creates the repo. langstar's own create-on-push sends "-/<repo>"
+# as the handle, which the API rejects, so letting langstar create it would fail here.
+langsmith prompt create --name "$repo" >/dev/null
+
+schema="$(mktemp)"
+trap 'rm -f "$schema"; cleanup' EXIT
+printf '%s' '{"type":"object","title":"Out","properties":{"answer":{"type":"string"}},"required":["answer"]}' >"$schema"
+
+# --schema matches the path cli/tests/prompt_structured_test.rs exercises; a push
+# without it got HTTP 400 "Manifest must have an id field" when this task was written.
+cargo run --quiet --bin langstar -- prompt push \
+  --owner - --repo "$repo" --template "$template" --input-variables question \
+  --schema "$schema" >/dev/null
+
+pulled="$(langsmith prompt pull -- "$handle")"
+
+# The manifest is nested JSON, so look for the template text anywhere in it.
+if ! printf '%s' "$pulled" | jq -e --arg t "$template" '[.. | strings | select(. == $t)] | length > 0' >/dev/null; then
+  echo "FAIL: the official CLI did not return the template langstar pushed" >&2
+  printf '%s\n' "$pulled" >&2
+  exit 1
+fi
+echo "OK: langsmith prompt pull returned the template pushed by langstar (${handle})"
diff --git a/mise.toml b/mise.toml
index 03b3fc5f..8f1be0b1 100644
--- a/mise.toml
+++ b/mise.toml
@@ -37,6 +37,15 @@ age = "1.3.2"
 git-cliff = "2.14.2"
 # GitHub CLI, used by the worktree cleanup task and the issue workflow skills.
 gh = "2.102.0"
+# Official LangChain tools, used as an outside check on langstar's results
+# (docs/dev/testing/official-tool-checks.md). Exact pins skip minimum_release_age,
+# so each was chosen to be older than five days.
+# https://github.com/langchain-ai/langsmith-cli
+"github:langchain-ai/langsmith-cli" = "0.3.0"
+# langgraph-cli installs through uv; the LangGraph Python SDK is a library with no
+# executable, so tasks run it with `uv run --with langgraph-sdk==$LANGGRAPH_SDK_VERSION`.
+uv = "0.12.23"
+"pipx:langgraph-cli" = "0.4.33"
 # rulesync is a Node CLI (v27 declares node >=22); without Node its executable
 # cannot run on a clean host.
 node = "22"
@@ -48,6 +57,8 @@ node = "22"
 enter = "lefthook install"
 
 [env]
+# The LangGraph Python SDK version that the check:* tasks run; see [tools] above.
+LANGGRAPH_SDK_VERSION = "0.4.6"
 # Test fixture names are not secrets, so they live here rather than in 1Password.
 REPOSITORY_OWNER = "codekiln"
 REPOSITORY_NAME = "langstar"

From a2f120efc88b3ff79c8ed42f9b7a3b8ef795ceb6 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:22:13 -0400
Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=A9=B9=20fix(testing):=20scope=20th?=
 =?UTF-8?q?e=20oracle=20task's=20credentials=20and=20delete=20only=20what?=
 =?UTF-8?q?=20it=20created?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#747 746.2-oracles Install official LangChain tools through mise to check langstar's results

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 mise-tasks/oracle/prompt-roundtrip | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 3abded25..34bdcdfe 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -6,26 +6,28 @@ set -euo pipefail
 # back, so a bug shared by langstar's push and pull cannot hide. The LangSmith
 # credentials come from fnox, so the check re-runs itself under `fnox exec`.
 # What each official tool can check: docs/dev/testing/official-tool-checks.md
-if [ -z "${LANGSMITH_API_KEY:-}" ]; then
+if [ -z "${LANGSMITH_API_KEY:-}" ] || [ -z "${LANGSMITH_ORGANIZATION_ID:-}" ] || [ -z "${LANGSMITH_WORKSPACE_ID:-}" ]; then
   exec fnox exec -- "$0" "$@"
 fi
 
-repo="oracle-roundtrip-$(date +%s)-$$"
+repo="oracle-roundtrip-$(date +%s)-$$-$RANDOM"
 template="Answer this question: {question}"
 # `-` is the owner handle of a private prompt in the workspace.
 handle="-/${repo}"
 
-cleanup() {
-  langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1 || echo "could not delete ${handle}; delete it by hand" >&2
-}
-trap cleanup EXIT
-
 # The official CLI creates the repo. langstar's own create-on-push sends "-/<repo>"
 # as the handle, which the API rejects, so letting langstar create it would fail here.
 langsmith prompt create --name "$repo" >/dev/null
 
+# Registered only after create succeeds, so a name collision can never delete
+# another run's prompt.
 schema="$(mktemp)"
-trap 'rm -f "$schema"; cleanup' EXIT
+cleanup() {
+  rm -f "$schema"
+  langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1 || echo "could not delete ${handle}; delete it by hand" >&2
+}
+trap cleanup EXIT
+
 printf '%s' '{"type":"object","title":"Out","properties":{"answer":{"type":"string"}},"required":["answer"]}' >"$schema"
 
 # --schema matches the path cli/tests/prompt_structured_test.rs exercises; a push

From 121d8b00129b8037bf4acf4c02819e73c9cbc3db Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:23:54 -0400
Subject: [PATCH 03/11] =?UTF-8?q?=F0=9F=A9=B9=20fix(testing):=20check=20th?=
 =?UTF-8?q?e=20pushed=20variable=20and=20schema,=20and=20correct=20the=20d?=
 =?UTF-8?q?oc=20and=20comments?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#747 746.2-oracles Install official LangChain tools through mise to check langstar's results

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 docs/dev/testing/official-tool-checks.md |  3 ++-
 mise-tasks/oracle/prompt-roundtrip       | 21 ++++++++++++---------
 mise.toml                                |  2 +-
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/docs/dev/testing/official-tool-checks.md b/docs/dev/testing/official-tool-checks.md
index 4f1ad2a1..900e0079 100644
--- a/docs/dev/testing/official-tool-checks.md
+++ b/docs/dev/testing/official-tool-checks.md
@@ -24,7 +24,8 @@ Bump a pin by hand and check that the release is older than five days: `minimum_
 | `runs` | `langsmith run`, `langsmith trace` |
 | `eval` | `langsmith evaluator`, `langsmith experiment` |
 | `queue` | none |
-| `deployment`, `assistant`, `graph` | `langgraph` (local server and build), `langgraph-sdk` (assistants, threads, runs on a deployment) |
+| `langsmith thread`, `langsmith hub` | no langstar command yet |
+| `deployment`, `assistant`, `graph` | `langgraph` (local server, build, `deploy list`/`delete`/`logs`/`revisions list`), `langgraph-sdk` (assistants, threads, runs on a deployment) |
 | `secrets`, `model-config`, `config` | none |
 
 The table says what a tool can reach, not what a check covers today. Run `langsmith <group> --help` to confirm a subcommand exists before writing a check.
diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 34bdcdfe..6e70bea5 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -6,8 +6,8 @@ set -euo pipefail
 # back, so a bug shared by langstar's push and pull cannot hide. The LangSmith
 # credentials come from fnox, so the check re-runs itself under `fnox exec`.
 # What each official tool can check: docs/dev/testing/official-tool-checks.md
-if [ -z "${LANGSMITH_API_KEY:-}" ] || [ -z "${LANGSMITH_ORGANIZATION_ID:-}" ] || [ -z "${LANGSMITH_WORKSPACE_ID:-}" ]; then
-  exec fnox exec -- "$0" "$@"
+if [ -z "${ORACLE_UNDER_FNOX:-}" ] && { [ -z "${LANGSMITH_API_KEY:-}" ] || [ -z "${LANGSMITH_ORGANIZATION_ID:-}" ] || [ -z "${LANGSMITH_WORKSPACE_ID:-}" ]; }; then
+  ORACLE_UNDER_FNOX=1 exec fnox exec -- "$0" "$@"
 fi
 
 repo="oracle-roundtrip-$(date +%s)-$$-$RANDOM"
@@ -38,10 +38,13 @@ cargo run --quiet --bin langstar -- prompt push \
 
 pulled="$(langsmith prompt pull -- "$handle")"
 
-# The manifest is nested JSON, so look for the template text anywhere in it.
-if ! printf '%s' "$pulled" | jq -e --arg t "$template" '[.. | strings | select(. == $t)] | length > 0' >/dev/null; then
-  echo "FAIL: the official CLI did not return the template langstar pushed" >&2
-  printf '%s\n' "$pulled" >&2
-  exit 1
-fi
-echo "OK: langsmith prompt pull returned the template pushed by langstar (${handle})"
+# The manifest is nested JSON, so look for each pushed value anywhere in it:
+# the template text, the input variable, and a property name from the schema.
+for needle in "$template" question answer; do
+  if ! printf '%s' "$pulled" | jq -e --arg t "$needle" '[.. | strings | select(. == $t)] + [.. | objects | keys[] | select(. == $t)] | length > 0' >/dev/null; then
+    echo "FAIL: the official CLI did not return '${needle}', which langstar pushed" >&2
+    printf '%s\n' "$pulled" >&2
+    exit 1
+  fi
+done
+echo "OK: langsmith prompt pull returned the template, variable and schema pushed by langstar (${handle})"
diff --git a/mise.toml b/mise.toml
index 8f1be0b1..0a396773 100644
--- a/mise.toml
+++ b/mise.toml
@@ -57,7 +57,7 @@ node = "22"
 enter = "lefthook install"
 
 [env]
-# The LangGraph Python SDK version that the check:* tasks run; see [tools] above.
+# The LangGraph Python SDK version that the oracle:* tasks will run; see [tools] above.
 LANGGRAPH_SDK_VERSION = "0.4.6"
 # Test fixture names are not secrets, so they live here rather than in 1Password.
 REPOSITORY_OWNER = "codekiln"

From 9d50a15b78e3c598599ab3d345bfe883bf172449 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:25:33 -0400
Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=93=9A=20docs(testing):=20link=20th?=
 =?UTF-8?q?e=20bugs=20the=20first=20check=20found=20and=20use=20plain=20wo?=
 =?UTF-8?q?rding?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#747 746.2-oracles Install official LangChain tools through mise to check langstar's results

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 docs/dev/testing/official-tool-checks.md | 8 ++++----
 mise-tasks/oracle/prompt-roundtrip       | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/dev/testing/official-tool-checks.md b/docs/dev/testing/official-tool-checks.md
index 900e0079..758e56cc 100644
--- a/docs/dev/testing/official-tool-checks.md
+++ b/docs/dev/testing/official-tool-checks.md
@@ -32,13 +32,13 @@ The table says what a tool can reach, not what a check covers today. Run `langsm
 
 ## Checks
 
-Each check is a mise task under `mise-tasks/oracle/`. It gets its credentials from `fnox exec` and cleans up what it creates.
+Each check is a mise task under `mise-tasks/oracle/`, named for the official tool acting as the reference result. It gets its credentials from `fnox exec` and cleans up what it creates.
 
 | Task | Checks |
 |------|--------|
 | `mise run oracle:prompt-roundtrip` | `langstar prompt push` against `langsmith prompt pull` |
 
-## Findings while writing the first check
+## Bugs the first check found
 
-- `langstar prompt push --owner -` for a repo that does not exist yet fails: the create-on-push step sends `-/<repo>` as the handle and the API answers 400. The task creates the repo with `langsmith prompt create` first.
-- A push without `--schema` to an empty repo got HTTP 400 "Manifest must have an id field". The task pushes with `--schema`, the path the integration tests use.
+- [#820](https://github.com/codekiln/langstar/issues/820): `langstar prompt push --owner -` to a new repo fails. The task creates the repo with `langsmith prompt create` first.
+- [#821](https://github.com/codekiln/langstar/issues/821): a push without `--schema` to an empty repo fails. The task pushes with `--schema`.
diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 6e70bea5..81e6b913 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -2,8 +2,8 @@
 #MISE description="Push a private prompt with langstar, read it back with the official langsmith CLI, then delete it"
 set -euo pipefail
 
-# Outside check for `langstar prompt push`: the official CLI reads the prompt
-# back, so a bug shared by langstar's push and pull cannot hide. The LangSmith
+# Check for `langstar prompt push`: the official CLI reads the prompt
+# back, so a bug that langstar's push and pull share still shows up. The LangSmith
 # credentials come from fnox, so the check re-runs itself under `fnox exec`.
 # What each official tool can check: docs/dev/testing/official-tool-checks.md
 if [ -z "${ORACLE_UNDER_FNOX:-}" ] && { [ -z "${LANGSMITH_API_KEY:-}" ] || [ -z "${LANGSMITH_ORGANIZATION_ID:-}" ] || [ -z "${LANGSMITH_WORKSPACE_ID:-}" ]; }; then
@@ -16,7 +16,7 @@ template="Answer this question: {question}"
 handle="-/${repo}"
 
 # The official CLI creates the repo. langstar's own create-on-push sends "-/<repo>"
-# as the handle, which the API rejects, so letting langstar create it would fail here.
+# as the handle, which the API rejects (https://github.com/codekiln/langstar/issues/820), so letting langstar create it would fail here.
 langsmith prompt create --name "$repo" >/dev/null
 
 # Registered only after create succeeds, so a name collision can never delete
@@ -31,7 +31,7 @@ trap cleanup EXIT
 printf '%s' '{"type":"object","title":"Out","properties":{"answer":{"type":"string"}},"required":["answer"]}' >"$schema"
 
 # --schema matches the path cli/tests/prompt_structured_test.rs exercises; a push
-# without it got HTTP 400 "Manifest must have an id field" when this task was written.
+# without it fails; see https://github.com/codekiln/langstar/issues/821.
 cargo run --quiet --bin langstar -- prompt push \
   --owner - --repo "$repo" --template "$template" --input-variables question \
   --schema "$schema" >/dev/null

From 37a68c8a3177e2227bbdbd5bb36991ac67016d79 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:30:41 -0400
Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=A9=B9=20fix(testing):=20pin=20jq?=
 =?UTF-8?q?=20and=20install=20the=20cleanup=20trap=20before=20mktemp?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#747 746.2-oracles Install official LangChain tools through mise to check langstar's results

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 mise-tasks/oracle/prompt-roundtrip | 5 +++--
 mise.toml                          | 2 ++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 81e6b913..59e6016b 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -21,12 +21,13 @@ langsmith prompt create --name "$repo" >/dev/null
 
 # Registered only after create succeeds, so a name collision can never delete
 # another run's prompt.
-schema="$(mktemp)"
+schema=""
 cleanup() {
-  rm -f "$schema"
+  [ -z "$schema" ] || rm -f "$schema"
   langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1 || echo "could not delete ${handle}; delete it by hand" >&2
 }
 trap cleanup EXIT
+schema="$(mktemp)"
 
 printf '%s' '{"type":"object","title":"Out","properties":{"answer":{"type":"string"}},"required":["answer"]}' >"$schema"
 
diff --git a/mise.toml b/mise.toml
index 0a396773..67496a28 100644
--- a/mise.toml
+++ b/mise.toml
@@ -46,6 +46,8 @@ gh = "2.102.0"
 # executable, so tasks run it with `uv run --with langgraph-sdk==$LANGGRAPH_SDK_VERSION`.
 uv = "0.12.23"
 "pipx:langgraph-cli" = "0.4.33"
+# The oracle:* tasks parse the official CLI's JSON with jq.
+jq = "1.8.1"
 # rulesync is a Node CLI (v27 declares node >=22); without Node its executable
 # cannot run on a clean host.
 node = "22"

From 0d614ff8c609e3f2d6a2005fd945e65a3494ae4b Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 13 Oct 2026 18:34:12 -0400
Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=A9=B9=20fix(testing):=20fail=20the?=
 =?UTF-8?q?=20task=20when=20it=20cannot=20delete=20its=20prompt?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#747 746.2-oracles Install official LangChain tools through mise to check langstar's results

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 mise-tasks/oracle/prompt-roundtrip | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 59e6016b..6ffc4504 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -2,7 +2,7 @@
 #MISE description="Push a private prompt with langstar, read it back with the official langsmith CLI, then delete it"
 set -euo pipefail
 
-# Check for `langstar prompt push`: the official CLI reads the prompt
+# This task checks `langstar prompt push`. The official CLI reads the prompt
 # back, so a bug that langstar's push and pull share still shows up. The LangSmith
 # credentials come from fnox, so the check re-runs itself under `fnox exec`.
 # What each official tool can check: docs/dev/testing/official-tool-checks.md
@@ -16,15 +16,22 @@ template="Answer this question: {question}"
 handle="-/${repo}"
 
 # The official CLI creates the repo. langstar's own create-on-push sends "-/<repo>"
-# as the handle, which the API rejects (https://github.com/codekiln/langstar/issues/820), so letting langstar create it would fail here.
+# as the handle, which the API rejects, so letting langstar create it would fail here.
+# https://github.com/codekiln/langstar/issues/820
 langsmith prompt create --name "$repo" >/dev/null
 
 # Registered only after create succeeds, so a name collision can never delete
 # another run's prompt.
 schema=""
+# A failed delete turns a passing run into a failure and keeps an earlier failure's status.
 cleanup() {
+  local status=$?
   [ -z "$schema" ] || rm -f "$schema"
-  langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1 || echo "could not delete ${handle}; delete it by hand" >&2
+  if ! langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1; then
+    echo "could not delete ${handle}; delete it by hand" >&2
+    [ "$status" -ne 0 ] || status=1
+  fi
+  exit "$status"
 }
 trap cleanup EXIT
 schema="$(mktemp)"
@@ -32,7 +39,7 @@ schema="$(mktemp)"
 printf '%s' '{"type":"object","title":"Out","properties":{"answer":{"type":"string"}},"required":["answer"]}' >"$schema"
 
 # --schema matches the path cli/tests/prompt_structured_test.rs exercises; a push
-# without it fails; see https://github.com/codekiln/langstar/issues/821.
+# without it fails (https://github.com/codekiln/langstar/issues/821).
 cargo run --quiet --bin langstar -- prompt push \
   --owner - --repo "$repo" --template "$template" --input-variables question \
   --schema "$schema" >/dev/null

From 7aef7ed0d7f11070273f982e2d0f2459f0ba8b52 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Wed, 14 Oct 2026 17:41:49 -0400
Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=A9=B9=20fix(oracle):=20point=20the?=
 =?UTF-8?q?=20official=20langsmith=20CLI=20at=20the=20endpoint=20langstar?=
 =?UTF-8?q?=20uses?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

prompt-roundtrip passes --api-url on every official CLI call, so a
LANGSMITH_ENDPOINT or saved profile cannot send it to a different server than
langstar's push.

#747 Pin the official LangChain tools and check prompt push against them
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 mise-tasks/oracle/prompt-roundtrip | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 6ffc4504..3d6dc91a 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -10,6 +10,12 @@ if [ -z "${ORACLE_UNDER_FNOX:-}" ] && { [ -z "${LANGSMITH_API_KEY:-}" ] || [ -z
   ORACLE_UNDER_FNOX=1 exec fnox exec -- "$0" "$@"
 fi
 
+# langstar's prompt commands always call this endpoint (LANGSMITH_API_BASE in
+# sdk/src/client.rs), but the official CLI would otherwise take LANGSMITH_ENDPOINT or a
+# saved profile. --api-url on every official call keeps both tools on the same server.
+api_url="https://api.smith.langchain.com"
+official=(langsmith --api-url "$api_url")
+
 repo="oracle-roundtrip-$(date +%s)-$$-$RANDOM"
 template="Answer this question: {question}"
 # `-` is the owner handle of a private prompt in the workspace.
@@ -18,7 +24,7 @@ handle="-/${repo}"
 # The official CLI creates the repo. langstar's own create-on-push sends "-/<repo>"
 # as the handle, which the API rejects, so letting langstar create it would fail here.
 # https://github.com/codekiln/langstar/issues/820
-langsmith prompt create --name "$repo" >/dev/null
+"${official[@]}" prompt create --name "$repo" >/dev/null
 
 # Registered only after create succeeds, so a name collision can never delete
 # another run's prompt.
@@ -27,7 +33,7 @@ schema=""
 cleanup() {
   local status=$?
   [ -z "$schema" ] || rm -f "$schema"
-  if ! langsmith prompt delete --yes -- "$handle" >/dev/null 2>&1; then
+  if ! "${official[@]}" prompt delete --yes -- "$handle" >/dev/null 2>&1; then
     echo "could not delete ${handle}; delete it by hand" >&2
     [ "$status" -ne 0 ] || status=1
   fi
@@ -44,7 +50,7 @@ cargo run --quiet --bin langstar -- prompt push \
   --owner - --repo "$repo" --template "$template" --input-variables question \
   --schema "$schema" >/dev/null
 
-pulled="$(langsmith prompt pull -- "$handle")"
+pulled="$("${official[@]}" prompt pull -- "$handle")"
 
 # The manifest is nested JSON, so look for each pushed value anywhere in it:
 # the template text, the input variable, and a property name from the schema.

From 72252ddf67c4d96f7365e64dab40244df143a585 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Fri, 30 Oct 2026 12:48:19 -0400
Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=A9=B9=20fix(oracle):=20compare=20t?=
 =?UTF-8?q?he=20pulled=20prompt's=20schema,=20template=20and=20variables?=
 =?UTF-8?q?=20at=20their=20manifest=20fields?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The roundtrip check searched for each pushed value anywhere in the pulled
JSON, so a changed property type or a dropped `required` still passed.
It now compares the schema with the pushed file and reads the template and
input variables at the fields langstar writes them to.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 mise-tasks/oracle/prompt-roundtrip | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 3d6dc91a..75bbcb67 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -52,13 +52,28 @@ cargo run --quiet --bin langstar -- prompt push \
 
 pulled="$("${official[@]}" prompt pull -- "$handle")"
 
-# The manifest is nested JSON, so look for each pushed value anywhere in it:
-# the template text, the input variable, and a property name from the schema.
-for needle in "$template" question answer; do
-  if ! printf '%s' "$pulled" | jq -e --arg t "$needle" '[.. | strings | select(. == $t)] + [.. | objects | keys[] | select(. == $t)] | length > 0' >/dev/null; then
-    echo "FAIL: the official CLI did not return '${needle}', which langstar pushed" >&2
+# `langsmith prompt pull` prints {owner, repo, commit_hash, description, manifest}, and
+# the manifest is the StructuredPrompt in LangChain's JSON form that langstar sent
+# (sdk/src/prompts.rs). Check each pushed value at the field it belongs in, and compare
+# the schema with the file as a whole, so a changed type or a dropped `required` fails.
+# The message nesting matches what `langstar prompt pull` reads in cli/src/commands/prompt.rs.
+check() {
+  local what="$1"
+  shift
+  if ! printf '%s' "$pulled" | jq -e "$@" >/dev/null; then
+    echo "FAIL: the official CLI returned a different ${what} than langstar pushed" >&2
     printf '%s\n' "$pulled" >&2
     exit 1
   fi
-done
-echo "OK: langsmith prompt pull returned the template, variable and schema pushed by langstar (${handle})"
+}
+# The filters below are jq programs; `$want` and `$t` are jq variables, not shell ones.
+# shellcheck disable=SC2016
+{
+  check "prompt class" '.manifest.id[-1] == "StructuredPrompt"'
+  check "schema" --slurpfile want "$schema" '.manifest.kwargs.schema_ == $want[0]'
+  check "list of input variables" '.manifest.kwargs.input_variables == ["question"]'
+  check "message count" '.manifest.kwargs.messages | length == 1'
+  check "template" --arg t "$template" '.manifest.kwargs.messages[0].kwargs.prompt.kwargs.template == $t'
+  check "list of template variables" '.manifest.kwargs.messages[0].kwargs.prompt.kwargs.input_variables == ["question"]'
+}
+echo "OK: langsmith prompt pull returned the template, variables and schema pushed by langstar (${handle})"

From 72d2d0e485cae87e7790e5b0a7eeccafbfc85412 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 3 Nov 2026 07:52:53 -0500
Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=93=9D=20docs(testing):=20say=20wha?=
 =?UTF-8?q?t=20an=20oracle=20check=20is=20and=20name=20each=20bug=20it=20f?=
 =?UTF-8?q?ound?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 docs/dev/testing/official-tool-checks.md | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/docs/dev/testing/official-tool-checks.md b/docs/dev/testing/official-tool-checks.md
index 758e56cc..06a42664 100644
--- a/docs/dev/testing/official-tool-checks.md
+++ b/docs/dev/testing/official-tool-checks.md
@@ -1,6 +1,6 @@
 # Checking langstar with the official LangChain tools
 
-A langstar command can be wrong in a way its own SDK cannot see, because the CLI and SDK share the same request code. The official tools are a second implementation: create with langstar, read back with the official tool, and compare. This extends [crud-lifecycle-pattern.md](crud-lifecycle-pattern.md).
+langstar's CLI and SDK send requests through the same code, so a bug in that code passes both sets of langstar tests. A check here creates something with langstar and reads it back with LangChain's own CLI, which LangChain wrote separately. This extends [crud-lifecycle-pattern.md](crud-lifecycle-pattern.md).
 
 ## The tools
 
@@ -12,7 +12,7 @@ A langstar command can be wrong in a way its own SDK cannot see, because the CLI
 | [langgraph-cli](https://pypi.org/project/langgraph-cli/) | `langgraph` |
 | [langgraph-sdk](https://pypi.org/project/langgraph-sdk/) | Python library; run it with `uv run --with "langgraph-sdk==$LANGGRAPH_SDK_VERSION"` |
 
-Bump a pin by hand and check that the release is older than five days: `minimum_release_age` does not apply to exact pins.
+mise's `minimum_release_age` setting holds back releases younger than five days, but it skips exact versions. Before you change a version here, check that the new release is at least five days old.
 
 ## Which tool checks which command
 
@@ -24,21 +24,22 @@ Bump a pin by hand and check that the release is older than five days: `minimum_
 | `runs` | `langsmith run`, `langsmith trace` |
 | `eval` | `langsmith evaluator`, `langsmith experiment` |
 | `queue` | none |
-| `langsmith thread`, `langsmith hub` | no langstar command yet |
 | `deployment`, `assistant`, `graph` | `langgraph` (local server, build, `deploy list`/`delete`/`logs`/`revisions list`), `langgraph-sdk` (assistants, threads, runs on a deployment) |
 | `secrets`, `model-config`, `config` | none |
 
-The table says what a tool can reach, not what a check covers today. Run `langsmith <group> --help` to confirm a subcommand exists before writing a check.
+The official `langsmith thread` and `langsmith hub` commands have no langstar command to check yet.
+
+Only `prompt` has a check today; the table lists what the official tools could check. Before writing a check, run `langsmith <group> --help` to see the subcommands.
 
 ## Checks
 
-Each check is a mise task under `mise-tasks/oracle/`, named for the official tool acting as the reference result. It gets its credentials from `fnox exec` and cleans up what it creates.
+Each check is a mise task under `mise-tasks/oracle/`. In testing, an oracle supplies the expected result, and here LangChain's own CLI is the oracle. Each task gets its credentials from `fnox exec` and cleans up what it creates.
 
 | Task | Checks |
 |------|--------|
 | `mise run oracle:prompt-roundtrip` | `langstar prompt push` against `langsmith prompt pull` |
 
-## Bugs the first check found
+## langstar bugs the prompt check found
 
-- [#820](https://github.com/codekiln/langstar/issues/820): `langstar prompt push --owner -` to a new repo fails. The task creates the repo with `langsmith prompt create` first.
-- [#821](https://github.com/codekiln/langstar/issues/821): a push without `--schema` to an empty repo fails. The task pushes with `--schema`.
+- [#820 prompt push --owner - fails when it creates a new repo](https://github.com/codekiln/langstar/issues/820): the task creates the repo with `langsmith prompt create` first.
+- [#821 prompt push without --schema to an empty repo fails](https://github.com/codekiln/langstar/issues/821): the task pushes with `--schema`.

From a3c2394319b6dc112bfb76b070e356484fdeac46 Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 3 Nov 2026 08:00:01 -0500
Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=93=9D=20docs(testing):=20say=20wha?=
 =?UTF-8?q?t=20the=20cleanup=20trap=20waits=20for=20and=20when=20a=20pin?=
 =?UTF-8?q?=20installs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 docs/dev/testing/official-tool-checks.md | 4 ++--
 mise-tasks/oracle/prompt-roundtrip       | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/dev/testing/official-tool-checks.md b/docs/dev/testing/official-tool-checks.md
index 06a42664..c398ee0d 100644
--- a/docs/dev/testing/official-tool-checks.md
+++ b/docs/dev/testing/official-tool-checks.md
@@ -1,6 +1,6 @@
 # Checking langstar with the official LangChain tools
 
-langstar's CLI and SDK send requests through the same code, so a bug in that code passes both sets of langstar tests. A check here creates something with langstar and reads it back with LangChain's own CLI, which LangChain wrote separately. This extends [crud-lifecycle-pattern.md](crud-lifecycle-pattern.md).
+langstar's CLI and SDK send requests through the same code, so a bug in that code passes both sets of langstar tests. A check here creates something with langstar and reads it back with LangChain's own CLI, which LangChain wrote separately. These checks add an outside reader to the create-then-read tests in [CRUD Lifecycle Testing Pattern](crud-lifecycle-pattern.md), where langstar's CLI and SDK check each other.
 
 ## The tools
 
@@ -12,7 +12,7 @@ langstar's CLI and SDK send requests through the same code, so a bug in that cod
 | [langgraph-cli](https://pypi.org/project/langgraph-cli/) | `langgraph` |
 | [langgraph-sdk](https://pypi.org/project/langgraph-sdk/) | Python library; run it with `uv run --with "langgraph-sdk==$LANGGRAPH_SDK_VERSION"` |
 
-mise's `minimum_release_age` setting holds back releases younger than five days, but it skips exact versions. Before you change a version here, check that the new release is at least five days old.
+mise's `minimum_release_age` setting holds back releases younger than five days only for a version range; an exact version installs at any age. Before you change a version here, check that the new release is at least five days old.
 
 ## Which tool checks which command
 
diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 75bbcb67..798636e6 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -26,8 +26,8 @@ handle="-/${repo}"
 # https://github.com/codekiln/langstar/issues/820
 "${official[@]}" prompt create --name "$repo" >/dev/null
 
-# Registered only after create succeeds, so a name collision can never delete
-# another run's prompt.
+# The script sets the cleanup trap only after `prompt create` succeeds, so a name
+# collision never deletes another run's prompt.
 schema=""
 # A failed delete turns a passing run into a failure and keeps an earlier failure's status.
 cleanup() {

From aad165a2c432d1c85cf4f359cac4be075c9b8a6d Mon Sep 17 00:00:00 2001
From: codekiln <140930+codekiln@users.noreply.github.com>
Date: Tue, 3 Nov 2026 08:06:00 -0500
Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=93=9D=20docs(testing):=20name=20th?=
 =?UTF-8?q?e=20cleanup=20function=20and=20the=20test=20that=20pushes=20wit?=
 =?UTF-8?q?h=20--schema?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 mise-tasks/oracle/prompt-roundtrip | 7 ++++---
 mise.toml                          | 5 +++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/mise-tasks/oracle/prompt-roundtrip b/mise-tasks/oracle/prompt-roundtrip
index 798636e6..f0fe6d64 100755
--- a/mise-tasks/oracle/prompt-roundtrip
+++ b/mise-tasks/oracle/prompt-roundtrip
@@ -29,7 +29,8 @@ handle="-/${repo}"
 # The script sets the cleanup trap only after `prompt create` succeeds, so a name
 # collision never deletes another run's prompt.
 schema=""
-# A failed delete turns a passing run into a failure and keeps an earlier failure's status.
+# cleanup removes the schema file and deletes the prompt. When the delete fails,
+# the task fails, and a run that had already failed keeps its own exit status.
 cleanup() {
   local status=$?
   [ -z "$schema" ] || rm -f "$schema"
@@ -44,8 +45,8 @@ schema="$(mktemp)"
 
 printf '%s' '{"type":"object","title":"Out","properties":{"answer":{"type":"string"}},"required":["answer"]}' >"$schema"
 
-# --schema matches the path cli/tests/prompt_structured_test.rs exercises; a push
-# without it fails (https://github.com/codekiln/langstar/issues/821).
+# langstar's own test, cli/tests/prompt_structured_test.rs, also pushes with --schema.
+# A push without --schema to an empty repo fails: https://github.com/codekiln/langstar/issues/821
 cargo run --quiet --bin langstar -- prompt push \
   --owner - --repo "$repo" --template "$template" --input-variables question \
   --schema "$schema" >/dev/null
diff --git a/mise.toml b/mise.toml
index 67496a28..72105685 100644
--- a/mise.toml
+++ b/mise.toml
@@ -38,8 +38,9 @@ git-cliff = "2.14.2"
 # GitHub CLI, used by the worktree cleanup task and the issue workflow skills.
 gh = "2.102.0"
 # Official LangChain tools, used as an outside check on langstar's results
-# (docs/dev/testing/official-tool-checks.md). Exact pins skip minimum_release_age,
-# so each was chosen to be older than five days.
+# (docs/dev/testing/official-tool-checks.md). mise's minimum_release_age holds back
+# new releases only for version ranges, so each exact version below is a release
+# that was more than five days old when it was pinned.
 # https://github.com/langchain-ai/langsmith-cli
 "github:langchain-ai/langsmith-cli" = "0.3.0"
 # langgraph-cli installs through uv; the LangGraph Python SDK is a library with no