From 100f814aead5e13a0a3087b2b59ac9e1a4ca8c69 Mon Sep 17 00:00:00 2001 From: Matt Brooker Date: Tue, 14 Jul 2026 17:20:32 -0400 Subject: [PATCH 1/3] feat(setup): require hosting-provider env var checklist in setup PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The instrument-integration setup run writes env vars to .env files only. Those never reach the deployed app, so a user could merge the PR and have production silently send no events, with nothing in the PR body warning them. Make the deployment target part of the prompt contract: detect it from repo markers (vercel.json/.vercel, netlify.toml, wrangler.toml, fly.toml), and always open the PR body with the env var keys to set and provider-specific steps. The sandbox can never read the provider's env, so the checklist is unconditional. Key names only — values stay out of the PR body. Drop WIZARD_PROMPT rather than extend it: its last consumer was removed in eae3b11b4 (#2016) when the enricher run replaced the wizard run, and @posthog/core is private, so nothing outside the repo can import it. The live path is buildPosthogSetupSuggestion, which is where the guidance lands. Co-Authored-By: Claude Fable 5 --- packages/core/src/setup/prompts.ts | 24 ++++++++++-- packages/core/src/setup/suggestions.test.ts | 41 ++++++++++++++++++++- packages/core/src/setup/suggestions.ts | 12 +++--- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/packages/core/src/setup/prompts.ts b/packages/core/src/setup/prompts.ts index bfe4aad109..d839caa2d7 100644 --- a/packages/core/src/setup/prompts.ts +++ b/packages/core/src/setup/prompts.ts @@ -1,10 +1,28 @@ import { BASE_CATEGORY_ENUM } from "@posthog/core/setup/types"; -export const WIZARD_PROMPT = `/instrument-integration +// Appended to every prompt that instruments an integration: writing env vars to +// .env files does nothing for the deployed app, and the sandbox can never verify +// the hosting provider's env, so the PR must always carry the checklist. +export const DEPLOYMENT_ENV_VAR_PROMPT = ` -After the integration is wired up, also instrument error tracking and session replay (run \`/instrument-error-tracking\`, then add session replay if the framework's posthog-js config supports it). +## Required: env var checklist in the PR body -Run autonomously with sensible defaults — do not ask the user questions. If the PostHog API key isn't already in the project's env files and you can't read it from the PostHog MCP server, leave a placeholder env var and note it in the PR body rather than blocking.`; +Adding env vars to \`.env\`, \`.env.local\` or \`.env.example\` only covers local development. It does not set them on the deployed app, so production will send no events until someone sets them in the hosting provider. You cannot read or set the hosting provider's env from here, so always assume they are missing and always include the section below. + +Detect the deployment target from repo-root markers: \`vercel.json\` or \`.vercel/\` → Vercel; \`netlify.toml\` → Netlify; \`wrangler.toml\` or \`wrangler.jsonc\` → Cloudflare; \`fly.toml\` → Fly.io. If nothing matches, call it "your hosting provider" and give the generic steps. + +Open the PR body — above the summary of changes — with a section titled "Before you merge: set environment variables in " containing: + +- Every env var key name the integration reads, exactly as it appears in the code (e.g. the PostHog public key and host vars for this framework). +- Steps for the detected provider: + - Vercel: \`vercel env add production\` per key, or Project Settings → Environment Variables. + - Netlify: \`netlify env:set \` per key, or Site configuration → Environment variables. + - Cloudflare: \`wrangler secret put \` per key, or \`[vars]\` in the wrangler config for non-secret values. + - Fly.io: \`fly secrets set =\` per key. + - Unknown provider: instruct the user to add each key wherever their production environment defines env vars, and to redeploy. +- A plain statement that until these are set in , the production deployment will send no events to PostHog. + +Never put secret values in the PR body — key names only. Point the user at their PostHog project settings for the value.`; const DISCOVERY_PROMPT_BASE = `You are analyzing this codebase to find the highest-value first tasks for the developer. diff --git a/packages/core/src/setup/suggestions.test.ts b/packages/core/src/setup/suggestions.test.ts index 610f19ff0e..b7840a9bac 100644 --- a/packages/core/src/setup/suggestions.test.ts +++ b/packages/core/src/setup/suggestions.test.ts @@ -67,7 +67,7 @@ describe("buildPosthogSetupSuggestion", () => { it("returns the install suggestion when not installed", () => { const task = buildPosthogSetupSuggestion("not_installed"); expect(task.id).toBe("posthog-setup"); - expect(task.prompt).toBe("/instrument-integration"); + expect(task.prompt?.startsWith("/instrument-integration")).toBe(true); }); it("returns the finish-init suggestion when installed but not initialized", () => { @@ -75,4 +75,43 @@ describe("buildPosthogSetupSuggestion", () => { expect(task.id).toBe("posthog-finish-init"); expect(task.prompt).toContain("skip install steps"); }); + + it.each(["not_installed", "installed_no_init"] as const)( + "requires the hosting-provider env var checklist in the PR body (%s)", + (state) => { + const prompt = buildPosthogSetupSuggestion(state).prompt ?? ""; + expect(prompt).toContain( + "Before you merge: set environment variables in ", + ); + expect(prompt).toContain("production will send no events"); + expect(prompt).toContain("Never put secret values in the PR body"); + }, + ); + + it.each([ + ["Vercel", "vercel env add production"], + ["Netlify", "netlify env:set "], + ["Cloudflare", "wrangler secret put "], + ["Fly.io", "fly secrets set ="], + ])("gives %s-specific env var steps", (provider, command) => { + const prompt = buildPosthogSetupSuggestion("not_installed").prompt ?? ""; + expect(prompt).toContain(provider); + expect(prompt).toContain(command); + }); + + it.each([ + ["vercel.json", "Vercel"], + ["netlify.toml", "Netlify"], + ["wrangler.toml", "Cloudflare"], + ["fly.toml", "Fly.io"], + ])("maps the %s marker to %s", (marker, provider) => { + const prompt = buildPosthogSetupSuggestion("not_installed").prompt ?? ""; + expect(prompt).toContain(`\`${marker}\``); + expect(prompt).toContain(provider); + }); + + it("falls back to a generic provider when no marker matches", () => { + const prompt = buildPosthogSetupSuggestion("not_installed").prompt ?? ""; + expect(prompt).toContain('call it "your hosting provider"'); + }); }); diff --git a/packages/core/src/setup/suggestions.ts b/packages/core/src/setup/suggestions.ts index fa406195c0..82462f85d4 100644 --- a/packages/core/src/setup/suggestions.ts +++ b/packages/core/src/setup/suggestions.ts @@ -1,3 +1,4 @@ +import { DEPLOYMENT_ENV_VAR_PROMPT } from "@posthog/core/setup/prompts"; import type { DiscoveredTask } from "@posthog/core/setup/types"; export interface StaleFlagPayload { @@ -62,8 +63,8 @@ export function buildPosthogSetupSuggestion( impact: "Without PostHog wired in, you have no visibility into how users interact with the product, no error or session-replay coverage, and no way to gate releases behind feature flags.", recommendation: - 'Click "Implement as new task" — the agent runs the bundled instrument-integration skill, sets up env vars, installs the SDK with your project\'s package manager, and opens a PR.', - prompt: "/instrument-integration", + 'Click "Implement as new task" — the agent runs the bundled instrument-integration skill, sets up env vars, installs the SDK with your project\'s package manager, and opens a PR listing the env vars you still need to set in your hosting provider.', + prompt: `/instrument-integration${DEPLOYMENT_ENV_VAR_PROMPT}`, }; } return { @@ -76,8 +77,9 @@ export function buildPosthogSetupSuggestion( impact: "Until init runs, all PostHog calls are no-ops — you'll see no events in the project, no error reports, and no session replays despite the SDK being installed.", recommendation: - 'Click "Implement as new task" — the agent adds the init call and provider component for your framework, sets up the public-token + host env vars, and opens a PR. The SDK package itself is left alone.', - prompt: - "/instrument-integration\n\nThe SDK is already declared in this repo — skip install steps and focus on adding the init call, provider, and env vars.", + 'Click "Implement as new task" — the agent adds the init call and provider component for your framework, sets up the public-token + host env vars, and opens a PR listing the env vars you still need to set in your hosting provider. The SDK package itself is left alone.', + prompt: `/instrument-integration + +The SDK is already declared in this repo — skip install steps and focus on adding the init call, provider, and env vars.${DEPLOYMENT_ENV_VAR_PROMPT}`, }; } From a904b3646e9fd414d1f2d145cc9766248d8d1b93 Mon Sep 17 00:00:00 2001 From: Matt Brooker Date: Tue, 14 Jul 2026 17:24:53 -0400 Subject: [PATCH 2/3] fix(setup): phrase env var checklist for non-PR runs too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The checklist assumed every run ends in a pull request. It doesn't. These suggestion prompts are prefilled into the task composer, where the user picks the workspace mode — cloud, worktree or local — and no mode opens a PR automatically. A local run following the old text had nowhere to put a section the prompt called mandatory, so the guidance was liable to be dropped in exactly the runs that still need it. Name a delivery target for both endings: PR body when a PR is opened, final message of the run when one isn't. The checklist itself is unchanged and stays unconditional — no run can read the hosting provider's env, so the vars are always assumed missing. Co-Authored-By: Claude Fable 5 --- packages/core/src/setup/prompts.ts | 20 +++++++++++++------- packages/core/src/setup/suggestions.test.ts | 19 ++++++++++++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/core/src/setup/prompts.ts b/packages/core/src/setup/prompts.ts index d839caa2d7..24b8e88de8 100644 --- a/packages/core/src/setup/prompts.ts +++ b/packages/core/src/setup/prompts.ts @@ -1,17 +1,23 @@ import { BASE_CATEGORY_ENUM } from "@posthog/core/setup/types"; -// Appended to every prompt that instruments an integration: writing env vars to -// .env files does nothing for the deployed app, and the sandbox can never verify -// the hosting provider's env, so the PR must always carry the checklist. +// Appended to every prompt that instruments an integration. Writing env vars to +// .env files does nothing for the deployed app, and no run can read the hosting +// provider's env, so the checklist is unconditional. Where it gets delivered is +// not: these prompts run in cloud, worktree and local modes (WorkspaceMode), and +// a PR is never opened automatically in any of them — hence the two-branch phrasing. export const DEPLOYMENT_ENV_VAR_PROMPT = ` -## Required: env var checklist in the PR body +## Required: hand off an env var checklist -Adding env vars to \`.env\`, \`.env.local\` or \`.env.example\` only covers local development. It does not set them on the deployed app, so production will send no events until someone sets them in the hosting provider. You cannot read or set the hosting provider's env from here, so always assume they are missing and always include the section below. +Adding env vars to \`.env\`, \`.env.local\` or \`.env.example\` only covers local development. It does not set them on the deployed app, so production will send no events until someone sets them in the hosting provider. You cannot read or set the hosting provider's env from here, so always assume they are missing and always hand off the checklist below. Detect the deployment target from repo-root markers: \`vercel.json\` or \`.vercel/\` → Vercel; \`netlify.toml\` → Netlify; \`wrangler.toml\` or \`wrangler.jsonc\` → Cloudflare; \`fly.toml\` → Fly.io. If nothing matches, call it "your hosting provider" and give the generic steps. -Open the PR body — above the summary of changes — with a section titled "Before you merge: set environment variables in " containing: +Deliver it wherever this run ends: +- If you open a pull request, its body must lead with the checklist — above the summary of changes — under the heading "Before you merge: set environment variables in ". +- If you are working against a local checkout and not opening a pull request, print the same checklist to the user as the final message of the run, under the heading "Before you deploy: set environment variables in ". + +Either way the checklist contains: - Every env var key name the integration reads, exactly as it appears in the code (e.g. the PostHog public key and host vars for this framework). - Steps for the detected provider: @@ -22,7 +28,7 @@ Open the PR body — above the summary of changes — with a section titled "Bef - Unknown provider: instruct the user to add each key wherever their production environment defines env vars, and to redeploy. - A plain statement that until these are set in , the production deployment will send no events to PostHog. -Never put secret values in the PR body — key names only. Point the user at their PostHog project settings for the value.`; +Never put secret values in a PR body — key names only. Point the user at their PostHog project settings for the value.`; const DISCOVERY_PROMPT_BASE = `You are analyzing this codebase to find the highest-value first tasks for the developer. diff --git a/packages/core/src/setup/suggestions.test.ts b/packages/core/src/setup/suggestions.test.ts index b7840a9bac..e847fac1a9 100644 --- a/packages/core/src/setup/suggestions.test.ts +++ b/packages/core/src/setup/suggestions.test.ts @@ -77,14 +77,27 @@ describe("buildPosthogSetupSuggestion", () => { }); it.each(["not_installed", "installed_no_init"] as const)( - "requires the hosting-provider env var checklist in the PR body (%s)", + "requires a hosting-provider env var checklist (%s)", + (state) => { + const prompt = buildPosthogSetupSuggestion(state).prompt ?? ""; + expect(prompt).toContain("production will send no events"); + expect(prompt).toContain("Never put secret values in a PR body"); + }, + ); + + // The prompt runs in cloud, worktree and local modes, and no mode opens a PR + // automatically — so it has to name a delivery target for both endings. + it.each(["not_installed", "installed_no_init"] as const)( + "routes the checklist to a PR body or the final message, per run type (%s)", (state) => { const prompt = buildPosthogSetupSuggestion(state).prompt ?? ""; expect(prompt).toContain( "Before you merge: set environment variables in ", ); - expect(prompt).toContain("production will send no events"); - expect(prompt).toContain("Never put secret values in the PR body"); + expect(prompt).toContain( + "Before you deploy: set environment variables in ", + ); + expect(prompt).toContain("not opening a pull request"); }, ); From bb4619e08ac1014da2caf92b1336478e05a9081a Mon Sep 17 00:00:00 2001 From: Matt Brooker Date: Tue, 21 Jul 2026 14:01:42 -0400 Subject: [PATCH 3/3] fix(setup): make provider placeholder substitution explicit in env var checklist Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LV8ErU3QavwTp2oMe9VSLS --- packages/core/src/setup/prompts.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/setup/prompts.ts b/packages/core/src/setup/prompts.ts index 24b8e88de8..a3ce543f32 100644 --- a/packages/core/src/setup/prompts.ts +++ b/packages/core/src/setup/prompts.ts @@ -1,6 +1,6 @@ import { BASE_CATEGORY_ENUM } from "@posthog/core/setup/types"; -// Appended to every prompt that instruments an integration. Writing env vars to +// Appended to every prompt that performs the initial PostHog SDK install/init. Writing env vars to // .env files does nothing for the deployed app, and no run can read the hosting // provider's env, so the checklist is unconditional. Where it gets delivered is // not: these prompts run in cloud, worktree and local modes (WorkspaceMode), and @@ -13,7 +13,7 @@ Adding env vars to \`.env\`, \`.env.local\` or \`.env.example\` only covers loca Detect the deployment target from repo-root markers: \`vercel.json\` or \`.vercel/\` → Vercel; \`netlify.toml\` → Netlify; \`wrangler.toml\` or \`wrangler.jsonc\` → Cloudflare; \`fly.toml\` → Fly.io. If nothing matches, call it "your hosting provider" and give the generic steps. -Deliver it wherever this run ends: +Deliver it wherever this run ends, replacing in the headings below with the detected provider name (or "your hosting provider" if none matched): - If you open a pull request, its body must lead with the checklist — above the summary of changes — under the heading "Before you merge: set environment variables in ". - If you are working against a local checkout and not opening a pull request, print the same checklist to the user as the final message of the run, under the heading "Before you deploy: set environment variables in ".