From ff5c7842e15d5512138606fb3be17019d56db19c Mon Sep 17 00:00:00 2001 From: engineer Date: Fri, 14 Aug 2026 08:35:57 -0700 Subject: [PATCH] tools(sentry): grade the noise gate against install-base uptake, not raw volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate ships inside the app binary, so it only runs on devices that took v0.4.14. Grading it on a raw event count is a measurement error in both directions: a still-high number at 30% uptake is the gate WORKING (~70% of baseline is the model's own prediction), and a dip from a quiet weekend is not efficacy. The scheduled re-reads on 08-17 / 08-21 / 09-05 would have hit the first one first. noise-gate-report.mjs folds Play's version share into the comparison expected_post = baseline x (1 - gated_share x 0.969) where 0.969 is measured, not guessed (90d replay in sentry-noise.test.ts), and grades the measured rate against that instead of against the 100%-uptake endpoint. It reports the endpoint separately, so 'is it on track today' and 'will it clear the 3,500/mo org gate' stop being the same question, and it inverts the model to print the IMPLIED on-device efficacy so the constant is checked rather than trusted. It refuses to grade two windows that look like results but are not: no client_discard/before_send (nothing ran the gate) and 0% Play share. Absence of evidence gets its own verdict, UNGRADED. Runs in CI because neither credential (Sentry org token, Play service account) exists outside GitHub Secrets — an agent picking up the 08-21 read locally is stuck otherwise. Weekly cron records the trend regardless. Verified against the live org: 1.2h after the production rollout it reads UNGRADED, before_send=0, 4.95/h vs a 4.71/h baseline — which is exactly right, no device has the build yet. Refs AGE-105 --- .../workflows/sentry-noise-gate-report.yml | 88 +++++ docs/analytics.md | 39 +++ scripts/noise-gate-report.mjs | 331 ++++++++++++++++++ scripts/noise-gate-report.test.mjs | 102 ++++++ scripts/play-version-share.mjs | 5 + 5 files changed, 565 insertions(+) create mode 100644 .github/workflows/sentry-noise-gate-report.yml create mode 100644 scripts/noise-gate-report.mjs create mode 100644 scripts/noise-gate-report.test.mjs diff --git a/.github/workflows/sentry-noise-gate-report.yml b/.github/workflows/sentry-noise-gate-report.yml new file mode 100644 index 00000000..ec9c2ed9 --- /dev/null +++ b/.github/workflows/sentry-noise-gate-report.yml @@ -0,0 +1,88 @@ +name: Sentry noise-gate report + +# Grades the AGE-105 noise gate against the number it promised, with install-base +# uptake folded in — see scripts/noise-gate-report.mjs for why a raw event count +# cannot grade a gate that ships inside an app binary. +# +# It lives in CI rather than on a laptop for one blunt reason: neither credential +# it needs (Sentry org read token, Play service account) exists outside GitHub +# Secrets, so an agent picking up this measurement locally is stuck. Dispatch this +# instead and read the run summary: +# +# gh workflow run "Sentry noise-gate report" -f post=2026-08-21T00:00:00Z..now +# gh run watch && gh run view # table is in the job summary +# +# The scheduled Monday run exists so the trend is recorded even if nobody asks. + +on: + workflow_dispatch: + inputs: + pre: + description: "Baseline window START..END (rate before the rollout reached devices)" + required: false + # Post-box-bot-fix, pre-mobile-rollout. The only clean baseline that + # measures mobile alone: AGE-55 silenced openclaw-box-bot at 06:19Z and + # v0.4.14 reached production at 14:22Z on the same day. + default: "2026-08-14T07:00:00Z..2026-08-14T14:00:00Z" + post: + description: "Measured window START..END (default: trailing 7d)" + required: false + default: "" + project: + description: "Sentry project slug" + required: false + default: "opencode-mobile" + schedule: + # Mondays 15:00 UTC. Weekly, not daily: a window under ~24h ranks sources but + # cannot certify a monthly rate, and Play vitals land with a multi-day lag. + - cron: "0 15 * * 1" + +permissions: + contents: read + +concurrency: + group: sentry-noise-gate-report-${{ github.ref }} + cancel-in-progress: false + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + + # The grading model is the part that can be wrong silently, so it is tested + # before it is trusted — in the same job that publishes the number. + - name: Test the grading model + run: node --test scripts/noise-gate-report.test.mjs + + - name: Report + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_PRODUCT_INTELLIGENCE_TOKEN || secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: ${{ secrets.SENTRY_ORG || 'vibetechnologies' }} + GOOGLE_PLAY_SERVICE_ACCOUNT_JSON: ${{ secrets.PLAY_STORE_SERVICE_ACCOUNT_JSON }} + run: | + set -euo pipefail + args=(--pre "${{ inputs.pre || '2026-08-14T07:00:00Z..2026-08-14T14:00:00Z' }}") + args+=(--project "${{ inputs.project || 'opencode-mobile' }}") + if [ -n "${{ inputs.post }}" ]; then args+=(--post "${{ inputs.post }}"); fi + node scripts/noise-gate-report.mjs "${args[@]}" + + - name: Raw org volume (per project, per outcome, with client_discard reasons) + if: always() + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_PRODUCT_INTELLIGENCE_TOKEN || secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: ${{ secrets.SENTRY_ORG || 'vibetechnologies' }} + run: | + set -euo pipefail + { + echo '' + echo '### Raw org volume' + echo '```' + node scripts/sentry-volume-report.mjs --by-reason || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/analytics.md b/docs/analytics.md index dbf47532..cc512c7a 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -165,6 +165,45 @@ production rollout at 14:22 UTC), two windows agreeing to within 0.2%: Mobile was 87% of the org's post-box-bot demand. Target: under ~1,500/month, which puts the org under the 3,500/month gate. +### Uptake is part of the measurement, not an excuse afterwards + +The gate ships **inside the app binary**, so it only runs on devices that installed v0.4.14. +A raw event count therefore cannot grade it. Replaying 90d of real events through the gate +drops 96.9% of them (~107/month) — but only at 100% install share, which Play never reaches +quickly and never reaches fully. + +The two symmetrical mistakes: + +- reading a still-high number at 30% uptake as "the gate failed" (it predicts ~70% of + baseline — that IS the gate working), and +- reading a dip caused by a quiet weekend, or by users simply opening the app less, as gate + efficacy. + +So grade against the uptake-adjusted expectation: + +``` +expected_post = baseline_rate x (1 - gated_share x 0.969) +``` + +`scripts/noise-gate-report.mjs` does exactly this: it pulls the Sentry rate +(`sentry-volume-report.mjs`) and the Play install share (`play-version-share.mjs`, +versionCode >= 150 = v0.4.14 = gated), prints measured-vs-expected-vs-100%-uptake, and +**refuses to grade** a window where `client_discard/before_send` is 0 or Play share is 0 — +neither of which is a pass or a failure, only an absence of evidence. It also inverts the +model to report the *implied* on-device efficacy, so the 96.9% constant is checked against +reality rather than assumed. + +Neither credential exists on a laptop, so run it in CI and read the job summary: + +```sh +gh workflow run "Sentry noise-gate report" -f post=2026-08-21T00:00:00Z..now +gh run view --log # or just open the run summary +``` + +It also runs itself weekly (Mondays 15:00 UTC) so the trend is recorded whether or not +anyone asks. Unit tests for the grading model live in `scripts/noise-gate-report.test.mjs` +and run in the same job that publishes the number. + ### The quota resets on the 4th — that is the real deadline Org-wide daily `accepted` shows a hard billing boundary: diff --git a/scripts/noise-gate-report.mjs b/scripts/noise-gate-report.mjs new file mode 100644 index 00000000..4f05d7d7 --- /dev/null +++ b/scripts/noise-gate-report.mjs @@ -0,0 +1,331 @@ +#!/usr/bin/env node +/** + * Did the Sentry noise gate actually cut the volume? — the honest version. + * + * WHY THIS EXISTS + * --------------- + * `scripts/sentry-volume-report.mjs` answers "how many events/month is the org + * consuming". That number alone CANNOT grade the gate (AGE-105), because the gate + * ships inside an app binary and only runs on devices that installed it. + * + * On 2026-08-14 the gate went to Play production (versionCode 151). Replaying 90d + * of this project's real events through the gate drops 96.9% of them, which + * predicts ~87 events/month — but ONLY once every user is on a gated build. Play + * uptake takes days-to-weeks and never reaches 100%. + * + * So a naive read on 2026-08-17 ("still 2,000/month, the gate failed") is a + * measurement error, not a finding: if a third of the install base has upgraded, + * the honest expectation is ~2/3 of the baseline, and 2,000 is a PASS. The + * reverse mistake is just as easy — a drop caused by fewer users opening the app + * over a weekend, read as gate efficacy. + * + * This script folds Play's version share into the comparison: + * + * expected_post = baseline_rate x (1 - gated_share x efficacy) + * + * and grades the measured post rate against THAT, not against the endpoint. It + * also reports the endpoint (`fullUptakeProjection`) so the answer to "will this + * clear the 3,500/month org gate when uptake completes" is separate from "is it + * on track today". + * + * NOT A SUBSTITUTE FOR GATE LIVENESS. `client_discard/before_send > 0` proves the + * gate is running on real devices and is independent of install share — it shows + * up first. This script surfaces it and refuses to grade a window without it. + * + * USAGE + * SENTRY_AUTH_TOKEN=... GOOGLE_PLAY_SERVICE_ACCOUNT_JSON=... \ + * node scripts/noise-gate-report.mjs \ + * --pre 2026-08-14T07:00:00Z..2026-08-14T14:00:00Z \ + * --post 2026-08-17T00:00:00Z..now + * + * --json machine-readable + * --no-play skip Play (grades at gated_share=0, i.e. "no credit for uptake") + * --project SLUG default opencode-mobile + * + * Neither credential lives on a laptop. Run it through + * `.github/workflows/sentry-noise-gate-report.yml` (workflow_dispatch), which has + * both as repo secrets and writes the table to the run summary. + * + * Exit code is 0 for any *readable* result — an unmet target is a finding to + * report, not a crash. Non-zero only when the data cannot be obtained. + */ + +import { execFileSync } from "node:child_process" +import { fileURLToPath } from "node:url" +import { dirname, join } from "node:path" + +const HERE = dirname(fileURLToPath(import.meta.url)) +const MONTH_HOURS = 730 + +/** + * First Play versionCode containing the gate. v0.4.14 = internal 150 / production + * 151 (publish-play-store.yml sets versionCode = github.run_number + 100, so the + * gradle number is meaningless here). Anything >= this has the gate. + */ +export const GATE_FIRST_VERSION_CODE = Number(process.env.GATE_FIRST_VERSION_CODE || 150) + +/** + * Share of this project's real events the gate hard-drops, measured — not guessed + * — by replaying the 90d census (648 events, 11 issues) of `opencode-mobile` + * through the gate's own precedence in `src/lib/sentry-noise.test.ts`. 96.9%. + * Deliberately conservative: it counts only the transport drop-list, ignoring the + * dedup/rate-cap layers, which can only remove more. + */ +export const GATE_EFFICACY = Number(process.env.GATE_EFFICACY || 0.969) + +/** Org-wide monthly budget from AGE-71, and this project's own target from AGE-105. */ +export const ORG_MONTHLY_GATE = 3500 +export const PROJECT_MONTHLY_TARGET = 1500 + +/** + * Tolerance on the expectation. The inputs are two independent samples (Sentry + * hours, Play daily vitals) of a diurnal process, so demanding equality would + * make every read a failure. 25% above expectation is the line. + */ +export const TOLERANCE = Number(process.env.NOISE_GATE_TOLERANCE || 0.25) + +/** + * The whole point of the script, isolated and pure so it can be tested without a + * network: what SHOULD the post-deploy rate be, given how much of the install + * base is actually running the gate? + * + * @param {object} p + * @param {number} p.baselinePerHour submitted events/hour before the rollout + * @param {number} p.actualPerHour submitted events/hour in the measured window + * @param {number} p.gatedShare 0..1 share of active users on a gated build + * @param {number} [p.efficacy] share of events the gate drops on a gated device + * @param {boolean} [p.gateLive] did client_discard/before_send appear at all + */ +export function grade({ baselinePerHour, actualPerHour, gatedShare, efficacy = GATE_EFFICACY, gateLive = true }) { + const share = Math.min(Math.max(gatedShare ?? 0, 0), 1) + const expectedPerHour = baselinePerHour * (1 - share * efficacy) + const ceilingPerHour = expectedPerHour * (1 + TOLERANCE) + const fullUptakePerHour = baselinePerHour * (1 - efficacy) + const observedReduction = baselinePerHour > 0 ? 1 - actualPerHour / baselinePerHour : 0 + // What the gate must be doing on gated devices for the observed number to hold. + // Inverts the model instead of trusting the constant: reduction = share x eff. + const impliedEfficacy = share > 0 ? Math.min(Math.max(observedReduction / share, 0), 1) : null + + let verdict + let because + if (!gateLive) { + verdict = "UNGRADED" + because = + "client_discard/before_send is 0 in this window — no device ran the gate, so nothing here measures it. " + + "Check the release actually reached users before reading the rate." + } else if (share === 0) { + verdict = "UNGRADED" + because = "Play reports 0% of active users on a gated build; the rate cannot be attributed to the gate yet." + } else if (actualPerHour <= ceilingPerHour) { + verdict = "ON_TRACK" + because = + `measured ${actualPerHour.toFixed(2)}/h is at or below the ${ceilingPerHour.toFixed(2)}/h ceiling implied by ` + + `${(share * 100).toFixed(1)}% uptake (expected ${expectedPerHour.toFixed(2)}/h +${(TOLERANCE * 100).toFixed(0)}%).` + } else { + verdict = "OFF_TRACK" + because = + `measured ${actualPerHour.toFixed(2)}/h exceeds the ${ceilingPerHour.toFixed(2)}/h ceiling for ` + + `${(share * 100).toFixed(1)}% uptake. Either the gate is not dropping what the replay said it would, or a new ` + + `noise class appeared. Name it — do not widen the drop-list blindly.` + } + + return { + verdict, + because, + gatedShare: share, + efficacy, + baselinePerMonth: baselinePerHour * MONTH_HOURS, + actualPerMonth: actualPerHour * MONTH_HOURS, + expectedPerMonth: expectedPerHour * MONTH_HOURS, + ceilingPerMonth: ceilingPerHour * MONTH_HOURS, + fullUptakeProjection: fullUptakePerHour * MONTH_HOURS, + observedReduction, + impliedEfficacy, + meetsProjectTargetAtFullUptake: fullUptakePerHour * MONTH_HOURS < PROJECT_MONTHLY_TARGET, + } +} + +/** + * Same question one level up: does the ORG clear its 3,500/month gate? + * + * `orgPerHour`/`projectPerHour` must come from the SAME window, and preferably the + * longest one available: the other projects here emit ~0.7 events/hour, so a 1-2h + * window routinely contains zero of them and would flatter the org total to "this + * project only". + */ +export function orgOutlook({ orgPerHour, projectPerHour, projectFullUptakePerMonth }) { + const otherPerMonth = Math.max(orgPerHour - projectPerHour, 0) * MONTH_HOURS + const projectedOrgPerMonth = otherPerMonth + projectFullUptakePerMonth + return { + orgPerMonthNow: orgPerHour * MONTH_HOURS, + otherProjectsPerMonth: otherPerMonth, + projectedOrgPerMonthAtFullUptake: projectedOrgPerMonth, + clearsOrgGate: projectedOrgPerMonth < ORG_MONTHLY_GATE, + headroom: ORG_MONTHLY_GATE - projectedOrgPerMonth, + } +} + +/** Share of active users on a build that contains the gate. */ +export function gatedShareFromPlay(playJson, firstGatedVersionCode = GATE_FIRST_VERSION_CODE) { + const versions = playJson?.versions ?? [] + const total = versions.reduce((s, v) => s + (v.users || 0), 0) + const gated = versions + .filter((v) => Number(v.versionCode) >= firstGatedVersionCode) + .reduce((s, v) => s + (v.users || 0), 0) + return { gated, total, share: total > 0 ? gated / total : 0, window: playJson?.window ?? null } +} + +function runJson(script, args) { + const out = execFileSync(process.execPath, [join(HERE, script), ...args], { + encoding: "utf8", + maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }) + return JSON.parse(out) +} + +function parseArgs(argv) { + const out = { project: "opencode-mobile", play: true } + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (a === "--pre") out.pre = argv[++i] + else if (a === "--post") out.post = argv[++i] + else if (a === "--project") out.project = argv[++i] + else if (a === "--json") out.json = true + else if (a === "--no-play") out.play = false + } + if (!out.pre) throw new Error("--pre START..END is required (the rate before the rollout reached devices)") + if (!out.post) out.post = `${new Date(Date.now() - 7 * 86_400_000).toISOString()}..now` + return out +} + +function pick(report, windowName, project) { + const w = report.windows.find((x) => x.name === windowName) + if (!w) throw new Error(`sentry-volume-report returned no window named ${windowName}`) + const p = w.projects.find((x) => x.project === project) + return { + hours: w.hours, + orgPerHour: w.orgSubmitted / w.hours, + perHour: p ? p.submittedPerHour : 0, + submitted: p ? p.submitted : 0, + gateDropped: p ? p.gateDropped : 0, + backoffDropped: p ? p.backoffDropped : 0, + } +} + +const money = (n) => Math.round(n).toLocaleString("en-US") + +async function main() { + const args = parseArgs(process.argv.slice(2)) + + const volume = runJson("sentry-volume-report.mjs", [ + "--json", + "--window", + `pre=${args.pre}`, + "--window", + `post=${args.post}`, + ]) + const pre = pick(volume, "pre", args.project) + const post = pick(volume, "post", args.project) + + let play = null + let shareInfo = { gated: 0, total: 0, share: 0, window: null } + if (args.play) { + play = runJson("play-version-share.mjs", ["--json"]) + shareInfo = gatedShareFromPlay(play) + } + + const g = grade({ + baselinePerHour: pre.perHour, + actualPerHour: post.perHour, + gatedShare: shareInfo.share, + gateLive: post.gateDropped > 0, + }) + // Other projects are a background rate, not something this gate changes, so + // estimate them from whichever window is long enough to contain any of them. + const bg = post.hours >= pre.hours ? post : pre + const org = orgOutlook({ + orgPerHour: bg.orgPerHour, + projectPerHour: bg.perHour, + projectFullUptakePerMonth: g.fullUptakeProjection, + }) + org.backgroundFrom = bg === post ? "post" : "pre" + org.backgroundHours = bg.hours + + const result = { + project: args.project, + generatedAt: new Date().toISOString(), + windows: { pre: { ...pre, spec: args.pre }, post: { ...post, spec: args.post } }, + uptake: shareInfo, + grade: g, + org, + } + + if (args.json) { + console.log(JSON.stringify(result, null, 2)) + return + } + + const lines = [] + lines.push(`## Noise-gate report — ${args.project}`) + lines.push("") + lines.push(`**${g.verdict}** — ${g.because}`) + lines.push("") + lines.push("| | events/month |") + lines.push("| --- | ---: |") + lines.push(`| baseline (pre, ${pre.hours}h) | ${money(g.baselinePerMonth)} |`) + lines.push(`| measured (post, ${post.hours}h) | ${money(g.actualPerMonth)} |`) + lines.push(`| expected at ${(g.gatedShare * 100).toFixed(1)}% uptake | ${money(g.expectedPerMonth)} |`) + lines.push(`| ceiling (+${(TOLERANCE * 100).toFixed(0)}%) | ${money(g.ceilingPerMonth)} |`) + lines.push(`| projection at 100% uptake | ${money(g.fullUptakeProjection)} |`) + lines.push(`| project target (AGE-105) | ${money(PROJECT_MONTHLY_TARGET)} |`) + lines.push("") + lines.push( + `Uptake: **${shareInfo.gated} of ${shareInfo.total}** active users on versionCode >= ${GATE_FIRST_VERSION_CODE}` + + ` = **${(shareInfo.share * 100).toFixed(1)}%**` + + (shareInfo.window ? ` (Play vitals ${shareInfo.window.start} → ${shareInfo.window.end})` : " (Play skipped)"), + ) + lines.push( + `Gate liveness: client_discard/before_send = **${post.gateDropped}** in the post window` + + ` (ratelimit_backoff ${post.backoffDropped} — that one is the org over quota, not us).`, + ) + if (g.impliedEfficacy !== null) { + lines.push( + `Implied on-device efficacy: **${(g.impliedEfficacy * 100).toFixed(1)}%** ` + + `(the replay predicted ${(g.efficacy * 100).toFixed(1)}%).`, + ) + } + lines.push("") + lines.push( + `Org outlook: other projects ${money(org.otherProjectsPerMonth)}/mo (from the ${org.backgroundFrom} window, ` + + `${org.backgroundHours}h) + this project at full uptake ${money(result.grade.fullUptakeProjection)}/mo = ` + + `**${money(org.projectedOrgPerMonthAtFullUptake)}/mo** vs the ${money(ORG_MONTHLY_GATE)}/mo org gate → ` + + `**${org.clearsOrgGate ? "clears" : "MISSES"}** (headroom ${money(org.headroom)}).`, + ) + lines.push("") + if (post.hours < 24) { + lines.push( + `⚠ The measured window is ${post.hours}h. Under ~24h a window can rank sources but cannot certify a monthly ` + + "rate — diurnal load is real. Treat the numbers above as directional.", + ) + lines.push("") + } + lines.push( + "Read `submitted` (accepted + rate_limited), never `accepted`: while the org is over quota, `accepted` is ~0 for " + + "every project and a broken org is indistinguishable from a fixed one.", + ) + + const text = lines.join("\n") + console.log(text) + if (process.env.GITHUB_STEP_SUMMARY) { + const { appendFileSync } = await import("node:fs") + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${text}\n`) + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((err) => { + console.error(`noise-gate-report: ${err.message}`) + process.exit(1) + }) +} diff --git a/scripts/noise-gate-report.test.mjs b/scripts/noise-gate-report.test.mjs new file mode 100644 index 00000000..963d241a --- /dev/null +++ b/scripts/noise-gate-report.test.mjs @@ -0,0 +1,102 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { + GATE_EFFICACY, + ORG_MONTHLY_GATE, + gatedShareFromPlay, + grade, + orgOutlook, +} from "./noise-gate-report.mjs" + +const MONTH_HOURS = 730 +const BASELINE = 4.71 // measured pre-rollout rate for opencode-mobile, 2026-08-14 07:00-14:00Z + +test("partial uptake is not gate failure — the mistake this script exists to prevent", () => { + // A third of users upgraded. A naive read ("still ~2,200/mo, target is 1,500") + // would call this a failure; with 33% uptake it is exactly on model. + const g = grade({ + baselinePerHour: BASELINE, + actualPerHour: BASELINE * (1 - 0.33 * GATE_EFFICACY), + gatedShare: 0.33, + }) + assert.equal(g.verdict, "ON_TRACK") + assert.ok(g.actualPerMonth > 2000, "the raw number is still far above the 1,500/mo target") + assert.ok(Math.abs(g.impliedEfficacy - GATE_EFFICACY) < 0.01) +}) + +test("full uptake at the replayed efficacy lands under the project target", () => { + const g = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE * (1 - GATE_EFFICACY), gatedShare: 1 }) + assert.equal(g.verdict, "ON_TRACK") + assert.ok(g.meetsProjectTargetAtFullUptake) + assert.ok(g.fullUptakeProjection < 150, `expected ~107/mo, got ${g.fullUptakeProjection}`) +}) + +test("a real regression still fails even when uptake is low enough to excuse a lot", () => { + // 10% uptake excuses almost nothing; volume that did not move at all is fine, + // but volume that GREW is a finding. + const flat = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE, gatedShare: 0.1 }) + assert.equal(flat.verdict, "ON_TRACK", "flat volume at 10% uptake is within tolerance") + const worse = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE * 1.6, gatedShare: 0.1 }) + assert.equal(worse.verdict, "OFF_TRACK") + assert.match(worse.because, /new\s+noise class|not dropping/) +}) + +test("a gate that does nothing on device is caught once uptake is high", () => { + const g = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE, gatedShare: 0.9 }) + assert.equal(g.verdict, "OFF_TRACK") + assert.equal(g.impliedEfficacy, 0, "0% of the drop happened, so implied efficacy is 0") +}) + +test("no before_send discards means the window measures nothing — refuse to grade it", () => { + const g = grade({ baselinePerHour: BASELINE, actualPerHour: 0.01, gatedShare: 0.9, gateLive: false }) + assert.equal(g.verdict, "UNGRADED") + assert.match(g.because, /before_send/) +}) + +test("zero uptake is ungraded, not a pass — a quiet weekend is not efficacy", () => { + const g = grade({ baselinePerHour: BASELINE, actualPerHour: 0.2, gatedShare: 0 }) + assert.equal(g.verdict, "UNGRADED") + assert.equal(g.impliedEfficacy, null) +}) + +test("uptake share counts only builds that contain the gate", () => { + const play = { + versions: [ + { versionCode: 151, users: 30 }, + { versionCode: 150, users: 10 }, + { versionCode: 149, users: 50 }, + { versionCode: 146, users: 10 }, + ], + window: { start: "2026-08-10", end: "2026-08-16" }, + } + const s = gatedShareFromPlay(play) + assert.equal(s.gated, 40) + assert.equal(s.total, 100) + assert.equal(s.share, 0.4) +}) + +test("no Play rows at all reads as 0% uptake, never as a divide-by-zero pass", () => { + const s = gatedShareFromPlay({ versions: [] }) + assert.equal(s.share, 0) + assert.equal(grade({ baselinePerHour: BASELINE, actualPerHour: 0, gatedShare: s.share }).verdict, "UNGRADED") +}) + +test("org outlook subtracts this project before projecting it forward", () => { + // org 5.43/h of which mobile is 4.71/h -> other projects 0.72/h ~= 526/mo + const g = grade({ baselinePerHour: BASELINE, actualPerHour: BASELINE, gatedShare: 0.5 }) + const o = orgOutlook({ + orgPerHour: 5.43, + projectPerHour: BASELINE, + projectFullUptakePerMonth: g.fullUptakeProjection, + }) + assert.ok(Math.abs(o.otherProjectsPerMonth - 0.72 * MONTH_HOURS) < 1) + assert.ok(o.clearsOrgGate) + assert.ok(o.projectedOrgPerMonthAtFullUptake < ORG_MONTHLY_GATE) +}) + +test("org gate can still be missed by other projects even with a perfect mobile gate", () => { + const o = orgOutlook({ orgPerHour: 6, projectPerHour: 0.5, projectFullUptakePerMonth: 100 }) + assert.equal(o.clearsOrgGate, false) + assert.ok(o.headroom < 0) +}) diff --git a/scripts/play-version-share.mjs b/scripts/play-version-share.mjs index 996ce7ac..74f5b818 100644 --- a/scripts/play-version-share.mjs +++ b/scripts/play-version-share.mjs @@ -88,6 +88,11 @@ const VERSION_NAMES = { 146: "0.4.12", // v0.4.13 (retry queue + mailto version stamp) — production dispatch run 49. 149: "0.4.13", + // v0.4.14 = the Sentry noise gate (AGE-105). 150 = internal (tag push, run 50), + // 151 = production (dispatch run 51, 2026-08-14 14:22Z). Both contain the gate, + // which is what scripts/noise-gate-report.mjs counts as "gated". + 150: "0.4.14", + 151: "0.4.14", }; const args = process.argv.slice(2);