From dd670d6cf357770687674d6a7ab5bc342162e495 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:57:35 -0700 Subject: [PATCH 1/5] fix: extract tier-2 events regardless of whether the metric publishes Found by a real 54-second session, which reported zero blinks. detectBlinks sat inside the branch that only ran when the blink metric passed its gates. The metric requires 60 seconds of frontal exposure; the session had 54, so it correctly abstained -- and took every blink actually observed down with it. A publication threshold was suppressing the extraction beneath it, which is the exact inversion of what Tier 2 is for. An abstaining metric must still leave its observations behind, or the substrate only records sessions that did not need it. Detection now runs whenever there are bins, and returns empty rather than reaching a percentile of an empty set when there are none. Also instruments what the diagnostics could not previously explain: - per-bin usable fraction and the largest gap between usable frames - an acceptance curve simulating a fractional frame gate - the voice lane, which had no diagnostics at all The acceptance curve exists because a fractional gate is NOT obviously the fix. Dropping a burst of unusable frames leaves a hole where they were, and the 200 ms gap rule may simply become the new binding constraint. The curve reports bins accepted, bins lost to that gap, and bins lost to sample count at each candidate threshold, so the threshold gets chosen from a real session instead of from my guess. Co-Authored-By: Claude Opus 5 (1M context) --- apps/capture-web/src/ambient-core-adapter.ts | 125 +++++++++ .../ambient-core/src/ambient-face.test.ts | 30 +++ packages/ambient-core/src/ambient-face.ts | 254 +++++++++++++++--- packages/ambient-core/src/ambient-metrics.ts | 9 +- packages/ambient-core/src/ambient-types.ts | 5 + packages/ambient-core/src/ambient-voice.ts | 97 ++++++- packages/ambient-core/src/index.ts | 3 + 7 files changed, 488 insertions(+), 35 deletions(-) diff --git a/apps/capture-web/src/ambient-core-adapter.ts b/apps/capture-web/src/ambient-core-adapter.ts index 05dfb0e..62bb6ab 100644 --- a/apps/capture-web/src/ambient-core-adapter.ts +++ b/apps/capture-web/src/ambient-core-adapter.ts @@ -1,5 +1,7 @@ import { finalizeAmbientMetrics, + type FaceScreeningDiagnostics, + type VoiceScreeningDiagnostics, type AmbientFaceCalibration, type AmbientFacialFrame, type AmbientMetricEvidence, @@ -130,6 +132,127 @@ function primaryTrack(outcome: AmbientMetricOutcome): string { * Counting only the metric's own events keeps the number honest and keeps * each gate answerable by its own evidence. */ +/** + * Prints why the session measured what it measured. + * + * A report full of abstentions looks identical whether the camera saw nobody or + * saw a face that never held still long enough for a bin to qualify, and those + * call for opposite fixes. Console only: no storage, no export, no contract + * surface. Counts and pose percentiles, never a per-frame series. + */ +function reportCaptureDiagnostics( + extraction: { diagnostics?: unknown; events?: unknown }, + faceFrameCount: number +): void { + const all = extraction.diagnostics as + | { face?: FaceScreeningDiagnostics; voice?: VoiceScreeningDiagnostics } + | undefined; + const diagnostics = all?.face; + const voice = all?.voice; + if (!diagnostics) return; + const events = extraction.events as + | { blinks?: readonly unknown[]; expressions?: readonly unknown[]; + pauses?: readonly unknown[]; speechRuns?: readonly unknown[] } + | undefined; + + const pct = (part: number, whole: number) => + whole > 0 ? `${((part / whole) * 100).toFixed(1)}%` : "n/a"; + const lines: string[] = [ + `frames delivered to extractor: ${faceFrameCount}`, + `frames passing every gate: ${diagnostics.usableFrameCount} (${pct(diagnostics.usableFrameCount, diagnostics.frameCount)})`, + `bins accepted: ${diagnostics.binsAccepted} of ${diagnostics.binsConsidered}` + ]; + const pose = diagnostics.pose; + if (pose) { + lines.push( + "absolute pose in degrees (limits yaw 7 / pitch 10 / roll 5):", + ` yaw p50 ${pose.yawP50.toFixed(1)} p95 ${pose.yawP95.toFixed(1)}`, + ` pitch p50 ${pose.pitchP50.toFixed(1)} p95 ${pose.pitchP95.toFixed(1)}`, + ` roll p50 ${pose.rollP50.toFixed(1)} p95 ${pose.rollP95.toFixed(1)}` + ); + } + const gates = Object.entries(diagnostics.frameGateFailures) + .sort(([, a], [, b]) => b - a); + lines.push( + gates.length === 0 + ? "no frame failed any gate" + : "frames failing each gate (one frame can fail several):" + ); + for (const [reason, count] of gates) { + lines.push(` ${reason.padEnd(20)} ${String(count).padStart(6)} ${pct(count, diagnostics.frameCount)}`); + } + const rejections = Object.entries(diagnostics.binRejections) + .sort(([, a], [, b]) => b - a); + lines.push( + rejections.length === 0 + ? "no bin was rejected" + : "bin rejections by first failing check:" + ); + for (const [reason, count] of rejections) { + lines.push(` ${reason.padEnd(20)} ${String(count).padStart(6)}`); + } + const curve = diagnostics.acceptanceCurve ?? []; + if (curve.length > 0) { + lines.push( + "if the all-or-nothing frame gate became fractional:", + " threshold bins lost to gap lost to sample count" + ); + for (const point of curve) { + lines.push( + ` ${point.threshold.toFixed(2).padStart(8)}` + + `${String(point.binsAccepted).padStart(7)}` + + `${String(point.lostToGap).padStart(14)}` + + `${String(point.lostToSampleCount).padStart(23)}` + ); + } + } + const perBin = diagnostics.bins ?? []; + if (perBin.length > 0) { + lines.push("per bin: usable fraction / largest gap between usable frames:"); + for (const bin of perBin) { + lines.push( + ` bin ${String(bin.index).padStart(3)} ` + + `${(bin.usableFraction * 100).toFixed(1).padStart(5)}% ` + + `${String(bin.usableFrameCount).padStart(4)}/${String(bin.frameCount).padEnd(4)} ` + + `gap ${bin.maxUsableGapMs.toFixed(0).padStart(5)} ms` + ); + } + } + if (voice) { + const req = voice.requirements; + lines.push( + "voice lane:", + ` frames ${voice.frameCount}, usable ${voice.usableFrameCount}, ` + + `speech-active ${voice.speechActiveFrameCount}, periodic ${voice.periodicFrameCount}`, + ` segments ${voice.segmentsAccepted} (needs ${req.minimumSegments})`, + ` eligible ${(voice.eligibleDurationMs / 1000).toFixed(1)}s ` + + `(needs ${(req.minimumEligibleSpanMs / 1000).toFixed(0)}s), ` + + `active speech ${(voice.activeSpeechMs / 1000).toFixed(1)}s ` + + `(needs ${(req.minimumActiveSpeechMs / 1000).toFixed(0)}s)` + ); + const vg = Object.entries(voice.frameGateFailures).sort(([, a], [, b]) => b - a); + if (vg.length > 0) { + lines.push(" frames failing each voice gate:"); + for (const [reason, count] of vg) { + lines.push(` ${reason.padEnd(22)} ${String(count).padStart(6)}`); + } + } + } + lines.push( + "tier-2 events:", + ` blinks ${events?.blinks?.length ?? 0}`, + ` expressions ${events?.expressions?.length ?? 0}`, + ` pauses ${events?.pauses?.length ?? 0}`, + ` speech runs ${events?.speechRuns?.length ?? 0}` + ); + // eslint-disable-next-line no-console + console.log( + `%cPhenoMetrix capture diagnostics%c\n${lines.join("\n")}`, + "font-weight:bold", + "font-weight:normal" + ); +} + function relevantEventCount( code: MetricCode, evidence: AmbientMetricEvidence @@ -456,6 +579,8 @@ export function buildAmbientObservation( calibration: input.faceCalibration } }); + + reportCaptureDiagnostics(extraction, input.faceFrames.length); const artifacts = extraction.outcomes.map((outcome) => outcomeArtifacts(outcome, input) ); diff --git a/packages/ambient-core/src/ambient-face.test.ts b/packages/ambient-core/src/ambient-face.test.ts index 2f75f74..c0a39d0 100644 --- a/packages/ambient-core/src/ambient-face.test.ts +++ b/packages/ambient-core/src/ambient-face.test.ts @@ -475,3 +475,33 @@ describe("blink events", () => { expect(left[0].depth).toBeLessThan(right[0].depth); }); }); + +describe("tier-2 events survive a withheld metric", () => { + it("extracts blinks from a session too short for the blink metric", () => { + // Regression from a real 54-second session: the blink metric requires 60 s + // of frontal exposure, and detection sat inside the branch that only ran + // when that gate passed. Every blink actually observed was discarded + // because a PUBLICATION threshold suppressed the EXTRACTION beneath it. + // Tier 2 exists so an abstaining metric still leaves its observations. + const frames = ambientFaceFrames(40_000); + const result = extractAmbientFaceMetrics(frames, OPTIONS); + + const rate = result.outcomes.find( + (outcome) => outcome.code === "ambient.face.blink_rate.bilateral" + ); + expect(rate?.status).toBe("withheld"); + if (rate?.status === "withheld") { + expect(rate.reasonCode).toBe("insufficient-exposure"); + } + + // The metric abstains; the events do not vanish with it. + expect((result.events?.blinks ?? []).length).toBeGreaterThan(0); + }); + + it("reports no blinks rather than throwing when nothing qualified", () => { + // No bins at all is a different statement from no blinks, and it must not + // reach the detector's percentile of an empty set. + const result = extractAmbientFaceMetrics([], OPTIONS); + expect(result.events?.blinks).toEqual([]); + }); +}); diff --git a/packages/ambient-core/src/ambient-face.ts b/packages/ambient-core/src/ambient-face.ts index d38a28f..5a5675a 100644 --- a/packages/ambient-core/src/ambient-face.ts +++ b/packages/ambient-core/src/ambient-face.ts @@ -114,6 +114,63 @@ interface BinScreening { bins: FacialBin[]; attributionFailureCount: number; qualityFailureCount: number; + diagnostics: FaceScreeningDiagnostics; +} + +/** + * Why a session measured what it measured, or why it measured nothing. + * + * Diagnostic only: no metric reads this, and it carries counts and pose + * statistics rather than any per-frame series. It exists because a report full + * of abstentions currently looks identical whether the camera saw nobody or saw + * a face that never held still enough for a bin to qualify -- and those call + * for opposite fixes. + */ +export interface FaceScreeningDiagnostics { + frameCount: number; + usableFrameCount: number; + /** Frames failing each gate. A frame can fail several, so these overlap. */ + frameGateFailures: Record; + /** Absolute pose in degrees across all frames carrying one. */ + pose: { + yawP50: number; yawP95: number; + pitchP50: number; pitchP95: number; + rollP50: number; rollP95: number; + } | null; + binsConsidered: number; + binsAccepted: number; + /** First failing check per rejected bin. */ + binRejections: Record; + /** + * Per bin, how much of it survived the frame gate and what that leaves. + * + * `maxUsableGapMs` is the largest hole between consecutive USABLE frames -- + * the gap that would exist if unusable frames were dropped rather than the + * whole bin. It is the reason a fractional gate is not obviously a fix: + * dropping a burst of bad frames leaves a hole, and the gap rule may simply + * become the new binding constraint. + */ + bins: Array<{ + index: number; + frameCount: number; + usableFrameCount: number; + usableFraction: number; + maxUsableGapMs: number; + }>; + /** + * Bins that WOULD qualify at each candidate frame-usability threshold, if the + * all-or-nothing rule were replaced by a fractional one. + * + * Simulates the full consequence, not just the fraction: a bin counts only if + * it also keeps enough usable frames and leaves no gap wider than the + * existing limit. This is what the threshold should be chosen from. + */ + acceptanceCurve: Array<{ + threshold: number; + binsAccepted: number; + lostToGap: number; + lostToSampleCount: number; + }>; } function finite(value: number): boolean { @@ -210,25 +267,49 @@ function calibratedSizeUsable( ); } +/** + * Every gate a frame failed, empty when it is usable. + * + * The boolean predicate is derived from this rather than duplicating it, so the + * diagnostic view and the measurement path can never disagree about why a frame + * was dropped. Without this the extractor rejects frames silently, and a session + * that abstains is indistinguishable from one that never saw a face. + */ +export function frameGateFailures( + frame: AmbientFacialFrame, + options: AmbientFaceExtractionOptions +): string[] { + const reasons: string[] = []; + const pose = frame.pose; + if (frame.faceCount !== 1) reasons.push("face-count"); + if (faceTrackSegmentId(frame) === null) reasons.push("no-track-id"); + if (!evaluateVisualQuality(frame, null).usable) reasons.push("image-quality"); + if (pose === null) { + reasons.push("no-pose"); + } else { + if (!finite(pose.yawDegrees) || + Math.abs(pose.yawDegrees) > AMBIENT_FACE_MAX_YAW_DEGREES) { + reasons.push("yaw"); + } + if (!finite(pose.pitchDegrees) || + Math.abs(pose.pitchDegrees) > AMBIENT_FACE_MAX_PITCH_DEGREES) { + reasons.push("pitch"); + } + if (!finite(pose.rollDegrees) || + Math.abs(pose.rollDegrees) > AMBIENT_FACE_MAX_ROLL_DEGREES) { + reasons.push("roll"); + } + } + if (!calibratedSizeUsable(frame, options)) reasons.push("face-scale"); + if (!completeGeometry(frame)) reasons.push("incomplete-geometry"); + return reasons; +} + function ambientFrameUsable( frame: AmbientFacialFrame, options: AmbientFaceExtractionOptions ): boolean { - const pose = frame.pose; - return ( - frame.faceCount === 1 && - faceTrackSegmentId(frame) !== null && - evaluateVisualQuality(frame, null).usable && - pose !== null && - finite(pose.yawDegrees) && - Math.abs(pose.yawDegrees) <= AMBIENT_FACE_MAX_YAW_DEGREES && - finite(pose.pitchDegrees) && - Math.abs(pose.pitchDegrees) <= AMBIENT_FACE_MAX_PITCH_DEGREES && - finite(pose.rollDegrees) && - Math.abs(pose.rollDegrees) <= AMBIENT_FACE_MAX_ROLL_DEGREES && - calibratedSizeUsable(frame, options) && - completeGeometry(frame) - ); + return frameGateFailures(frame, options).length === 0; } function mouthWidth(frame: AmbientFacialFrame): number { @@ -314,10 +395,22 @@ function binValues( function qualifyBin( index: number, frames: AmbientFacialFrame[], - options: AmbientFaceExtractionOptions + options: AmbientFaceExtractionOptions, + onReject?: (reason: string) => void ): FacialBin | null { - if (frames.length < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) return null; - if (!frames.every((frame) => ambientFrameUsable(frame, options))) return null; + const reject = (reason: string): null => { + onReject?.(reason); + return null; + }; + if (frames.length < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { + return reject("too-few-frames"); + } + if (!frames.every((frame) => ambientFrameUsable(frame, options))) { + // All-or-nothing by design: one unusable frame discards the whole bin. In + // real capture this is the gate most likely to dominate, which is exactly + // why it now says so. + return reject("frame-gate"); + } const processorRefs = new Set(frames.map((frame) => frame.processorRef)); const trackSegmentIds = new Set(frames.map(faceTrackSegmentId)); const epochs = new Set(frames.map((frame) => frame.captureEpoch)); @@ -326,7 +419,7 @@ function qualifyBin( trackSegmentIds.size !== 1 || epochs.size !== 1 ) { - return null; + return reject("mixed-provenance"); } const gaps = frames .slice(1) @@ -336,7 +429,7 @@ function qualifyBin( (gap) => gap <= 0 || gap > AMBIENT_FACE_MAX_FRAME_GAP_MS ) ) { - return null; + return reject("frame-gap"); } const actualSpanMs = frames.at(-1)!.tMs - frames[0].tMs; const stepMs = nominalStepMs(frames); @@ -350,7 +443,7 @@ function qualifyBin( actualSpanMs < AMBIENT_FACE_MIN_BIN_SPAN_MS || durationMs < AMBIENT_FACE_MIN_BIN_DATA_MS ) { - return null; + return reject("short-bin"); } const sizes = frames.map((frame) => Math.sqrt( @@ -364,7 +457,7 @@ function qualifyBin( sizeP10 <= 0 || sizeP90 / sizeP10 > AMBIENT_FACE_MAX_WITHIN_BIN_SIZE_RATIO ) { - return null; + return reject("scale-drift"); } const startMs = options.sessionStartedAtMs + index * AMBIENT_FACE_BIN_MS; const processorRef = frames[0].processorRef; @@ -411,13 +504,92 @@ function screenBins( bucket.push(frame); buckets.set(index, bucket); } - const bins = [...buckets.entries()] - .sort(([left], [right]) => left - right) - .flatMap(([index, bucket]) => { - const bin = qualifyBin(index, bucket, options); - return bin ? [bin] : []; + const gateFailures: Record = {}; + let usableFrameCount = 0; + for (const frame of frames) { + const failures = frameGateFailures(frame, options); + if (failures.length === 0) usableFrameCount += 1; + for (const reason of failures) { + gateFailures[reason] = (gateFailures[reason] ?? 0) + 1; + } + } + const poses = frames + .map((frame) => frame.pose) + .filter((pose): pose is NonNullable => pose !== null); + const absAt = (pick: (p: NonNullable) => number, q: number) => + percentile(poses.map((pose) => Math.abs(pick(pose))), q); + + const binRejections: Record = {}; + const entries = [...buckets.entries()].sort(([left], [right]) => left - right); + + const binStats = entries.map(([index, bucket]) => { + const usable = bucket.filter((frame) => ambientFrameUsable(frame, options)); + let maxUsableGapMs = 0; + for (let position = 1; position < usable.length; position += 1) { + maxUsableGapMs = Math.max( + maxUsableGapMs, + usable[position].tMs - usable[position - 1].tMs + ); + } + return { + index, + frameCount: bucket.length, + usableFrameCount: usable.length, + usableFraction: + bucket.length > 0 ? usable.length / bucket.length : 0, + maxUsableGapMs + }; + }); + + const acceptanceCurve = [1, 0.98, 0.95, 0.9, 0.85, 0.8].map((threshold) => { + let binsAccepted = 0; + let lostToGap = 0; + let lostToSampleCount = 0; + for (const bin of binStats) { + if (bin.usableFraction < threshold) continue; + if (bin.usableFrameCount < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { + lostToSampleCount += 1; + continue; + } + if (bin.maxUsableGapMs > AMBIENT_FACE_MAX_FRAME_GAP_MS) { + lostToGap += 1; + continue; + } + binsAccepted += 1; + } + return { threshold, binsAccepted, lostToGap, lostToSampleCount }; + }); + const bins = entries.flatMap(([index, bucket]) => { + const bin = qualifyBin(index, bucket, options, (reason) => { + binRejections[reason] = (binRejections[reason] ?? 0) + 1; }); - return { bins, attributionFailureCount, qualityFailureCount }; + return bin ? [bin] : []; + }); + return { + bins, + attributionFailureCount, + qualityFailureCount, + diagnostics: { + frameCount: frames.length, + usableFrameCount, + frameGateFailures: gateFailures, + pose: poses.length > 0 + ? { + yawP50: absAt((pose) => pose.yawDegrees, 0.5), + yawP95: absAt((pose) => pose.yawDegrees, 0.95), + pitchP50: absAt((pose) => pose.pitchDegrees, 0.5), + pitchP95: absAt((pose) => pose.pitchDegrees, 0.95), + rollP50: absAt((pose) => pose.rollDegrees, 0.5), + rollP95: absAt((pose) => pose.rollDegrees, 0.95) + } + : null, + binsConsidered: entries.length, + binsAccepted: bins.length, + binRejections, + bins: binStats, + acceptanceCurve + } + }; } function evidenceFor( @@ -918,6 +1090,25 @@ export function extractAmbientFaceMetrics( detail: "Bilateral blink rate requires a P95 frame gap no greater than 75 ms." }; } + /* + * Detection runs whenever there are bins to run it on, independent of whether + * the blink METRIC publishes. + * + * This used to sit inside the else branch below, so a session that failed the + * 60-second exposure gate discarded every blink it had actually observed. A + * 54-second session with a face in frame throughout reported zero blinks -- + * not because none occurred, but because a publication threshold suppressed + * the extraction feeding it. Tier 2 exists precisely so an abstaining metric + * still leaves its observations behind. + */ + const blinks = + screening.bins.length > 0 + ? detectBlinks(screening.bins) + : { count: 0, perBinCounts: [] as number[], events: [] as BinnedBlink[] }; + blinkEvents.push( + ...blinks.events.map(({ binIndex: _binIndex, ...event }) => event) + ); + if (blinkFailure) { outcomes.push( withheldOutcome( @@ -929,10 +1120,6 @@ export function extractAmbientFaceMetrics( ) ); } else { - const blinks = detectBlinks(screening.bins); - blinkEvents.push( - ...blinks.events.map(({ binIndex: _binIndex, ...event }) => event) - ); const blinkEvidence = evidenceFor(inRange, screening.bins, { frontalExposureMs, blinkCount: blinks.count @@ -1181,6 +1368,7 @@ export function extractAmbientFaceMetrics( // Already fully computed for the summary and previously reduced to two // integers before anything could see them. expressions: expressionSummary?.events ?? [] - } + }, + diagnostics: screening.diagnostics }; } diff --git a/packages/ambient-core/src/ambient-metrics.ts b/packages/ambient-core/src/ambient-metrics.ts index 62a212e..aec297e 100644 --- a/packages/ambient-core/src/ambient-metrics.ts +++ b/packages/ambient-core/src/ambient-metrics.ts @@ -40,6 +40,13 @@ export function finalizeAmbientMetrics( return { outcomes, ignoredFrameCount: - voice.ignoredFrameCount + face.ignoredFrameCount + voice.ignoredFrameCount + face.ignoredFrameCount, + events: { + blinks: face.events?.blinks ?? [], + expressions: face.events?.expressions ?? [], + pauses: voice.events?.pauses ?? [], + speechRuns: voice.events?.speechRuns ?? [] + }, + diagnostics: { face: face.diagnostics, voice: voice.diagnostics } }; } diff --git a/packages/ambient-core/src/ambient-types.ts b/packages/ambient-core/src/ambient-types.ts index 0ea8bbd..b362720 100644 --- a/packages/ambient-core/src/ambient-types.ts +++ b/packages/ambient-core/src/ambient-types.ts @@ -256,6 +256,11 @@ export interface AmbientExtractionResult< * and found nothing, which is a different statement. */ events?: Partial; + /** + * Why this extractor measured what it measured. Diagnostic only; no metric, + * outcome, or report reads it. + */ + diagnostics?: unknown; } export interface AmbientSessionExtractionInput { diff --git a/packages/ambient-core/src/ambient-voice.ts b/packages/ambient-core/src/ambient-voice.ts index 2784012..d0fc7eb 100644 --- a/packages/ambient-core/src/ambient-voice.ts +++ b/packages/ambient-core/src/ambient-voice.ts @@ -94,6 +94,33 @@ interface VoiceSegment { sourceWindowRef: string; } +/** + * Why the voice lane measured what it measured. + * + * The face lane's equivalent showed that a report of abstentions says nothing + * about whether the camera saw a face. The same was true here and worse: a + * silent voice lane could mean the participant did not speak, or that every + * frame failed an acquisition gate, and nothing distinguished them. + */ +export interface VoiceScreeningDiagnostics { + frameCount: number; + /** Frames passing every acquisition gate. */ + usableFrameCount: number; + speechActiveFrameCount: number; + periodicFrameCount: number; + /** Frames failing each gate; a frame can fail several. */ + frameGateFailures: Record; + segmentsAccepted: number; + eligibleDurationMs: number; + activeSpeechMs: number; + /** What the extractor needs before any timing metric can publish. */ + requirements: { + minimumSegments: number; + minimumEligibleSpanMs: number; + minimumActiveSpeechMs: number; + }; +} + interface RunDurations { speechRunsMs: number[]; pausesMs: number[]; @@ -107,6 +134,43 @@ function finite(value: number): boolean { return Number.isFinite(value); } +/** Every acquisition gate a voice frame failed; empty when usable. */ +export function voiceFrameGateFailures(frame: AmbientVoiceFrame): string[] { + const reasons: string[] = []; + const active = frame.speechActive; + if (frame.taskContext !== AMBIENT_VOICE_TASK_CONTEXT) reasons.push("task-context"); + if (typeof frame.trackSegmentId !== "string" || frame.trackSegmentId.length === 0) { + reasons.push("no-track-id"); + } + if (!finite(frame.tMs)) reasons.push("no-timestamp"); + if (!finite(frame.sampleRateHz) || frame.sampleRateHz < AMBIENT_VOICE_MIN_SAMPLE_RATE_HZ) { + reasons.push("sample-rate"); + } + if (!finite(frame.blockGapMs) || frame.blockGapMs > AMBIENT_VOICE_MAX_GAP_MS) { + reasons.push("block-gap"); + } + if (!finite(frame.lostBlockFraction) || + frame.lostBlockFraction > AMBIENT_VOICE_MAX_LOST_BLOCK_FRACTION) { + reasons.push("lost-blocks"); + } + if (!finite(frame.clippedSampleFraction) || + frame.clippedSampleFraction > AMBIENT_VOICE_MAX_CLIPPED_FRACTION) { + reasons.push("clipping"); + } + if (!finite(frame.dcOffset) || + Math.abs(frame.dcOffset) > AMBIENT_VOICE_MAX_ABSOLUTE_DC_OFFSET) { + reasons.push("dc-offset"); + } + if (active && (!finite(frame.snrDb) || frame.snrDb < AMBIENT_VOICE_MIN_SPEECH_SNR_DB)) { + reasons.push("speech-snr"); + } + const blocking = frame.qualityReasons.filter((reason) => + FRAME_BLOCKING_REASONS.has(reason) + ); + for (const reason of blocking) reasons.push(`quality:${reason}`); + return reasons; +} + function timingFrameUsable(frame: AmbientVoiceFrame): boolean { const active = frame.speechActive; return ( @@ -958,9 +1022,40 @@ export function extractAmbientVoiceMetrics( } return outcome; }); + const voiceGateFailures: Record = {}; + let usableVoiceFrames = 0; + for (const frame of inRange) { + const failures = voiceFrameGateFailures(frame); + if (failures.length === 0) usableVoiceFrames += 1; + for (const reason of failures) { + voiceGateFailures[reason] = (voiceGateFailures[reason] ?? 0) + 1; + } + } + const voiceDiagnostics: VoiceScreeningDiagnostics = { + frameCount: inRange.length, + usableFrameCount: usableVoiceFrames, + speechActiveFrameCount: inRange.filter((frame) => frame.speechActive).length, + periodicFrameCount: inRange.filter((frame) => frame.periodic).length, + frameGateFailures: voiceGateFailures, + segmentsAccepted: segments.length, + eligibleDurationMs: segments.reduce( + (total, segment) => total + segment.durationMs, + 0 + ), + activeSpeechMs: segments.reduce( + (total, segment) => total + segment.activeDurationMs, + 0 + ), + requirements: { + minimumSegments: AMBIENT_VOICE_MIN_SEGMENTS, + minimumEligibleSpanMs: AMBIENT_VOICE_TIMING_MIN_MS, + minimumActiveSpeechMs: AMBIENT_VOICE_ACTIVE_MIN_MS + } + }; return { outcomes: ordered, ignoredFrameCount, - events: { pauses: pauseEvents, speechRuns: speechRunEvents } + events: { pauses: pauseEvents, speechRuns: speechRunEvents }, + diagnostics: voiceDiagnostics }; } diff --git a/packages/ambient-core/src/index.ts b/packages/ambient-core/src/index.ts index a4400ca..afd9a55 100644 --- a/packages/ambient-core/src/index.ts +++ b/packages/ambient-core/src/index.ts @@ -112,6 +112,9 @@ export { type ExpressionEvent, type ExpressionSummary } from "./expression-events.js"; +export { frameGateFailures } from "./ambient-face.js"; +export type { FaceScreeningDiagnostics } from "./ambient-face.js"; +export type { VoiceScreeningDiagnostics } from "./ambient-voice.js"; export type { BlinkEventRecord, ExpressionEventRecord, From 079aed8bd29e1e00aaa614283429e718c578bc7a Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:05:57 -0700 Subject: [PATCH 2/5] fix: detect tier-2 events on the raw stream, not the pose-gated bins A real 70-second session with a face in frame throughout produced 0 of 14 qualifying bins and, consequently, not one blink or expression. All 27 metrics correctly reported Not measurable. The reporting was right; the coupling was not. Blink and expression detection read screening.bins, so they inherited a gate built for a different purpose. The pose limits exist so CROSS-FRAME GEOMETRIC COMPARISON stays valid -- comparing one corner against the other, measuring asymmetry. A blink is not that. It is a relative aperture change within one eye over about 150 ms, during which head pose is essentially constant, so it survives a 15-degree turn intact. Holding detection to a standard designed for asymmetry cost that session everything it contained. Detection now runs on a looser stream: attribution, image quality, and geometry completeness, without the pose and calibrated-scale gates. Every event carries poseWithinMeasurementLimits, so rate and timing can use all of them while anything comparing the two sides can filter to the trustworthy set. Detectors emit a Detected* shape and whatever holds the pose stream annotates it -- geometry alone cannot assert that flag. Published metrics are untouched. The blink rate stays bin-derived and remains explicitly a rate over pose-qualified windows; these events answer the separate question of what the session actually contained. The session's other findings are recorded but not yet acted on: a fractional frame gate will not rescue Tier 3 on its own, because at the ~50% usable fraction the data shows, bins clear the sample-count minimum and then die on the 200 ms gap rule instead -- one of them by 2 ms. Also renames the e2e mount path and the audio worklet processor, which the PhenoMetrix rename missed because neither matched its patterns. Co-Authored-By: Claude Opus 5 (1M context) --- apps/capture-web/e2e/ambient-smoke.spec.ts | 2 +- apps/capture-web/e2e/static-server.ts | 2 +- apps/capture-web/playwright.config.ts | 4 +- apps/capture-web/public/asset-manifest.json | 2 +- .../public/voice-capture-worklet.js | 4 +- apps/capture-web/src/static-assets.test.ts | 6 +- apps/capture-web/src/voice-capture.ts | 2 +- apps/capture-web/src/voice-worklet.test.ts | 4 +- .../ambient-core/src/ambient-face.test.ts | 44 ++++++ packages/ambient-core/src/ambient-face.ts | 130 ++++++++++++++++-- .../ambient-core/src/expression-events.ts | 4 +- packages/ambient-core/src/index.ts | 2 + packages/ambient-core/src/kinematic-events.ts | 38 +++++ 13 files changed, 221 insertions(+), 23 deletions(-) diff --git a/apps/capture-web/e2e/ambient-smoke.spec.ts b/apps/capture-web/e2e/ambient-smoke.spec.ts index c605ee1..1e76289 100644 --- a/apps/capture-web/e2e/ambient-smoke.spec.ts +++ b/apps/capture-web/e2e/ambient-smoke.spec.ts @@ -5,7 +5,7 @@ import { resolveLateAudio } from "./ambient-browser-fixture.js"; -const appUrl = "/phenometric/"; +const appUrl = "/phenometrix/"; async function consentAndStart(page: Page): Promise { await page.locator("#consent-checkbox").check(); diff --git a/apps/capture-web/e2e/static-server.ts b/apps/capture-web/e2e/static-server.ts index 7947e56..5ffb4c7 100644 --- a/apps/capture-web/e2e/static-server.ts +++ b/apps/capture-web/e2e/static-server.ts @@ -6,7 +6,7 @@ import { fileURLToPath } from "node:url"; const dist = resolve( fileURLToPath(new URL("../dist", import.meta.url)) ); -const mount = "/phenometric/"; +const mount = "/phenometrix/"; const contentTypes: Record = { ".css": "text/css; charset=utf-8", ".html": "text/html; charset=utf-8", diff --git a/apps/capture-web/playwright.config.ts b/apps/capture-web/playwright.config.ts index 4f4cc9d..47d1d85 100644 --- a/apps/capture-web/playwright.config.ts +++ b/apps/capture-web/playwright.config.ts @@ -6,14 +6,14 @@ export default defineConfig({ fullyParallel: false, workers: 1, use: { - baseURL: "http://127.0.0.1:4173/phenometric/", + baseURL: "http://127.0.0.1:4173/phenometrix/", channel: "chrome", headless: true, trace: "retain-on-failure" }, webServer: { command: "pnpm exec tsx e2e/static-server.ts", - url: "http://127.0.0.1:4173/phenometric/", + url: "http://127.0.0.1:4173/phenometrix/", reuseExistingServer: false, timeout: 30_000 } diff --git a/apps/capture-web/public/asset-manifest.json b/apps/capture-web/public/asset-manifest.json index 68eeb6f..df6f8ce 100644 --- a/apps/capture-web/public/asset-manifest.json +++ b/apps/capture-web/public/asset-manifest.json @@ -7,7 +7,7 @@ }, "voiceWorklet": { "path": "voice-capture-worklet.js", - "sha256": "7003f2dbbf8cbf234f1f8e15195cca36b1fe0a014944b6593c69441ba3945332" + "sha256": "a80e57bea25a7b8991f5cdecbb96bef2b85c58fb0443d80091f96e15c12b168a" }, "visionWasmScript": { "path": "mediapipe/vision_wasm_internal.js", diff --git a/apps/capture-web/public/voice-capture-worklet.js b/apps/capture-web/public/voice-capture-worklet.js index 00d41f1..7e8328b 100644 --- a/apps/capture-web/public/voice-capture-worklet.js +++ b/apps/capture-web/public/voice-capture-worklet.js @@ -60,7 +60,7 @@ class PhenoMetrixVoiceCaptureProcessor extends AudioWorkletProcessor { this.absoluteSampleIndex += this.blockSamples; this.dataPort.postMessage( { - schemaVersion: "phenometric.voice-worklet-message.v1", + schemaVersion: "phenometrix.voice-worklet-message.v1", type: "pcm-block", captureEpoch: this.captureEpoch, sequence: this.sequence, @@ -97,6 +97,6 @@ class PhenoMetrixVoiceCaptureProcessor extends AudioWorkletProcessor { } registerProcessor( - "phenometric-voice-capture", + "phenometrix-voice-capture", PhenoMetrixVoiceCaptureProcessor ); diff --git a/apps/capture-web/src/static-assets.test.ts b/apps/capture-web/src/static-assets.test.ts index ac008b6..2f672d3 100644 --- a/apps/capture-web/src/static-assets.test.ts +++ b/apps/capture-web/src/static-assets.test.ts @@ -12,14 +12,14 @@ describe("static asset manifest", () => { expect( resolveAssetUrl( "models/face_landmarker.task", - "https://example.test/tools/phenometric/index.html" + "https://example.test/tools/phenometrix/index.html" ) - ).toBe("https://example.test/tools/phenometric/models/face_landmarker.task"); + ).toBe("https://example.test/tools/phenometrix/models/face_landmarker.task"); }); it("rejects root-relative and escaping paths", () => { const template = { - schemaVersion: "phenometric.static-assets.v1", + schemaVersion: "phenometrix.static-assets.v1", assets: {} }; expect(() => diff --git a/apps/capture-web/src/voice-capture.ts b/apps/capture-web/src/voice-capture.ts index 3180433..1846f31 100644 --- a/apps/capture-web/src/voice-capture.ts +++ b/apps/capture-web/src/voice-capture.ts @@ -63,7 +63,7 @@ export async function startVoiceCapturePipeline( options.audioContext.createMediaStreamSource(options.stream); const worklet = new AudioWorkletNode( options.audioContext, - "phenometric-voice-capture", + "phenometrix-voice-capture", { numberOfInputs: 1, numberOfOutputs: 1, diff --git a/apps/capture-web/src/voice-worklet.test.ts b/apps/capture-web/src/voice-worklet.test.ts index 9344d22..447e261 100644 --- a/apps/capture-web/src/voice-worklet.test.ts +++ b/apps/capture-web/src/voice-worklet.test.ts @@ -47,7 +47,7 @@ function loadProcessor(): new () => { name: string, processor: typeof registered ) => { - expect(name).toBe("phenometric-voice-capture"); + expect(name).toBe("phenometrix-voice-capture"); registered = processor; }, sampleRate: 48_000, @@ -96,7 +96,7 @@ describe("voice capture AudioWorklet", () => { renderBlock(processor, 0.25); expect(blocks).toHaveLength(1); expect(blocks[0]).toMatchObject({ - schemaVersion: "phenometric.voice-worklet-message.v1", + schemaVersion: "phenometrix.voice-worklet-message.v1", type: "pcm-block", captureEpoch: 7, sequence: 1, diff --git a/packages/ambient-core/src/ambient-face.test.ts b/packages/ambient-core/src/ambient-face.test.ts index c0a39d0..4c1c4c2 100644 --- a/packages/ambient-core/src/ambient-face.test.ts +++ b/packages/ambient-core/src/ambient-face.test.ts @@ -505,3 +505,47 @@ describe("tier-2 events survive a withheld metric", () => { expect(result.events?.blinks).toEqual([]); }); }); + +describe("tier-2 events survive a session no bin qualifies", () => { + /** Blinking normally, but pitched past the 10-degree limit throughout. */ + function pitchedAwayFrames(): AmbientFacialFrame[] { + return ambientFaceFrames(70_000, 30, (frame) => ({ + pose: { yawDegrees: 2, pitchDegrees: 14, rollDegrees: 2 } + })); + } + + it("records blinks when every bin fails the pose gate", () => { + // Reproduces a real 70-second session: a face in frame the whole time, a + // systematic pose offset from a low-mounted camera, 0 of 14 bins accepted, + // and consequently not one blink recorded. Detection had been reading the + // qualifying bins, so it inherited a gate built for cross-frame geometric + // comparison -- a standard a blink does not need. + const result = extractAmbientFaceMetrics(pitchedAwayFrames(), OPTIONS); + + expect( + result.outcomes.every((outcome) => outcome.status === "withheld") + ).toBe(true); + expect((result.events?.blinks ?? []).length).toBeGreaterThan(0); + }); + + it("marks those events as outside the measurement pose limits", () => { + // The events exist, and they say plainly that a left-versus-right + // comparison of them would not be trustworthy. Rate and timing can use + // them; asymmetry should not. + const result = extractAmbientFaceMetrics(pitchedAwayFrames(), OPTIONS); + const blinks = result.events?.blinks ?? []; + expect(blinks.length).toBeGreaterThan(0); + expect( + blinks.every((blink) => blink.poseWithinMeasurementLimits === false) + ).toBe(true); + }); + + it("flags events as within limits when the pose is good", () => { + const result = extractAmbientFaceMetrics(ambientFaceFrames(), OPTIONS); + const blinks = result.events?.blinks ?? []; + expect(blinks.length).toBeGreaterThan(0); + expect( + blinks.every((blink) => blink.poseWithinMeasurementLimits === true) + ).toBe(true); + }); +}); diff --git a/packages/ambient-core/src/ambient-face.ts b/packages/ambient-core/src/ambient-face.ts index 5a5675a..3e4b209 100644 --- a/packages/ambient-core/src/ambient-face.ts +++ b/packages/ambient-core/src/ambient-face.ts @@ -12,6 +12,8 @@ import { } from "./expression-events.js"; import type { BlinkEventRecord, + DetectedBlink, + ExpressionEventRecord, SubjectSide } from "./kinematic-events.js"; import { @@ -312,6 +314,53 @@ function ambientFrameUsable( return frameGateFailures(frame, options).length === 0; } +/** + * Whether a frame can support Tier-2 event detection. + * + * Deliberately looser than {@link ambientFrameUsable}: it drops the pose and + * calibrated-scale gates and keeps attribution, image quality, and geometry + * completeness. + * + * Those pose limits exist so that CROSS-FRAME GEOMETRIC COMPARISON stays valid + * -- comparing one corner against the other, measuring asymmetry. A blink is + * not that. It is a relative aperture change within one eye over about 150 ms, + * during which the head pose is essentially constant, so it survives a 15 degree + * turn intact. Holding event detection to a standard designed for a different + * measurement cost a real 70-second session every blink and expression it + * contained. + * + * Events carry {@link BlinkEventRecord.poseWithinMeasurementLimits} so a + * consumer that DOES need geometric comparability can still filter to the + * stricter set. + */ +function tier2FrameUsable(frame: AmbientFacialFrame): boolean { + return ( + frame.faceCount === 1 && + faceTrackSegmentId(frame) !== null && + evaluateVisualQuality(frame, null).usable && + completeGeometry(frame) + ); +} + +/** Whether every frame spanning an event stayed inside the Tier-3 pose limits. */ +function poseWithinLimits( + frames: readonly AmbientFacialFrame[], + startMs: number, + endMs: number, + options: AmbientFaceExtractionOptions +): boolean { + const spanning = frames.filter( + (frame) => frame.tMs >= startMs && frame.tMs <= endMs + ); + if (spanning.length === 0) return false; + return spanning.every((frame) => { + const failures = frameGateFailures(frame, options); + return !failures.some((reason) => + reason === "yaw" || reason === "pitch" || reason === "roll" + ); + }); +} + function mouthWidth(frame: AmbientFacialFrame): number { const corners = frame.mouthCorners!; return Math.hypot( @@ -786,8 +835,19 @@ function p95Gaps(bins: readonly FacialBin[]): number { return gaps.length > 0 ? percentile(gaps, 0.95) : Number.POSITIVE_INFINITY; } +/** + * The detector needs only an ordered group of frames and an index to attribute + * results to. A qualifying bin satisfies this, and so does the raw frame stream + * wrapped as a single group -- which is what lets Tier-2 extraction run without + * inheriting the pose gating a bin implies. + */ +interface FrameGroup { + index: number; + frames: AmbientFacialFrame[]; +} + /** One eye's blink, plus the bin it peaked in so per-bin rates survive. */ -interface BinnedBlink extends BlinkEventRecord { +interface BinnedBlink extends DetectedBlink { binIndex: number; } @@ -805,7 +865,7 @@ interface BinnedBlink extends BlinkEventRecord { * waveform, three findings, none of them recoverable from a count. */ function detectBlinksForEye( - bins: readonly FacialBin[], + bins: readonly FrameGroup[], side: SubjectSide, apertureOf: (frame: AmbientFacialFrame) => number ): BinnedBlink[] { @@ -924,7 +984,7 @@ function detectBlinksForEye( } /** Every blink from both eyes, ordered by the moment of maximum closure. */ -function detectBlinkEvents(bins: readonly FacialBin[]): BinnedBlink[] { +function detectBlinkEvents(bins: readonly FrameGroup[]): BinnedBlink[] { return [ ...detectBlinksForEye(bins, "left", (frame) => frame.eyeAperture!.left), ...detectBlinksForEye(bins, "right", (frame) => frame.eyeAperture!.right) @@ -939,7 +999,7 @@ function detectBlinkEvents(bins: readonly FacialBin[]): BinnedBlink[] { * closure now exists as an event without inflating that count. */ function detectBlinks( - bins: readonly FacialBin[] + bins: readonly FrameGroup[] ): { count: number; perBinCounts: number[]; events: BinnedBlink[] } { const events = detectBlinkEvents(bins); const perBinCounts = bins.map(() => 0); @@ -981,6 +1041,10 @@ export function extractAmbientFaceMetrics( // outcomes. An outcome carries one value by construction, so a series cannot // travel inside it. const blinkEvents: BlinkEventRecord[] = []; + // Ordered, attribution- and geometry-complete frames without the pose gate. + const tier2Frames = [...inRange] + .filter(tier2FrameUsable) + .sort((left, right) => left.tMs - right.tMs); const screening = screenBins(inRange, options); const evidence = evidenceFor(inRange, screening.bins); const failure = commonFailure(screening, options); @@ -1105,9 +1169,31 @@ export function extractAmbientFaceMetrics( screening.bins.length > 0 ? detectBlinks(screening.bins) : { count: 0, perBinCounts: [] as number[], events: [] as BinnedBlink[] }; - blinkEvents.push( - ...blinks.events.map(({ binIndex: _binIndex, ...event }) => event) - ); + + /* + * Tier-2 events come from the LOOSE stream, not the qualifying bins. + * + * The published blink rate above stays bin-derived and unchanged: it is + * explicitly a rate over pose-qualified windows. These events answer a + * different question -- what did the session actually contain -- and a + * session whose bins all failed the pose gate still contained blinks. + */ + if (tier2Frames.length > 0) { + for (const event of detectBlinkEvents([ + { index: 0, frames: tier2Frames } + ])) { + const { binIndex: _binIndex, ...record } = event; + blinkEvents.push({ + ...record, + poseWithinMeasurementLimits: poseWithinLimits( + tier2Frames, + record.onsetMs, + record.offsetMs, + options + ) + }); + } + } if (blinkFailure) { outcomes.push( @@ -1151,6 +1237,34 @@ export function extractAmbientFaceMetrics( ) : null; + /* + * Expressions are detected on the loose stream for the same reason blinks are: + * a mouth movement is recoverable at a head angle that would invalidate a + * left-versus-right comparison of it. The per-side excursion METRICS below + * still come from the qualifying bins; these events are the record of what + * the session contained. + */ + const looseExpressions = + tier2Frames.length > 0 + ? summarizeExpressions( + tier2Frames, + tier2Frames.length > 1 + ? tier2Frames.at(-1)!.tMs - tier2Frames[0].tMs + : 0 + ) + : null; + const expressionEvents: ExpressionEventRecord[] = ( + looseExpressions?.events ?? [] + ).map((event) => ({ + ...event, + poseWithinMeasurementLimits: poseWithinLimits( + tier2Frames, + event.startMs, + event.endMs, + options + ) + })); + const expressionEvidence = evidenceFor(inRange, screening.bins, { expressionEventCount: expressionSummary?.eventCount, coupledExpressionEventCount: expressionSummary?.synkinesisEventCount @@ -1367,7 +1481,7 @@ export function extractAmbientFaceMetrics( blinks: blinkEvents, // Already fully computed for the summary and previously reduced to two // integers before anything could see them. - expressions: expressionSummary?.events ?? [] + expressions: expressionEvents }, diagnostics: screening.diagnostics }; diff --git a/packages/ambient-core/src/expression-events.ts b/packages/ambient-core/src/expression-events.ts index e8c8213..7de742c 100644 --- a/packages/ambient-core/src/expression-events.ts +++ b/packages/ambient-core/src/expression-events.ts @@ -1,4 +1,4 @@ -import type { ExpressionEventRecord } from "./kinematic-events.js"; +import type { DetectedExpression } from "./kinematic-events.js"; /** * Spontaneous facial expression events. * @@ -79,7 +79,7 @@ export interface ExpressionBaseline { * because the detector's callers predate the record type; the fields are the * same set. */ -export type ExpressionEvent = ExpressionEventRecord; +export type ExpressionEvent = DetectedExpression; function finite(value: number | null | undefined): value is number { return typeof value === "number" && Number.isFinite(value); diff --git a/packages/ambient-core/src/index.ts b/packages/ambient-core/src/index.ts index afd9a55..93523ff 100644 --- a/packages/ambient-core/src/index.ts +++ b/packages/ambient-core/src/index.ts @@ -117,6 +117,8 @@ export type { FaceScreeningDiagnostics } from "./ambient-face.js"; export type { VoiceScreeningDiagnostics } from "./ambient-voice.js"; export type { BlinkEventRecord, + DetectedBlink, + DetectedExpression, ExpressionEventRecord, PauseEventRecord, PauseKind, diff --git a/packages/ambient-core/src/kinematic-events.ts b/packages/ambient-core/src/kinematic-events.ts index 09347e9..48c2223 100644 --- a/packages/ambient-core/src/kinematic-events.ts +++ b/packages/ambient-core/src/kinematic-events.ts @@ -64,6 +64,17 @@ export interface BlinkEventRecord { closedDwellMs: number; /** Frames that contributed. A low count means the phases are coarse. */ frameCount: number; + /** + * Whether every frame spanning this event stayed inside the pose limits the + * session metrics require. + * + * Detection runs on a looser stream than measurement, because a blink or a + * mouth movement is recoverable at a head angle that would invalidate a + * left-versus-right geometric comparison. Rate and timing can use every + * event; anything comparing the two sides should use only those flagged + * true. + */ + poseWithinMeasurementLimits: boolean; } /** @@ -95,6 +106,17 @@ export interface ExpressionEventRecord { /** Lid aperture change at peak, per side, for oculo-oral coupling. */ lidApertureDeltaLeft: number; lidApertureDeltaRight: number; + /** + * Whether every frame spanning this event stayed inside the pose limits the + * session metrics require. + * + * Detection runs on a looser stream than measurement, because a blink or a + * mouth movement is recoverable at a head angle that would invalidate a + * left-versus-right geometric comparison. Rate and timing can use every + * event; anything comparing the two sides should use only those flagged + * true. + */ + poseWithinMeasurementLimits: boolean; } /** What ended a stretch of speech, which decides what the silence measures. */ @@ -157,3 +179,19 @@ export interface SessionEventRecords { pauses: readonly PauseEventRecord[]; speechRuns: readonly SpeechRunEventRecord[]; } + +/* + * What a detector produces, before pose context is attached. + * + * The detectors work on geometry alone and have no view of the pose limits the + * session metrics impose, so they must not assert that field. Whatever holds + * both the events and the pose stream fills it in. + */ +export type DetectedBlink = Omit< + BlinkEventRecord, + "poseWithinMeasurementLimits" +>; +export type DetectedExpression = Omit< + ExpressionEventRecord, + "poseWithinMeasurementLimits" +>; From 6d4e247a84427383f12128ad8828feed525a6ad3 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:11:09 -0700 Subject: [PATCH 3/5] fix: restore the worklet processor name and two corrupted schema identifiers Live telemetry stopped after the previous commit. The worklet is served from public/ at a fixed unhashed URL, so browsers cache it hard: a stale copy registers "phenometric-voice-capture" while the fresh bundle asks for "phenometrix-voice-capture", AudioWorkletNode throws, and the voice lane dies without a visible error. The 70-second session before that commit produced 6972 voice frames, so the break is squarely that rename. Renaming it was wrong on principle, not just in effect. It is a runtime registration identifier -- the same category as the "phenometric..vN" schema strings that were deliberately left alone during the rename, for exactly this reason. The rule was stated and then applied inconsistently. The worklet file is restored byte-for-byte from before the rename, so a browser holding a cached copy matches the manifest again rather than failing its integrity check. The explanation lives in voice-capture.ts, which Vite content-hashes, instead of in the file that must not change. The same blanket substitution had also corrupted two wire identifiers into "phenometrix.static-assets.v1" and "phenometrix.voice-worklet-message.v1". Both restored; all twenty schema identifiers verified intact. Co-Authored-By: Claude Opus 5 (1M context) --- apps/capture-web/public/asset-manifest.json | 2 +- apps/capture-web/public/voice-capture-worklet.js | 4 ++-- apps/capture-web/src/static-assets.test.ts | 2 +- apps/capture-web/src/voice-capture.ts | 13 ++++++++++++- apps/capture-web/src/voice-worklet.test.ts | 4 ++-- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/apps/capture-web/public/asset-manifest.json b/apps/capture-web/public/asset-manifest.json index df6f8ce..68eeb6f 100644 --- a/apps/capture-web/public/asset-manifest.json +++ b/apps/capture-web/public/asset-manifest.json @@ -7,7 +7,7 @@ }, "voiceWorklet": { "path": "voice-capture-worklet.js", - "sha256": "a80e57bea25a7b8991f5cdecbb96bef2b85c58fb0443d80091f96e15c12b168a" + "sha256": "7003f2dbbf8cbf234f1f8e15195cca36b1fe0a014944b6593c69441ba3945332" }, "visionWasmScript": { "path": "mediapipe/vision_wasm_internal.js", diff --git a/apps/capture-web/public/voice-capture-worklet.js b/apps/capture-web/public/voice-capture-worklet.js index 7e8328b..00d41f1 100644 --- a/apps/capture-web/public/voice-capture-worklet.js +++ b/apps/capture-web/public/voice-capture-worklet.js @@ -60,7 +60,7 @@ class PhenoMetrixVoiceCaptureProcessor extends AudioWorkletProcessor { this.absoluteSampleIndex += this.blockSamples; this.dataPort.postMessage( { - schemaVersion: "phenometrix.voice-worklet-message.v1", + schemaVersion: "phenometric.voice-worklet-message.v1", type: "pcm-block", captureEpoch: this.captureEpoch, sequence: this.sequence, @@ -97,6 +97,6 @@ class PhenoMetrixVoiceCaptureProcessor extends AudioWorkletProcessor { } registerProcessor( - "phenometrix-voice-capture", + "phenometric-voice-capture", PhenoMetrixVoiceCaptureProcessor ); diff --git a/apps/capture-web/src/static-assets.test.ts b/apps/capture-web/src/static-assets.test.ts index 2f672d3..638bccc 100644 --- a/apps/capture-web/src/static-assets.test.ts +++ b/apps/capture-web/src/static-assets.test.ts @@ -19,7 +19,7 @@ describe("static asset manifest", () => { it("rejects root-relative and escaping paths", () => { const template = { - schemaVersion: "phenometrix.static-assets.v1", + schemaVersion: "phenometric.static-assets.v1", assets: {} }; expect(() => diff --git a/apps/capture-web/src/voice-capture.ts b/apps/capture-web/src/voice-capture.ts index 1846f31..dd493b5 100644 --- a/apps/capture-web/src/voice-capture.ts +++ b/apps/capture-web/src/voice-capture.ts @@ -63,7 +63,18 @@ export async function startVoiceCapturePipeline( options.audioContext.createMediaStreamSource(options.stream); const worklet = new AudioWorkletNode( options.audioContext, - "phenometrix-voice-capture", + /* + * Deliberately NOT renamed with the rest of PhenoMetrix. + * + * This is a runtime registration identifier, the same category as the + * "phenometric..vN" schema strings that were left alone for the same + * reason. Worse, the worklet is served from public/ at a fixed unhashed + * URL, so browsers cache it hard: a stale copy registers the old name while + * a fresh bundle asks for the new one, AudioWorkletNode throws, and the + * voice lane dies silently -- live telemetry simply stops. Renaming it was + * cosmetic and cost exactly that. + */ + "phenometric-voice-capture", { numberOfInputs: 1, numberOfOutputs: 1, diff --git a/apps/capture-web/src/voice-worklet.test.ts b/apps/capture-web/src/voice-worklet.test.ts index 447e261..9344d22 100644 --- a/apps/capture-web/src/voice-worklet.test.ts +++ b/apps/capture-web/src/voice-worklet.test.ts @@ -47,7 +47,7 @@ function loadProcessor(): new () => { name: string, processor: typeof registered ) => { - expect(name).toBe("phenometrix-voice-capture"); + expect(name).toBe("phenometric-voice-capture"); registered = processor; }, sampleRate: 48_000, @@ -96,7 +96,7 @@ describe("voice capture AudioWorklet", () => { renderBlock(processor, 0.25); expect(blocks).toHaveLength(1); expect(blocks[0]).toMatchObject({ - schemaVersion: "phenometrix.voice-worklet-message.v1", + schemaVersion: "phenometric.voice-worklet-message.v1", type: "pcm-block", captureEpoch: 7, sequence: 1, From 6b758eef1d12c68aea69f3511a3938df1b9643c3 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Sat, 25 Jul 2026 14:15:47 -0700 Subject: [PATCH 4/5] fix: drop unusable frames from a bin instead of dropping the bin Across three real sessions the all-or-nothing frame gate was the single reason nothing was ever measurable: it discarded 82% of one session's bins to exclude 22% of its frames, and 15 of 15 in another. One glance away cost the surrounding five seconds. No new threshold replaces it, because the pack already carried one. minimumDataPerBinMs of 4000 in a 5000 ms bin IS an 80% requirement; the all-or-nothing rule was redundant with it and far stricter. Dropping bad frames and letting the published requirement do its job makes the code enforce what the pack always said. Every retained frame is individually pose-valid, so the geometry stays sound -- the bin rests on less of it, which is precisely what minimumDataPerBinMs and minimumSamplesPerBin exist to bound. durationMs had to become honest first. It summed raw inter-frame gaps, so two frames 200 ms apart contributed 200 ms of "data" while carrying two samples -- harmless while the gate guaranteed no holes, wrong the moment it stopped. Each retained frame now represents one nominal step of observation. CORRECTS AN EARLIER PROJECTION. The acceptance curve reported that a fractional gate at 0.80 would have published metrics from the measured session. It would not have: the simulation omitted the actualSpanMs >= 4800 check. Losing frames from a bin EDGE shortens the usable span one-for-one, and the span rule allows 200 ms of slack against the data rule's 1000 ms. The curve now simulates span too, and reports bins lost to it. So the honest scope of this change is narrower than claimed: it recovers bins that lost frames in a SCATTERED pattern, and nothing else. An edge burst is rejected by span; a mid-bin burst is rejected by the gap rule. The measured sessions lost frames in multi-second bursts, so this alone will not make them measurable. The tests say so explicitly rather than using a loss pattern that flatters the change. Co-Authored-By: Claude Opus 5 (1M context) --- apps/capture-web/src/ambient-core-adapter.ts | 8 ++- .../ambient-core/src/ambient-face.test.ts | 63 +++++++++++++++++++ packages/ambient-core/src/ambient-face.ts | 62 ++++++++++++++---- 3 files changed, 117 insertions(+), 16 deletions(-) diff --git a/apps/capture-web/src/ambient-core-adapter.ts b/apps/capture-web/src/ambient-core-adapter.ts index 62bb6ab..c5af6aa 100644 --- a/apps/capture-web/src/ambient-core-adapter.ts +++ b/apps/capture-web/src/ambient-core-adapter.ts @@ -195,14 +195,15 @@ function reportCaptureDiagnostics( if (curve.length > 0) { lines.push( "if the all-or-nothing frame gate became fractional:", - " threshold bins lost to gap lost to sample count" + " threshold bins lost to gap lost to span lost to samples" ); for (const point of curve) { lines.push( ` ${point.threshold.toFixed(2).padStart(8)}` + `${String(point.binsAccepted).padStart(7)}` + `${String(point.lostToGap).padStart(14)}` + - `${String(point.lostToSampleCount).padStart(23)}` + `${String(point.lostToSpan ?? 0).padStart(15)}` + + `${String(point.lostToSampleCount).padStart(17)}` ); } } @@ -214,7 +215,8 @@ function reportCaptureDiagnostics( ` bin ${String(bin.index).padStart(3)} ` + `${(bin.usableFraction * 100).toFixed(1).padStart(5)}% ` + `${String(bin.usableFrameCount).padStart(4)}/${String(bin.frameCount).padEnd(4)} ` + - `gap ${bin.maxUsableGapMs.toFixed(0).padStart(5)} ms` + `gap ${bin.maxUsableGapMs.toFixed(0).padStart(5)} ms ` + + `span ${(bin.usableSpanMs ?? 0).toFixed(0).padStart(5)} ms` ); } } diff --git a/packages/ambient-core/src/ambient-face.test.ts b/packages/ambient-core/src/ambient-face.test.ts index 4c1c4c2..e569bed 100644 --- a/packages/ambient-core/src/ambient-face.test.ts +++ b/packages/ambient-core/src/ambient-face.test.ts @@ -549,3 +549,66 @@ describe("tier-2 events survive a session no bin qualifies", () => { ).toBe(true); }); }); + +describe("bins survive brief pose excursions", () => { + /** + * A session where the head leaves the pose limits for a burst in each bin. + * `badFramesPerBin` out of 150 are pushed past the yaw limit, contiguously, + * so the excursion looks like a real glance away rather than noise. + */ + function withExcursions(badFramesPerBin: number): AmbientFacialFrame[] { + // Scattered rather than contiguous, and never at a bin edge. A contiguous + // edge loss shortens the usable SPAN one-for-one, and the span rule allows + // only 200 ms of slack against the data rule's 1000 ms -- so an edge burst + // is rejected by span no matter how much data survives. Mid-bin bursts are + // rejected by the gap rule for the same reason. Only scattered loss is + // recoverable, which is exactly what this change buys and no more. + const stride = Math.max(2, Math.floor(148 / badFramesPerBin)); + return ambientFaceFrames(70_000, 30, (frame, index) => { + const positionInBin = index % 150; + const dropped = + positionInBin > 0 && + positionInBin < 149 && + positionInBin % stride === 0; + return dropped + ? { pose: { yawDegrees: 18, pitchDegrees: 2, rollDegrees: 1 } } + : {}; + }); + } + + it("accepts a bin that keeps enough analyzed data", () => { + // 15 of 150 frames lost: 135 remain, about 4.5 s of the 5 s bin, clearing + // minimumDataPerBinMs. The all-or-nothing rule discarded this entirely. + const result = extractAmbientFaceMetrics(withExcursions(15), OPTIONS); + const measured = result.outcomes.filter( + (outcome) => outcome.status === "measured" + ); + expect(measured.length).toBeGreaterThan(0); + }); + + it("still rejects a bin that lost too much of its data", () => { + // Every other frame lost: about 2.5 s of a 5 s bin, well under the 4 s the + // pack requires. The threshold is the published requirement, not a new + // constant. + const result = extractAmbientFaceMetrics(withExcursions(74), OPTIONS); + const eyeLeft = result.outcomes.find( + (outcome) => outcome.code === "ambient.face.eye_aperture.left" + ); + expect(eyeLeft?.status).toBe("withheld"); + }); + + it("measures only from frames that individually passed the pose gate", () => { + // The retained frames are all pose-valid, so geometry stays sound; the bin + // simply rests on less of it. A bin's sample count must reflect what was + // actually used, not what arrived. + const result = extractAmbientFaceMetrics(withExcursions(15), OPTIONS); + const eyeLeft = result.outcomes.find( + (outcome) => outcome.code === "ambient.face.eye_aperture.left" + ); + expect(eyeLeft?.status).toBe("measured"); + expect(eyeLeft?.evidence.samplesPerBin).toBeLessThan(150); + expect(eyeLeft?.evidence.samplesPerBin).toBeGreaterThanOrEqual( + 80 + ); + }); +}); diff --git a/packages/ambient-core/src/ambient-face.ts b/packages/ambient-core/src/ambient-face.ts index 3e4b209..c110a13 100644 --- a/packages/ambient-core/src/ambient-face.ts +++ b/packages/ambient-core/src/ambient-face.ts @@ -158,6 +158,8 @@ export interface FaceScreeningDiagnostics { usableFrameCount: number; usableFraction: number; maxUsableGapMs: number; + /** Wall-clock extent of the usable frames; the span rule tests this. */ + usableSpanMs: number; }>; /** * Bins that WOULD qualify at each candidate frame-usability threshold, if the @@ -172,6 +174,7 @@ export interface FaceScreeningDiagnostics { binsAccepted: number; lostToGap: number; lostToSampleCount: number; + lostToSpan: number; }>; } @@ -443,7 +446,7 @@ function binValues( function qualifyBin( index: number, - frames: AmbientFacialFrame[], + candidateFrames: readonly AmbientFacialFrame[], options: AmbientFaceExtractionOptions, onReject?: (reason: string) => void ): FacialBin | null { @@ -451,14 +454,29 @@ function qualifyBin( onReject?.(reason); return null; }; + /* + * Unusable frames are dropped; the bin is not. + * + * This used to require EVERY frame to pass, which in real capture discarded + * 82% of a session's bins to exclude 22% of its frames -- one glance away + * costing the surrounding five seconds. Measured across three real sessions, + * it was the single reason nothing was ever measurable. + * + * No new threshold replaces it, because the pack already carries one: + * `minimumDataPerBinMs` of 4000 in a 5000 ms bin IS an 80% requirement. The + * all-or-nothing rule was redundant with it and far stricter. Dropping bad + * frames and letting the published requirement do its job makes the code + * enforce what the pack always said. + * + * Every retained frame is individually pose-valid, so the geometry stays + * sound -- the bin simply rests on less of it, which is exactly what + * `minimumDataPerBinMs` and `minimumSamplesPerBin` exist to bound. + */ + const frames = candidateFrames.filter((frame) => + ambientFrameUsable(frame, options) + ); if (frames.length < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { - return reject("too-few-frames"); - } - if (!frames.every((frame) => ambientFrameUsable(frame, options))) { - // All-or-nothing by design: one unusable frame discards the whole bin. In - // real capture this is the gate most likely to dominate, which is exactly - // why it now says so. - return reject("frame-gate"); + return reject("too-few-usable-frames"); } const processorRefs = new Set(frames.map((frame) => frame.processorRef)); const trackSegmentIds = new Set(frames.map(faceTrackSegmentId)); @@ -482,11 +500,19 @@ function qualifyBin( } const actualSpanMs = frames.at(-1)!.tMs - frames[0].tMs; const stepMs = nominalStepMs(frames); + /* + * How much of this bin was actually ANALYZED, not how much time elapsed + * across it. + * + * Summing raw inter-frame gaps counted a hole as data: two frames 200 ms + * apart contributed 200 ms while carrying two samples. That was harmless + * while the frame gate guaranteed no holes and wrong the moment it stopped. + * Each retained frame now represents one nominal step of observation, which + * is what `minimumDataPerBinMs` is checked against. + */ const durationMs = Math.min( AMBIENT_FACE_BIN_MS, - Math.round( - (gaps.reduce((total, gap) => total + gap, 0) + stepMs) * 1_000 - ) / 1_000 + Math.round(frames.length * stepMs * 1_000) / 1_000 ); if ( actualSpanMs < AMBIENT_FACE_MIN_BIN_SPAN_MS || @@ -586,7 +612,9 @@ function screenBins( usableFrameCount: usable.length, usableFraction: bucket.length > 0 ? usable.length / bucket.length : 0, - maxUsableGapMs + maxUsableGapMs, + usableSpanMs: + usable.length > 1 ? usable.at(-1)!.tMs - usable[0].tMs : 0 }; }); @@ -594,6 +622,7 @@ function screenBins( let binsAccepted = 0; let lostToGap = 0; let lostToSampleCount = 0; + let lostToSpan = 0; for (const bin of binStats) { if (bin.usableFraction < threshold) continue; if (bin.usableFrameCount < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { @@ -604,9 +633,16 @@ function screenBins( lostToGap += 1; continue; } + // Omitting this made an earlier projection optimistic: losing frames from + // a bin EDGE shortens the usable span one-for-one, and the span rule is + // far tighter than the data rule -- 200 ms of slack against 1000 ms. + if (bin.usableSpanMs < AMBIENT_FACE_MIN_BIN_SPAN_MS) { + lostToSpan += 1; + continue; + } binsAccepted += 1; } - return { threshold, binsAccepted, lostToGap, lostToSampleCount }; + return { threshold, binsAccepted, lostToGap, lostToSampleCount, lostToSpan }; }); const bins = entries.flatMap(([index, bucket]) => { const bin = qualifyBin(index, bucket, options, (reason) => { From c3558f55aed13ab96000985e443d1a114e91e190 Mon Sep 17 00:00:00 2001 From: Logan Nye <87274608+logannye@users.noreply.github.com> Date: Sat, 25 Jul 2026 14:21:27 -0700 Subject: [PATCH 5/5] feat: gate pose on deviation from the session's resting pose Three measured sessions failed the face lane on pose, and the third made the cause plain: median pitch 7.4 degrees against a limit of 10, with the median sitting at the ceiling before any head movement at all. That is a camera mounted below eye level, which is constant and harmless, and the gate could not tell it from the subject turning away, which is neither. Limits now measure deviation from the session's resting pose -- the per-axis median, computed from the session itself so it needs no capture-path or contract change and adapts to how the participant actually sat. How far that reference may itself sit from frontal is bounded per axis, on geometric grounds rather than preference: yaw 10 rotation about the vertical axis foreshortens one side of the face and not the other, so a constant offset biases every left-versus-right measurement this system exists to make pitch 20 symmetric across the midline; moves both sides together and leaves asymmetry largely alone. Also the axis camera placement actually offsets roll 15 in-plane, and already cancelled by aligning the coordinate x-axis to the inter-eye line before anything is measured A session outside those bounds has no admissible reference and falls back to frontal rather than accepting an arbitrary baseline. Fixes an incomplete decoupling from two commits ago: evaluateVisualQuality carries a SECOND pose gate of its own, looser but still absolute, and Tier-2 detection read only its `.usable` flag. Events still vanished past 15 degrees for the same reason and with the same consequence. Pose is now excluded explicitly there while lighting, sharpness, and framing remain required, since those do corrupt the landmarks a blink is measured from. Pack moves to 3.2.0 and the policy fields are renamed from maximumAbsolute* to maximumYawDeviationDegrees and siblings, because the old names now describe something the code does not do. Face measurements under 3.1.0 and 3.2.0 are not interchangeable. Also adds a copy-diagnostics button to the report view. Calibration is an iterate-and-rerun loop and console-only output cost a hand-copy or the whole session each round. Clipboard on an explicit click is the same act as selecting the text by hand: no storage, no file, no network. Co-Authored-By: Claude Opus 5 (1M context) --- apps/capture-web/index.html | 1 + apps/capture-web/src/ambient-core-adapter.ts | 40 ++++- .../ambient-core/src/ambient-face.test.ts | 20 ++- packages/ambient-core/src/ambient-face.ts | 161 +++++++++++++++--- packages/contracts/src/ambient-protocol.ts | 13 +- packages/contracts/src/protocol.ts | 22 ++- 6 files changed, 221 insertions(+), 36 deletions(-) diff --git a/apps/capture-web/index.html b/apps/capture-web/index.html index b68888e..9b23b50 100644 --- a/apps/capture-web/index.html +++ b/apps/capture-web/index.html @@ -174,6 +174,7 @@

Ambient session measurement report

+
diff --git a/apps/capture-web/src/ambient-core-adapter.ts b/apps/capture-web/src/ambient-core-adapter.ts index c5af6aa..d5e2a2b 100644 --- a/apps/capture-web/src/ambient-core-adapter.ts +++ b/apps/capture-web/src/ambient-core-adapter.ts @@ -247,12 +247,44 @@ function reportCaptureDiagnostics( ` pauses ${events?.pauses?.length ?? 0}`, ` speech runs ${events?.speechRuns?.length ?? 0}` ); + const report = `PhenoMetrix capture diagnostics\n${lines.join("\n")}`; // eslint-disable-next-line no-console - console.log( - `%cPhenoMetrix capture diagnostics%c\n${lines.join("\n")}`, - "font-weight:bold", - "font-weight:normal" + console.log(report); + publishDiagnosticsReport(report); +} + +/** + * Makes the diagnostics copyable from the finish screen. + * + * Calibration is an iterate-and-rerun loop, and console-only output made every + * round cost a hand-copy or the whole session. Clipboard on an explicit click + * is the same act as selecting the text by hand: no storage, no file, no + * network, and nothing retained after the page closes. + */ +function publishDiagnosticsReport(report: string): void { + // The adapter is exercised headlessly in unit tests, where there is no DOM + // and no clipboard. Diagnostics are a browser affordance, not part of what + // this function computes. + if (typeof document === "undefined") return; + const button = document.querySelector( + "#copy-diagnostics" ); + if (!button || typeof navigator === "undefined" || !navigator.clipboard) { + return; + } + button.hidden = false; + button.onclick = () => { + void navigator.clipboard + .writeText(report) + .then(() => { + button.textContent = "Diagnostics copied"; + }) + .catch(() => { + // Clipboard permission can be refused; say so rather than appearing to + // have worked. + button.textContent = "Copy failed - select the console output"; + }); + }; } function relevantEventCount( diff --git a/packages/ambient-core/src/ambient-face.test.ts b/packages/ambient-core/src/ambient-face.test.ts index e569bed..2372cb0 100644 --- a/packages/ambient-core/src/ambient-face.test.ts +++ b/packages/ambient-core/src/ambient-face.test.ts @@ -218,9 +218,17 @@ describe("extractAmbientFaceMetrics", () => { }); it("rejects bins beyond strict ambient pose, gap, and sample thresholds", () => { + // Pose limits measure DEVIATION from the session's resting pose, so a + // constant offset no longer rejects anything -- that is the laptop-camera + // case, and treating it as head movement was the defect. Rejection now + // requires the head to actually move relative to how it sat. const pose = extractAmbientFaceMetrics( - ambientFaceFrames(30_000, 30, () => ({ - pose: { yawDegrees: 7.001, pitchDegrees: 0, rollDegrees: 0 } + ambientFaceFrames(30_000, 30, (frame, index) => ({ + pose: { + yawDegrees: index % 2 === 0 ? 0 : 14.001, + pitchDegrees: 0, + rollDegrees: 0 + } })), OPTIONS ); @@ -509,8 +517,12 @@ describe("tier-2 events survive a withheld metric", () => { describe("tier-2 events survive a session no bin qualifies", () => { /** Blinking normally, but pitched past the 10-degree limit throughout. */ function pitchedAwayFrames(): AmbientFacialFrame[] { - return ambientFaceFrames(70_000, 30, (frame) => ({ - pose: { yawDegrees: 2, pitchDegrees: 14, rollDegrees: 2 } + // Yaw beyond the RESTING bound, so the session has no admissible reference + // and gating falls back to frontal -- every frame then fails. A constant + // pitch offset would no longer do this, because that is precisely the + // camera-placement case the resting pose now absorbs. + return ambientFaceFrames(70_000, 30, () => ({ + pose: { yawDegrees: 25, pitchDegrees: 2, rollDegrees: 2 } })); } diff --git a/packages/ambient-core/src/ambient-face.ts b/packages/ambient-core/src/ambient-face.ts index c110a13..28cf5e1 100644 --- a/packages/ambient-core/src/ambient-face.ts +++ b/packages/ambient-core/src/ambient-face.ts @@ -39,9 +39,87 @@ export const AMBIENT_FACE_MIN_SAMPLES_PER_BIN = 80; export const AMBIENT_FACE_MAX_FRAME_GAP_MS = 200; export const AMBIENT_FACE_MIN_BINS = 3; export const AMBIENT_FACE_MIN_SPAN_MS = 30_000; +/* + * Pose limits are DEVIATION FROM THE SESSION'S RESTING POSE, not from frontal. + * + * A laptop camera sits below eye level, so a seated participant reads as + * several degrees of constant pitch that no amount of sitting still removes. A + * measured session showed a median pitch of 7.4 against a limit of 10 -- half + * the session at the ceiling before any head movement at all. Gating on + * absolute angle conflates "the camera is mounted low", which is constant and + * harmless, with "the subject turned away", which is neither. + * + * The resting pose is the session median, and how far IT may sit from frontal + * is bounded separately below, per axis, because the three rotations do not + * bias measurement equally. + */ export const AMBIENT_FACE_MAX_YAW_DEGREES = 7; export const AMBIENT_FACE_MAX_PITCH_DEGREES = 10; export const AMBIENT_FACE_MAX_ROLL_DEGREES = 5; + +/* + * How far the resting pose itself may sit from frontal. + * + * These differ per axis on geometric grounds, not preference: + * + * YAW is rotation about the vertical axis, so it foreshortens one side of the + * face and not the other. A constant yaw offset therefore biases every + * left-versus-right measurement this system exists to make. Kept tight. + * + * PITCH is rotation about the horizontal axis and is symmetric across the + * midline: it moves both sides together and leaves asymmetry largely alone. + * This is also the axis camera placement actually offsets. Generous. + * + * ROLL is in-plane, and the coordinate system already cancels it by aligning + * its x-axis to the inter-eye line before measuring anything. Generous. + */ +export const AMBIENT_FACE_MAX_RESTING_YAW_DEGREES = 10; +export const AMBIENT_FACE_MAX_RESTING_PITCH_DEGREES = 20; +export const AMBIENT_FACE_MAX_RESTING_ROLL_DEGREES = 15; + +/** The pose a session is measured relative to. */ +export interface RestingPose { + yawDegrees: number; + pitchDegrees: number; + rollDegrees: number; +} + +/** + * Session resting pose: the median of each axis across frames carrying one. + * + * Self-calibrating rather than taken from the calibration step, so it needs no + * capture-path or contract change and adapts to how the participant actually + * sat. Returns null when it falls outside the resting bounds -- a session spent + * genuinely turned away has no usable reference, and measuring deviation from a + * bad baseline would silently accept the whole thing. + */ +export function restingPose( + frames: readonly AmbientFacialFrame[] +): RestingPose | null { + const poses = frames + .map((frame) => frame.pose) + .filter((pose): pose is NonNullable => pose !== null) + .filter( + (pose) => + finite(pose.yawDegrees) && + finite(pose.pitchDegrees) && + finite(pose.rollDegrees) + ); + if (poses.length === 0) return null; + const resting = { + yawDegrees: median(poses.map((pose) => pose.yawDegrees)), + pitchDegrees: median(poses.map((pose) => pose.pitchDegrees)), + rollDegrees: median(poses.map((pose) => pose.rollDegrees)) + }; + if ( + Math.abs(resting.yawDegrees) > AMBIENT_FACE_MAX_RESTING_YAW_DEGREES || + Math.abs(resting.pitchDegrees) > AMBIENT_FACE_MAX_RESTING_PITCH_DEGREES || + Math.abs(resting.rollDegrees) > AMBIENT_FACE_MAX_RESTING_ROLL_DEGREES + ) { + return null; + } + return resting; +} export const AMBIENT_FACE_MAX_CALIBRATION_SIZE_DELTA = 0.2; export const AMBIENT_FACE_MAX_WITHIN_BIN_SIZE_RATIO = 1.15; export const AMBIENT_BLINK_MIN_EXPOSURE_MS = 60_000; @@ -139,6 +217,12 @@ export interface FaceScreeningDiagnostics { pitchP50: number; pitchP95: number; rollP50: number; rollP95: number; } | null; + /** + * The pose every frame in this session was judged relative to, or null when + * no reference inside the resting bounds existed and gating fell back to + * frontal. + */ + restingPose: RestingPose | null; binsConsidered: number; binsAccepted: number; /** First failing check per rejected bin. */ @@ -282,7 +366,8 @@ function calibratedSizeUsable( */ export function frameGateFailures( frame: AmbientFacialFrame, - options: AmbientFaceExtractionOptions + options: AmbientFaceExtractionOptions, + resting: RestingPose | null = null ): string[] { const reasons: string[] = []; const pose = frame.pose; @@ -292,16 +377,27 @@ export function frameGateFailures( if (pose === null) { reasons.push("no-pose"); } else { + // Deviation from the session's resting pose. A null reference means the + // session had none inside the resting bounds, and it falls back to frontal + // rather than accepting an arbitrary baseline. + const reference = resting ?? { + yawDegrees: 0, + pitchDegrees: 0, + rollDegrees: 0 + }; if (!finite(pose.yawDegrees) || - Math.abs(pose.yawDegrees) > AMBIENT_FACE_MAX_YAW_DEGREES) { + Math.abs(pose.yawDegrees - reference.yawDegrees) > + AMBIENT_FACE_MAX_YAW_DEGREES) { reasons.push("yaw"); } if (!finite(pose.pitchDegrees) || - Math.abs(pose.pitchDegrees) > AMBIENT_FACE_MAX_PITCH_DEGREES) { + Math.abs(pose.pitchDegrees - reference.pitchDegrees) > + AMBIENT_FACE_MAX_PITCH_DEGREES) { reasons.push("pitch"); } if (!finite(pose.rollDegrees) || - Math.abs(pose.rollDegrees) > AMBIENT_FACE_MAX_ROLL_DEGREES) { + Math.abs(pose.rollDegrees - reference.rollDegrees) > + AMBIENT_FACE_MAX_ROLL_DEGREES) { reasons.push("roll"); } } @@ -312,9 +408,10 @@ export function frameGateFailures( function ambientFrameUsable( frame: AmbientFacialFrame, - options: AmbientFaceExtractionOptions + options: AmbientFaceExtractionOptions, + resting: RestingPose | null = null ): boolean { - return frameGateFailures(frame, options).length === 0; + return frameGateFailures(frame, options, resting).length === 0; } /** @@ -337,11 +434,22 @@ function ambientFrameUsable( * stricter set. */ function tier2FrameUsable(frame: AmbientFacialFrame): boolean { - return ( - frame.faceCount === 1 && - faceTrackSegmentId(frame) !== null && - evaluateVisualQuality(frame, null).usable && - completeGeometry(frame) + if ( + frame.faceCount !== 1 || + faceTrackSegmentId(frame) === null || + !completeGeometry(frame) + ) { + return false; + } + // The image-quality assessment carries a SECOND pose gate of its own, looser + // than the extractor's but still absolute. Reading only `.usable` here made + // the decoupling incomplete: events still vanished once the head passed 15 + // degrees, for the same reason and with the same consequence. Pose is + // excluded explicitly; everything else -- lighting, sharpness, framing -- is + // still required, because those DO corrupt the landmarks a blink is measured + // from. + return evaluateVisualQuality(frame, null).reasonCodes.every( + (reason) => reason === "pose-out-of-range" ); } @@ -350,14 +458,15 @@ function poseWithinLimits( frames: readonly AmbientFacialFrame[], startMs: number, endMs: number, - options: AmbientFaceExtractionOptions + options: AmbientFaceExtractionOptions, + resting: RestingPose | null ): boolean { const spanning = frames.filter( (frame) => frame.tMs >= startMs && frame.tMs <= endMs ); if (spanning.length === 0) return false; return spanning.every((frame) => { - const failures = frameGateFailures(frame, options); + const failures = frameGateFailures(frame, options, resting); return !failures.some((reason) => reason === "yaw" || reason === "pitch" || reason === "roll" ); @@ -448,6 +557,7 @@ function qualifyBin( index: number, candidateFrames: readonly AmbientFacialFrame[], options: AmbientFaceExtractionOptions, + resting: RestingPose | null, onReject?: (reason: string) => void ): FacialBin | null { const reject = (reason: string): null => { @@ -473,7 +583,7 @@ function qualifyBin( * `minimumDataPerBinMs` and `minimumSamplesPerBin` exist to bound. */ const frames = candidateFrames.filter((frame) => - ambientFrameUsable(frame, options) + ambientFrameUsable(frame, options, resting) ); if (frames.length < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { return reject("too-few-usable-frames"); @@ -565,13 +675,16 @@ function screenBins( options: AmbientFaceExtractionOptions ): BinScreening { const buckets = new Map(); + // One reference for the whole session, so every bin is judged against the + // same baseline rather than drifting with local head position. + const resting = restingPose(frames); let attributionFailureCount = 0; let qualityFailureCount = 0; for (const frame of frames) { if (frame.faceCount !== 1 || faceTrackSegmentId(frame) === null) { attributionFailureCount += 1; } - if (!ambientFrameUsable(frame, options)) qualityFailureCount += 1; + if (!ambientFrameUsable(frame, options, resting)) qualityFailureCount += 1; const index = Math.floor( (frame.tMs - options.sessionStartedAtMs) / AMBIENT_FACE_BIN_MS ); @@ -582,7 +695,7 @@ function screenBins( const gateFailures: Record = {}; let usableFrameCount = 0; for (const frame of frames) { - const failures = frameGateFailures(frame, options); + const failures = frameGateFailures(frame, options, resting); if (failures.length === 0) usableFrameCount += 1; for (const reason of failures) { gateFailures[reason] = (gateFailures[reason] ?? 0) + 1; @@ -598,7 +711,9 @@ function screenBins( const entries = [...buckets.entries()].sort(([left], [right]) => left - right); const binStats = entries.map(([index, bucket]) => { - const usable = bucket.filter((frame) => ambientFrameUsable(frame, options)); + const usable = bucket.filter((frame) => + ambientFrameUsable(frame, options, resting) + ); let maxUsableGapMs = 0; for (let position = 1; position < usable.length; position += 1) { maxUsableGapMs = Math.max( @@ -645,7 +760,7 @@ function screenBins( return { threshold, binsAccepted, lostToGap, lostToSampleCount, lostToSpan }; }); const bins = entries.flatMap(([index, bucket]) => { - const bin = qualifyBin(index, bucket, options, (reason) => { + const bin = qualifyBin(index, bucket, options, resting, (reason) => { binRejections[reason] = (binRejections[reason] ?? 0) + 1; }); return bin ? [bin] : []; @@ -668,6 +783,7 @@ function screenBins( rollP95: absAt((pose) => pose.rollDegrees, 0.95) } : null, + restingPose: resting, binsConsidered: entries.length, binsAccepted: bins.length, binRejections, @@ -1077,6 +1193,9 @@ export function extractAmbientFaceMetrics( // outcomes. An outcome carries one value by construction, so a series cannot // travel inside it. const blinkEvents: BlinkEventRecord[] = []; + // Same reference the bin screener uses, so "within measurement limits" on an + // event means the same thing it means for a bin. + const sessionRestingPose = restingPose(inRange); // Ordered, attribution- and geometry-complete frames without the pose gate. const tier2Frames = [...inRange] .filter(tier2FrameUsable) @@ -1225,7 +1344,8 @@ export function extractAmbientFaceMetrics( tier2Frames, record.onsetMs, record.offsetMs, - options + options, + sessionRestingPose ) }); } @@ -1297,7 +1417,8 @@ export function extractAmbientFaceMetrics( tier2Frames, event.startMs, event.endMs, - options + options, + sessionRestingPose ) })); diff --git a/packages/contracts/src/ambient-protocol.ts b/packages/contracts/src/ambient-protocol.ts index 58c03f1..324992b 100644 --- a/packages/contracts/src/ambient-protocol.ts +++ b/packages/contracts/src/ambient-protocol.ts @@ -43,10 +43,10 @@ const rawProtocolPack = { // 3.0.0: brow geometry and per-eye closure added (5 face metrics). // Sessions measured under different packs are not comparable, and the // content digest below makes that structurally visible rather than implicit. - version: "3.1.0", + version: "3.2.0", // SHA-256 of the canonical pack content with this field omitted. contentSha256: - "5d3ccce909f84b5a1f37e47f8d7e0b08f8cbcf6fea5cd2de09c3cf31261335a8", + "b12f5977a9bc1b23bbafbe6b01c5a5c9b4d535fa9ae2dd4fe5722ea49624ad11", status: "nonclinical-prototype", maximumSessionDurationMs: 300_000, supportedTarget: { @@ -90,9 +90,12 @@ const rawProtocolPack = { minimumSamplesPerBin: 80, minimumBinSpanMs: 4_800, maximumFrameGapMs: 200, - maximumAbsoluteYawDegrees: 7, - maximumAbsolutePitchDegrees: 10, - maximumAbsoluteRollDegrees: 5, + maximumYawDeviationDegrees: 7, + maximumPitchDeviationDegrees: 10, + maximumRollDeviationDegrees: 5, + maximumRestingYawDegrees: 10, + maximumRestingPitchDegrees: 20, + maximumRestingRollDegrees: 15, maximumCalibrationScaleDeviation: 0.2, maximumWithinBinScaleRatio: 1.15, minimumBins: 3, diff --git a/packages/contracts/src/protocol.ts b/packages/contracts/src/protocol.ts index afcfe34..b9e1b2f 100644 --- a/packages/contracts/src/protocol.ts +++ b/packages/contracts/src/protocol.ts @@ -122,9 +122,25 @@ export const CaptureQualityPolicyV1Schema = z minimumSamplesPerBin: z.literal(80), minimumBinSpanMs: z.literal(4_800), maximumFrameGapMs: z.literal(200), - maximumAbsoluteYawDegrees: z.literal(7), - maximumAbsolutePitchDegrees: z.literal(10), - maximumAbsoluteRollDegrees: z.literal(5), + /* + * Pose limits are DEVIATION from the session's resting pose, not from + * frontal. A camera mounted below eye level reads as constant pitch + * that no amount of sitting still removes, and gating on absolute angle + * conflated that with the subject turning away. + * + * The resting bounds below cap how far that reference may itself sit + * from frontal, and differ per axis on geometric grounds: yaw + * foreshortens one side of the face and so biases every left-versus- + * right measurement; pitch is symmetric across the midline; roll is + * already cancelled by aligning the coordinate x-axis to the inter-eye + * line. + */ + maximumYawDeviationDegrees: z.literal(7), + maximumPitchDeviationDegrees: z.literal(10), + maximumRollDeviationDegrees: z.literal(5), + maximumRestingYawDegrees: z.literal(10), + maximumRestingPitchDegrees: z.literal(20), + maximumRestingRollDegrees: z.literal(15), maximumCalibrationScaleDeviation: z.literal(0.2), maximumWithinBinScaleRatio: z.literal(1.15), minimumBins: z.literal(3),