diff --git a/apps/capture-web/e2e/ambient-smoke.spec.ts b/apps/capture-web/e2e/ambient-smoke.spec.ts index c605ee1..1e76289 100644 --- a/apps/capture-web/e2e/ambient-smoke.spec.ts +++ b/apps/capture-web/e2e/ambient-smoke.spec.ts @@ -5,7 +5,7 @@ import { resolveLateAudio } from "./ambient-browser-fixture.js"; -const appUrl = "/phenometric/"; +const appUrl = "/phenometrix/"; async function consentAndStart(page: Page): Promise { await page.locator("#consent-checkbox").check(); diff --git a/apps/capture-web/e2e/static-server.ts b/apps/capture-web/e2e/static-server.ts index 7947e56..5ffb4c7 100644 --- a/apps/capture-web/e2e/static-server.ts +++ b/apps/capture-web/e2e/static-server.ts @@ -6,7 +6,7 @@ import { fileURLToPath } from "node:url"; const dist = resolve( fileURLToPath(new URL("../dist", import.meta.url)) ); -const mount = "/phenometric/"; +const mount = "/phenometrix/"; const contentTypes: Record = { ".css": "text/css; charset=utf-8", ".html": "text/html; charset=utf-8", diff --git a/apps/capture-web/index.html b/apps/capture-web/index.html index b68888e..9b23b50 100644 --- a/apps/capture-web/index.html +++ b/apps/capture-web/index.html @@ -174,6 +174,7 @@

Ambient session measurement report

+
diff --git a/apps/capture-web/playwright.config.ts b/apps/capture-web/playwright.config.ts index 4f4cc9d..47d1d85 100644 --- a/apps/capture-web/playwright.config.ts +++ b/apps/capture-web/playwright.config.ts @@ -6,14 +6,14 @@ export default defineConfig({ fullyParallel: false, workers: 1, use: { - baseURL: "http://127.0.0.1:4173/phenometric/", + baseURL: "http://127.0.0.1:4173/phenometrix/", channel: "chrome", headless: true, trace: "retain-on-failure" }, webServer: { command: "pnpm exec tsx e2e/static-server.ts", - url: "http://127.0.0.1:4173/phenometric/", + url: "http://127.0.0.1:4173/phenometrix/", reuseExistingServer: false, timeout: 30_000 } diff --git a/apps/capture-web/src/ambient-core-adapter.ts b/apps/capture-web/src/ambient-core-adapter.ts index 05dfb0e..d5e2a2b 100644 --- a/apps/capture-web/src/ambient-core-adapter.ts +++ b/apps/capture-web/src/ambient-core-adapter.ts @@ -1,5 +1,7 @@ import { finalizeAmbientMetrics, + type FaceScreeningDiagnostics, + type VoiceScreeningDiagnostics, type AmbientFaceCalibration, type AmbientFacialFrame, type AmbientMetricEvidence, @@ -130,6 +132,161 @@ function primaryTrack(outcome: AmbientMetricOutcome): string { * Counting only the metric's own events keeps the number honest and keeps * each gate answerable by its own evidence. */ +/** + * Prints why the session measured what it measured. + * + * A report full of abstentions looks identical whether the camera saw nobody or + * saw a face that never held still long enough for a bin to qualify, and those + * call for opposite fixes. Console only: no storage, no export, no contract + * surface. Counts and pose percentiles, never a per-frame series. + */ +function reportCaptureDiagnostics( + extraction: { diagnostics?: unknown; events?: unknown }, + faceFrameCount: number +): void { + const all = extraction.diagnostics as + | { face?: FaceScreeningDiagnostics; voice?: VoiceScreeningDiagnostics } + | undefined; + const diagnostics = all?.face; + const voice = all?.voice; + if (!diagnostics) return; + const events = extraction.events as + | { blinks?: readonly unknown[]; expressions?: readonly unknown[]; + pauses?: readonly unknown[]; speechRuns?: readonly unknown[] } + | undefined; + + const pct = (part: number, whole: number) => + whole > 0 ? `${((part / whole) * 100).toFixed(1)}%` : "n/a"; + const lines: string[] = [ + `frames delivered to extractor: ${faceFrameCount}`, + `frames passing every gate: ${diagnostics.usableFrameCount} (${pct(diagnostics.usableFrameCount, diagnostics.frameCount)})`, + `bins accepted: ${diagnostics.binsAccepted} of ${diagnostics.binsConsidered}` + ]; + const pose = diagnostics.pose; + if (pose) { + lines.push( + "absolute pose in degrees (limits yaw 7 / pitch 10 / roll 5):", + ` yaw p50 ${pose.yawP50.toFixed(1)} p95 ${pose.yawP95.toFixed(1)}`, + ` pitch p50 ${pose.pitchP50.toFixed(1)} p95 ${pose.pitchP95.toFixed(1)}`, + ` roll p50 ${pose.rollP50.toFixed(1)} p95 ${pose.rollP95.toFixed(1)}` + ); + } + const gates = Object.entries(diagnostics.frameGateFailures) + .sort(([, a], [, b]) => b - a); + lines.push( + gates.length === 0 + ? "no frame failed any gate" + : "frames failing each gate (one frame can fail several):" + ); + for (const [reason, count] of gates) { + lines.push(` ${reason.padEnd(20)} ${String(count).padStart(6)} ${pct(count, diagnostics.frameCount)}`); + } + const rejections = Object.entries(diagnostics.binRejections) + .sort(([, a], [, b]) => b - a); + lines.push( + rejections.length === 0 + ? "no bin was rejected" + : "bin rejections by first failing check:" + ); + for (const [reason, count] of rejections) { + lines.push(` ${reason.padEnd(20)} ${String(count).padStart(6)}`); + } + const curve = diagnostics.acceptanceCurve ?? []; + if (curve.length > 0) { + lines.push( + "if the all-or-nothing frame gate became fractional:", + " threshold bins lost to gap lost to span lost to samples" + ); + for (const point of curve) { + lines.push( + ` ${point.threshold.toFixed(2).padStart(8)}` + + `${String(point.binsAccepted).padStart(7)}` + + `${String(point.lostToGap).padStart(14)}` + + `${String(point.lostToSpan ?? 0).padStart(15)}` + + `${String(point.lostToSampleCount).padStart(17)}` + ); + } + } + const perBin = diagnostics.bins ?? []; + if (perBin.length > 0) { + lines.push("per bin: usable fraction / largest gap between usable frames:"); + for (const bin of perBin) { + lines.push( + ` bin ${String(bin.index).padStart(3)} ` + + `${(bin.usableFraction * 100).toFixed(1).padStart(5)}% ` + + `${String(bin.usableFrameCount).padStart(4)}/${String(bin.frameCount).padEnd(4)} ` + + `gap ${bin.maxUsableGapMs.toFixed(0).padStart(5)} ms ` + + `span ${(bin.usableSpanMs ?? 0).toFixed(0).padStart(5)} ms` + ); + } + } + if (voice) { + const req = voice.requirements; + lines.push( + "voice lane:", + ` frames ${voice.frameCount}, usable ${voice.usableFrameCount}, ` + + `speech-active ${voice.speechActiveFrameCount}, periodic ${voice.periodicFrameCount}`, + ` segments ${voice.segmentsAccepted} (needs ${req.minimumSegments})`, + ` eligible ${(voice.eligibleDurationMs / 1000).toFixed(1)}s ` + + `(needs ${(req.minimumEligibleSpanMs / 1000).toFixed(0)}s), ` + + `active speech ${(voice.activeSpeechMs / 1000).toFixed(1)}s ` + + `(needs ${(req.minimumActiveSpeechMs / 1000).toFixed(0)}s)` + ); + const vg = Object.entries(voice.frameGateFailures).sort(([, a], [, b]) => b - a); + if (vg.length > 0) { + lines.push(" frames failing each voice gate:"); + for (const [reason, count] of vg) { + lines.push(` ${reason.padEnd(22)} ${String(count).padStart(6)}`); + } + } + } + lines.push( + "tier-2 events:", + ` blinks ${events?.blinks?.length ?? 0}`, + ` expressions ${events?.expressions?.length ?? 0}`, + ` pauses ${events?.pauses?.length ?? 0}`, + ` speech runs ${events?.speechRuns?.length ?? 0}` + ); + const report = `PhenoMetrix capture diagnostics\n${lines.join("\n")}`; + // eslint-disable-next-line no-console + console.log(report); + publishDiagnosticsReport(report); +} + +/** + * Makes the diagnostics copyable from the finish screen. + * + * Calibration is an iterate-and-rerun loop, and console-only output made every + * round cost a hand-copy or the whole session. Clipboard on an explicit click + * is the same act as selecting the text by hand: no storage, no file, no + * network, and nothing retained after the page closes. + */ +function publishDiagnosticsReport(report: string): void { + // The adapter is exercised headlessly in unit tests, where there is no DOM + // and no clipboard. Diagnostics are a browser affordance, not part of what + // this function computes. + if (typeof document === "undefined") return; + const button = document.querySelector( + "#copy-diagnostics" + ); + if (!button || typeof navigator === "undefined" || !navigator.clipboard) { + return; + } + button.hidden = false; + button.onclick = () => { + void navigator.clipboard + .writeText(report) + .then(() => { + button.textContent = "Diagnostics copied"; + }) + .catch(() => { + // Clipboard permission can be refused; say so rather than appearing to + // have worked. + button.textContent = "Copy failed - select the console output"; + }); + }; +} + function relevantEventCount( code: MetricCode, evidence: AmbientMetricEvidence @@ -456,6 +613,8 @@ export function buildAmbientObservation( calibration: input.faceCalibration } }); + + reportCaptureDiagnostics(extraction, input.faceFrames.length); const artifacts = extraction.outcomes.map((outcome) => outcomeArtifacts(outcome, input) ); diff --git a/apps/capture-web/src/static-assets.test.ts b/apps/capture-web/src/static-assets.test.ts index ac008b6..638bccc 100644 --- a/apps/capture-web/src/static-assets.test.ts +++ b/apps/capture-web/src/static-assets.test.ts @@ -12,9 +12,9 @@ describe("static asset manifest", () => { expect( resolveAssetUrl( "models/face_landmarker.task", - "https://example.test/tools/phenometric/index.html" + "https://example.test/tools/phenometrix/index.html" ) - ).toBe("https://example.test/tools/phenometric/models/face_landmarker.task"); + ).toBe("https://example.test/tools/phenometrix/models/face_landmarker.task"); }); it("rejects root-relative and escaping paths", () => { diff --git a/apps/capture-web/src/voice-capture.ts b/apps/capture-web/src/voice-capture.ts index 3180433..dd493b5 100644 --- a/apps/capture-web/src/voice-capture.ts +++ b/apps/capture-web/src/voice-capture.ts @@ -63,6 +63,17 @@ export async function startVoiceCapturePipeline( options.audioContext.createMediaStreamSource(options.stream); const worklet = new AudioWorkletNode( options.audioContext, + /* + * Deliberately NOT renamed with the rest of PhenoMetrix. + * + * This is a runtime registration identifier, the same category as the + * "phenometric..vN" schema strings that were left alone for the same + * reason. Worse, the worklet is served from public/ at a fixed unhashed + * URL, so browsers cache it hard: a stale copy registers the old name while + * a fresh bundle asks for the new one, AudioWorkletNode throws, and the + * voice lane dies silently -- live telemetry simply stops. Renaming it was + * cosmetic and cost exactly that. + */ "phenometric-voice-capture", { numberOfInputs: 1, diff --git a/packages/ambient-core/src/ambient-face.test.ts b/packages/ambient-core/src/ambient-face.test.ts index 2f75f74..2372cb0 100644 --- a/packages/ambient-core/src/ambient-face.test.ts +++ b/packages/ambient-core/src/ambient-face.test.ts @@ -218,9 +218,17 @@ describe("extractAmbientFaceMetrics", () => { }); it("rejects bins beyond strict ambient pose, gap, and sample thresholds", () => { + // Pose limits measure DEVIATION from the session's resting pose, so a + // constant offset no longer rejects anything -- that is the laptop-camera + // case, and treating it as head movement was the defect. Rejection now + // requires the head to actually move relative to how it sat. const pose = extractAmbientFaceMetrics( - ambientFaceFrames(30_000, 30, () => ({ - pose: { yawDegrees: 7.001, pitchDegrees: 0, rollDegrees: 0 } + ambientFaceFrames(30_000, 30, (frame, index) => ({ + pose: { + yawDegrees: index % 2 === 0 ? 0 : 14.001, + pitchDegrees: 0, + rollDegrees: 0 + } })), OPTIONS ); @@ -475,3 +483,144 @@ describe("blink events", () => { expect(left[0].depth).toBeLessThan(right[0].depth); }); }); + +describe("tier-2 events survive a withheld metric", () => { + it("extracts blinks from a session too short for the blink metric", () => { + // Regression from a real 54-second session: the blink metric requires 60 s + // of frontal exposure, and detection sat inside the branch that only ran + // when that gate passed. Every blink actually observed was discarded + // because a PUBLICATION threshold suppressed the EXTRACTION beneath it. + // Tier 2 exists so an abstaining metric still leaves its observations. + const frames = ambientFaceFrames(40_000); + const result = extractAmbientFaceMetrics(frames, OPTIONS); + + const rate = result.outcomes.find( + (outcome) => outcome.code === "ambient.face.blink_rate.bilateral" + ); + expect(rate?.status).toBe("withheld"); + if (rate?.status === "withheld") { + expect(rate.reasonCode).toBe("insufficient-exposure"); + } + + // The metric abstains; the events do not vanish with it. + expect((result.events?.blinks ?? []).length).toBeGreaterThan(0); + }); + + it("reports no blinks rather than throwing when nothing qualified", () => { + // No bins at all is a different statement from no blinks, and it must not + // reach the detector's percentile of an empty set. + const result = extractAmbientFaceMetrics([], OPTIONS); + expect(result.events?.blinks).toEqual([]); + }); +}); + +describe("tier-2 events survive a session no bin qualifies", () => { + /** Blinking normally, but pitched past the 10-degree limit throughout. */ + function pitchedAwayFrames(): AmbientFacialFrame[] { + // Yaw beyond the RESTING bound, so the session has no admissible reference + // and gating falls back to frontal -- every frame then fails. A constant + // pitch offset would no longer do this, because that is precisely the + // camera-placement case the resting pose now absorbs. + return ambientFaceFrames(70_000, 30, () => ({ + pose: { yawDegrees: 25, pitchDegrees: 2, rollDegrees: 2 } + })); + } + + it("records blinks when every bin fails the pose gate", () => { + // Reproduces a real 70-second session: a face in frame the whole time, a + // systematic pose offset from a low-mounted camera, 0 of 14 bins accepted, + // and consequently not one blink recorded. Detection had been reading the + // qualifying bins, so it inherited a gate built for cross-frame geometric + // comparison -- a standard a blink does not need. + const result = extractAmbientFaceMetrics(pitchedAwayFrames(), OPTIONS); + + expect( + result.outcomes.every((outcome) => outcome.status === "withheld") + ).toBe(true); + expect((result.events?.blinks ?? []).length).toBeGreaterThan(0); + }); + + it("marks those events as outside the measurement pose limits", () => { + // The events exist, and they say plainly that a left-versus-right + // comparison of them would not be trustworthy. Rate and timing can use + // them; asymmetry should not. + const result = extractAmbientFaceMetrics(pitchedAwayFrames(), OPTIONS); + const blinks = result.events?.blinks ?? []; + expect(blinks.length).toBeGreaterThan(0); + expect( + blinks.every((blink) => blink.poseWithinMeasurementLimits === false) + ).toBe(true); + }); + + it("flags events as within limits when the pose is good", () => { + const result = extractAmbientFaceMetrics(ambientFaceFrames(), OPTIONS); + const blinks = result.events?.blinks ?? []; + expect(blinks.length).toBeGreaterThan(0); + expect( + blinks.every((blink) => blink.poseWithinMeasurementLimits === true) + ).toBe(true); + }); +}); + +describe("bins survive brief pose excursions", () => { + /** + * A session where the head leaves the pose limits for a burst in each bin. + * `badFramesPerBin` out of 150 are pushed past the yaw limit, contiguously, + * so the excursion looks like a real glance away rather than noise. + */ + function withExcursions(badFramesPerBin: number): AmbientFacialFrame[] { + // Scattered rather than contiguous, and never at a bin edge. A contiguous + // edge loss shortens the usable SPAN one-for-one, and the span rule allows + // only 200 ms of slack against the data rule's 1000 ms -- so an edge burst + // is rejected by span no matter how much data survives. Mid-bin bursts are + // rejected by the gap rule for the same reason. Only scattered loss is + // recoverable, which is exactly what this change buys and no more. + const stride = Math.max(2, Math.floor(148 / badFramesPerBin)); + return ambientFaceFrames(70_000, 30, (frame, index) => { + const positionInBin = index % 150; + const dropped = + positionInBin > 0 && + positionInBin < 149 && + positionInBin % stride === 0; + return dropped + ? { pose: { yawDegrees: 18, pitchDegrees: 2, rollDegrees: 1 } } + : {}; + }); + } + + it("accepts a bin that keeps enough analyzed data", () => { + // 15 of 150 frames lost: 135 remain, about 4.5 s of the 5 s bin, clearing + // minimumDataPerBinMs. The all-or-nothing rule discarded this entirely. + const result = extractAmbientFaceMetrics(withExcursions(15), OPTIONS); + const measured = result.outcomes.filter( + (outcome) => outcome.status === "measured" + ); + expect(measured.length).toBeGreaterThan(0); + }); + + it("still rejects a bin that lost too much of its data", () => { + // Every other frame lost: about 2.5 s of a 5 s bin, well under the 4 s the + // pack requires. The threshold is the published requirement, not a new + // constant. + const result = extractAmbientFaceMetrics(withExcursions(74), OPTIONS); + const eyeLeft = result.outcomes.find( + (outcome) => outcome.code === "ambient.face.eye_aperture.left" + ); + expect(eyeLeft?.status).toBe("withheld"); + }); + + it("measures only from frames that individually passed the pose gate", () => { + // The retained frames are all pose-valid, so geometry stays sound; the bin + // simply rests on less of it. A bin's sample count must reflect what was + // actually used, not what arrived. + const result = extractAmbientFaceMetrics(withExcursions(15), OPTIONS); + const eyeLeft = result.outcomes.find( + (outcome) => outcome.code === "ambient.face.eye_aperture.left" + ); + expect(eyeLeft?.status).toBe("measured"); + expect(eyeLeft?.evidence.samplesPerBin).toBeLessThan(150); + expect(eyeLeft?.evidence.samplesPerBin).toBeGreaterThanOrEqual( + 80 + ); + }); +}); diff --git a/packages/ambient-core/src/ambient-face.ts b/packages/ambient-core/src/ambient-face.ts index d38a28f..28cf5e1 100644 --- a/packages/ambient-core/src/ambient-face.ts +++ b/packages/ambient-core/src/ambient-face.ts @@ -12,6 +12,8 @@ import { } from "./expression-events.js"; import type { BlinkEventRecord, + DetectedBlink, + ExpressionEventRecord, SubjectSide } from "./kinematic-events.js"; import { @@ -37,9 +39,87 @@ export const AMBIENT_FACE_MIN_SAMPLES_PER_BIN = 80; export const AMBIENT_FACE_MAX_FRAME_GAP_MS = 200; export const AMBIENT_FACE_MIN_BINS = 3; export const AMBIENT_FACE_MIN_SPAN_MS = 30_000; +/* + * Pose limits are DEVIATION FROM THE SESSION'S RESTING POSE, not from frontal. + * + * A laptop camera sits below eye level, so a seated participant reads as + * several degrees of constant pitch that no amount of sitting still removes. A + * measured session showed a median pitch of 7.4 against a limit of 10 -- half + * the session at the ceiling before any head movement at all. Gating on + * absolute angle conflates "the camera is mounted low", which is constant and + * harmless, with "the subject turned away", which is neither. + * + * The resting pose is the session median, and how far IT may sit from frontal + * is bounded separately below, per axis, because the three rotations do not + * bias measurement equally. + */ export const AMBIENT_FACE_MAX_YAW_DEGREES = 7; export const AMBIENT_FACE_MAX_PITCH_DEGREES = 10; export const AMBIENT_FACE_MAX_ROLL_DEGREES = 5; + +/* + * How far the resting pose itself may sit from frontal. + * + * These differ per axis on geometric grounds, not preference: + * + * YAW is rotation about the vertical axis, so it foreshortens one side of the + * face and not the other. A constant yaw offset therefore biases every + * left-versus-right measurement this system exists to make. Kept tight. + * + * PITCH is rotation about the horizontal axis and is symmetric across the + * midline: it moves both sides together and leaves asymmetry largely alone. + * This is also the axis camera placement actually offsets. Generous. + * + * ROLL is in-plane, and the coordinate system already cancels it by aligning + * its x-axis to the inter-eye line before measuring anything. Generous. + */ +export const AMBIENT_FACE_MAX_RESTING_YAW_DEGREES = 10; +export const AMBIENT_FACE_MAX_RESTING_PITCH_DEGREES = 20; +export const AMBIENT_FACE_MAX_RESTING_ROLL_DEGREES = 15; + +/** The pose a session is measured relative to. */ +export interface RestingPose { + yawDegrees: number; + pitchDegrees: number; + rollDegrees: number; +} + +/** + * Session resting pose: the median of each axis across frames carrying one. + * + * Self-calibrating rather than taken from the calibration step, so it needs no + * capture-path or contract change and adapts to how the participant actually + * sat. Returns null when it falls outside the resting bounds -- a session spent + * genuinely turned away has no usable reference, and measuring deviation from a + * bad baseline would silently accept the whole thing. + */ +export function restingPose( + frames: readonly AmbientFacialFrame[] +): RestingPose | null { + const poses = frames + .map((frame) => frame.pose) + .filter((pose): pose is NonNullable => pose !== null) + .filter( + (pose) => + finite(pose.yawDegrees) && + finite(pose.pitchDegrees) && + finite(pose.rollDegrees) + ); + if (poses.length === 0) return null; + const resting = { + yawDegrees: median(poses.map((pose) => pose.yawDegrees)), + pitchDegrees: median(poses.map((pose) => pose.pitchDegrees)), + rollDegrees: median(poses.map((pose) => pose.rollDegrees)) + }; + if ( + Math.abs(resting.yawDegrees) > AMBIENT_FACE_MAX_RESTING_YAW_DEGREES || + Math.abs(resting.pitchDegrees) > AMBIENT_FACE_MAX_RESTING_PITCH_DEGREES || + Math.abs(resting.rollDegrees) > AMBIENT_FACE_MAX_RESTING_ROLL_DEGREES + ) { + return null; + } + return resting; +} export const AMBIENT_FACE_MAX_CALIBRATION_SIZE_DELTA = 0.2; export const AMBIENT_FACE_MAX_WITHIN_BIN_SIZE_RATIO = 1.15; export const AMBIENT_BLINK_MIN_EXPOSURE_MS = 60_000; @@ -114,6 +194,72 @@ interface BinScreening { bins: FacialBin[]; attributionFailureCount: number; qualityFailureCount: number; + diagnostics: FaceScreeningDiagnostics; +} + +/** + * Why a session measured what it measured, or why it measured nothing. + * + * Diagnostic only: no metric reads this, and it carries counts and pose + * statistics rather than any per-frame series. It exists because a report full + * of abstentions currently looks identical whether the camera saw nobody or saw + * a face that never held still enough for a bin to qualify -- and those call + * for opposite fixes. + */ +export interface FaceScreeningDiagnostics { + frameCount: number; + usableFrameCount: number; + /** Frames failing each gate. A frame can fail several, so these overlap. */ + frameGateFailures: Record; + /** Absolute pose in degrees across all frames carrying one. */ + pose: { + yawP50: number; yawP95: number; + pitchP50: number; pitchP95: number; + rollP50: number; rollP95: number; + } | null; + /** + * The pose every frame in this session was judged relative to, or null when + * no reference inside the resting bounds existed and gating fell back to + * frontal. + */ + restingPose: RestingPose | null; + binsConsidered: number; + binsAccepted: number; + /** First failing check per rejected bin. */ + binRejections: Record; + /** + * Per bin, how much of it survived the frame gate and what that leaves. + * + * `maxUsableGapMs` is the largest hole between consecutive USABLE frames -- + * the gap that would exist if unusable frames were dropped rather than the + * whole bin. It is the reason a fractional gate is not obviously a fix: + * dropping a burst of bad frames leaves a hole, and the gap rule may simply + * become the new binding constraint. + */ + bins: Array<{ + index: number; + frameCount: number; + usableFrameCount: number; + usableFraction: number; + maxUsableGapMs: number; + /** Wall-clock extent of the usable frames; the span rule tests this. */ + usableSpanMs: number; + }>; + /** + * Bins that WOULD qualify at each candidate frame-usability threshold, if the + * all-or-nothing rule were replaced by a fractional one. + * + * Simulates the full consequence, not just the fraction: a bin counts only if + * it also keeps enough usable frames and leaves no gap wider than the + * existing limit. This is what the threshold should be chosen from. + */ + acceptanceCurve: Array<{ + threshold: number; + binsAccepted: number; + lostToGap: number; + lostToSampleCount: number; + lostToSpan: number; + }>; } function finite(value: number): boolean { @@ -210,25 +356,121 @@ function calibratedSizeUsable( ); } +/** + * Every gate a frame failed, empty when it is usable. + * + * The boolean predicate is derived from this rather than duplicating it, so the + * diagnostic view and the measurement path can never disagree about why a frame + * was dropped. Without this the extractor rejects frames silently, and a session + * that abstains is indistinguishable from one that never saw a face. + */ +export function frameGateFailures( + frame: AmbientFacialFrame, + options: AmbientFaceExtractionOptions, + resting: RestingPose | null = null +): string[] { + const reasons: string[] = []; + const pose = frame.pose; + if (frame.faceCount !== 1) reasons.push("face-count"); + if (faceTrackSegmentId(frame) === null) reasons.push("no-track-id"); + if (!evaluateVisualQuality(frame, null).usable) reasons.push("image-quality"); + if (pose === null) { + reasons.push("no-pose"); + } else { + // Deviation from the session's resting pose. A null reference means the + // session had none inside the resting bounds, and it falls back to frontal + // rather than accepting an arbitrary baseline. + const reference = resting ?? { + yawDegrees: 0, + pitchDegrees: 0, + rollDegrees: 0 + }; + if (!finite(pose.yawDegrees) || + Math.abs(pose.yawDegrees - reference.yawDegrees) > + AMBIENT_FACE_MAX_YAW_DEGREES) { + reasons.push("yaw"); + } + if (!finite(pose.pitchDegrees) || + Math.abs(pose.pitchDegrees - reference.pitchDegrees) > + AMBIENT_FACE_MAX_PITCH_DEGREES) { + reasons.push("pitch"); + } + if (!finite(pose.rollDegrees) || + Math.abs(pose.rollDegrees - reference.rollDegrees) > + AMBIENT_FACE_MAX_ROLL_DEGREES) { + reasons.push("roll"); + } + } + if (!calibratedSizeUsable(frame, options)) reasons.push("face-scale"); + if (!completeGeometry(frame)) reasons.push("incomplete-geometry"); + return reasons; +} + function ambientFrameUsable( frame: AmbientFacialFrame, - options: AmbientFaceExtractionOptions + options: AmbientFaceExtractionOptions, + resting: RestingPose | null = null ): boolean { - const pose = frame.pose; - return ( - frame.faceCount === 1 && - faceTrackSegmentId(frame) !== null && - evaluateVisualQuality(frame, null).usable && - pose !== null && - finite(pose.yawDegrees) && - Math.abs(pose.yawDegrees) <= AMBIENT_FACE_MAX_YAW_DEGREES && - finite(pose.pitchDegrees) && - Math.abs(pose.pitchDegrees) <= AMBIENT_FACE_MAX_PITCH_DEGREES && - finite(pose.rollDegrees) && - Math.abs(pose.rollDegrees) <= AMBIENT_FACE_MAX_ROLL_DEGREES && - calibratedSizeUsable(frame, options) && - completeGeometry(frame) + return frameGateFailures(frame, options, resting).length === 0; +} + +/** + * Whether a frame can support Tier-2 event detection. + * + * Deliberately looser than {@link ambientFrameUsable}: it drops the pose and + * calibrated-scale gates and keeps attribution, image quality, and geometry + * completeness. + * + * Those pose limits exist so that CROSS-FRAME GEOMETRIC COMPARISON stays valid + * -- comparing one corner against the other, measuring asymmetry. A blink is + * not that. It is a relative aperture change within one eye over about 150 ms, + * during which the head pose is essentially constant, so it survives a 15 degree + * turn intact. Holding event detection to a standard designed for a different + * measurement cost a real 70-second session every blink and expression it + * contained. + * + * Events carry {@link BlinkEventRecord.poseWithinMeasurementLimits} so a + * consumer that DOES need geometric comparability can still filter to the + * stricter set. + */ +function tier2FrameUsable(frame: AmbientFacialFrame): boolean { + if ( + frame.faceCount !== 1 || + faceTrackSegmentId(frame) === null || + !completeGeometry(frame) + ) { + return false; + } + // The image-quality assessment carries a SECOND pose gate of its own, looser + // than the extractor's but still absolute. Reading only `.usable` here made + // the decoupling incomplete: events still vanished once the head passed 15 + // degrees, for the same reason and with the same consequence. Pose is + // excluded explicitly; everything else -- lighting, sharpness, framing -- is + // still required, because those DO corrupt the landmarks a blink is measured + // from. + return evaluateVisualQuality(frame, null).reasonCodes.every( + (reason) => reason === "pose-out-of-range" + ); +} + +/** Whether every frame spanning an event stayed inside the Tier-3 pose limits. */ +function poseWithinLimits( + frames: readonly AmbientFacialFrame[], + startMs: number, + endMs: number, + options: AmbientFaceExtractionOptions, + resting: RestingPose | null +): boolean { + const spanning = frames.filter( + (frame) => frame.tMs >= startMs && frame.tMs <= endMs ); + if (spanning.length === 0) return false; + return spanning.every((frame) => { + const failures = frameGateFailures(frame, options, resting); + return !failures.some((reason) => + reason === "yaw" || reason === "pitch" || reason === "roll" + ); + }); } function mouthWidth(frame: AmbientFacialFrame): number { @@ -313,11 +555,39 @@ function binValues( function qualifyBin( index: number, - frames: AmbientFacialFrame[], - options: AmbientFaceExtractionOptions + candidateFrames: readonly AmbientFacialFrame[], + options: AmbientFaceExtractionOptions, + resting: RestingPose | null, + onReject?: (reason: string) => void ): FacialBin | null { - if (frames.length < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) return null; - if (!frames.every((frame) => ambientFrameUsable(frame, options))) return null; + const reject = (reason: string): null => { + onReject?.(reason); + return null; + }; + /* + * Unusable frames are dropped; the bin is not. + * + * This used to require EVERY frame to pass, which in real capture discarded + * 82% of a session's bins to exclude 22% of its frames -- one glance away + * costing the surrounding five seconds. Measured across three real sessions, + * it was the single reason nothing was ever measurable. + * + * No new threshold replaces it, because the pack already carries one: + * `minimumDataPerBinMs` of 4000 in a 5000 ms bin IS an 80% requirement. The + * all-or-nothing rule was redundant with it and far stricter. Dropping bad + * frames and letting the published requirement do its job makes the code + * enforce what the pack always said. + * + * Every retained frame is individually pose-valid, so the geometry stays + * sound -- the bin simply rests on less of it, which is exactly what + * `minimumDataPerBinMs` and `minimumSamplesPerBin` exist to bound. + */ + const frames = candidateFrames.filter((frame) => + ambientFrameUsable(frame, options, resting) + ); + if (frames.length < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { + return reject("too-few-usable-frames"); + } const processorRefs = new Set(frames.map((frame) => frame.processorRef)); const trackSegmentIds = new Set(frames.map(faceTrackSegmentId)); const epochs = new Set(frames.map((frame) => frame.captureEpoch)); @@ -326,7 +596,7 @@ function qualifyBin( trackSegmentIds.size !== 1 || epochs.size !== 1 ) { - return null; + return reject("mixed-provenance"); } const gaps = frames .slice(1) @@ -336,21 +606,29 @@ function qualifyBin( (gap) => gap <= 0 || gap > AMBIENT_FACE_MAX_FRAME_GAP_MS ) ) { - return null; + return reject("frame-gap"); } const actualSpanMs = frames.at(-1)!.tMs - frames[0].tMs; const stepMs = nominalStepMs(frames); + /* + * How much of this bin was actually ANALYZED, not how much time elapsed + * across it. + * + * Summing raw inter-frame gaps counted a hole as data: two frames 200 ms + * apart contributed 200 ms while carrying two samples. That was harmless + * while the frame gate guaranteed no holes and wrong the moment it stopped. + * Each retained frame now represents one nominal step of observation, which + * is what `minimumDataPerBinMs` is checked against. + */ const durationMs = Math.min( AMBIENT_FACE_BIN_MS, - Math.round( - (gaps.reduce((total, gap) => total + gap, 0) + stepMs) * 1_000 - ) / 1_000 + Math.round(frames.length * stepMs * 1_000) / 1_000 ); if ( actualSpanMs < AMBIENT_FACE_MIN_BIN_SPAN_MS || durationMs < AMBIENT_FACE_MIN_BIN_DATA_MS ) { - return null; + return reject("short-bin"); } const sizes = frames.map((frame) => Math.sqrt( @@ -364,7 +642,7 @@ function qualifyBin( sizeP10 <= 0 || sizeP90 / sizeP10 > AMBIENT_FACE_MAX_WITHIN_BIN_SIZE_RATIO ) { - return null; + return reject("scale-drift"); } const startMs = options.sessionStartedAtMs + index * AMBIENT_FACE_BIN_MS; const processorRef = frames[0].processorRef; @@ -397,13 +675,16 @@ function screenBins( options: AmbientFaceExtractionOptions ): BinScreening { const buckets = new Map(); + // One reference for the whole session, so every bin is judged against the + // same baseline rather than drifting with local head position. + const resting = restingPose(frames); let attributionFailureCount = 0; let qualityFailureCount = 0; for (const frame of frames) { if (frame.faceCount !== 1 || faceTrackSegmentId(frame) === null) { attributionFailureCount += 1; } - if (!ambientFrameUsable(frame, options)) qualityFailureCount += 1; + if (!ambientFrameUsable(frame, options, resting)) qualityFailureCount += 1; const index = Math.floor( (frame.tMs - options.sessionStartedAtMs) / AMBIENT_FACE_BIN_MS ); @@ -411,13 +692,105 @@ function screenBins( bucket.push(frame); buckets.set(index, bucket); } - const bins = [...buckets.entries()] - .sort(([left], [right]) => left - right) - .flatMap(([index, bucket]) => { - const bin = qualifyBin(index, bucket, options); - return bin ? [bin] : []; + const gateFailures: Record = {}; + let usableFrameCount = 0; + for (const frame of frames) { + const failures = frameGateFailures(frame, options, resting); + if (failures.length === 0) usableFrameCount += 1; + for (const reason of failures) { + gateFailures[reason] = (gateFailures[reason] ?? 0) + 1; + } + } + const poses = frames + .map((frame) => frame.pose) + .filter((pose): pose is NonNullable => pose !== null); + const absAt = (pick: (p: NonNullable) => number, q: number) => + percentile(poses.map((pose) => Math.abs(pick(pose))), q); + + const binRejections: Record = {}; + const entries = [...buckets.entries()].sort(([left], [right]) => left - right); + + const binStats = entries.map(([index, bucket]) => { + const usable = bucket.filter((frame) => + ambientFrameUsable(frame, options, resting) + ); + let maxUsableGapMs = 0; + for (let position = 1; position < usable.length; position += 1) { + maxUsableGapMs = Math.max( + maxUsableGapMs, + usable[position].tMs - usable[position - 1].tMs + ); + } + return { + index, + frameCount: bucket.length, + usableFrameCount: usable.length, + usableFraction: + bucket.length > 0 ? usable.length / bucket.length : 0, + maxUsableGapMs, + usableSpanMs: + usable.length > 1 ? usable.at(-1)!.tMs - usable[0].tMs : 0 + }; + }); + + const acceptanceCurve = [1, 0.98, 0.95, 0.9, 0.85, 0.8].map((threshold) => { + let binsAccepted = 0; + let lostToGap = 0; + let lostToSampleCount = 0; + let lostToSpan = 0; + for (const bin of binStats) { + if (bin.usableFraction < threshold) continue; + if (bin.usableFrameCount < AMBIENT_FACE_MIN_SAMPLES_PER_BIN) { + lostToSampleCount += 1; + continue; + } + if (bin.maxUsableGapMs > AMBIENT_FACE_MAX_FRAME_GAP_MS) { + lostToGap += 1; + continue; + } + // Omitting this made an earlier projection optimistic: losing frames from + // a bin EDGE shortens the usable span one-for-one, and the span rule is + // far tighter than the data rule -- 200 ms of slack against 1000 ms. + if (bin.usableSpanMs < AMBIENT_FACE_MIN_BIN_SPAN_MS) { + lostToSpan += 1; + continue; + } + binsAccepted += 1; + } + return { threshold, binsAccepted, lostToGap, lostToSampleCount, lostToSpan }; + }); + const bins = entries.flatMap(([index, bucket]) => { + const bin = qualifyBin(index, bucket, options, resting, (reason) => { + binRejections[reason] = (binRejections[reason] ?? 0) + 1; }); - return { bins, attributionFailureCount, qualityFailureCount }; + return bin ? [bin] : []; + }); + return { + bins, + attributionFailureCount, + qualityFailureCount, + diagnostics: { + frameCount: frames.length, + usableFrameCount, + frameGateFailures: gateFailures, + pose: poses.length > 0 + ? { + yawP50: absAt((pose) => pose.yawDegrees, 0.5), + yawP95: absAt((pose) => pose.yawDegrees, 0.95), + pitchP50: absAt((pose) => pose.pitchDegrees, 0.5), + pitchP95: absAt((pose) => pose.pitchDegrees, 0.95), + rollP50: absAt((pose) => pose.rollDegrees, 0.5), + rollP95: absAt((pose) => pose.rollDegrees, 0.95) + } + : null, + restingPose: resting, + binsConsidered: entries.length, + binsAccepted: bins.length, + binRejections, + bins: binStats, + acceptanceCurve + } + }; } function evidenceFor( @@ -614,8 +987,19 @@ function p95Gaps(bins: readonly FacialBin[]): number { return gaps.length > 0 ? percentile(gaps, 0.95) : Number.POSITIVE_INFINITY; } +/** + * The detector needs only an ordered group of frames and an index to attribute + * results to. A qualifying bin satisfies this, and so does the raw frame stream + * wrapped as a single group -- which is what lets Tier-2 extraction run without + * inheriting the pose gating a bin implies. + */ +interface FrameGroup { + index: number; + frames: AmbientFacialFrame[]; +} + /** One eye's blink, plus the bin it peaked in so per-bin rates survive. */ -interface BinnedBlink extends BlinkEventRecord { +interface BinnedBlink extends DetectedBlink { binIndex: number; } @@ -633,7 +1017,7 @@ interface BinnedBlink extends BlinkEventRecord { * waveform, three findings, none of them recoverable from a count. */ function detectBlinksForEye( - bins: readonly FacialBin[], + bins: readonly FrameGroup[], side: SubjectSide, apertureOf: (frame: AmbientFacialFrame) => number ): BinnedBlink[] { @@ -752,7 +1136,7 @@ function detectBlinksForEye( } /** Every blink from both eyes, ordered by the moment of maximum closure. */ -function detectBlinkEvents(bins: readonly FacialBin[]): BinnedBlink[] { +function detectBlinkEvents(bins: readonly FrameGroup[]): BinnedBlink[] { return [ ...detectBlinksForEye(bins, "left", (frame) => frame.eyeAperture!.left), ...detectBlinksForEye(bins, "right", (frame) => frame.eyeAperture!.right) @@ -767,7 +1151,7 @@ function detectBlinkEvents(bins: readonly FacialBin[]): BinnedBlink[] { * closure now exists as an event without inflating that count. */ function detectBlinks( - bins: readonly FacialBin[] + bins: readonly FrameGroup[] ): { count: number; perBinCounts: number[]; events: BinnedBlink[] } { const events = detectBlinkEvents(bins); const perBinCounts = bins.map(() => 0); @@ -809,6 +1193,13 @@ export function extractAmbientFaceMetrics( // outcomes. An outcome carries one value by construction, so a series cannot // travel inside it. const blinkEvents: BlinkEventRecord[] = []; + // Same reference the bin screener uses, so "within measurement limits" on an + // event means the same thing it means for a bin. + const sessionRestingPose = restingPose(inRange); + // Ordered, attribution- and geometry-complete frames without the pose gate. + const tier2Frames = [...inRange] + .filter(tier2FrameUsable) + .sort((left, right) => left.tMs - right.tMs); const screening = screenBins(inRange, options); const evidence = evidenceFor(inRange, screening.bins); const failure = commonFailure(screening, options); @@ -918,6 +1309,48 @@ export function extractAmbientFaceMetrics( detail: "Bilateral blink rate requires a P95 frame gap no greater than 75 ms." }; } + /* + * Detection runs whenever there are bins to run it on, independent of whether + * the blink METRIC publishes. + * + * This used to sit inside the else branch below, so a session that failed the + * 60-second exposure gate discarded every blink it had actually observed. A + * 54-second session with a face in frame throughout reported zero blinks -- + * not because none occurred, but because a publication threshold suppressed + * the extraction feeding it. Tier 2 exists precisely so an abstaining metric + * still leaves its observations behind. + */ + const blinks = + screening.bins.length > 0 + ? detectBlinks(screening.bins) + : { count: 0, perBinCounts: [] as number[], events: [] as BinnedBlink[] }; + + /* + * Tier-2 events come from the LOOSE stream, not the qualifying bins. + * + * The published blink rate above stays bin-derived and unchanged: it is + * explicitly a rate over pose-qualified windows. These events answer a + * different question -- what did the session actually contain -- and a + * session whose bins all failed the pose gate still contained blinks. + */ + if (tier2Frames.length > 0) { + for (const event of detectBlinkEvents([ + { index: 0, frames: tier2Frames } + ])) { + const { binIndex: _binIndex, ...record } = event; + blinkEvents.push({ + ...record, + poseWithinMeasurementLimits: poseWithinLimits( + tier2Frames, + record.onsetMs, + record.offsetMs, + options, + sessionRestingPose + ) + }); + } + } + if (blinkFailure) { outcomes.push( withheldOutcome( @@ -929,10 +1362,6 @@ export function extractAmbientFaceMetrics( ) ); } else { - const blinks = detectBlinks(screening.bins); - blinkEvents.push( - ...blinks.events.map(({ binIndex: _binIndex, ...event }) => event) - ); const blinkEvidence = evidenceFor(inRange, screening.bins, { frontalExposureMs, blinkCount: blinks.count @@ -964,6 +1393,35 @@ export function extractAmbientFaceMetrics( ) : null; + /* + * Expressions are detected on the loose stream for the same reason blinks are: + * a mouth movement is recoverable at a head angle that would invalidate a + * left-versus-right comparison of it. The per-side excursion METRICS below + * still come from the qualifying bins; these events are the record of what + * the session contained. + */ + const looseExpressions = + tier2Frames.length > 0 + ? summarizeExpressions( + tier2Frames, + tier2Frames.length > 1 + ? tier2Frames.at(-1)!.tMs - tier2Frames[0].tMs + : 0 + ) + : null; + const expressionEvents: ExpressionEventRecord[] = ( + looseExpressions?.events ?? [] + ).map((event) => ({ + ...event, + poseWithinMeasurementLimits: poseWithinLimits( + tier2Frames, + event.startMs, + event.endMs, + options, + sessionRestingPose + ) + })); + const expressionEvidence = evidenceFor(inRange, screening.bins, { expressionEventCount: expressionSummary?.eventCount, coupledExpressionEventCount: expressionSummary?.synkinesisEventCount @@ -1180,7 +1638,8 @@ export function extractAmbientFaceMetrics( blinks: blinkEvents, // Already fully computed for the summary and previously reduced to two // integers before anything could see them. - expressions: expressionSummary?.events ?? [] - } + expressions: expressionEvents + }, + diagnostics: screening.diagnostics }; } diff --git a/packages/ambient-core/src/ambient-metrics.ts b/packages/ambient-core/src/ambient-metrics.ts index 62a212e..aec297e 100644 --- a/packages/ambient-core/src/ambient-metrics.ts +++ b/packages/ambient-core/src/ambient-metrics.ts @@ -40,6 +40,13 @@ export function finalizeAmbientMetrics( return { outcomes, ignoredFrameCount: - voice.ignoredFrameCount + face.ignoredFrameCount + voice.ignoredFrameCount + face.ignoredFrameCount, + events: { + blinks: face.events?.blinks ?? [], + expressions: face.events?.expressions ?? [], + pauses: voice.events?.pauses ?? [], + speechRuns: voice.events?.speechRuns ?? [] + }, + diagnostics: { face: face.diagnostics, voice: voice.diagnostics } }; } diff --git a/packages/ambient-core/src/ambient-types.ts b/packages/ambient-core/src/ambient-types.ts index 0ea8bbd..b362720 100644 --- a/packages/ambient-core/src/ambient-types.ts +++ b/packages/ambient-core/src/ambient-types.ts @@ -256,6 +256,11 @@ export interface AmbientExtractionResult< * and found nothing, which is a different statement. */ events?: Partial; + /** + * Why this extractor measured what it measured. Diagnostic only; no metric, + * outcome, or report reads it. + */ + diagnostics?: unknown; } export interface AmbientSessionExtractionInput { diff --git a/packages/ambient-core/src/ambient-voice.ts b/packages/ambient-core/src/ambient-voice.ts index 2784012..d0fc7eb 100644 --- a/packages/ambient-core/src/ambient-voice.ts +++ b/packages/ambient-core/src/ambient-voice.ts @@ -94,6 +94,33 @@ interface VoiceSegment { sourceWindowRef: string; } +/** + * Why the voice lane measured what it measured. + * + * The face lane's equivalent showed that a report of abstentions says nothing + * about whether the camera saw a face. The same was true here and worse: a + * silent voice lane could mean the participant did not speak, or that every + * frame failed an acquisition gate, and nothing distinguished them. + */ +export interface VoiceScreeningDiagnostics { + frameCount: number; + /** Frames passing every acquisition gate. */ + usableFrameCount: number; + speechActiveFrameCount: number; + periodicFrameCount: number; + /** Frames failing each gate; a frame can fail several. */ + frameGateFailures: Record; + segmentsAccepted: number; + eligibleDurationMs: number; + activeSpeechMs: number; + /** What the extractor needs before any timing metric can publish. */ + requirements: { + minimumSegments: number; + minimumEligibleSpanMs: number; + minimumActiveSpeechMs: number; + }; +} + interface RunDurations { speechRunsMs: number[]; pausesMs: number[]; @@ -107,6 +134,43 @@ function finite(value: number): boolean { return Number.isFinite(value); } +/** Every acquisition gate a voice frame failed; empty when usable. */ +export function voiceFrameGateFailures(frame: AmbientVoiceFrame): string[] { + const reasons: string[] = []; + const active = frame.speechActive; + if (frame.taskContext !== AMBIENT_VOICE_TASK_CONTEXT) reasons.push("task-context"); + if (typeof frame.trackSegmentId !== "string" || frame.trackSegmentId.length === 0) { + reasons.push("no-track-id"); + } + if (!finite(frame.tMs)) reasons.push("no-timestamp"); + if (!finite(frame.sampleRateHz) || frame.sampleRateHz < AMBIENT_VOICE_MIN_SAMPLE_RATE_HZ) { + reasons.push("sample-rate"); + } + if (!finite(frame.blockGapMs) || frame.blockGapMs > AMBIENT_VOICE_MAX_GAP_MS) { + reasons.push("block-gap"); + } + if (!finite(frame.lostBlockFraction) || + frame.lostBlockFraction > AMBIENT_VOICE_MAX_LOST_BLOCK_FRACTION) { + reasons.push("lost-blocks"); + } + if (!finite(frame.clippedSampleFraction) || + frame.clippedSampleFraction > AMBIENT_VOICE_MAX_CLIPPED_FRACTION) { + reasons.push("clipping"); + } + if (!finite(frame.dcOffset) || + Math.abs(frame.dcOffset) > AMBIENT_VOICE_MAX_ABSOLUTE_DC_OFFSET) { + reasons.push("dc-offset"); + } + if (active && (!finite(frame.snrDb) || frame.snrDb < AMBIENT_VOICE_MIN_SPEECH_SNR_DB)) { + reasons.push("speech-snr"); + } + const blocking = frame.qualityReasons.filter((reason) => + FRAME_BLOCKING_REASONS.has(reason) + ); + for (const reason of blocking) reasons.push(`quality:${reason}`); + return reasons; +} + function timingFrameUsable(frame: AmbientVoiceFrame): boolean { const active = frame.speechActive; return ( @@ -958,9 +1022,40 @@ export function extractAmbientVoiceMetrics( } return outcome; }); + const voiceGateFailures: Record = {}; + let usableVoiceFrames = 0; + for (const frame of inRange) { + const failures = voiceFrameGateFailures(frame); + if (failures.length === 0) usableVoiceFrames += 1; + for (const reason of failures) { + voiceGateFailures[reason] = (voiceGateFailures[reason] ?? 0) + 1; + } + } + const voiceDiagnostics: VoiceScreeningDiagnostics = { + frameCount: inRange.length, + usableFrameCount: usableVoiceFrames, + speechActiveFrameCount: inRange.filter((frame) => frame.speechActive).length, + periodicFrameCount: inRange.filter((frame) => frame.periodic).length, + frameGateFailures: voiceGateFailures, + segmentsAccepted: segments.length, + eligibleDurationMs: segments.reduce( + (total, segment) => total + segment.durationMs, + 0 + ), + activeSpeechMs: segments.reduce( + (total, segment) => total + segment.activeDurationMs, + 0 + ), + requirements: { + minimumSegments: AMBIENT_VOICE_MIN_SEGMENTS, + minimumEligibleSpanMs: AMBIENT_VOICE_TIMING_MIN_MS, + minimumActiveSpeechMs: AMBIENT_VOICE_ACTIVE_MIN_MS + } + }; return { outcomes: ordered, ignoredFrameCount, - events: { pauses: pauseEvents, speechRuns: speechRunEvents } + events: { pauses: pauseEvents, speechRuns: speechRunEvents }, + diagnostics: voiceDiagnostics }; } diff --git a/packages/ambient-core/src/expression-events.ts b/packages/ambient-core/src/expression-events.ts index e8c8213..7de742c 100644 --- a/packages/ambient-core/src/expression-events.ts +++ b/packages/ambient-core/src/expression-events.ts @@ -1,4 +1,4 @@ -import type { ExpressionEventRecord } from "./kinematic-events.js"; +import type { DetectedExpression } from "./kinematic-events.js"; /** * Spontaneous facial expression events. * @@ -79,7 +79,7 @@ export interface ExpressionBaseline { * because the detector's callers predate the record type; the fields are the * same set. */ -export type ExpressionEvent = ExpressionEventRecord; +export type ExpressionEvent = DetectedExpression; function finite(value: number | null | undefined): value is number { return typeof value === "number" && Number.isFinite(value); diff --git a/packages/ambient-core/src/index.ts b/packages/ambient-core/src/index.ts index a4400ca..93523ff 100644 --- a/packages/ambient-core/src/index.ts +++ b/packages/ambient-core/src/index.ts @@ -112,8 +112,13 @@ export { type ExpressionEvent, type ExpressionSummary } from "./expression-events.js"; +export { frameGateFailures } from "./ambient-face.js"; +export type { FaceScreeningDiagnostics } from "./ambient-face.js"; +export type { VoiceScreeningDiagnostics } from "./ambient-voice.js"; export type { BlinkEventRecord, + DetectedBlink, + DetectedExpression, ExpressionEventRecord, PauseEventRecord, PauseKind, diff --git a/packages/ambient-core/src/kinematic-events.ts b/packages/ambient-core/src/kinematic-events.ts index 09347e9..48c2223 100644 --- a/packages/ambient-core/src/kinematic-events.ts +++ b/packages/ambient-core/src/kinematic-events.ts @@ -64,6 +64,17 @@ export interface BlinkEventRecord { closedDwellMs: number; /** Frames that contributed. A low count means the phases are coarse. */ frameCount: number; + /** + * Whether every frame spanning this event stayed inside the pose limits the + * session metrics require. + * + * Detection runs on a looser stream than measurement, because a blink or a + * mouth movement is recoverable at a head angle that would invalidate a + * left-versus-right geometric comparison. Rate and timing can use every + * event; anything comparing the two sides should use only those flagged + * true. + */ + poseWithinMeasurementLimits: boolean; } /** @@ -95,6 +106,17 @@ export interface ExpressionEventRecord { /** Lid aperture change at peak, per side, for oculo-oral coupling. */ lidApertureDeltaLeft: number; lidApertureDeltaRight: number; + /** + * Whether every frame spanning this event stayed inside the pose limits the + * session metrics require. + * + * Detection runs on a looser stream than measurement, because a blink or a + * mouth movement is recoverable at a head angle that would invalidate a + * left-versus-right geometric comparison. Rate and timing can use every + * event; anything comparing the two sides should use only those flagged + * true. + */ + poseWithinMeasurementLimits: boolean; } /** What ended a stretch of speech, which decides what the silence measures. */ @@ -157,3 +179,19 @@ export interface SessionEventRecords { pauses: readonly PauseEventRecord[]; speechRuns: readonly SpeechRunEventRecord[]; } + +/* + * What a detector produces, before pose context is attached. + * + * The detectors work on geometry alone and have no view of the pose limits the + * session metrics impose, so they must not assert that field. Whatever holds + * both the events and the pose stream fills it in. + */ +export type DetectedBlink = Omit< + BlinkEventRecord, + "poseWithinMeasurementLimits" +>; +export type DetectedExpression = Omit< + ExpressionEventRecord, + "poseWithinMeasurementLimits" +>; diff --git a/packages/contracts/src/ambient-protocol.ts b/packages/contracts/src/ambient-protocol.ts index 58c03f1..324992b 100644 --- a/packages/contracts/src/ambient-protocol.ts +++ b/packages/contracts/src/ambient-protocol.ts @@ -43,10 +43,10 @@ const rawProtocolPack = { // 3.0.0: brow geometry and per-eye closure added (5 face metrics). // Sessions measured under different packs are not comparable, and the // content digest below makes that structurally visible rather than implicit. - version: "3.1.0", + version: "3.2.0", // SHA-256 of the canonical pack content with this field omitted. contentSha256: - "5d3ccce909f84b5a1f37e47f8d7e0b08f8cbcf6fea5cd2de09c3cf31261335a8", + "b12f5977a9bc1b23bbafbe6b01c5a5c9b4d535fa9ae2dd4fe5722ea49624ad11", status: "nonclinical-prototype", maximumSessionDurationMs: 300_000, supportedTarget: { @@ -90,9 +90,12 @@ const rawProtocolPack = { minimumSamplesPerBin: 80, minimumBinSpanMs: 4_800, maximumFrameGapMs: 200, - maximumAbsoluteYawDegrees: 7, - maximumAbsolutePitchDegrees: 10, - maximumAbsoluteRollDegrees: 5, + maximumYawDeviationDegrees: 7, + maximumPitchDeviationDegrees: 10, + maximumRollDeviationDegrees: 5, + maximumRestingYawDegrees: 10, + maximumRestingPitchDegrees: 20, + maximumRestingRollDegrees: 15, maximumCalibrationScaleDeviation: 0.2, maximumWithinBinScaleRatio: 1.15, minimumBins: 3, diff --git a/packages/contracts/src/protocol.ts b/packages/contracts/src/protocol.ts index afcfe34..b9e1b2f 100644 --- a/packages/contracts/src/protocol.ts +++ b/packages/contracts/src/protocol.ts @@ -122,9 +122,25 @@ export const CaptureQualityPolicyV1Schema = z minimumSamplesPerBin: z.literal(80), minimumBinSpanMs: z.literal(4_800), maximumFrameGapMs: z.literal(200), - maximumAbsoluteYawDegrees: z.literal(7), - maximumAbsolutePitchDegrees: z.literal(10), - maximumAbsoluteRollDegrees: z.literal(5), + /* + * Pose limits are DEVIATION from the session's resting pose, not from + * frontal. A camera mounted below eye level reads as constant pitch + * that no amount of sitting still removes, and gating on absolute angle + * conflated that with the subject turning away. + * + * The resting bounds below cap how far that reference may itself sit + * from frontal, and differ per axis on geometric grounds: yaw + * foreshortens one side of the face and so biases every left-versus- + * right measurement; pitch is symmetric across the midline; roll is + * already cancelled by aligning the coordinate x-axis to the inter-eye + * line. + */ + maximumYawDeviationDegrees: z.literal(7), + maximumPitchDeviationDegrees: z.literal(10), + maximumRollDeviationDegrees: z.literal(5), + maximumRestingYawDegrees: z.literal(10), + maximumRestingPitchDegrees: z.literal(20), + maximumRestingRollDegrees: z.literal(15), maximumCalibrationScaleDeviation: z.literal(0.2), maximumWithinBinScaleRatio: z.literal(1.15), minimumBins: z.literal(3),