diff --git a/web/analytics/flows-funnel.txt b/web/analytics/flows-funnel.txt index 63b20b4..696e69c 100644 --- a/web/analytics/flows-funnel.txt +++ b/web/analytics/flows-funnel.txt @@ -8,6 +8,33 @@ the handoff payload and is preserved by parseFlowHandoff, which is what makes the cross-app report possible; a handoff carrying no usable journey still deploys and is counted with journey_id = null. +IDENTITY CONTRACT (marketing -> Google -> Cloud) +Marketing never calls identify: the visitor there is anonymous, and that is +correct. Continuity rests on two mechanisms, in this order. +1. Shared browser storage. Both apps are served from one origin in production + (marketing at agentrelay.com/, Cloud at agentrelay.com/cloud) and posthog-js + persists its anonymous distinct_id and session under the project token, so + the same anonymous person survives the handoff and the Google round trip. + This requires both apps to use the SAME project key; a different key is a + different project and no amount of identify will join the funnel. +2. The handoff payload. cloudConnectionsHref (web/lib/flow-onboarding.ts) puts + the marketing anonymous distinct_id in the fragment as analytics.distinctId + beside analytics.journeyId, and cloud_handoff_started reports whether it was + sent as distinct_id_sent. Only PostHog's anonymous device id travels: no + email, name, repository, ticket, or anything typed into onboarding. It is in + the fragment, not the query, so it never reaches OAuth state or server logs. + It is absent when PostHog is not configured or the visitor opted out, so + distinct_id_sent = false is an expected state and not a failure. +Cloud then: reports handoff_distinct_id_matches on cloud_import_received (did +mechanism 1 hold?), calls identify(userId) on the first authenticated view +before google_login_completed so pre-auth anonymous events merge into the user, +and calls alias(analytics.distinctId) once per handoff when the marketing id +differs from the anonymous id Cloud itself held — the repair for a handoff whose +storage did not carry over (preview hosts, dev, blocked or partitioned storage). +A server-side cloud_auth event at the Google callback records signup versus +login on the user's own distinct_id, and reset() on logout keeps a shared +machine from bleeding one person's identity into the next visitor. + PRIMARY QUESTIONS 1. Which step loses the most journeys? 2. Where do people hesitate, change their answers, go back, or ask for help? @@ -20,8 +47,12 @@ Use event property journey_id as the correlation key, not event counts. Filter funnel_version = 2 and exclude localhost, development, and staff traffic. Marketing and Cloud must send events to the same PostHog project for joined reports. A custom journey property does not automatically merge PostHog person identities: -use a query/report grouped by journey_id for the cross-app funnel. Standard -person-based funnels are appropriate only when identity continuity is verified. +a report grouped by journey_id remains the reading that does not depend on +identity at all, and stays the safe default for auditing the numbers. +Person-based funnels are now valid once identify is verified in production (see +IDENTITY CONTRACT above): confirm in PostHog that a $identify event follows +Google sign-in and that cloud_auth arrives on the same distinct_id as the +user id. Until that check is done, prefer the journey_id grouping. Acquisition (optional starting steps): flows_landing_viewed @@ -36,13 +67,25 @@ Core setup, in order: flows_onboarding_step_viewed stage = connections Cloud branch (emitted by Cloud except where marked): - flows_onboarding_cloud_handoff_started marketing side, beacon transport - flows_onboarding_cloud_import_received from_fragment, has_journey + flows_onboarding_cloud_handoff_started marketing side, beacon transport; + distinct_id_sent + flows_onboarding_cloud_import_received from_fragment, has_journey, + handoff_distinct_id_matches flows_onboarding_google_redirect_started + $identify Cloud, first authenticated view, + before google_login_completed + $create_alias Cloud, only when the marketing + distinct id did not carry over + cloud_auth Cloud server side at the Google + callback; action = signup/login, + provider, signup_source; NOT + prefixed flows_onboarding_ flows_onboarding_google_login_completed a stored (not fragment) handoff on an authenticated view: the return trip flows_onboarding_cloud_flow_saved flow_id - flows_onboarding_cloud_dashboard_viewed from = run_complete/run_panel/deployed + flows_onboarding_cloud_dashboard_viewed from = run_complete/run_panel/deployed; + deployed is the terminal step of the + cross-app funnel flows_onboarding_connection_setup_clicked not implemented flows_onboarding_connection_setup_viewed not implemented @@ -106,6 +149,12 @@ configured_filters: filter field names only, such as github.labels; never values other_agent_selected, has_extra_instructions: booleans only. handoff_id: correlates a particular Cloud attempt. flow_id: correlates a successfully saved Cloud flow. +distinct_id_sent: whether the handoff carried the marketing anonymous PostHog + distinct id. False when PostHog is unconfigured or the visitor opted out. +handoff_distinct_id_matches: Cloud side. True when Cloud's own anonymous + distinct_id equals the one the handoff carried, which means shared same-origin + storage survived the trip; false means Cloud aliased to repair it; null when + no id was carried or no PostHog client was available to compare. PostHog's normal device/browser/referrer/session properties remain available. Cloud events additionally carry surface=cloud, cloud_step, cloud_step_index, @@ -168,9 +217,15 @@ INTERPRETATION AND COVERAGE LIMITS - Session replay depends on PostHog project enablement and sampling. Inputs are masked and code/configuration surfaces are blocked. Analytics do not collect raw repository names, tickets, source filters, generated code, or write-in names. -- Both web apps need NEXT_PUBLIC_POSTHOG_KEY and the intended host at build time. - PostHog project ingestion, replay enablement, and reports need a post-deployment - check by an authorized operator. None were verified against production here. +- Both web apps need NEXT_PUBLIC_POSTHOG_KEY and the intended host, and both + must use the SAME project key: marketing's GitHub Actions variable has to equal + the Cloud project key. Different keys mean different projects and no joined + funnel, whatever identify does. PostHog project ingestion, replay enablement, + and reports need a post-deployment check by an authorized operator. None were + verified against production here. +- The carried distinct id proves only what the browser reported. An alias merges + two anonymous people into one user; it cannot recover events PostHog never + received because the SDK was absent, blocked, or opted out. RELEASE VALIDATION Use a nonproduction PostHog project first. Complete one fresh Cloud journey, diff --git a/web/app/flows/onboarding/FactoryBuilder.tsx b/web/app/flows/onboarding/FactoryBuilder.tsx index 9f78829..c61ca24 100644 --- a/web/app/flows/onboarding/FactoryBuilder.tsx +++ b/web/app/flows/onboarding/FactoryBuilder.tsx @@ -68,7 +68,7 @@ export function FactoryBuilder() { const questionHeading = useRef(null); const copyTimeout = useRef | null>(null); const previousStep = useRef(null); - const { track, getJourneyId, markOutcome } = useFlowAnalytics(draft, FLOW_STAGES[Math.max(0, routeStep + 1)], hydrated && !loadingStage); + const { track, getJourneyId, getDistinctId, markOutcome } = useFlowAnalytics(draft, FLOW_STAGES[Math.max(0, routeStep + 1)], hydrated && !loadingStage); const fieldStart = useRef>({}); const storageStatus = useRef({ restored: false, failed: false, reported: false }); const ready = draft.step === 3; @@ -282,7 +282,7 @@ export function FactoryBuilder() { :

Let’s run your first flow.

Your software factory is built. Choose where to put it to work.

- +
track('help_toggled', { section: 'review_flow', open: event.currentTarget.open })}> Review your flow
diff --git a/web/app/flows/onboarding/RunOptions.tsx b/web/app/flows/onboarding/RunOptions.tsx index d96e237..9197c20 100644 --- a/web/app/flows/onboarding/RunOptions.tsx +++ b/web/app/flows/onboarding/RunOptions.tsx @@ -8,9 +8,9 @@ import { LOCAL_INSTALL, LOCAL_RUN, localKitArchive } from '../../../lib/flow-loc import type { FlowTrack } from '../../../lib/flow-analytics'; import s from './onboarding.module.css'; -export function RunOptions({ draft, chosen, setDestination, onTrack, getJourneyId, markOutcome, onNotice }: { +export function RunOptions({ draft, chosen, setDestination, onTrack, getJourneyId, getDistinctId, markOutcome, onNotice }: { chosen: 'cloud' | 'local'; setDestination: (destination: 'cloud' | 'local') => void; - draft: FactoryDraft; onTrack: FlowTrack; getJourneyId: () => string | undefined; markOutcome: (outcome: 'cloud_handoff' | 'local_kit_downloaded') => void; onNotice: (message: string) => void; + draft: FactoryDraft; onTrack: FlowTrack; getJourneyId: () => string | undefined; getDistinctId: () => string | undefined; markOutcome: (outcome: 'cloud_handoff' | 'local_kit_downloaded') => void; onNotice: (message: string) => void; }) { const [copied, setCopied] = useState(''); const [signingIn, setSigningIn] = useState(false); @@ -24,8 +24,9 @@ export function RunOptions({ draft, chosen, setDestination, onTrack, getJourneyI setSigningIn(true); try { const handoffId = crypto.randomUUID(); - const href = cloudConnectionsHref(draft, handoffId, getJourneyId()); - onTrack('cloud_handoff_started', { handoff_id: handoffId, destination: 'cloud' }); + const distinctId = getDistinctId(); + const href = cloudConnectionsHref(draft, handoffId, getJourneyId(), distinctId); + onTrack('cloud_handoff_started', { handoff_id: handoffId, destination: 'cloud', distinct_id_sent: Boolean(distinctId) }); markOutcome('cloud_handoff'); window.location.assign(href); } catch { diff --git a/web/app/flows/onboarding/useFlowAnalytics.ts b/web/app/flows/onboarding/useFlowAnalytics.ts index d580942..1904603 100644 --- a/web/app/flows/onboarding/useFlowAnalytics.ts +++ b/web/app/flows/onboarding/useFlowAnalytics.ts @@ -44,6 +44,16 @@ export function useFlowAnalytics(draft: FactoryDraft, stage: FlowStage, enabled: }; }, []); const track: FlowTrack = useCallback((event, properties = {}) => tracker.current?.track(event, properties), []); - return { track, getJourneyId: () => ph && !ph.has_opted_out_capturing() ? tracker.current?.id : undefined, + // PostHog's anonymous device id for this visitor, carried into the Cloud + // handoff so Cloud can join the two apps' people even when same-origin + // browser storage did not survive the trip. Never sent when PostHog is + // absent or the visitor opted out. + const getDistinctId = useCallback(() => { + try { + if (!process.env.NEXT_PUBLIC_POSTHOG_KEY || !ph || ph.has_opted_out_capturing()) return undefined; + return ph.get_distinct_id() || undefined; + } catch { return undefined; } + }, [ph]); + return { track, getJourneyId: () => ph && !ph.has_opted_out_capturing() ? tracker.current?.id : undefined, getDistinctId, markOutcome: (outcome: 'cloud_handoff' | 'local_kit_downloaded') => tracker.current?.markOutcome(outcome) }; } diff --git a/web/lib/flow-onboarding.ts b/web/lib/flow-onboarding.ts index b8bce45..c4e6f8d 100644 --- a/web/lib/flow-onboarding.ts +++ b/web/lib/flow-onboarding.ts @@ -84,12 +84,18 @@ export function cloudBlockedReason(draft: FactoryDraft): string { return isMarkdownOnly(draft) ? MARKDOWN_ONLY_CLOUD_NOTE : ''; } -export function cloudConnectionsHref(draft: FactoryDraft, handoffId: string, journeyId?: string): string { +export function cloudConnectionsHref(draft: FactoryDraft, handoffId: string, journeyId?: string, distinctId?: string): string { if (!canContinue(draft, 2)) throw new Error('Choose a workflow before continuing to Cloud.'); // The fragment is read only by Cloud's browser deploy page, which keeps it in // localStorage across Google sign-in. Source code and ticket filters must not // enter OAuth state, cookies, or server access logs. - const payload = { version: 1, handoffId, ...(journeyId ? { analytics: { journeyId } } : {}), name: 'Software factory', source: factorySource({ ...draft, step: 3 }), + // distinctId is PostHog's anonymous device identifier for this visitor, and + // nothing else: no email, name, or anything typed into onboarding. Both apps + // are same-origin in production, so Cloud normally reads the same anonymous + // id from shared browser storage. Carrying it here lets Cloud measure whether + // that continuity actually held and merge the two people when it did not. + const analytics = { ...(journeyId ? { journeyId } : {}), ...(distinctId ? { distinctId } : {}) }; + const payload = { version: 1, handoffId, ...(Object.keys(analytics).length ? { analytics } : {}), name: 'Software factory', source: factorySource({ ...draft, step: 3 }), workflow: draft.workflow, preview: flowPreview(draft), sources: draft.sources, sourceSettings: draft.sourceSettings, agents: draft.agents, otherAgent: draft.otherAgent, otherAgentSelected: otherAgentIsSelected(draft), task: draft.task }; const base = process.env.NEXT_PUBLIC_CLOUD_URL || 'https://agentrelay.com/cloud'; diff --git a/web/lib/test/flow-analytics.test.ts b/web/lib/test/flow-analytics.test.ts index e2b997b..5d81d22 100644 --- a/web/lib/test/flow-analytics.test.ts +++ b/web/lib/test/flow-analytics.test.ts @@ -86,6 +86,27 @@ describe('safe analytics payloads and correlation', () => { expect(JSON.parse(decodeURIComponent(url.hash.slice(1))).analytics).toEqual({ journeyId: id }); expect(JSON.parse(decodeURIComponent(new URL(cloudConnectionsHref(draft, freshId)).hash.slice(1))).analytics).toBeUndefined(); }); + + it('carries the anonymous PostHog distinct id beside the journey, and only when given', () => { + // Both apps are same-origin in production, so Cloud usually reads the same + // anonymous distinct_id from shared storage. Sending it in the handoff is + // what lets Cloud tell that continuity held, and alias the two people when + // it did not. It is a device id, never anything the visitor typed. + const draft = { ...DEFAULT_FACTORY, workflow: 'simple' as const, sources: ['markdown' as const], agents: ['claude' as const], step: 3 }; + const distinctId = '0198f0aa-1c2d-7c3e-8f10-b2c3d4e5f607'; + const analytics = (href: string) => JSON.parse(decodeURIComponent(new URL(href).hash.slice(1))).analytics; + expect(analytics(cloudConnectionsHref(draft, freshId, id, distinctId))).toEqual({ journeyId: id, distinctId }); + // A journey can expire or be opted out of while PostHog still has an id, + // and PostHog can be absent while the journey exists; neither may drag the + // other into the payload or leave an empty analytics object behind. + expect(analytics(cloudConnectionsHref(draft, freshId, undefined, distinctId))).toEqual({ distinctId }); + expect(analytics(cloudConnectionsHref(draft, freshId, id))).toEqual({ journeyId: id }); + expect(analytics(cloudConnectionsHref(draft, freshId, id, ''))).toEqual({ journeyId: id }); + expect(analytics(cloudConnectionsHref(draft, freshId, '', ''))).toBeUndefined(); + // The id must stay in the fragment: a query would reach Cloud's server + // access logs and OAuth state. + expect(new URL(cloudConnectionsHref(draft, freshId, id, distinctId)).search).toBe(''); + }); }); describe('mounted journey expiration', () => {