diff --git a/.changeset/experimentation-client.md b/.changeset/experimentation-client.md new file mode 100644 index 0000000000..5d1679f7d2 --- /dev/null +++ b/.changeset/experimentation-client.md @@ -0,0 +1,5 @@ +--- +'eas-cli': minor +--- + +Add experimentation client. diff --git a/packages/eas-cli/package.json b/packages/eas-cli/package.json index 3a5657ce18..7cb64ace49 100644 --- a/packages/eas-cli/package.json +++ b/packages/eas-cli/package.json @@ -51,6 +51,7 @@ "@expo/eas-build-job": "25.0.0", "@expo/eas-json": "25.0.0", "@expo/env": "^1.0.0", + "@expo/experimentation": "0.0.5", "@expo/json-file": "8.3.3", "@expo/logger": "25.0.0", "@expo/multipart-body-parser": "2.0.0", diff --git a/packages/eas-cli/src/__tests__/commands/utils-test.ts b/packages/eas-cli/src/__tests__/commands/utils-test.ts new file mode 100644 index 0000000000..6a2e9f7f79 --- /dev/null +++ b/packages/eas-cli/src/__tests__/commands/utils-test.ts @@ -0,0 +1,25 @@ +import EasCommand from '../../commandUtils/EasCommand'; +import { mockCommandContext } from './utils'; + +jest.mock('fs'); + +class TestCommand extends EasCommand { + static override contextDefinition = { + ...this.ContextOptions.LoggedIn, + ...this.ContextOptions.Experimentation, + }; + + async runAsync(): Promise {} +} + +describe(mockCommandContext, () => { + it('provides a disabled experimentation client that returns defaults', async () => { + const ctx = mockCommandContext(TestCommand, {}); + const client = await ctx.experimentation.getClientAsync(); + expect(client.getUserNamespace('any').getParam('variant', 'control')).toBe('control'); + expect(client.getAccountNamespace('any', { accountId: 'a' }).getParam('enabled', true)).toBe( + true + ); + expect(client.getDeviceNamespace('any').getParam('count', 3)).toBe(3); + }); +}); diff --git a/packages/eas-cli/src/__tests__/commands/utils.ts b/packages/eas-cli/src/__tests__/commands/utils.ts index 5f5e41b13f..e4a0ece68d 100644 --- a/packages/eas-cli/src/__tests__/commands/utils.ts +++ b/packages/eas-cli/src/__tests__/commands/utils.ts @@ -7,6 +7,9 @@ import { v4 as uuidv4 } from 'uuid'; import { ContextInput, ContextOutput } from '../../commandUtils/EasCommand'; import { DynamicConfigContextFn } from '../../commandUtils/context/DynamicProjectConfigContextField'; +import { ExperimentOverrides } from '../../experimentation/ExperimentOverrides'; +import { ExperimentationClient } from '../../experimentation/ExperimentationClient'; +import { LazyExperimentationClient } from '../../experimentation/LazyExperimentationClient'; import { AppFragment, Role, @@ -96,6 +99,11 @@ export function mockCommandContext< if (contextKey === 'sessionManager') { result.sessionManager = {}; } + if (contextKey === 'experimentation') { + result.experimentation = LazyExperimentationClient.fromClient( + ExperimentationClient.createDisabled(ExperimentOverrides.empty()) + ); + } if (contextKey === 'projectConfig') { result.projectConfig = { exp: overrides.exp ?? getMockExpoConfig(), diff --git a/packages/eas-cli/src/analytics/AnalyticsManager.ts b/packages/eas-cli/src/analytics/AnalyticsManager.ts index dd7a1b0ecb..59432a3dbb 100644 --- a/packages/eas-cli/src/analytics/AnalyticsManager.ts +++ b/packages/eas-cli/src/analytics/AnalyticsManager.ts @@ -21,12 +21,17 @@ export type AnalyticsEvent = | SubmissionEvent | MetadataEvent | BillingEvent - | SimulatorEvent; + | SimulatorEvent + | ExperimentationEvent; export enum CommandEvent { ACTION = 'action', // generic event type which is used to determine the 'daily active user' stat, include an `action: eas ${subcommand}` property inside of the event properties object } +export enum ExperimentationEvent { + EXPERIMENT_VIEWED = 'Experiment Viewed', +} + export enum BillingEvent { SUBSCRIBE_COMMAND = 'billing cli subscribe command', } @@ -105,6 +110,8 @@ export interface Analytics { export interface AnalyticsWithOrchestration extends Analytics { setActor(actor: Actor): void; flushAsync(): Promise; + /** The persisted RudderStack anonymous ID, or null when analytics is disabled. */ + getDeviceId(): string | null; } const USER_SETTINGS_KEY_AMPLITUDE_ENABLED = 'amplitudeEnabled'; @@ -183,6 +190,9 @@ class NoOpAnalytics implements AnalyticsWithOrchestration { logEvent(): void {} setActor(): void {} async flushAsync(): Promise {} + getDeviceId(): null { + return null; + } } const RudderstackAnalyticsConfig = @@ -234,6 +244,10 @@ class RudderstackAnalytics implements AnalyticsWithOrchestration { this.identifiedActor = actor; } + public getDeviceId(): string { + return this.persistentDeviceId; + } + public logEvent(name: AnalyticsEvent, properties: AnalyticsEventProperties): void { const userId = this.identifiedActor?.id; const deviceId = this.persistentDeviceId; diff --git a/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts b/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts index 0847588f19..86744c08b4 100644 --- a/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts +++ b/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts @@ -140,3 +140,55 @@ describe(isAnalyticsOptedOutAsync, () => { await expect(isAnalyticsOptedOutAsync()).resolves.toBe(true); }); }); + +describe('getDeviceId', () => { + // Backs UserSettings with an in-memory store so a value written by createAnalyticsAsync is read back. + function useInMemoryUserSettings(initial: Record): void { + const stored: Record = { ...initial }; + (userSettingsMock.setAsync as jest.Mock).mockImplementation( + async (key: string, value: unknown) => { + stored[key] = value; + return {}; + } + ); + (userSettingsMock.getAsync as jest.Mock).mockImplementation( + async (key: string, defaultValue: unknown) => (key in stored ? stored[key] : defaultValue) + ); + } + + it('returns the persisted analytics device ID', async () => { + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBe('persistent-device-id'); + }); + + it('creates and persists a device ID when none is stored', async () => { + userSettingsMock.getAsync.mockImplementation(async (key, defaultValue) => + key === 'analyticsDeviceId' ? null : defaultValue + ); + const analytics = await createAnalyticsAsync(); + const deviceId = analytics.getDeviceId(); + expect(deviceId).toMatch(/^[0-9a-f-]{36}$/); + expect(userSettingsMock.setAsync).toHaveBeenCalledWith('analyticsDeviceId', deviceId); + }); + + it('is null when DISABLE_EAS_ANALYTICS is set', async () => { + useInMemoryUserSettings({ analyticsDeviceId: 'persistent-device-id' }); + process.env.DISABLE_EAS_ANALYTICS = '1'; + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBeNull(); + }); + + it('is null when running behind https_proxy', async () => { + process.env.https_proxy = 'http://proxy.local:8080'; + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBeNull(); + }); + + it('is null when analytics were turned off with eas analytics off', async () => { + userSettingsMock.getAsync.mockImplementation(async (key, defaultValue) => + key === 'analyticsEnabled' ? false : defaultValue + ); + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBeNull(); + }); +}); diff --git a/packages/eas-cli/src/commandUtils/EasCommand.ts b/packages/eas-cli/src/commandUtils/EasCommand.ts index b1337dcf53..8b152e1607 100644 --- a/packages/eas-cli/src/commandUtils/EasCommand.ts +++ b/packages/eas-cli/src/commandUtils/EasCommand.ts @@ -7,6 +7,7 @@ import nullthrows from 'nullthrows'; import AnalyticsContextField from './context/AnalyticsContextField'; import ContextField from './context/ContextField'; import DynamicLoggedInContextField from './context/DynamicLoggedInContextField'; +import ExperimentationContextField from './context/ExperimentationContextField'; import { DynamicPrivateProjectConfigContextField, DynamicPublicProjectConfigContextField, @@ -148,6 +149,14 @@ export default abstract class EasCommand extends Command { ProjectId: { projectId: new ProjectIdContextField(), }, + /** + * Experimentation client for user, account, and device experiments. + * Declare a login field first so user experiments see the actor. + * Account experiments take the account ID from the command. + */ + Experimentation: { + experimentation: new ExperimentationContextField(), + }, }; /** diff --git a/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts b/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts index a3c74cd36d..63ab364413 100644 --- a/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts +++ b/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts @@ -48,6 +48,7 @@ const analytics: AnalyticsWithOrchestration = { logEvent: jest.fn((): void => {}), setActor: jest.fn((): void => {}), flushAsync: jest.fn(async (): Promise => {}), + getDeviceId: jest.fn((): string | null => null), }; beforeEach(() => { diff --git a/packages/eas-cli/src/commandUtils/context/ExperimentationContextField.ts b/packages/eas-cli/src/commandUtils/context/ExperimentationContextField.ts new file mode 100644 index 0000000000..4ddbe4c6de --- /dev/null +++ b/packages/eas-cli/src/commandUtils/context/ExperimentationContextField.ts @@ -0,0 +1,111 @@ +import ContextField, { ContextOptions } from './ContextField'; +import { createGraphqlClient } from './contextUtils/createGraphqlClient'; +import { AnalyticsWithOrchestration } from '../../analytics/AnalyticsManager'; +import { ExperimentOverrides } from '../../experimentation/ExperimentOverrides'; +import { + ExperimentationClient, + createLoggerConfig, +} from '../../experimentation/ExperimentationClient'; +import { LazyExperimentationClient } from '../../experimentation/LazyExperimentationClient'; +import { ExperimentationConfigQuery } from '../../graphql/queries/ExperimentationConfigQuery'; +import Log from '../../log'; + +/** Caps how long the config fetch and the actor lookup can delay a command. */ +const REQUEST_TIMEOUT_MS = 3000; + +const CI_ENVIRONMENT_FLAGS = ['EAS_BUILD', 'CI']; + +function isRunningInCi(): boolean { + return CI_ENVIRONMENT_FLAGS.some(flag => !!process.env[flag]); +} + +/** + * Never prompts, throws, writes files, or reads the app config. A failed config fetch disables + * all scopes. CI runs skip the fetch and get defaults. + * The config fetch runs on the first `getClientAsync` call, so commands that never read a variant + * make no request. + */ +export default class ExperimentationContextField extends ContextField { + // One handle per command run, so repeated getContextAsync calls cannot log duplicate exposures. + private readonly clientByAnalytics = new WeakMap< + AnalyticsWithOrchestration, + LazyExperimentationClient + >(); + + async getValueAsync(options: ContextOptions): Promise { + let client = this.clientByAnalytics.get(options.analytics); + if (!client) { + client = new LazyExperimentationClient(() => this.buildClientAsync(options)); + this.clientByAnalytics.set(options.analytics, client); + } + return client; + } + + private async buildClientAsync({ + sessionManager, + analytics, + }: ContextOptions): Promise { + const overrides = ExperimentOverrides.fromEnv(); + + if (isRunningInCi()) { + Log.debug('[experimentation] Running in CI. Experiments return defaults.'); + return ExperimentationClient.createDisabled(overrides); + } + + const deviceId = analytics.getDeviceId(); + if (deviceId === null) { + Log.debug('[experimentation] Analytics is disabled. Experiments return defaults.'); + return ExperimentationClient.createDisabled(overrides); + } + + try { + const graphqlClient = createGraphqlClient( + { + accessToken: sessionManager.getAccessToken(), + sessionSecret: sessionManager.getSessionSecret(), + }, + { requestTimeoutMs: REQUEST_TIMEOUT_MS } + ); + + // The session manager skips the request once an actor is cached. Without a cached actor + // (for example, a stale session) it makes its own request, so it needs the same cap. + const [actorResult, configsResult] = await Promise.allSettled([ + sessionManager.getUserAsync({ requestTimeoutMs: REQUEST_TIMEOUT_MS }), + ExperimentationConfigQuery.getConfigsAsync(graphqlClient), + ]); + + if (configsResult.status === 'rejected') { + Log.debug( + '[experimentation] Failed to fetch the experiment config. Experiments return defaults.', + errorMessage(configsResult.reason) + ); + return ExperimentationClient.createDisabled(overrides); + } + + const configs = configsResult.value; + return new ExperimentationClient( + { + user: configs.userConfig, + account: configs.accountConfig, + device: configs.deviceConfig, + }, + { + user: actorResult.status === 'fulfilled' ? (actorResult.value?.id ?? null) : null, + device: deviceId, + }, + overrides, + createLoggerConfig(analytics) + ); + } catch (error) { + Log.debug( + '[experimentation] Failed to initialize experiments. Experiments return defaults.', + errorMessage(error) + ); + return ExperimentationClient.createDisabled(overrides); + } + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/eas-cli/src/commandUtils/context/__tests__/ExperimentationContextField-test.ts b/packages/eas-cli/src/commandUtils/context/__tests__/ExperimentationContextField-test.ts new file mode 100644 index 0000000000..25180e14fb --- /dev/null +++ b/packages/eas-cli/src/commandUtils/context/__tests__/ExperimentationContextField-test.ts @@ -0,0 +1,289 @@ +import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; + +import { AnalyticsWithOrchestration } from '../../../analytics/AnalyticsManager'; +import { TEST_DEFINITION } from '../../../experimentation/__tests__/fixtures'; +import { ExperimentationConfigQuery } from '../../../graphql/queries/ExperimentationConfigQuery'; +import Log from '../../../log'; +import SessionManager from '../../../user/SessionManager'; +import { Actor } from '../../../user/User'; +import { ContextOptions } from '../ContextField'; +import { ExperimentationClient } from '../../../experimentation/ExperimentationClient'; +import ExperimentationContextField from '../ExperimentationContextField'; +import { createGraphqlClient } from '../contextUtils/createGraphqlClient'; + +jest.mock('../../../env', () => ({ + __esModule: true, + default: { experimentOverride: undefined as string | undefined }, +})); +jest.mock('../../../graphql/queries/ExperimentationConfigQuery'); +jest.mock('../../../log'); +jest.mock('../contextUtils/createGraphqlClient'); + +const mockEnv = jest.requireMock('../../../env').default as { experimentOverride?: string }; +const getConfigsAsyncMock = jest.mocked(ExperimentationConfigQuery.getConfigsAsync); +const fakeGraphqlClient = { fake: true } as any; + +const actor = { __typename: 'User', id: 'user-1' } as unknown as Actor; +const emptyDefinition = { experiments: [], namespaces: [] }; +const allConfigs = { + userConfig: TEST_DEFINITION, + accountConfig: TEST_DEFINITION, + deviceConfig: TEST_DEFINITION, +}; + +function createAnalytics(deviceId: string | null = 'device-1'): AnalyticsWithOrchestration & { + logEvent: jest.Mock; +} { + return { + logEvent: jest.fn(), + setActor: jest.fn(), + flushAsync: jest.fn(), + getDeviceId: jest.fn(() => deviceId), + }; +} + +function createSessionManager({ loggedOut = false }: { loggedOut?: boolean } = {}): { + sessionManager: SessionManager; + mocked: SessionManager; +} { + const mocked = mock(); + when(mocked.getAccessToken()).thenReturn(null); + when(mocked.getSessionSecret()).thenReturn(loggedOut ? null : 'secret'); + when(mocked.getUserAsync(anything())).thenResolve(loggedOut ? undefined : actor); + return { sessionManager: instance(mocked), mocked }; +} + +function options(overrides: Partial = {}): ContextOptions { + return { + nonInteractive: false, + sessionManager: createSessionManager().sessionManager, + analytics: createAnalytics(), + ...overrides, + } as ContextOptions; +} + +/** Resolves the context and then loads the client, as a command does on its first variant read. */ +async function loadClientAsync( + contextOptions: ContextOptions = options() +): Promise { + const lazyClient = await new ExperimentationContextField().getValueAsync(contextOptions); + return await lazyClient.getClientAsync(); +} + +async function flushPromisesAsync(): Promise { + await new Promise(resolve => setImmediate(resolve)); +} + +afterAll(() => { + process.env = ORIGINAL_ENV; +}); + +const ORIGINAL_ENV = process.env; + +beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...ORIGINAL_ENV }; + delete process.env.CI; + delete process.env.EAS_BUILD; + mockEnv.experimentOverride = undefined; + jest.mocked(createGraphqlClient).mockReturnValue(fakeGraphqlClient); + getConfigsAsyncMock.mockResolvedValue(allConfigs); +}); + +describe(ExperimentationContextField, () => { + it('returns a disabled client and fetches nothing when analytics is disabled', async () => { + const { sessionManager, mocked } = createSessionManager(); + const client = await loadClientAsync( + options({ analytics: createAnalytics(null), sessionManager }) + ); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + expect(createGraphqlClient).not.toHaveBeenCalled(); + verify(mocked.getUserAsync(anything())).never(); + }); + + it.each(['CI', 'EAS_BUILD'])( + 'returns a disabled client and fetches nothing when %s is set', + async flag => { + process.env[flag] = '1'; + const { sessionManager, mocked } = createSessionManager(); + const client = await loadClientAsync(options({ sessionManager })); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + expect(createGraphqlClient).not.toHaveBeenCalled(); + verify(mocked.getUserAsync(anything())).never(); + } + ); + + it('still honors EAS_EXPERIMENT_OVERRIDE in CI', async () => { + process.env.CI = 'true'; + mockEnv.experimentOverride = 'cli-full:variant=treatment'; + const client = await loadClientAsync(options()); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('treatment'); + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + }); + + it('still honors EAS_EXPERIMENT_OVERRIDE when analytics is disabled', async () => { + mockEnv.experimentOverride = 'cli-full:variant=treatment'; + const client = await loadClientAsync(options({ analytics: createAnalytics(null) })); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('treatment'); + }); + + it('buckets each scope on its own unit and logs exposures through analytics', async () => { + const analytics = createAnalytics('device-1'); + const client = await loadClientAsync(options({ analytics })); + + client.getUserNamespace('cli-full').getParam('variant', 'control'); + client + .getAccountNamespace('cli-full', { accountId: 'account-1' }) + .getParam('variant', 'control'); + client.getDeviceNamespace('cli-full').getParam('variant', 'control'); + + const units = analytics.logEvent.mock.calls.map(([, properties]) => properties.unit); + expect(units).toEqual(['user-1', 'account-1', 'device-1']); + expect(createGraphqlClient).toHaveBeenCalledWith( + { accessToken: null, sessionSecret: 'secret' }, + { requestTimeoutMs: 3000 } + ); + }); + + it('gives logged-out users defaults for user experiments but still runs device experiments', async () => { + const analytics = createAnalytics('device-1'); + const { sessionManager } = createSessionManager({ loggedOut: true }); + const client = await loadClientAsync(options({ analytics, sessionManager })); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(['control', 'treatment']).toContain( + client.getDeviceNamespace('cli-full').getParam('variant', 'x') + ); + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + }); + + it('fails open to defaults when the config fetch rejects', async () => { + getConfigsAsyncMock.mockRejectedValue(new Error('offline')); + const analytics = createAnalytics(); + const client = await loadClientAsync(options({ analytics })); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + expect(Log.debug).toHaveBeenCalledWith( + expect.stringContaining('Failed to fetch'), + expect.anything() + ); + expect(Log.error).not.toHaveBeenCalled(); + expect(Log.warn).not.toHaveBeenCalled(); + }); + + it('returns defaults and logs nothing for empty configs, as the server sends for robots', async () => { + getConfigsAsyncMock.mockResolvedValue({ + userConfig: emptyDefinition, + accountConfig: emptyDefinition, + deviceConfig: emptyDefinition, + }); + const analytics = createAnalytics(); + const client = await loadClientAsync(options({ analytics })); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('fails open to defaults when the server config is invalid', async () => { + getConfigsAsyncMock.mockResolvedValue({ + ...allConfigs, + userConfig: { + experiments: TEST_DEFINITION.experiments, + namespaces: [ + { + name: 'too-many', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'cli-test', numSegments: 20 }, + ], + }, + ], + }, + }); + const client = await loadClientAsync(options()); + + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith( + expect.stringContaining('Failed to initialize'), + expect.anything() + ); + }); + + it('logs account exposures on the account ID the command passes', async () => { + const analytics = createAnalytics(); + const client = await loadClientAsync(options({ analytics })); + client + .getAccountNamespace('cli-full', { accountId: 'picked-account' }) + .getParam('variant', 'x'); + expect(analytics.logEvent).toHaveBeenCalledWith( + 'Experiment Viewed', + expect.objectContaining({ unit: 'picked-account' }) + ); + }); + + it('caps the actor lookup with the same timeout as the config fetch', async () => { + const { sessionManager, mocked } = createSessionManager(); + await loadClientAsync(options({ sessionManager })); + + verify(mocked.getUserAsync(deepEqual({ requestTimeoutMs: 3000 }))).once(); + expect(createGraphqlClient).toHaveBeenCalledWith(expect.anything(), { requestTimeoutMs: 3000 }); + }); + + it('runs the actor lookup and the config fetch in parallel', async () => { + let resolveConfigs!: (value: typeof allConfigs) => void; + let resolveActor!: (value: Actor) => void; + getConfigsAsyncMock.mockReturnValue(new Promise(resolve => (resolveConfigs = resolve))); + const mocked = mock(); + when(mocked.getAccessToken()).thenReturn(null); + when(mocked.getSessionSecret()).thenReturn('secret'); + when(mocked.getUserAsync(anything())).thenReturn( + new Promise(resolve => (resolveActor = resolve)) + ); + + const pending = loadClientAsync(options({ sessionManager: instance(mocked) })); + await flushPromisesAsync(); + + expect(getConfigsAsyncMock).toHaveBeenCalledTimes(1); + verify(mocked.getUserAsync(anything())).once(); + + resolveConfigs(allConfigs); + resolveActor(actor); + await expect(pending).resolves.toBeDefined(); + }); + + it('fetches nothing until a command asks for the client', async () => { + const { sessionManager, mocked } = createSessionManager(); + const lazyClient = await new ExperimentationContextField().getValueAsync( + options({ sessionManager }) + ); + + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + expect(createGraphqlClient).not.toHaveBeenCalled(); + verify(mocked.getUserAsync(anything())).never(); + + await lazyClient.getClientAsync(); + expect(getConfigsAsyncMock).toHaveBeenCalledTimes(1); + }); + + it('caches one client per analytics instance so repeated context calls do not log duplicate exposures', async () => { + const field = new ExperimentationContextField(); + const analytics = createAnalytics(); + const first = await field.getValueAsync(options({ analytics })); + const second = await field.getValueAsync(options({ analytics })); + const other = await field.getValueAsync(options({ analytics: createAnalytics() })); + + expect(second).toBe(first); + expect(other).not.toBe(first); + expect(await second.getClientAsync()).toBe(await first.getClientAsync()); + expect(await other.getClientAsync()).not.toBe(await first.getClientAsync()); + expect(getConfigsAsyncMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/eas-cli/src/commandUtils/context/contextUtils/__tests__/createGraphqlClient-test.ts b/packages/eas-cli/src/commandUtils/context/contextUtils/__tests__/createGraphqlClient-test.ts new file mode 100644 index 0000000000..c3d0f3c389 --- /dev/null +++ b/packages/eas-cli/src/commandUtils/context/contextUtils/__tests__/createGraphqlClient-test.ts @@ -0,0 +1,124 @@ +import { createRequest } from '@urql/core'; +import http from 'http'; +import { AddressInfo } from 'net'; + +import { createGraphqlClient } from '../createGraphqlClient'; + +jest.mock('../../../../api', () => ({ + getExpoApiBaseUrl: () => process.env.CREATE_GRAPHQL_CLIENT_TEST_URL, +})); + +// A plain string keeps this document out of GraphQL codegen, which scans gql tags in src/**. +const TEST_QUERY = 'query CreateGraphqlClientTest { __typename }'; + +function fetchOptionsOf(client: ReturnType): RequestInit { + const operation = client.createRequestOperation('query', createRequest(TEST_QUERY, {})); + const { fetchOptions } = operation.context; + return typeof fetchOptions === 'function' ? fetchOptions() : (fetchOptions ?? {}); +} + +/** Serves one GraphQL response after `delayMs`. Returns the base URL. */ +async function startSlowServerAsync(server: http.Server, delayMs: number): Promise { + server.on('request', (_req, res) => { + setTimeout(() => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ data: { __typename: 'RootQuery' } })); + }, delayMs); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return `http://127.0.0.1:${port}`; +} + +describe(createGraphqlClient, () => { + it('sends the access token as a bearer header', () => { + const init = fetchOptionsOf(createGraphqlClient({ accessToken: 'token', sessionSecret: null })); + expect(init.headers).toEqual({ authorization: 'Bearer token' }); + }); + + it('sends the session secret when there is no access token', () => { + const init = fetchOptionsOf( + createGraphqlClient({ accessToken: null, sessionSecret: 'secret' }) + ); + expect(init.headers).toEqual({ 'expo-session': 'secret' }); + }); + + it('keeps the auth headers when a request timeout is set', () => { + const init = fetchOptionsOf( + createGraphqlClient({ accessToken: 'token', sessionSecret: null }, { requestTimeoutMs: 50 }) + ); + expect(init.headers).toEqual({ authorization: 'Bearer token' }); + }); + + describe('against a slow server', () => { + const server = http.createServer(); + const serverDelayMs = 2000; + + beforeAll(async () => { + process.env.CREATE_GRAPHQL_CLIENT_TEST_URL = await startSlowServerAsync( + server, + serverDelayMs + ); + }); + + afterAll(async () => { + delete process.env.CREATE_GRAPHQL_CLIENT_TEST_URL; + server.closeAllConnections(); + await new Promise(resolve => { + server.close(() => { + resolve(); + }); + }); + }); + + it('aborts the request when the timeout elapses', async () => { + // Only a real request proves the timeout works. + const client = createGraphqlClient( + { accessToken: 'token', sessionSecret: null }, + { requestTimeoutMs: 100 } + ); + const start = Date.now(); + const result = await client.query(TEST_QUERY, {}, { noRetry: true }).toPromise(); + expect(Date.now() - start).toBeLessThan(serverDelayMs); + expect(result.data).toBeUndefined(); + expect(result.error?.networkError?.name).toBe('TimeoutError'); + expect(result.error?.networkError?.message).toBe('Request timed out after 100 ms'); + }); + + it('waits for the response when no timeout is set', async () => { + const client = createGraphqlClient({ accessToken: 'token', sessionSecret: null }); + const result = await client.query(TEST_QUERY, {}, { noRetry: true }).toPromise(); + expect(result.error).toBeUndefined(); + expect(result.data).toEqual({ __typename: 'RootQuery' }); + }); + }); + + describe('against a closed port', () => { + beforeAll(async () => { + // Listen once to reserve a free port, then close so the port refuses connections. + const server = http.createServer(); + process.env.CREATE_GRAPHQL_CLIENT_TEST_URL = await startSlowServerAsync(server, 0); + await new Promise(resolve => { + server.close(() => { + resolve(); + }); + }); + }); + + afterAll(() => { + delete process.env.CREATE_GRAPHQL_CLIENT_TEST_URL; + }); + + it('passes through fetch errors that are not timeouts', async () => { + const client = createGraphqlClient( + { accessToken: 'token', sessionSecret: null }, + { requestTimeoutMs: 5000 } + ); + const result = await client.query(TEST_QUERY, {}, { noRetry: true }).toPromise(); + expect(result.data).toBeUndefined(); + expect(result.error?.networkError).toBeDefined(); + expect(result.error?.networkError?.name).not.toBe('TimeoutError'); + expect(result.error?.networkError?.message).toContain('ECONNREFUSED'); + }); + }); +}); diff --git a/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts b/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts index 633e090453..98272e203d 100644 --- a/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts +++ b/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts @@ -1,6 +1,10 @@ import { Client, cacheExchange, createClient as createUrqlClient, fetchExchange } from '@urql/core'; import { retryExchange } from '@urql/exchange-retry'; -import fetch from 'node-fetch'; +import fetch, { + RequestInfo as NodeFetchRequestInfo, + RequestInit as NodeFetchRequestInit, + Response as NodeFetchResponse, +} from 'node-fetch'; import { getExpoApiBaseUrl } from '../../../api'; import { httpsProxyAgent } from '../../../fetch'; @@ -8,10 +12,46 @@ import { httpsProxyAgent } from '../../../fetch'; // Extend the urql Client to allow for future customizations or type additions ahead of upstream type support. export interface ExpoGraphqlClient extends Client {} -export function createGraphqlClient(authInfo: { - accessToken: string | null; - sessionSecret: string | null; -}): ExpoGraphqlClient { +export function createGraphqlClient( + authInfo: { + accessToken: string | null; + sessionSecret: string | null; + }, + options: { + /** Request timeout. Kept at the client level: a per-operation `fetchOptions` in urql would replace the auth headers. */ + requestTimeoutMs?: number; + } = {} +): ExpoGraphqlClient { + const { requestTimeoutMs } = options; + // urql overwrites `fetchOptions.signal`, so the timeout must be added inside fetch. + const fetchWithTimeoutAsync = async ( + url: NodeFetchRequestInfo, + init?: NodeFetchRequestInit + ): Promise => { + if (!requestTimeoutMs) { + return await fetch(url, init); + } + const timeoutSignal = AbortSignal.timeout(requestTimeoutMs); + try { + return await fetch(url, { + ...init, + signal: AbortSignal.any([ + ...(init?.signal ? [init.signal as AbortSignal] : []), + timeoutSignal, + ]), + }); + } catch (error) { + if (timeoutSignal.aborted) { + const timeoutError = new Error(`Request timed out after ${requestTimeoutMs} ms`, { + cause: error, + }); + timeoutError.name = 'TimeoutError'; + throw timeoutError; + } + throw error; + } + }; + return createUrqlClient({ url: getExpoApiBaseUrl() + '/graphql', exchanges: [ @@ -29,7 +69,7 @@ export function createGraphqlClient(authInfo: { fetchExchange, ], // @ts-expect-error Type 'typeof fetch' is not assignable to type '(input: RequestInfo, init?: RequestInit | undefined) => Promise'. - fetch, + fetch: fetchWithTimeoutAsync, fetchOptions: (): RequestInit => { const headers: Record = {}; if (authInfo.accessToken) { diff --git a/packages/eas-cli/src/commands/account/__tests__/login.test.ts b/packages/eas-cli/src/commands/account/__tests__/login.test.ts index 30717e51a9..ada438fd5d 100644 --- a/packages/eas-cli/src/commands/account/__tests__/login.test.ts +++ b/packages/eas-cli/src/commands/account/__tests__/login.test.ts @@ -28,6 +28,7 @@ const sessionManager = new SessionManager({ logEvent: jest.fn(), setActor: jest.fn(), flushAsync: jest.fn(), + getDeviceId: jest.fn(), }); const originalIsTTY = process.stdin.isTTY; const originalCI = process.env.CI; diff --git a/packages/eas-cli/src/commands/analytics.ts b/packages/eas-cli/src/commands/analytics.ts index 5975137389..e147acfbf8 100644 --- a/packages/eas-cli/src/commands/analytics.ts +++ b/packages/eas-cli/src/commands/analytics.ts @@ -4,7 +4,8 @@ import EasCommand from '../commandUtils/EasCommand'; import Log from '../log'; export default class AnalyticsView extends EasCommand { - static override description = 'display or change analytics settings'; + static override description = + 'display or change analytics settings (disabling analytics also disables experiments)'; static override args = { STATUS: Args.string({ diff --git a/packages/eas-cli/src/env.ts b/packages/eas-cli/src/env.ts index 42ff6eb234..55258d6a89 100644 --- a/packages/eas-cli/src/env.ts +++ b/packages/eas-cli/src/env.ts @@ -21,4 +21,10 @@ export default { * Comma separated list of feature gate keys of feature gates to evaluate override to false. */ featureGateDisable: process.env.EAS_FG_DISABLE, + /** + * Comma separated list of experiment overrides in the form `namespace:param=value`. + * Overrides apply in every scope, even when analytics is disabled. A namespace with any override + * returns defaults for its other params and logs no exposure. + */ + experimentOverride: process.env.EAS_EXPERIMENT_OVERRIDE, }; diff --git a/packages/eas-cli/src/experimentation/AnalyticsExposureLogger.ts b/packages/eas-cli/src/experimentation/AnalyticsExposureLogger.ts new file mode 100644 index 0000000000..84359f2f29 --- /dev/null +++ b/packages/eas-cli/src/experimentation/AnalyticsExposureLogger.ts @@ -0,0 +1,18 @@ +import type { ExposureData, ExposureLogger } from '@expo/experimentation'; + +import { Analytics, ExperimentationEvent } from '../analytics/AnalyticsManager'; + +export class AnalyticsExposureLogger implements ExposureLogger> { + constructor(private readonly analytics: Analytics) {} + + public logExposure(data: ExposureData>): void { + const { variant } = data.params; + this.analytics.logEvent(ExperimentationEvent.EXPERIMENT_VIEWED, { + experimentName: data.name, + // Analytics properties cannot be undefined. Fall back to all params when there is no variant. + variationName: variant !== undefined ? String(variant) : JSON.stringify(data.params), + // The unit is always a single value (actor ID, account ID, or device ID). + unit: String(data.unit[0]), + }); + } +} diff --git a/packages/eas-cli/src/experimentation/ExperimentNamespace.ts b/packages/eas-cli/src/experimentation/ExperimentNamespace.ts new file mode 100644 index 0000000000..7b68d85c17 --- /dev/null +++ b/packages/eas-cli/src/experimentation/ExperimentNamespace.ts @@ -0,0 +1,79 @@ +import type { Namespace } from '@expo/experimentation'; + +import { ExperimentOverrides } from './ExperimentOverrides'; +import Log from '../log'; + +export type ExperimentNamespace = { + getParam(paramName: string, defaultValue: T): T; + isOverridden: boolean; +}; + +export const FALLBACK_NAMESPACE: ExperimentNamespace = { + getParam(_paramName, defaultValue) { + return defaultValue; + }, + isOverridden: false, +}; + +/** Serves overrides without consulting the real namespace, so no exposure is logged. */ +export class OverrideFallbackNamespace implements ExperimentNamespace { + public readonly isOverridden = true; + + constructor( + private readonly namespaceName: string, + private readonly overrides: ExperimentOverrides + ) {} + + public getParam(paramName: string, defaultValue: T): T { + if (this.overrides.hasOverride(this.namespaceName, paramName)) { + return coerceOverrideValue( + this.overrides.getOverride(this.namespaceName, paramName), + defaultValue + ); + } + return defaultValue; + } +} + +/** Returns the default instead of throwing when the library rejects a param on first read. */ +export class SafeNamespace implements ExperimentNamespace { + public readonly isOverridden = false; + + constructor( + private readonly namespace: Namespace, + private readonly namespaceName: string + ) {} + + public getParam(paramName: string, defaultValue: T): T { + try { + return this.namespace.getParam(paramName, defaultValue); + } catch (error) { + Log.debug( + `[experimentation] Failed to read param "${paramName}" from namespace "${this.namespaceName}". Returning the default.`, + error + ); + return defaultValue; + } + } +} + +export function coerceOverrideValue(overrideValue: string | undefined, defaultValue: T): T { + if (overrideValue === undefined || overrideValue.trim() === '') { + return defaultValue; + } + if (typeof defaultValue === 'boolean') { + const normalized = overrideValue.toLowerCase(); + if (normalized === 'true') { + return true as T; + } + if (normalized === 'false') { + return false as T; + } + return defaultValue; + } + if (typeof defaultValue === 'number') { + const parsed = Number(overrideValue); + return (Number.isFinite(parsed) ? parsed : defaultValue) as T; + } + return overrideValue as T; +} diff --git a/packages/eas-cli/src/experimentation/ExperimentOverrides.ts b/packages/eas-cli/src/experimentation/ExperimentOverrides.ts new file mode 100644 index 0000000000..c9409c8d06 --- /dev/null +++ b/packages/eas-cli/src/experimentation/ExperimentOverrides.ts @@ -0,0 +1,73 @@ +import env from '../env'; + +export type ExperimentOverrideMap = ReadonlyMap>; + +/** + * Parses `EAS_EXPERIMENT_OVERRIDE`: comma-separated `namespace:param=value` entries. Values may + * contain `:` and `=` but not `,`. Overrides are keyed by namespace name and apply in every scope. + * + * @example EAS_EXPERIMENT_OVERRIDE="onboarding:variant=treatment,cli-hints:enabled=true" + */ +export function parseExperimentOverrides(rawValue: string | undefined): ExperimentOverrideMap { + const overrideMap = new Map>(); + if (!rawValue) { + return overrideMap; + } + + for (const entry of rawValue.split(',')) { + const override = entry.trim(); + if (!override) { + continue; + } + + const firstEqualsIndex = override.indexOf('='); + if (firstEqualsIndex === -1) { + continue; + } + const namespaceAndParam = override.slice(0, firstEqualsIndex); + const value = override.slice(firstEqualsIndex + 1).trim(); + + const firstColonIndex = namespaceAndParam.indexOf(':'); + if (firstColonIndex === -1) { + continue; + } + const namespace = namespaceAndParam.slice(0, firstColonIndex).trim(); + const param = namespaceAndParam.slice(firstColonIndex + 1).trim(); + if (!namespace || !param) { + continue; + } + + let params = overrideMap.get(namespace); + if (!params) { + params = new Map(); + overrideMap.set(namespace, params); + } + params.set(param, value); + } + + return overrideMap; +} + +export class ExperimentOverrides { + constructor(private readonly overrideMap: ExperimentOverrideMap) {} + + public static fromEnv(): ExperimentOverrides { + return new ExperimentOverrides(parseExperimentOverrides(env.experimentOverride)); + } + + public static empty(): ExperimentOverrides { + return new ExperimentOverrides(new Map()); + } + + public hasNamespaceOverride(namespace: string): boolean { + return this.overrideMap.has(namespace); + } + + public hasOverride(namespace: string, param: string): boolean { + return this.overrideMap.get(namespace)?.has(param) ?? false; + } + + public getOverride(namespace: string, param: string): string | undefined { + return this.overrideMap.get(namespace)?.get(param); + } +} diff --git a/packages/eas-cli/src/experimentation/ExperimentationClient.ts b/packages/eas-cli/src/experimentation/ExperimentationClient.ts new file mode 100644 index 0000000000..4773214efd --- /dev/null +++ b/packages/eas-cli/src/experimentation/ExperimentationClient.ts @@ -0,0 +1,175 @@ +import { + type ExperimentUnit, + type ExperimentationDefinition, + type LoggerConfig, + type Namespace, + NoOpExposureLogger, + parseExperimentationConfig, +} from '@expo/experimentation'; +import { createHash } from 'node:crypto'; + +import { AnalyticsExposureLogger } from './AnalyticsExposureLogger'; +import { + type ExperimentNamespace, + FALLBACK_NAMESPACE, + OverrideFallbackNamespace, + SafeNamespace, +} from './ExperimentNamespace'; +import { ExperimentOverrides } from './ExperimentOverrides'; +import { Analytics } from '../analytics/AnalyticsManager'; +import Log from '../log'; + +export type ExperimentationScope = 'user' | 'account' | 'device'; + +export type ExperimentationConfigs = Record; + +/** Commands pass the account unit on each `getAccountNamespace` call. */ +export type ExperimentationUnits = Record, string | null>; + +export const EMPTY_EXPERIMENTATION_DEFINITION: ExperimentationDefinition = { + experiments: [], + namespaces: [], +}; + +const EMPTY_CONFIGS: ExperimentationConfigs = { + user: EMPTY_EXPERIMENTATION_DEFINITION, + account: EMPTY_EXPERIMENTATION_DEFINITION, + device: EMPTY_EXPERIMENTATION_DEFINITION, +}; + +const NULL_UNITS: ExperimentationUnits = { user: null, device: null }; + +function sha1(input: string): string { + return createHash('sha1').update(input).digest('hex'); +} + +export function createLoggerConfig(analytics: Analytics): LoggerConfig { + return { + flavorMap: new Map([ + ['no-op', new NoOpExposureLogger()], + ['rudderstack', new AnalyticsExposureLogger(analytics)], + ]), + fallbackLoggerType: 'no-op', + }; +} + +const NO_OP_LOGGER_CONFIG: LoggerConfig = { + flavorMap: new Map([['no-op', new NoOpExposureLogger()]]), + fallbackLoggerType: 'no-op', +}; + +/** Cached per (unit, name): the library deduplicates exposures per Namespace instance. */ +class ScopedNamespaces { + private readonly factories: ReadonlyMap Namespace>; + private readonly instances = new Map>(); + + constructor( + definition: ExperimentationDefinition, + loggerConfig: LoggerConfig, + defaultUnit: string | null + ) { + this.factories = parseExperimentationConfig(definition, { sha1Fn: sha1 }, loggerConfig); + // Build every namespace now so an invalid server config throws here, in the fail-open path. + if (defaultUnit) { + for (const name of this.factories.keys()) { + this.getNamespace(name, defaultUnit); + } + } + } + + public getNamespace(name: string, unit: string): Namespace | null { + const factory = this.factories.get(name); + if (!factory) { + return null; + } + let byName = this.instances.get(unit); + if (!byName) { + byName = new Map(); + this.instances.set(unit, byName); + } + let namespace = byName.get(name); + if (!namespace) { + namespace = factory([unit]); + byName.set(name, namespace); + } + return namespace; + } +} + +/** + * Resolves experiment params for the current CLI run. Units: user is the actor ID, account is the + * ID the command passes to `getAccountNamespace`, device is the analytics device ID. + * User and account units are shared across surfaces. The device unit is specific to the CLI, so a + * device namespace must not be shared with another surface. + * The first `getParam` on a namespace logs an exposure, so call it only where the variant is used. + */ +export class ExperimentationClient { + private readonly scopes: Record; + + constructor( + configs: ExperimentationConfigs, + private readonly units: ExperimentationUnits, + private readonly overrides: ExperimentOverrides, + loggerConfig: LoggerConfig + ) { + this.scopes = { + user: new ScopedNamespaces(configs.user, loggerConfig, units.user), + account: new ScopedNamespaces(configs.account, loggerConfig, null), + device: new ScopedNamespaces(configs.device, loggerConfig, units.device), + }; + } + + /** Env overrides still apply, so QA can force a variant even with analytics disabled. */ + public static createDisabled( + overrides: ExperimentOverrides = ExperimentOverrides.fromEnv() + ): ExperimentationClient { + return new ExperimentationClient(EMPTY_CONFIGS, NULL_UNITS, overrides, NO_OP_LOGGER_CONFIG); + } + + public getUserNamespace(name: string): ExperimentNamespace { + return this.resolve('user', name, this.units.user); + } + + /** Takes the account the command already resolved, so no extra lookup or config read happens here. */ + public getAccountNamespace( + name: string, + { accountId }: { accountId: string } + ): ExperimentNamespace { + return this.resolve('account', name, accountId); + } + + public getDeviceNamespace(name: string): ExperimentNamespace { + return this.resolve('device', name, this.units.device); + } + + private resolve( + scope: ExperimentationScope, + name: string, + unit: string | null + ): ExperimentNamespace { + if (this.overrides.hasNamespaceOverride(name)) { + return new OverrideFallbackNamespace(name, this.overrides); + } + if (!unit) { + return FALLBACK_NAMESPACE; + } + + let namespace: Namespace | null; + try { + namespace = this.scopes[scope].getNamespace(name, unit); + } catch (error) { + Log.debug( + `[experimentation] Failed to build namespace "${name}" in the ${scope} config. Returning defaults.`, + error + ); + return FALLBACK_NAMESPACE; + } + if (!namespace) { + Log.debug( + `[experimentation] Namespace "${name}" not found in the ${scope} config. Returning defaults.` + ); + return FALLBACK_NAMESPACE; + } + return new SafeNamespace(namespace, name); + } +} diff --git a/packages/eas-cli/src/experimentation/LazyExperimentationClient.ts b/packages/eas-cli/src/experimentation/LazyExperimentationClient.ts new file mode 100644 index 0000000000..6806ff704c --- /dev/null +++ b/packages/eas-cli/src/experimentation/LazyExperimentationClient.ts @@ -0,0 +1,20 @@ +import { ExperimentationClient } from './ExperimentationClient'; + +/** + * Defers building the experimentation client, and the config fetch behind it, until a command + * first needs a variant. + */ +export class LazyExperimentationClient { + private pending: Promise | null = null; + + constructor(private readonly loadAsync: () => Promise) {} + + public static fromClient(client: ExperimentationClient): LazyExperimentationClient { + return new LazyExperimentationClient(async () => client); + } + + public async getClientAsync(): Promise { + this.pending ??= this.loadAsync(); + return await this.pending; + } +} diff --git a/packages/eas-cli/src/experimentation/__tests__/AnalyticsExposureLogger-test.ts b/packages/eas-cli/src/experimentation/__tests__/AnalyticsExposureLogger-test.ts new file mode 100644 index 0000000000..cdb43b7fc5 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/AnalyticsExposureLogger-test.ts @@ -0,0 +1,124 @@ +import RudderAnalytics from '@expo/rudder-sdk-node'; + +import UserSettings from '../../user/UserSettings'; +import { createAnalyticsAsync } from '../../analytics/AnalyticsManager'; +import { AnalyticsExposureLogger } from '../AnalyticsExposureLogger'; +import { createMockAnalytics } from './fixtures'; + +const mockIdentify = jest.fn(); +const mockTrack = jest.fn(); +const mockFlush = jest.fn(); + +jest.mock('@expo/rudder-sdk-node', () => + jest.fn().mockImplementation(() => ({ + identify: mockIdentify, + track: mockTrack, + flush: mockFlush, + })) +); +jest.mock('../../user/UserSettings', () => ({ + __esModule: true, + default: { + deleteKeyAsync: jest.fn(), + getAsync: jest.fn(), + setAsync: jest.fn(), + }, +})); +jest.mock('../../analytics/agent', () => ({ getAgentTelemetryContext: jest.fn() })); +jest.mock('../../analytics/sandbox', () => ({ getSandboxTelemetryContext: jest.fn() })); + +const originalDisableEasAnalytics = process.env.DISABLE_EAS_ANALYTICS; +const originalHttpsProxy = process.env.https_proxy; + +afterAll(() => { + if (originalDisableEasAnalytics === undefined) { + delete process.env.DISABLE_EAS_ANALYTICS; + } else { + process.env.DISABLE_EAS_ANALYTICS = originalDisableEasAnalytics; + } + if (originalHttpsProxy === undefined) { + delete process.env.https_proxy; + } else { + process.env.https_proxy = originalHttpsProxy; + } +}); + +const exposure = { + name: 'cli-test', + time: 1700000000, + salt: 'cli-full.cli-test.cli-test', +}; + +describe(AnalyticsExposureLogger, () => { + it('logs Experiment Viewed with experimentName, variationName, and unit', () => { + const analytics = createMockAnalytics(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: ['account-1'], + params: { variant: 'treatment' }, + }); + expect(analytics.logEvent).toHaveBeenCalledWith('Experiment Viewed', { + experimentName: 'cli-test', + variationName: 'treatment', + unit: 'account-1', + }); + }); + + it('never sends an undefined variationName', () => { + const analytics = createMockAnalytics(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: ['account-1'], + params: { enabled: true, count: 2 }, + }); + expect(analytics.logEvent).toHaveBeenCalledWith( + 'Experiment Viewed', + expect.objectContaining({ variationName: JSON.stringify({ enabled: true, count: 2 }) }) + ); + }); + + it('sends numeric units and variants as strings', () => { + const analytics = createMockAnalytics(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: [42], + params: { variant: 3 }, + }); + expect(analytics.logEvent).toHaveBeenCalledWith( + 'Experiment Viewed', + expect.objectContaining({ unit: '42', variationName: '3' }) + ); + }); + + it('reaches RudderStack with the CLI source and anonymous ID', async () => { + jest.mocked(RudderAnalytics).mockClear(); + mockTrack.mockClear(); + jest + .mocked(UserSettings.getAsync) + .mockImplementation(async (key, defaultValue) => + key === 'analyticsDeviceId' ? 'persistent-device-id' : defaultValue + ); + delete process.env.DISABLE_EAS_ANALYTICS; + delete process.env.https_proxy; + + const analytics = await createAnalyticsAsync(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: ['account-1'], + params: { variant: 'treatment' }, + }); + + expect(mockTrack).toHaveBeenCalledWith( + expect.objectContaining({ + event: 'Experiment Viewed', + anonymousId: 'persistent-device-id', + properties: expect.objectContaining({ + experimentName: 'cli-test', + variationName: 'treatment', + unit: 'account-1', + source: 'eas cli', + }), + }) + ); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/ExperimentNamespace-test.ts b/packages/eas-cli/src/experimentation/__tests__/ExperimentNamespace-test.ts new file mode 100644 index 0000000000..749ec23395 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/ExperimentNamespace-test.ts @@ -0,0 +1,74 @@ +import type { Namespace } from '@expo/experimentation'; + +import Log from '../../log'; +import { + FALLBACK_NAMESPACE, + OverrideFallbackNamespace, + SafeNamespace, + coerceOverrideValue, +} from '../ExperimentNamespace'; +import { ExperimentOverrides, parseExperimentOverrides } from '../ExperimentOverrides'; + +jest.mock('../../log'); + +describe(coerceOverrideValue, () => { + it('coerces booleans only from "true" and "false"', () => { + expect(coerceOverrideValue('false', true)).toBe(false); + expect(coerceOverrideValue('TRUE', false)).toBe(true); + expect(coerceOverrideValue('1', false)).toBe(false); + expect(coerceOverrideValue('yes', true)).toBe(true); + }); + + it('coerces finite numbers and falls back otherwise', () => { + expect(coerceOverrideValue('2', 1)).toBe(2); + expect(coerceOverrideValue('2.5', 1)).toBe(2.5); + expect(coerceOverrideValue('x', 1)).toBe(1); + expect(coerceOverrideValue('Infinity', 1)).toBe(1); + }); + + it('passes strings through and returns the default for empty input', () => { + expect(coerceOverrideValue('b', 'a')).toBe('b'); + expect(coerceOverrideValue('', 'a')).toBe('a'); + expect(coerceOverrideValue(' ', 'a')).toBe('a'); + expect(coerceOverrideValue(undefined, 'a')).toBe('a'); + }); +}); + +describe('FALLBACK_NAMESPACE', () => { + it('returns the default for every param', () => { + expect(FALLBACK_NAMESPACE.getParam('variant', 'control')).toBe('control'); + expect(FALLBACK_NAMESPACE.isOverridden).toBe(false); + }); +}); + +describe(OverrideFallbackNamespace, () => { + const overrides = new ExperimentOverrides(parseExperimentOverrides('ns:variant=treatment')); + const namespace = new OverrideFallbackNamespace('ns', overrides); + + it('returns the coerced override for overridden params', () => { + expect(namespace.getParam('variant', 'control')).toBe('treatment'); + expect(namespace.isOverridden).toBe(true); + }); + + it('returns the default for params without an override', () => { + expect(namespace.getParam('enabled', false)).toBe(false); + }); +}); + +describe(SafeNamespace, () => { + it('delegates to the wrapped namespace', () => { + const inner = { getParam: jest.fn().mockReturnValue('treatment') } as unknown as Namespace; + expect(new SafeNamespace(inner, 'ns').getParam('variant', 'control')).toBe('treatment'); + expect(inner.getParam).toHaveBeenCalledWith('variant', 'control'); + }); + + it('returns the default and logs at debug level when the wrapped namespace throws', () => { + const inner = { + getParam: jest.fn(() => { + throw new Error('bad operator'); + }), + } as unknown as Namespace; + expect(new SafeNamespace(inner, 'ns').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalled(); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/ExperimentOverrides-test.ts b/packages/eas-cli/src/experimentation/__tests__/ExperimentOverrides-test.ts new file mode 100644 index 0000000000..15e25d2f46 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/ExperimentOverrides-test.ts @@ -0,0 +1,72 @@ +import { ExperimentOverrides, parseExperimentOverrides } from '../ExperimentOverrides'; + +jest.mock('../../env', () => ({ + __esModule: true, + default: { experimentOverride: 'from-env:variant=treatment' }, +})); + +describe(parseExperimentOverrides, () => { + it('parses a single override', () => { + const map = parseExperimentOverrides('pricing:variant=treatment'); + expect(map.get('pricing')?.get('variant')).toBe('treatment'); + }); + + it('parses multiple overrides and multiple params per namespace', () => { + const map = parseExperimentOverrides('a:variant=x,b:limit=3,a:enabled=true'); + expect(map.get('a')?.get('variant')).toBe('x'); + expect(map.get('a')?.get('enabled')).toBe('true'); + expect(map.get('b')?.get('limit')).toBe('3'); + }); + + it('returns an empty map for undefined or empty input', () => { + expect(parseExperimentOverrides(undefined).size).toBe(0); + expect(parseExperimentOverrides('').size).toBe(0); + expect(parseExperimentOverrides(' , ').size).toBe(0); + }); + + it('skips entries without "=" or ":" and entries with an empty namespace or param', () => { + const map = parseExperimentOverrides( + 'bogus,ns:x,variant=1,:variant=1,ns:=1,ok:variant=control' + ); + expect([...map.keys()]).toEqual(['ok']); + }); + + it('keeps "=" and ":" inside values', () => { + const map = parseExperimentOverrides('ns:url=https://expo.dev/a=b'); + expect(map.get('ns')?.get('url')).toBe('https://expo.dev/a=b'); + }); + + it('trims whitespace and tolerates a trailing comma', () => { + const map = parseExperimentOverrides(' ns : variant = treatment , '); + expect(map.get('ns')?.get('variant')).toBe('treatment'); + expect(map.size).toBe(1); + }); + + it('lets later duplicates win', () => { + const map = parseExperimentOverrides('ns:variant=a,ns:variant=b'); + expect(map.get('ns')?.get('variant')).toBe('b'); + }); +}); + +describe(ExperimentOverrides, () => { + it('exposes namespace and param lookups', () => { + const overrides = new ExperimentOverrides(parseExperimentOverrides('ns:variant=treatment')); + expect(overrides.hasNamespaceOverride('ns')).toBe(true); + expect(overrides.hasNamespaceOverride('other')).toBe(false); + expect(overrides.hasOverride('ns', 'variant')).toBe(true); + expect(overrides.hasOverride('ns', 'enabled')).toBe(false); + expect(overrides.hasOverride('other', 'variant')).toBe(false); + expect(overrides.getOverride('ns', 'variant')).toBe('treatment'); + expect(overrides.getOverride('ns', 'enabled')).toBeUndefined(); + expect(overrides.getOverride('other', 'variant')).toBeUndefined(); + }); + + it('reads EAS_EXPERIMENT_OVERRIDE through env', () => { + const overrides = ExperimentOverrides.fromEnv(); + expect(overrides.getOverride('from-env', 'variant')).toBe('treatment'); + }); + + it('empty() has no overrides', () => { + expect(ExperimentOverrides.empty().hasNamespaceOverride('from-env')).toBe(false); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/ExperimentationClient-test.ts b/packages/eas-cli/src/experimentation/__tests__/ExperimentationClient-test.ts new file mode 100644 index 0000000000..59861712b4 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/ExperimentationClient-test.ts @@ -0,0 +1,376 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; + +import Log from '../../log'; +import { + EMPTY_EXPERIMENTATION_DEFINITION, + ExperimentationClient, + ExperimentationConfigs, + ExperimentationUnits, + createLoggerConfig, +} from '../ExperimentationClient'; +import { ExperimentOverrides, parseExperimentOverrides } from '../ExperimentOverrides'; +import { TEST_DEFINITION, createMockAnalytics, findUnitsByValue } from './fixtures'; +import { FALLBACK_NAMESPACE } from '../ExperimentNamespace'; + +jest.mock('../../log'); +jest.mock('../../env', () => ({ + __esModule: true, + default: { experimentOverride: 'cli-full:variant=from-env' }, +})); + +function createClient({ + configs = {}, + units = {}, + overrides = ExperimentOverrides.empty(), + analytics = createMockAnalytics(), +}: { + configs?: Partial; + units?: Partial; + overrides?: ExperimentOverrides; + analytics?: ReturnType; +} = {}): { client: ExperimentationClient; analytics: ReturnType } { + const client = new ExperimentationClient( + { + user: configs.user ?? EMPTY_EXPERIMENTATION_DEFINITION, + account: configs.account ?? EMPTY_EXPERIMENTATION_DEFINITION, + device: configs.device ?? EMPTY_EXPERIMENTATION_DEFINITION, + }, + { user: units.user ?? null, device: units.device ?? null }, + overrides, + createLoggerConfig(analytics) + ); + return { client, analytics }; +} + +function userClientForUnit(unit: string): ExperimentationClient { + return createClient({ configs: { user: TEST_DEFINITION }, units: { user: unit } }).client; +} + +const unitsByVariant = findUnitsByValue( + unit => userClientForUnit(unit).getUserNamespace('cli-full'), + 'variant', + 'control', + ['control', 'treatment'] +); +const controlUnit = unitsByVariant.get('control')!; +const treatmentUnit = unitsByVariant.get('treatment')!; + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe(findUnitsByValue, () => { + it('throws when no unit produces an expected value', () => { + expect(() => findUnitsByValue(() => FALLBACK_NAMESPACE, 'variant', 'x', ['never'])).toThrow( + 'Could not find units for all of: never' + ); + }); +}); + +describe(ExperimentationClient, () => { + describe('assignment', () => { + it('gives the same unit the same variant across clients', () => { + const first = userClientForUnit(treatmentUnit).getUserNamespace('cli-full'); + const second = userClientForUnit(treatmentUnit).getUserNamespace('cli-full'); + expect(first.getParam('variant', 'control')).toBe('treatment'); + expect(second.getParam('variant', 'control')).toBe('treatment'); + expect( + userClientForUnit(controlUnit).getUserNamespace('cli-full').getParam('variant', 'x') + ).toBe('control'); + }); + + it('returns the default and logs at debug level for an unknown namespace', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + }); + expect(client.getUserNamespace('missing').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith(expect.stringContaining('"missing"')); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('returns the default and logs no exposure when the scope has no unit', () => { + const { client, analytics } = createClient({ configs: { user: TEST_DEFINITION } }); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + expect(Log.debug).not.toHaveBeenCalled(); + }); + + it('treats an empty string unit like a missing unit', () => { + const { client, analytics } = createClient({ configs: { account: TEST_DEFINITION } }); + expect( + client.getAccountNamespace('cli-full', { accountId: '' }).getParam('variant', 'control') + ).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('buckets account experiments on the account ID the command passes', () => { + const { client, analytics } = createClient({ configs: { account: TEST_DEFINITION } }); + expect( + client.getAccountNamespace('cli-full', { accountId: controlUnit }).getParam('variant', 'x') + ).toBe('control'); + expect( + client + .getAccountNamespace('cli-full', { accountId: treatmentUnit }) + .getParam('variant', 'x') + ).toBe('treatment'); + expect(analytics.logEvent).toHaveBeenCalledTimes(2); + }); + + it('resolves the same namespace name independently per scope', () => { + const scoped = (choice: string): ExperimentationDefinition => ({ + experiments: [ + { + experimentName: 'shared', + paramDefinitions: { variant: { type: 'uniform_choice', choices: [choice] } }, + exposureLoggerType: 'no-op', + }, + ], + namespaces: [ + { + name: 'shared', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'shared', numSegments: 10 }, + ], + }, + ], + }); + const { client } = createClient({ + configs: { + user: scoped('from-user'), + account: scoped('from-account'), + device: scoped('from-device'), + }, + units: { user: 'u', device: 'd' }, + }); + expect(client.getUserNamespace('shared').getParam('variant', 'x')).toBe('from-user'); + expect( + client.getAccountNamespace('shared', { accountId: 'a' }).getParam('variant', 'x') + ).toBe('from-account'); + expect(client.getDeviceNamespace('shared').getParam('variant', 'x')).toBe('from-device'); + }); + + it('returns the default and logs no exposure for a unit outside the allocated segments', () => { + const unallocated = findUnitsByValue( + unit => userClientForUnit(unit).getUserNamespace('cli-half'), + 'variant', + 'not-allocated', + ['not-allocated'] + ).get('not-allocated')!; + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: unallocated }, + }); + expect(client.getUserNamespace('cli-half').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + }); + + describe('exposure logging', () => { + it('logs one exposure per namespace per client', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + }); + const namespace = client.getUserNamespace('cli-full'); + namespace.getParam('variant', 'control'); + namespace.getParam('enabled', false); + client.getUserNamespace('cli-full').getParam('variant', 'control'); + + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + expect(analytics.logEvent).toHaveBeenCalledWith('Experiment Viewed', { + experimentName: 'cli-test', + variationName: 'treatment', + unit: treatmentUnit, + }); + }); + + it('logs again for a second client with the same unit, so clients must be cached per command', () => { + const analytics = createMockAnalytics(); + createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + analytics, + }) + .client.getUserNamespace('cli-full') + .getParam('variant', 'control'); + createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + analytics, + }) + .client.getUserNamespace('cli-full') + .getParam('variant', 'control'); + expect(analytics.logEvent).toHaveBeenCalledTimes(2); + }); + + it('logs nothing for experiments that use the no-op logger', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: 'unit-1' }, + }); + expect(['a', 'b']).toContain( + client.getUserNamespace('cli-quiet-ns').getParam('variant', 'x') + ); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('falls back to the no-op logger for an unknown exposureLoggerType', () => { + const definition: ExperimentationDefinition = { + experiments: [ + { + experimentName: 'odd', + paramDefinitions: { variant: { type: 'uniform_choice', choices: ['only'] } }, + exposureLoggerType: 'something-else', + }, + ], + namespaces: [ + { + name: 'odd', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'odd', numSegments: 10 }, + ], + }, + ], + }; + const { client, analytics } = createClient({ + configs: { user: definition }, + units: { user: 'u' }, + }); + expect(client.getUserNamespace('odd').getParam('variant', 'x')).toBe('only'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + }); + + describe('overrides', () => { + it('returns the override instead of the assigned variant and logs no exposure', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: controlUnit }, + overrides: new ExperimentOverrides(parseExperimentOverrides('cli-full:variant=treatment')), + }); + const namespace = client.getUserNamespace('cli-full'); + expect(namespace.isOverridden).toBe(true); + expect(namespace.getParam('variant', 'control')).toBe('treatment'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('returns the default for other params of an overridden namespace without logging an exposure', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: controlUnit }, + overrides: new ExperimentOverrides(parseExperimentOverrides('cli-full:variant=treatment')), + }); + expect(client.getUserNamespace('cli-full').getParam('enabled', false)).toBe(false); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('applies overrides to namespaces that are not in the server config', () => { + const { client } = createClient({ + overrides: new ExperimentOverrides( + parseExperimentOverrides('not-deployed:variant=treatment') + ), + }); + expect(client.getUserNamespace('not-deployed').getParam('variant', 'control')).toBe( + 'treatment' + ); + }); + + it('applies the same override in every scope', () => { + const { client } = createClient({ + overrides: new ExperimentOverrides(parseExperimentOverrides('ns:enabled=true')), + }); + expect(client.getUserNamespace('ns').getParam('enabled', false)).toBe(true); + expect(client.getAccountNamespace('ns', { accountId: 'a' }).getParam('enabled', false)).toBe( + true + ); + expect(client.getDeviceNamespace('ns').getParam('enabled', false)).toBe(true); + }); + }); + + describe('invalid server config', () => { + it('returns the default without throwing when a param uses an unknown operator', () => { + const definition = { + experiments: [ + { + experimentName: 'broken', + paramDefinitions: { variant: { type: 'not-an-operator' } }, + exposureLoggerType: 'rudderstack', + }, + ], + namespaces: [ + { + name: 'broken', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'broken', numSegments: 10 }, + ], + }, + ], + } as unknown as ExperimentationDefinition; + const { client } = createClient({ configs: { user: definition }, units: { user: 'u' } }); + expect(client.getUserNamespace('broken').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalled(); + }); + + const overAllocated: ExperimentationDefinition = { + experiments: TEST_DEFINITION.experiments, + namespaces: [ + { + name: 'too-many', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'cli-test', numSegments: 20 }, + ], + }, + ], + }; + + it('throws during construction when a namespace over-allocates segments', () => { + expect(() => + createClient({ configs: { user: overAllocated }, units: { user: 'u' } }) + ).toThrow(/requested 20 segments/); + }); + + it('returns the default when a namespace fails to build for an account ID', () => { + // Account namespaces have no default unit, so the failure surfaces on lookup, not construction. + const { client, analytics } = createClient({ configs: { account: overAllocated } }); + expect( + client.getAccountNamespace('too-many', { accountId: 'a' }).getParam('variant', 'control') + ).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith( + expect.stringContaining('Failed to build namespace'), + expect.anything() + ); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + }); + + describe('createDisabled', () => { + it('returns defaults for every scope and logs no exposure', () => { + const client = ExperimentationClient.createDisabled(ExperimentOverrides.empty()); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).not.toHaveBeenCalled(); + // The account scope always has a unit, so the empty config reports the namespace as missing. + expect( + client.getAccountNamespace('cli-full', { accountId: 'a' }).getParam('variant', 'control') + ).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith(expect.stringContaining('not found')); + }); + + it('reads overrides from the environment by default', () => { + const client = ExperimentationClient.createDisabled(); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('from-env'); + }); + + it('still honors overrides', () => { + const client = ExperimentationClient.createDisabled( + new ExperimentOverrides(parseExperimentOverrides('cli-full:variant=treatment')) + ); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('treatment'); + }); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/fixtures.ts b/packages/eas-cli/src/experimentation/__tests__/fixtures.ts new file mode 100644 index 0000000000..4aeb03ed14 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/fixtures.ts @@ -0,0 +1,79 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; + +import { Analytics } from '../../analytics/AnalyticsManager'; +import { ExperimentNamespace } from '../ExperimentNamespace'; + +/** + * - `cli-full`: every unit is in the `cli-test` experiment (50% control, 50% treatment). + * - `cli-half`: half of the units are in `cli-test`, the rest get defaults. + * - `cli-quiet-ns`: experiment with the `no-op` exposure logger. + */ +export const TEST_DEFINITION: ExperimentationDefinition = { + experiments: [ + { + experimentName: 'cli-test', + paramDefinitions: { + variant: { type: 'uniform_choice', choices: ['control', 'treatment'] }, + enabled: { type: 'bernoulli', p: 0.5 }, + }, + exposureLoggerType: 'rudderstack', + }, + { + experimentName: 'cli-quiet', + paramDefinitions: { + variant: { type: 'uniform_choice', choices: ['a', 'b'] }, + }, + exposureLoggerType: 'no-op', + }, + ], + namespaces: [ + { + name: 'cli-full', + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'cli-test full', experimentName: 'cli-test', numSegments: 1000 }, + ], + }, + { + name: 'cli-half', + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'cli-test half', experimentName: 'cli-test', numSegments: 500 }, + ], + }, + { + name: 'cli-quiet-ns', + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'cli-quiet full', experimentName: 'cli-quiet', numSegments: 1000 }, + ], + }, + ], +}; + +export function createMockAnalytics(): Analytics & { logEvent: jest.Mock } { + return { logEvent: jest.fn() }; +} + +/** + * Finds one unit per expected value by probing the real library, instead of hard-coding hashes. + */ +export function findUnitsByValue( + getNamespaceForUnit: (unit: string) => ExperimentNamespace, + paramName: string, + defaultValue: T, + expectedValues: T[] +): Map { + const found = new Map(); + for (let i = 0; i < 1000 && found.size < expectedValues.length; i++) { + const unit = `unit-${i}`; + const value = getNamespaceForUnit(unit).getParam(paramName, defaultValue); + if (expectedValues.includes(value) && !found.has(value)) { + found.set(value, unit); + } + } + if (found.size !== expectedValues.length) { + throw new Error(`Could not find units for all of: ${expectedValues.join(', ')}`); + } + return found; +} diff --git a/packages/eas-cli/src/graphql/generated.ts b/packages/eas-cli/src/graphql/generated.ts index 5d74aa90fa..ff340be4f1 100644 --- a/packages/eas-cli/src/graphql/generated.ts +++ b/packages/eas-cli/src/graphql/generated.ts @@ -16098,6 +16098,11 @@ export type EnvironmentVariablesSharedWithSensitiveQueryVariables = Exact<{ export type EnvironmentVariablesSharedWithSensitiveQuery = { __typename?: 'RootQuery', app: { __typename?: 'AppQuery', byId: { __typename?: 'App', id: string, ownerAccount: { __typename?: 'Account', id: string, environmentVariablesIncludingSensitive: Array<{ __typename?: 'EnvironmentVariableWithSecret', id: string, name: string, value?: string | null, environments?: Array | null, createdAt: any, updatedAt: any, scope: EnvironmentVariableScope, visibility: EnvironmentVariableVisibility, type: EnvironmentSecretType, valueWithFileContent?: string | null }> } } } }; +export type ExperimentationConfigsQueryVariables = Exact<{ [key: string]: never; }>; + + +export type ExperimentationConfigsQuery = { __typename?: 'RootQuery', experimentation: { __typename?: 'ExperimentationQuery', userConfig: any, accountConfig: any, deviceConfig: any } }; + export type FingerprintsByAppIdQueryVariables = Exact<{ appId: Scalars['String']['input']; after?: InputMaybe; diff --git a/packages/eas-cli/src/graphql/queries/ExperimentationConfigQuery.ts b/packages/eas-cli/src/graphql/queries/ExperimentationConfigQuery.ts new file mode 100644 index 0000000000..4bfabb1993 --- /dev/null +++ b/packages/eas-cli/src/graphql/queries/ExperimentationConfigQuery.ts @@ -0,0 +1,66 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; +import { OperationResult } from '@urql/core'; +import gql from 'graphql-tag'; + +import { ExpoGraphqlClient } from '../../commandUtils/context/contextUtils/createGraphqlClient'; +import { EMPTY_EXPERIMENTATION_DEFINITION } from '../../experimentation/ExperimentationClient'; +import { ExperimentationConfigsQuery, ExperimentationConfigsQueryVariables } from '../generated'; + +export type ExperimentationConfigsResult = { + userConfig: ExperimentationDefinition; + accountConfig: ExperimentationDefinition; + deviceConfig: ExperimentationDefinition; +}; + +/** Like `withErrorHandlingAsync`, but never logs. Callers fail open. */ +async function quietQueryAsync(promise: Promise>): Promise { + const { data, error } = await promise; + if (error) { + throw error; + } + if (!data) { + throw new Error('Returned query result data is null!'); + } + return data; +} + +/** The server sends an untyped JSONObject. Anything but the expected shape means no experiments. */ +function toExperimentationDefinition(value: unknown): ExperimentationDefinition { + if ( + value && + typeof value === 'object' && + Array.isArray((value as { experiments?: unknown }).experiments) && + Array.isArray((value as { namespaces?: unknown }).namespaces) + ) { + return value as ExperimentationDefinition; + } + return EMPTY_EXPERIMENTATION_DEFINITION; +} + +export const ExperimentationConfigQuery = { + async getConfigsAsync(graphqlClient: ExpoGraphqlClient): Promise { + const data = await quietQueryAsync( + graphqlClient + .query( + gql` + query ExperimentationConfigs { + experimentation { + userConfig + accountConfig + deviceConfig + } + } + `, + {}, + // A failing network must not delay the command. + { additionalTypenames: ['ExperimentationQuery'], noRetry: true } + ) + .toPromise() + ); + return { + userConfig: toExperimentationDefinition(data.experimentation.userConfig), + accountConfig: toExperimentationDefinition(data.experimentation.accountConfig), + deviceConfig: toExperimentationDefinition(data.experimentation.deviceConfig), + }; + }, +}; diff --git a/packages/eas-cli/src/graphql/queries/UserQuery.ts b/packages/eas-cli/src/graphql/queries/UserQuery.ts index 99c94a90d2..2048e1bce5 100644 --- a/packages/eas-cli/src/graphql/queries/UserQuery.ts +++ b/packages/eas-cli/src/graphql/queries/UserQuery.ts @@ -12,7 +12,10 @@ import { AccountFragmentNode } from '../types/Account'; import { MeActorFragmentNode } from '../types/Actor'; export const UserQuery = { - async currentUserAsync(graphqlClient: ExpoGraphqlClient): Promise { + async currentUserAsync( + graphqlClient: ExpoGraphqlClient, + { noRetry = false }: { noRetry?: boolean } = {} + ): Promise { const data = await withErrorHandlingAsync( graphqlClient .query( @@ -29,6 +32,7 @@ export const UserQuery = { {}, { additionalTypenames: ['User', 'SSOUser'], + noRetry, } ) .toPromise() diff --git a/packages/eas-cli/src/graphql/queries/__tests__/ExperimentationConfigQuery-test.ts b/packages/eas-cli/src/graphql/queries/__tests__/ExperimentationConfigQuery-test.ts new file mode 100644 index 0000000000..30c4b292b6 --- /dev/null +++ b/packages/eas-cli/src/graphql/queries/__tests__/ExperimentationConfigQuery-test.ts @@ -0,0 +1,89 @@ +import { CombinedError } from '@urql/core'; +import { print } from 'graphql'; + +import { ExpoGraphqlClient } from '../../../commandUtils/context/contextUtils/createGraphqlClient'; +import Log from '../../../log'; +import { ExperimentationConfigQuery } from '../ExperimentationConfigQuery'; + +jest.mock('../../../log'); + +const definition = { + experiments: [{ experimentName: 'x', paramDefinitions: {}, exposureLoggerType: 'rudderstack' }], + namespaces: [], +}; + +function makeGraphqlClient(result: { data?: unknown; error?: CombinedError }): ExpoGraphqlClient & { + query: jest.Mock; +} { + return { + query: jest.fn().mockReturnValue({ toPromise: jest.fn().mockResolvedValue(result) }), + } as unknown as ExpoGraphqlClient & { query: jest.Mock }; +} + +describe('ExperimentationConfigQuery.getConfigsAsync', () => { + it('returns the three configs', async () => { + const graphqlClient = makeGraphqlClient({ + data: { + experimentation: { + userConfig: definition, + accountConfig: definition, + deviceConfig: definition, + }, + }, + }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).resolves.toEqual({ + userConfig: definition, + accountConfig: definition, + deviceConfig: definition, + }); + }); + + it('replaces malformed configs with an empty definition', async () => { + const graphqlClient = makeGraphqlClient({ + data: { + experimentation: { + userConfig: { experiments: 'nope', namespaces: [] }, + accountConfig: null, + deviceConfig: definition, + }, + }, + }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).resolves.toEqual({ + userConfig: { experiments: [], namespaces: [] }, + accountConfig: { experiments: [], namespaces: [] }, + deviceConfig: definition, + }); + }); + + it('disables retries and selects only the config fields', async () => { + const graphqlClient = makeGraphqlClient({ + data: { experimentation: { userConfig: {}, accountConfig: {}, deviceConfig: {} } }, + }); + await ExperimentationConfigQuery.getConfigsAsync(graphqlClient); + + const [document, variables, context] = graphqlClient.query.mock.calls[0]; + const printed = print(document); + expect(printed).toContain('experimentation'); + expect(printed).toContain('userConfig'); + expect(printed).toContain('accountConfig'); + expect(printed).toContain('deviceConfig'); + expect(printed).not.toContain('deviceExperimentationUnit'); + expect(variables).toEqual({}); + expect(context).toEqual({ additionalTypenames: ['ExperimentationQuery'], noRetry: true }); + }); + + it('rejects on a GraphQL error without logging', async () => { + const error = new CombinedError({ networkError: new Error('offline') }); + const graphqlClient = makeGraphqlClient({ error }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).rejects.toBe(error); + expect(Log.error).not.toHaveBeenCalled(); + expect(Log.warn).not.toHaveBeenCalled(); + }); + + it('rejects when the result has no data', async () => { + const graphqlClient = makeGraphqlClient({ data: null }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).rejects.toThrow( + 'Returned query result data is null!' + ); + }); +}); diff --git a/packages/eas-cli/src/graphql/queries/__tests__/UserQuery-test.ts b/packages/eas-cli/src/graphql/queries/__tests__/UserQuery-test.ts new file mode 100644 index 0000000000..74c5f20af4 --- /dev/null +++ b/packages/eas-cli/src/graphql/queries/__tests__/UserQuery-test.ts @@ -0,0 +1,74 @@ +import http from 'http'; +import { AddressInfo } from 'net'; + +import { createGraphqlClient } from '../../../commandUtils/context/contextUtils/createGraphqlClient'; +import { UserQuery } from '../UserQuery'; + +jest.mock('../../../api', () => ({ + getExpoApiBaseUrl: () => process.env.USER_QUERY_TEST_URL, +})); + +describe(UserQuery.currentUserAsync, () => { + describe('against a slow server', () => { + const server = http.createServer(); + const serverDelayMs = 2000; + const requestTimeoutMs = 100; + let requestCount = 0; + // Aborted requests leave their response timers pending; clear them so the worker can exit. + const responseTimers: NodeJS.Timeout[] = []; + + beforeAll(async () => { + server.on('request', (_req, res) => { + requestCount += 1; + responseTimers.push( + setTimeout(() => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ data: { meActor: null } })); + }, serverDelayMs) + ); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + process.env.USER_QUERY_TEST_URL = `http://127.0.0.1:${port}`; + }); + + beforeEach(() => { + requestCount = 0; + }); + + afterAll(async () => { + delete process.env.USER_QUERY_TEST_URL; + for (const timer of responseTimers) { + clearTimeout(timer); + } + server.closeAllConnections(); + await new Promise(resolve => { + server.close(() => { + resolve(); + }); + }); + }); + + it('fails after one timed-out request when retries are disabled', async () => { + const client = createGraphqlClient( + { accessToken: 'token', sessionSecret: null }, + { requestTimeoutMs } + ); + await expect(UserQuery.currentUserAsync(client, { noRetry: true })).rejects.toThrow( + `Request timed out after ${requestTimeoutMs} ms` + ); + expect(requestCount).toBe(1); + }); + + it('retries a timed-out request by default', async () => { + const client = createGraphqlClient( + { accessToken: 'token', sessionSecret: null }, + { requestTimeoutMs } + ); + await expect(UserQuery.currentUserAsync(client)).rejects.toThrow( + `Request timed out after ${requestTimeoutMs} ms` + ); + expect(requestCount).toBe(2); + }, 15000); + }); +}); diff --git a/packages/eas-cli/src/user/SessionManager.ts b/packages/eas-cli/src/user/SessionManager.ts index 43ff17e53d..d59ea629eb 100644 --- a/packages/eas-cli/src/user/SessionManager.ts +++ b/packages/eas-cli/src/user/SessionManager.ts @@ -87,13 +87,23 @@ export default class SessionManager { await this.setSessionAsync(undefined); } - public async getUserAsync(): Promise { + /** + * @param requestTimeoutMs caps the actor lookup when no actor is cached yet. Callers that must + * not delay a command past a deadline pass it; the default has no timeout. A capped lookup + * also skips the retry exchange, since a retry would double the wait. + */ + public async getUserAsync({ requestTimeoutMs }: { requestTimeoutMs?: number } = {}): Promise< + Actor | undefined + > { if (!this.currentActor && (this.getAccessToken() || this.getSessionSecret())) { const authenticationInfo = { accessToken: this.getAccessToken(), sessionSecret: this.getSessionSecret(), }; - const actor = await UserQuery.currentUserAsync(createGraphqlClient(authenticationInfo)); + const actor = await UserQuery.currentUserAsync( + createGraphqlClient(authenticationInfo, { requestTimeoutMs }), + { noRetry: requestTimeoutMs !== undefined } + ); this.currentActor = actor ?? undefined; if (actor) { this.analytics.setActor(actor); diff --git a/packages/eas-cli/src/user/__tests__/SessionManager-test.ts b/packages/eas-cli/src/user/__tests__/SessionManager-test.ts index 5840df0f79..d4d92703ea 100644 --- a/packages/eas-cli/src/user/__tests__/SessionManager-test.ts +++ b/packages/eas-cli/src/user/__tests__/SessionManager-test.ts @@ -5,6 +5,8 @@ import path from 'path'; import { ApiV2Error } from '../../ApiV2Error'; import { AnalyticsWithOrchestration } from '../../analytics/AnalyticsManager'; import { ApiV2Client } from '../../api'; +import { createGraphqlClient } from '../../commandUtils/context/contextUtils/createGraphqlClient'; +import { UserQuery } from '../../graphql/queries/UserQuery'; import Log from '../../log'; import { promptAsync } from '../../prompts'; import { getStateJsonPath } from '../../utils/paths'; @@ -17,16 +19,17 @@ jest.mock('../../log'); jest.mock('fs'); jest.mock('../../graphql/queries/UserQuery', () => ({ UserQuery: { - currentUserAsync: async () => ({ + currentUserAsync: jest.fn(async () => ({ __typename: 'User', username: 'USERNAME', id: 'USER_ID', - }), + })), }, })); jest.mock('../fetchSessionSecretAndUser'); jest.mock('../fetchSessionSecretAndUserFromBrowserAuthFlow'); jest.mock('../../api'); +jest.mock('../../commandUtils/context/contextUtils/createGraphqlClient'); const authStub: any = { sessionSecret: 'SESSION_SECRET', @@ -41,6 +44,7 @@ const analytics: AnalyticsWithOrchestration = { logEvent: jest.fn((): void => {}), setActor: jest.fn((): void => {}), flushAsync: jest.fn(async (): Promise => {}), + getDeviceId: jest.fn((): string | null => null), }; beforeEach(() => { @@ -131,6 +135,36 @@ describe(SessionManager, () => { }); expect(await sessionManager.getUserAsync()).toMatchObject({ __typename: 'User' }); }); + + it('creates the GraphQL client without a timeout and keeps retries by default', async () => { + process.env.EXPO_TOKEN = 'accesstoken'; + const sessionManager = new SessionManager(analytics); + await sessionManager.getUserAsync(); + expect(createGraphqlClient).toHaveBeenCalledWith( + { accessToken: 'accesstoken', sessionSecret: null }, + { requestTimeoutMs: undefined } + ); + expect(jest.mocked(UserQuery.currentUserAsync).mock.calls[0][1]).toEqual({ noRetry: false }); + }); + + it('forwards the request timeout to the GraphQL client and disables retries', async () => { + process.env.EXPO_TOKEN = 'accesstoken'; + const sessionManager = new SessionManager(analytics); + await sessionManager.getUserAsync({ requestTimeoutMs: 1234 }); + expect(createGraphqlClient).toHaveBeenCalledWith( + { accessToken: 'accesstoken', sessionSecret: null }, + { requestTimeoutMs: 1234 } + ); + expect(jest.mocked(UserQuery.currentUserAsync).mock.calls[0][1]).toEqual({ noRetry: true }); + }); + + it('skips the request once an actor is cached', async () => { + process.env.EXPO_TOKEN = 'accesstoken'; + const sessionManager = new SessionManager(analytics); + await sessionManager.getUserAsync(); + await sessionManager.getUserAsync({ requestTimeoutMs: 1234 }); + expect(createGraphqlClient).toHaveBeenCalledTimes(1); + }); }); describe('loginAsync', () => { diff --git a/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts b/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts index 1cf4370fc2..361a0eab0e 100644 --- a/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts +++ b/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts @@ -18,7 +18,12 @@ const now = Date.parse('2026-09-29T12:00:00Z'); let directory: string; function manager(): SessionManager { - return new SessionManager({ logEvent: jest.fn(), setActor: jest.fn(), flushAsync: jest.fn() }); + return new SessionManager({ + logEvent: jest.fn(), + setActor: jest.fn(), + flushAsync: jest.fn(), + getDeviceId: jest.fn(), + }); } async function startAsync(): Promise { diff --git a/yarn.lock b/yarn.lock index 654637da7d..3a45fdfb4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2854,6 +2854,15 @@ __metadata: languageName: node linkType: hard +"@expo/experimentation@npm:0.0.5": + version: 0.0.5 + resolution: "@expo/experimentation@npm:0.0.5" + dependencies: + nullthrows: "npm:^1.1.1" + checksum: 10c0/bc054e1bcc0583f2a09e8b56d2f95bb62cf2f4eaf78edac4eb709c73cfd1fdb93626b948f4184f6dc8096a44b6c12c97109a84622bb56620484b4e61856007fd + languageName: node + linkType: hard + "@expo/image-utils@npm:^0.7.6": version: 0.7.6 resolution: "@expo/image-utils@npm:0.7.6" @@ -10736,6 +10745,7 @@ __metadata: "@expo/eas-build-job": "npm:25.0.0" "@expo/eas-json": "npm:25.0.0" "@expo/env": "npm:^1.0.0" + "@expo/experimentation": "npm:0.0.5" "@expo/json-file": "npm:8.3.3" "@expo/logger": "npm:25.0.0" "@expo/multipart-body-parser": "npm:2.0.0"