Sample size: 2 × 2 designs, A/B tests and repeated trials

src/content/guideIndex.ts

@@ -83,7 +83,7 @@ export const PAGE_ENTRIES: { path: string; title: string; what: string; keywords
     path: '/sample-size',
     title: 'How many participants?',
     what: 'Work out the sample size for your study, step by step.',
-    keywords: 'sample size how many participants people power analysis gpower g*power n',
+    keywords: 'sample size how many participants people power analysis gpower g*power n ab test a/b 2x2 factorial',
   },
   {
     path: '/workspace',

src/pages/SampleSize.test.tsx

@@ -192,6 +192,49 @@ describe('sample size page', () => {
     expect(document.querySelector('.ss-meaning')!.textContent).toContain('f = 0.16');
   });
 
+  test('a before-and-after study with a control group plans for the interaction in a 2 × 2 mixed design', async () => {
+    const { pick, next, fill } = setup();
+    await pick(/^Two factors at once/);
+    await pick(/How many people do I need/);
+    await pick(/One factor between people, one within/);
+    await pick(/Whether one factor's effect depends on the other/);
+    await pick(/^r = \.50/);
+    await pick(/I can guess the scores/);
+    await fill(/Effect of one factor at the first level/, '5');
+    await fill(/The same effect at the other level/, '0');
+    await fill(/Standard deviation of the scores within one cell/, '10');
+    // (5 − 0) / 10 = 0.5 SD, f = 0.5 / √(8 × (1 − .5)) = 0.25.
+    expect(document.querySelector('.ss-meaning')!.textContent).toContain('f = 0.25');
+    await next();
+    await pick(/^80%/);
+    await pick(/α = \.05/);
+    expect(question()).toBe('How many people will you lose?');
+    await pick(/^None/);
+    // λ = N·f² with df2 = N − 2: 128 people, 64 per group.
+    expect(report()).toContain('128 participants (64 per group, each measured in both conditions) are needed to detect an effect of f = 0.25 (with a correlation of r = .50 between conditions)');
+    expect(report()).toContain('the F test for the interaction in a 2 × 2 mixed ANOVA');
+    expect(trail()).toContain('One factor between, one within');
+  });
+
+  test('2 × 2 in plain words: interactions are described by their shape, and between designs skip the correlation', async () => {
+    const { pick } = setup();
+    await pick(/^Two factors at once/);
+    await pick(/How many people do I need/);
+    await pick(/Each person is in one combination only/);
+    await pick(/Whether one factor's effect depends on the other/);
+    expect(question()).toBe('How big is the effect you want to be able to find?');
+    await pick(/Choose a size in plain words/);
+    const disappears = screen.getByRole('button', { name: /The effect disappears \(f = 0.13\)/ });
+    expect(disappears.textContent).toContain('Recommended');
+    // f = 0.5 / 4 = 0.125: λ = N/64 reaches 80% power at 508 people (127 per cell), as in R.
+    expect(disappears.textContent).toContain('Needs about 508 people');
+    await pick(/The effect disappears/);
+    await pick(/^80%/);
+    await pick(/α = \.05/);
+    await pick(/^None/);
+    expect(report()).toContain('508 participants (127 in each of the four cells)');
+  });
+
   test("the browser's Back button goes back one question, keeping the answers", async () => {
     const { user, pick } = setup();
     await pick(/^Compare two separate groups/);

src/pages/sampleSizeDesigns.ts

@@ -1,4 +1,4 @@
-import { fFromEta2, type DesignId } from '../stats/power';
+import { factorialF, fFromEta2, type DesignId, type Layout, type Which } from '../stats/power';
 
 /** Everything the sample size page says about one design. */
 export type Design = {
@@ -23,7 +23,7 @@ export const DESIGNS: Design[] = [
   {
     id: 'two-groups',
     title: 'Compare two separate groups',
-    example: 'Do remote and office workers differ in wellbeing?',
+    example: 'Do remote and office workers differ in wellbeing? Or an A/B test on a score, such as time on a page.',
     test: 'independent-samples t-test',
     lessonId: '12-1',
     symbol: 'd',
@@ -62,6 +62,20 @@ export const DESIGNS: Design[] = [
     assumes:
       'Every person does every condition, scores are roughly normal, and sphericity holds: the differences between each pair of conditions vary about equally. This is the formula G*Power uses for "ANOVA: repeated measures, within factors" with ε = 1. When sphericity fails, the Greenhouse-Geisser correction costs power, so plan for more people. The same test is the F test for condition in a mixed model with a random intercept per person (Lesson 14-4). The plan is for the overall F test; follow-up comparisons between pairs of conditions have their own, often lower, power.',
   },
+  {
+    id: 'factorial',
+    title: 'Two factors at once, like a 2 × 2 design',
+    example: 'Does a training raise wellbeing more than a waiting list, measured before and after? Or: do photo and text ads work differently for young and old viewers?',
+    test: '2 × 2 ANOVA',
+    lessonId: '13-2',
+    symbol: 'f',
+    effectName: "Cohen's f",
+    effectMeaning: 'the size of one effect relative to the variation left after the other effects.',
+    benchmarks: [0.1, 0.25, 0.4],
+    defaultEffect: 0.25,
+    assumes:
+      'Two factors with two levels each, the same number of people in every cell or group, and roughly normal scores with the same spread in every cell. For factors within people, the correlation between conditions is the same for every pair. Each effect in a 2 × 2 design has 1 degree of freedom, so its F test is a t-test on one contrast; for a design with both factors between people this is G*Power\'s "ANOVA: fixed effects, special, main effects and interactions". A factor with three or more levels, or several stimuli per condition, needs a simulation or the Superpower package. If each condition repeats the same trial several times, plan on each person\'s average over those trials.',
+  },
   {
     id: 'one-sample',
     title: 'Compare one group with a fixed value',
@@ -133,7 +147,7 @@ export const DESIGNS: Design[] = [
   {
     id: 'proportions',
     title: 'Compare a yes/no outcome between two groups',
-    example: 'Do fewer mentored employees leave the company?',
+    example: 'Do fewer mentored employees leave the company? Or an A/B test: do more visitors click with version B than with version A?',
     test: 'test of two proportions (a chi-square test on a 2 × 2 table)',
     lessonId: '09-3',
     symbol: 'p₂',
@@ -173,7 +187,7 @@ export type FieldValues = Record<string, number>;
  * What to ask for each design and route, and how it becomes the effect size.
  * `rows` and `cols` (chi-square) come from the earlier table-size question.
  */
-export function effectInputs(id: DesignId, route: 'matter' | 'research'): { fields: EffectField[]; compute: (v: FieldValues, raw: Record<string, string>) => number } {
+export function effectInputs(id: DesignId, route: 'matter' | 'research', factors?: { layout: Layout; which: Which }): { fields: EffectField[]; compute: (v: FieldValues, raw: Record<string, string>) => number } {
   const sd: EffectField = { key: 'sd', label: 'Standard deviation of the outcome', hint: 'How much scores vary within one group. Take it from earlier studies that used the same measure, or from the scale\'s manual. Not sure? Think of the range where almost everyone\'s scores fall and divide it by 4: if almost everyone scores between 30 and 70, the SD is about 10.', placeholder: 'e.g. 10' };
   const r2Full: EffectField = { key: 'r2', label: 'Expected R² of the full model', hint: 'What all predictors together explain, controls included. If unsure, take what the controls explain in earlier studies and add the change.', placeholder: 'e.g. .30' };
   switch (id) {
@@ -220,6 +234,32 @@ export function effectInputs(id: DesignId, route: 'matter' | 'research'): { fiel
             // λ = n(k − 1)·ηp²/(1 − ηp²), written as G*Power's f for the chosen correlation.
             compute: (v) => Math.sqrt((v.eta2 / (1 - v.eta2)) * ((v.k - 1) * (1 - v.rho)) / v.k),
           };
+    case 'factorial': {
+      const { layout, which } = factors ?? { layout: 'between', which: 'interaction' };
+      const cellSd: EffectField = { ...sd, label: 'Standard deviation of the scores within one cell', hint: 'A cell is one combination of the two factors. How much people differ from each other there, from earlier studies or a pilot. Not sure? Think of the range where almost everyone\'s scores fall and divide it by 4: if almost everyone scores between 30 and 70, the SD is about 10.' };
+      if (route === 'research') {
+        return {
+          fields: [{ key: 'eta2', label: 'Partial η² of the same effect in a similar study', hint: 'Only from a study with the same layout (the same factors between or within people): partial η² from another layout is not comparable.', placeholder: 'e.g. .04' }],
+          compute: (v) => fFromEta2(v.eta2),
+        };
+      }
+      return which === 'interaction'
+        ? {
+            fields: [
+              { key: 'eff1', label: 'Effect of one factor at the first level of the other', hint: 'A difference in the outcome\'s own units. In a pre-post design with a control group: the expected change in the training group, for example 5 points.', placeholder: 'e.g. 5' },
+              { key: 'eff2', label: 'The same effect at the other level', hint: 'For example 0 points of change in the waiting-list group. Use a minus sign if the effect goes the other way there.', placeholder: 'e.g. 0' },
+              cellSd,
+            ],
+            compute: (v) => factorialF(layout, which, (v.eff1 - v.eff2) / v.sd, v.rho),
+          }
+        : {
+            fields: [
+              { key: 'diff', label: 'Smallest difference between the two levels of the factor that would matter', hint: 'Averaged over the other factor, in the outcome\'s own units, for example 3 points.', placeholder: 'e.g. 3' },
+              cellSd,
+            ],
+            compute: (v) => factorialF(layout, which, v.diff / v.sd, v.rho),
+          };
+    }
     case 'anova':
       return route === 'matter'
         ? {
@@ -267,7 +307,8 @@ export function effectInputs(id: DesignId, route: 'matter' | 'research'): { fiel
 /** Effect sizes in plain words, for a student with no numbers to go on. */
 export type Preset = { value: number; label: string; note: string; recommended?: boolean };
 
-export function presets(design: Design): Preset[] {
+export function presets(design: Design, factors?: { layout: Layout; which: Which; rho?: number }): Preset[] {
+  if (design.id === 'factorial' && factors) return factorialPresets(factors.layout, factors.which, factors.rho);
   const [s, m, l] = design.benchmarks ?? [0, 0, 0];
   const d = design.symbol === 'd' || design.symbol === 'dz';
   const cohen: Preset[] = [
@@ -284,6 +325,24 @@ export function presets(design: Design): Preset[] {
   return cohen;
 }
 
+/** 2 × 2 effects in plain words: a main effect as Cohen's d, an interaction as the shape of the two simple effects. */
+function factorialPresets(layout: Layout, which: Which, rho = 0.5): Preset[] {
+  const f = (delta: number) => factorialF(layout, which, delta, rho);
+  const show = (delta: number) => `f = ${f(delta).toFixed(2)}`;
+  if (which === 'interaction') {
+    return [
+      { value: f(0.5), label: `The effect disappears (${show(0.5)})`, note: 'One factor has a medium effect (d = 0.50) at one level of the other factor and none at the other level. For example: the training helps, the waiting list does not.', recommended: true },
+      { value: f(0.25), label: `The effect halves (${show(0.25)})`, note: 'A medium effect (d = 0.50) at one level shrinks to a small one (d = 0.25) at the other. Common, and very hard to detect.' },
+      { value: f(1), label: `The effect reverses (${show(1)})`, note: 'A medium effect (d = 0.50) one way at one level, and the same effect the other way at the other level. Rare; choose it only with a strong reason.' },
+    ];
+  }
+  return [
+    { value: f(0.2), label: `Small: subtle (d = 0.20, ${show(0.2)})`, note: 'The two levels differ by a fifth of a standard deviation, averaged over the other factor. Real, but hard to notice.' },
+    { value: f(0.5), label: `Medium: noticeable (d = 0.50, ${show(0.5)})`, note: 'Half a standard deviation. Common in student projects, but real effects are often smaller.' },
+    { value: f(0.8), label: `Large: obvious (d = 0.80, ${show(0.8)})`, note: 'Most people would notice it without any statistics. Rarely realistic; choose it only with a strong reason.' },
+  ];
+}
+
 function fmt(design: Design, value: number): string {
   return design.id === 'correlation' ? value.toFixed(2).replace(/^0\./, '.') : value.toFixed(2);
 }

src/stats/power.itest.ts

@@ -30,6 +30,9 @@ const PLANS: Plan[] = [
   { design: 'anova', effect: 0.2, alpha: 0.05, sides: 2, k: 4 },
   { design: 'repeated', effect: 0.25, alpha: 0.05, sides: 2, k: 4, rho: 0.5 },
   { design: 'repeated', effect: 0.15, alpha: 0.01, sides: 2, k: 3, rho: 0.7 },
+  { design: 'factorial', effect: 0.25, alpha: 0.05, sides: 2, layout: 'between', which: 'main' },
+  { design: 'factorial', effect: 0.15, alpha: 0.05, sides: 2, layout: 'mixed', which: 'interaction', rho: 0.5 },
+  { design: 'factorial', effect: 0.2, alpha: 0.01, sides: 2, layout: 'within', which: 'interaction', rho: 0.7 },
   { design: 'correlation', effect: 0.25, alpha: 0.05, sides: 2 },
   { design: 'correlation', effect: 0.2, alpha: 0.05, sides: 1 },
   { design: 'regression', effect: 0.1, alpha: 0.05, sides: 2, predictors: 4 },
@@ -43,14 +46,14 @@ const PLANS: Plan[] = [
 async function runAndRead(code: string, plan: Plan, solveFor: 'n' | 'effect'): Promise<number> {
   const lines = code.split('\n');
   // The regression code ends with the number of people; every other call returns a power.htest.
-  // The repeated-measures code is base R and ends with its answer.
-  if (plan.design === 'repeated') return webR.evalRNumber(`local({\n${code}\n})`);
+  // The repeated-measures and 2 × 2 code is base R and ends with its answer.
+  if (plan.design === 'repeated' || plan.design === 'factorial') return webR.evalRNumber(`local({\n${code}\n})`);
   if (solveFor === 'n' && (plan.design === 'regression' || plan.design === 'r2-change')) {
     return webR.evalRNumber(`local({\n${lines.filter((l) => !/^res$|^#/.test(l)).join('\n')}\n})`);
   }
   const field = solveFor === 'n'
     ? (plan.design === 'chi-square' ? 'N' : 'n')
-    : ({ 'two-groups': 'd', paired: 'd', repeated: 'f', 'one-sample': 'd', anova: 'f', correlation: 'r', regression: 'f2', 'r2-change': 'f2', proportions: 'p2', 'chi-square': 'w' } as const)[plan.design];
+    : ({ 'two-groups': 'd', paired: 'd', repeated: 'f', factorial: 'f', 'one-sample': 'd', anova: 'f', correlation: 'r', regression: 'f2', 'r2-change': 'f2', proportions: 'p2', 'chi-square': 'w' } as const)[plan.design];
   const call = lines.filter((l) => !l.startsWith('library')).join('\n');
   const value = await webR.evalRNumber(`local({ library(pwr); (${call})$${field} })`);
   return solveFor === 'n' ? Math.ceil(value) : value;

src/stats/power.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from 'vitest';
-import { dzFromD, f2FromR2, fFromEta2, fFromMeans, powerAt, rCode, solveEffect, solveN, totalN, wFromV, type Plan } from './power';
+import { dzFromD, f2FromR2, factorialF, fFromEta2, fFromMeans, powerAt, rCode, solveEffect, solveN, totalN, wFromV, type Layout, type Plan, type Which } from './power';
 
 // Printed by the pwr package (1.3) and base R's power.prop.test, with n
 // rounded up: ceiling(pwr.t.test(d = 0.5, power = 0.8)$n) and so on.
@@ -138,6 +138,31 @@ describe('repeated measures match G*Power', () => {
   });
 });
 
+describe('2 × 2 designs', () => {
+  const plan = (layout: Layout, which: Which, effect: number, rho = 0.5): Plan => ({ design: 'factorial', effect, alpha: 0.05, sides: 2, layout, which, rho });
+  test('between: G*Power\'s "fixed effects, special" gives 128 people for f = .25 (32 per cell)', () => {
+    expect(solveN(plan('between', 'main', 0.25), 0.8)).toBe(32);
+    expect(totalN(plan('between', 'main', 0.25), 32)).toBe(128);
+  });
+  test('power, from R\'s pf, and within simulation error of a t-test on the contrast (2000 runs)', () => {
+    // Simple effects of 0.5 and 0 SD, r = .5: simulated power .475 (mixed, 30 per group) and .582 (within, 40 people).
+    expect(powerAt(plan('mixed', 'interaction', factorialF('mixed', 'interaction', 0.5)), 30)).toBeCloseTo(0.4778965, 6);
+    expect(powerAt(plan('within', 'interaction', factorialF('within', 'interaction', 0.5)), 40)).toBeCloseTo(0.5874031, 6);
+    expect(powerAt(plan('mixed', 'main-between', factorialF('mixed', 'main-between', 0.5)), 40)).toBeCloseTo(0.7224485, 6);
+  });
+  test('a within main effect of two conditions is the paired t-test', () => {
+    // Mixed, within factor: each person's change, dz = d / √(2(1 − r)).
+    const f = factorialF('mixed', 'main-within', 0.5, 0.5);
+    expect(f).toBeCloseTo(0.5, 10);
+  });
+  test('an interaction that makes an effect disappear needs about four times the people of that effect alone', () => {
+    const simple = solveN({ design: 'two-groups', effect: 0.5, alpha: 0.05, sides: 2 }, 0.8)! * 2;
+    const interaction = totalN(plan('between', 'interaction', 0.125), solveN(plan('between', 'interaction', 0.125), 0.8)!);
+    expect(interaction / simple).toBeGreaterThan(3.8);
+    expect(interaction / simple).toBeLessThan(4.2);
+  });
+});
+
 describe('planning helpers', () => {
   test('the solved n reaches the power and one fewer does not', () => {
     const plan: Plan = { design: 'two-groups', effect: 0.5, alpha: 0.05, sides: 2 };