diff --git a/src/content/guideIndex.ts b/src/content/guideIndex.ts index 59b5c41..72fd05b 100644 --- a/src/content/guideIndex.ts +++ b/src/content/guideIndex.ts @@ -83,7 +83,7 @@ export const PAGE_ENTRIES: { path: string; title: string; what: string; keywords path: '/sample-size', title: 'How many participants?', what: 'Work out the sample size for your study, step by step.', - keywords: 'sample size how many participants people power analysis gpower g*power n', + keywords: 'sample size how many participants people power analysis gpower g*power n ab test a/b 2x2 factorial', }, { path: '/workspace', diff --git a/src/pages/SampleSize.test.tsx b/src/pages/SampleSize.test.tsx index 1086f4d..e0ef74a 100644 --- a/src/pages/SampleSize.test.tsx +++ b/src/pages/SampleSize.test.tsx @@ -192,6 +192,49 @@ describe('sample size page', () => { expect(document.querySelector('.ss-meaning')!.textContent).toContain('f = 0.16'); }); + test('a before-and-after study with a control group plans for the interaction in a 2 × 2 mixed design', async () => { + const { pick, next, fill } = setup(); + await pick(/^Two factors at once/); + await pick(/How many people do I need/); + await pick(/One factor between people, one within/); + await pick(/Whether one factor's effect depends on the other/); + await pick(/^r = \.50/); + await pick(/I can guess the scores/); + await fill(/Effect of one factor at the first level/, '5'); + await fill(/The same effect at the other level/, '0'); + await fill(/Standard deviation of the scores within one cell/, '10'); + // (5 − 0) / 10 = 0.5 SD, f = 0.5 / √(8 × (1 − .5)) = 0.25. + expect(document.querySelector('.ss-meaning')!.textContent).toContain('f = 0.25'); + await next(); + await pick(/^80%/); + await pick(/α = \.05/); + expect(question()).toBe('How many people will you lose?'); + await pick(/^None/); + // λ = N·f² with df2 = N − 2: 128 people, 64 per group. + expect(report()).toContain('128 participants (64 per group, each measured in both conditions) are needed to detect an effect of f = 0.25 (with a correlation of r = .50 between conditions)'); + expect(report()).toContain('the F test for the interaction in a 2 × 2 mixed ANOVA'); + expect(trail()).toContain('One factor between, one within'); + }); + + test('2 × 2 in plain words: interactions are described by their shape, and between designs skip the correlation', async () => { + const { pick } = setup(); + await pick(/^Two factors at once/); + await pick(/How many people do I need/); + await pick(/Each person is in one combination only/); + await pick(/Whether one factor's effect depends on the other/); + expect(question()).toBe('How big is the effect you want to be able to find?'); + await pick(/Choose a size in plain words/); + const disappears = screen.getByRole('button', { name: /The effect disappears \(f = 0.13\)/ }); + expect(disappears.textContent).toContain('Recommended'); + // f = 0.5 / 4 = 0.125: λ = N/64 reaches 80% power at 508 people (127 per cell), as in R. + expect(disappears.textContent).toContain('Needs about 508 people'); + await pick(/The effect disappears/); + await pick(/^80%/); + await pick(/α = \.05/); + await pick(/^None/); + expect(report()).toContain('508 participants (127 in each of the four cells)'); + }); + test("the browser's Back button goes back one question, keeping the answers", async () => { const { user, pick } = setup(); await pick(/^Compare two separate groups/); diff --git a/src/pages/SampleSize.tsx b/src/pages/SampleSize.tsx index f0d1906..3e7c005 100644 --- a/src/pages/SampleSize.tsx +++ b/src/pages/SampleSize.tsx @@ -2,7 +2,7 @@ import { useEffect, useId, useMemo, useRef, useState, type ReactNode } from 'rea import { Link, useSearchParams } from 'react-router-dom'; import AvatarTip from '../components/AvatarTip'; import { pnorm } from '../stats/distributions'; -import { HAS_SIDES, minN, PER_GROUP, powerAt, rCode, solveEffect, solveN, stimulusSimCode, totalN, type DesignId, type Plan } from '../stats/power'; +import { HAS_SIDES, minN, PER_GROUP, powerAt, rCode, solveEffect, solveN, stimulusSimCode, totalN, type DesignId, type Layout, type Plan, type Which } from '../stats/power'; import { DESIGNS, designById, effectInputs, presets, type Design, type EffectRoute } from './sampleSizeDesigns'; import './ModelChooser.css'; import './SampleSize.css'; @@ -31,6 +31,8 @@ function meaning(design: Design, effect: number, p1: number): string { return `If scores are normal, about ${pct(pnorm(effect))} of people score on the expected side of the fixed value.`; case 'repeated': return `Within one condition, the conditions would explain ${pct(effect ** 2 / (1 + effect ** 2), 1)} of the variance in the scores (η², leaving out the differences between people).`; + case 'factorial': + return `This effect explains ${pct(effect ** 2 / (1 + effect ** 2), 1)} of the variance the other effects leave (partial η²).`; case 'anova': return `The groups explain ${pct(effect ** 2 / (1 + effect ** 2), 1)} of the variance in the outcome (η²).`; case 'correlation': @@ -49,22 +51,41 @@ function meaning(design: Design, effect: number, p1: number): string { function peopleText(design: Design, plan: Plan, n: number): string { const total = totalN(plan, n); if (design.id === 'paired') return `${total} participants, each measured twice`; + if (design.id === 'factorial') { + if (plan.layout === 'within') return `${total} participants, each in all four conditions`; + if (plan.layout === 'mixed') return `${total} participants (${n} per group, each measured in both conditions)`; + return `${total} participants (${n} in each of the four cells)`; + } if (PER_GROUP[design.id]) return `${total} participants (${n} per group)`; return `${total} participants`; } -function testText(design: Design, sides: 1 | 2): string { +const WHICH_TEXT: Record = { + interaction: 'interaction', + main: 'main effect', + 'main-within': 'main effect of the within-person factor', + 'main-between': 'main effect of the between-person factor', +}; +const LAYOUT_TEXT: Record = { + between: 'a 2 × 2 between-subjects ANOVA', + within: 'a 2 × 2 within-subjects (repeated-measures) ANOVA', + mixed: 'a 2 × 2 mixed ANOVA (one factor between, one within people)', +}; + +function testText(design: Design, sides: 1 | 2, plan?: Plan): string { + if (design.id === 'factorial' && plan) return `the F test for the ${WHICH_TEXT[plan.which ?? 'interaction']} in ${LAYOUT_TEXT[plan.layout ?? 'between']}`; return HAS_SIDES[design.id] ? `a ${sides === 2 ? 'two' : 'one'}-sided ${design.test}` : `the ${design.test}`; } function effectText(design: Design, plan: Plan, value: number): string { if (design.id === 'proportions') return `a difference between ${pct(plan.p1 ?? 0, 1)} and ${pct(value, 1)}`; - if (design.id === 'repeated') return `an effect of f = ${formatEffect(design, value)} (with a correlation of r = ${noZero((plan.rho ?? 0.5).toFixed(2))} between conditions)`; + if (design.id === 'repeated' || (design.id === 'factorial' && plan.layout !== 'between')) return `an effect of f = ${formatEffect(design, value)} (with a correlation of r = ${noZero((plan.rho ?? 0.5).toFixed(2))} between conditions)`; return `an effect of ${design.symbol} = ${formatEffect(design, value)}`; } function tool(design: Design): string { - if (design.id === 'repeated') return 'the formula G*Power uses for repeated measures (Faul et al., 2007), computed in R'; + if (design.id === 'repeated') return 'the formula G*Power uses for repeated measures (Faul et al., 2007), computed in R,'; + if (design.id === 'factorial') return 'the noncentral F distribution, as in G*Power (Faul et al., 2007), computed in R,'; return design.id === 'proportions' ? 'the power.prop.test() function in R' : 'the pwr package in R (Champely, 2020)'; } @@ -103,7 +124,7 @@ function PowerCurve({ points, xLabel, target, mark, markLabel }: { ); } -type StepId = 'design' | 'mode' | 'n' | 'k' | 'rho' | 'stimuli' | 'nstim' | 'predictors' | 'added' | 'table' | 'p1' | 'route' | 'effect' | 'power' | 'alpha' | 'tests' | 'sides' | 'dropout'; +type StepId = 'design' | 'mode' | 'layout' | 'which' | 'n' | 'k' | 'rho' | 'stimuli' | 'nstim' | 'predictors' | 'added' | 'table' | 'p1' | 'route' | 'effect' | 'power' | 'alpha' | 'tests' | 'sides' | 'dropout'; type AlphaChoice = 'standard' | 'strict' | 'bonferroni'; type Answers = { @@ -113,6 +134,8 @@ type Answers = { rho?: number; /** Several stimuli (pictures, words, items) per condition. */ stimuli?: boolean; + layout?: Layout; + which?: Which; effect?: number; power?: number; alpha?: AlphaChoice; @@ -129,7 +152,9 @@ function stepsFor(a: Answers): StepId[] { if (!a.design) return steps; steps.push('mode'); if (!a.mode) return steps; + if (a.design === 'factorial') steps.push('layout', 'which'); if (a.mode === 'effect') steps.push('n'); + if (a.design === 'factorial' && a.layout !== 'between') steps.push('rho'); if (a.design === 'anova' || a.design === 'repeated') steps.push('k'); if (a.design === 'repeated') { steps.push('rho', 'stimuli'); @@ -150,8 +175,9 @@ function stepsFor(a: Answers): StepId[] { const alphaOf = (a: Answers) => (a.alpha === 'strict' ? 0.01 : a.alpha === 'bonferroni' ? 0.05 / parse(a.text.tests) : 0.05); const alphaLabel = (x: number) => noZero(String(Number(x.toPrecision(3)))); -function unitLabel(design: Design): string { +function unitLabel(design: Design, layout?: Layout): string { if (design.id === 'paired') return 'people, each measured twice'; + if (design.id === 'factorial') return layout === 'between' ? 'people in each of the four cells' : layout === 'mixed' ? 'people per group' : 'people in total'; return PER_GROUP[design.id] ? 'people per group' : 'people in total'; } @@ -161,7 +187,7 @@ function inputStep(step: StepId, a: Answers, design: Design): { fields: { key: s const whole = (key: string, lo: number, hi: number) => Number.isInteger(parse(t[key])) && parse(t[key]) >= lo && parse(t[key]) <= hi; switch (step) { case 'n': - return { fields: [{ key: 'n', label: `Number of ${unitLabel(design)}` }], error: whole('n', 2, 1_000_000) ? undefined : 'Enter a whole number of people.' }; + return { fields: [{ key: 'n', label: `Number of ${unitLabel(design, a.layout)}` }], error: whole('n', 2, 1_000_000) ? undefined : 'Enter a whole number of people.' }; case 'k': return { fields: [{ key: 'k', label: design.id === 'repeated' ? 'Number of conditions' : 'Number of groups' }], error: whole('k', 2, 50) ? undefined : 'Enter a whole number from 2 to 50.' }; case 'predictors': @@ -271,6 +297,8 @@ export default function SampleSize() { df: (parse(t.rows) - 1) * (parse(t.cols) - 1), p1: parse(t.p1) / 100, rho: a.rho, + layout: a.layout, + which: a.which, }; }, [answers]); @@ -329,6 +357,7 @@ export default function SampleSize() { const p: Plan = { design: design.id, effect, alpha: 0.05, sides: 2, k: parse(t.k), predictors: parse(t.predictors), added: parse(t.added), df: (parse(t.rows) - 1) * (parse(t.cols) - 1), p1: parse(t.p1) / 100, rho: answers.rho, + layout: answers.layout, which: answers.which, }; const n = solveN(p, 0.8); return n === null ? null : totalN(p, n); @@ -340,7 +369,9 @@ export default function SampleSize() { switch (s) { case 'design': return design.title; case 'mode': return answers.mode === 'n' ? 'Plan a new study' : 'Sample size is fixed'; - case 'n': return `${t.n} ${unitLabel(design)}`; + case 'layout': return { between: 'Each person in one cell', within: 'Each person in all four', mixed: 'One factor between, one within' }[answers.layout ?? 'between']; + case 'which': return WHICH_TEXT[answers.which ?? 'interaction'].replace(/^./, (c) => c.toUpperCase()); + case 'n': return `${t.n} ${unitLabel(design, answers.layout)}`; case 'k': return `${t.k} ${design.id === 'repeated' ? 'conditions' : 'groups'}`; case 'rho': return `r = ${noZero((answers.rho ?? 0.5).toFixed(2))} between conditions`; case 'stimuli': return answers.stimuli ? 'Several stimuli per condition' : 'One score per condition'; @@ -389,12 +420,12 @@ export default function SampleSize() { ); } else if (step === 'design') { question = 'What will you test?'; - help = <>Pick the comparison your research question makes. Not sure? Which model should I use? helps you choose. Repeated measures, with or without several stimuli per condition, are under "Measure the same people in several conditions". Logistic regression, mediation and models with more random factors are not here: plan those by simulation, as Lesson 8-3 does.; + help = <>Pick the comparison your research question makes. Not sure? Which model should I use? helps you choose. An A/B test is two separate groups: choose the yes/no comparison for clicks or sign-ups, and the first option for a score. Repeated measures, with or without several stimuli per condition, are under "Measure the same people in several conditions"; a before-and-after study with a control group is a 2 × 2 design. Logistic regression, mediation and models with more random factors are not here: plan those by simulation, as Lesson 8-3 does.; body = ( ({ value: d.id, label: d.title, note: `e.g. ${d.example} Analysis: ${d.test}.` }))} chosen={answers.design} - onPick={(value) => answer(value === answers.design ? {} : { design: value, route: undefined, effect: undefined, text: { ...answers.text, ...Object.fromEntries(['diff', 'sd', 'r', 'es', 'spread', 'eta2', 'r2', 'dr2', 'v', 'p2'].map((k) => [k, ''])), rpre: '.5' } })} + onPick={(value) => answer(value === answers.design ? {} : { design: value, route: undefined, effect: undefined, text: { ...answers.text, ...Object.fromEntries(['diff', 'sd', 'r', 'es', 'spread', 'eta2', 'r2', 'dr2', 'v', 'p2', 'eff1', 'eff2'].map((k) => [k, ''])), rpre: '.5' } })} /> ); } else if (step === 'mode') { @@ -440,7 +471,7 @@ export default function SampleSize() { ); body = ( { + options={presets(design, answers.layout && answers.which ? { layout: answers.layout, which: answers.which, rho: answers.rho } : undefined).map((p) => { const people = previewPeople(p.value); return { value: p.value, label: p.label, note: <>{p.note} {people === null ? 'Needs more than a million people.' : `Needs about ${people} people.`}, tag: p.recommended ? 'Recommended' : undefined }; })} @@ -450,7 +481,7 @@ export default function SampleSize() { ); } else if (step === 'effect') { const route = design.id === 'proportions' ? 'matter' : (answers.route as 'matter' | 'research'); - const { fields, compute } = effectInputs(design.id, route); + const { fields, compute } = effectInputs(design.id, route, answers.layout && answers.which ? { layout: answers.layout, which: answers.which } : undefined); const values = Object.fromEntries(Object.entries(answers.text).map(([k, v]) => [k, parse(v)])); const missing = fields.some((f) => !f.optional && (answers.text[f.key] ?? '').trim() === ''); const effect = missing ? NaN : compute(values, answers.text); @@ -486,9 +517,42 @@ export default function SampleSize() { ); + } else if (step === 'layout') { + question = 'How do people take part in the four combinations?'; + help = 'A 2 × 2 design crosses two factors with two levels each, which gives four combinations (cells). A factor is between people when each person gets one level, and within people when each person gets both.'; + body = ( + + options={[ + { value: 'mixed', label: 'One factor between people, one within', note: 'For example training versus waiting list (between), each measured before and after (within). The usual design for a study with a control group.', tag: 'Most common' }, + { value: 'within', label: 'Each person does all four combinations', note: 'For example every participant sees photo and text ads, each in two colours. Needs the fewest people.' }, + { value: 'between', label: 'Each person is in one combination only', note: 'For example four versions of a questionnaire, each person answers one. Needs the most people.' }, + ]} + chosen={answers.layout} + onPick={(value) => answer({ layout: value, which: undefined, effect: undefined })} + /> + ); + } else if (step === 'which') { + question = 'Which effect is your question about?'; + help = 'Plan for the effect your hypothesis is about. An interaction asks whether the effect of one factor depends on the other; it needs far more people than a main effect of the same size, often four times as many or more.'; + const main: Option[] = answers.layout === 'mixed' + ? [ + { value: 'main-within', label: 'The effect of the within-person factor', note: 'Averaged over both groups. For example: do scores change from before to after, whichever group people are in?' }, + { value: 'main-between', label: 'The difference between the groups', note: 'Averaged over both conditions. Usually the least interesting question in a before-and-after study.' }, + ] + : [{ value: 'main', label: 'The effect of one factor, averaged over the other', note: 'A main effect. For example: do photo ads work better than text ads, whatever their colour?' }]; + body = ( + + options={[ + { value: 'interaction', label: 'Whether one factor\'s effect depends on the other', note: answers.layout === 'mixed' ? 'The interaction. For example: does the training group improve more than the waiting-list group? In a before-and-after study with a control group, this is the test that answers your question.' : 'The interaction. For example: do photo ads work better than text ads for young viewers, but not for old ones?', tag: 'Most common' }, + ...main, + ]} + chosen={answers.which} + onPick={(value) => answer({ which: value, effect: undefined })} + /> + ); } else if (step === 'rho') { question = 'How strongly do a person\'s scores in different conditions go together?'; - help = 'People who score high in one condition usually score high in the others too. The stronger that correlation, the fewer people you need, because every person is compared with themselves. Take it from a pilot or an earlier study with the same measure if you can.'; + help = 'People who score high in one condition usually score high in the others too. The stronger that correlation, the fewer people you need, because every person is compared with themselves. Take it from a pilot or an earlier study with the same measure if you can. If people do each condition several times (repeated trials), plan on their average per condition: averages over many trials are more reliable, so they usually correlate more strongly.'; body = ( How to report it

An a priori power analysis with {tool(design)} showed that {peopleText(design, result.plan, result.n)}{design.id === 'paired' && ','} are - needed to detect {effectText(design, result.plan, result.plan.effect)} with {pct(power)} power in {testText(design, result.plan.sides)} at + needed to detect {effectText(design, result.plan, result.plan.effect)} with {pct(power)} power in {testText(design, result.plan.sides, result.plan)} at α = {alphaText}.{' '} {answers.route === 'unknown' ? '[Say that this is a conventional value, and why no better estimate was available.]' : '[Say where the effect size comes from.]'} {result.drop > 0 && <> To allow for {pct(result.drop)} dropout, we will recruit {totalN(result.plan, result.recruitUnit)} participants.} @@ -677,7 +741,7 @@ export default function SampleSize() {

{design.id === 'proportions' ? pct(result.effect, 1) : formatEffect(design, result.effect)} {design.id === 'proportions' ? 'in the second group' : `smallest ${design.symbol} with ${pct(power)} power`}

- With {peopleText(design, result.plan, result.n)}, {testText(design, result.plan.sides)} at α = {alphaText} has {pct(power)} power + With {peopleText(design, result.plan, result.n)}, {testText(design, result.plan.sides, result.plan)} at α = {alphaText} has {pct(power)} power for {design.id === 'proportions' ? `a difference between ${pct(result.plan.p1 ?? 0, 1)} and ${pct(result.effect, 1)} or larger` : `effects of ${design.symbol} = ${formatEffect(design, result.effect)} or larger`}. Smaller effects may well go unnoticed, so a non-significant result says little about them.

@@ -692,7 +756,7 @@ export default function SampleSize() {

How to report it

A sensitivity power analysis with {tool(design)} showed that with {peopleText(design, result.plan, result.n)},{' '} - {testText(design, result.plan.sides)} at α = {alphaText} has {pct(power)} power to detect{' '} + {testText(design, result.plan.sides, result.plan)} at α = {alphaText} has {pct(power)} power to detect{' '} {design.id === 'proportions' ? `a difference between ${pct(result.plan.p1 ?? 0, 1)} and ${pct(result.effect, 1)}` : `effects of ${design.symbol} = ${formatEffect(design, result.effect)}`} or larger.

@@ -701,12 +765,12 @@ export default function SampleSize() { {code && ( <>

Check it in R

-

The same calculation in R. It runs in the R Workspace; {design.id === 'proportions' || design.id === 'repeated' ? 'it needs only base R' : 'the first run downloads the pwr package'}.

+

The same calculation in R. It runs in the R Workspace; {design.id === 'proportions' || design.id === 'repeated' || design.id === 'factorial' ? 'it needs only base R' : 'the first run downloads the pwr package'}.

{code}
- {answers.mode === 'n' && design.id !== 'regression' && design.id !== 'r2-change' && design.id !== 'repeated' && ( + {answers.mode === 'n' && design.id !== 'regression' && design.id !== 'r2-change' && design.id !== 'repeated' && design.id !== 'factorial' && (

R reports n unrounded{PER_GROUP[design.id] ? ' and per group' : ''}; round it up to whole people.

)} diff --git a/src/pages/sampleSizeDesigns.ts b/src/pages/sampleSizeDesigns.ts index 0437b74..cc1a4b6 100644 --- a/src/pages/sampleSizeDesigns.ts +++ b/src/pages/sampleSizeDesigns.ts @@ -1,4 +1,4 @@ -import { fFromEta2, type DesignId } from '../stats/power'; +import { factorialF, fFromEta2, type DesignId, type Layout, type Which } from '../stats/power'; /** Everything the sample size page says about one design. */ export type Design = { @@ -23,7 +23,7 @@ export const DESIGNS: Design[] = [ { id: 'two-groups', title: 'Compare two separate groups', - example: 'Do remote and office workers differ in wellbeing?', + example: 'Do remote and office workers differ in wellbeing? Or an A/B test on a score, such as time on a page.', test: 'independent-samples t-test', lessonId: '12-1', symbol: 'd', @@ -62,6 +62,20 @@ export const DESIGNS: Design[] = [ assumes: 'Every person does every condition, scores are roughly normal, and sphericity holds: the differences between each pair of conditions vary about equally. This is the formula G*Power uses for "ANOVA: repeated measures, within factors" with ε = 1. When sphericity fails, the Greenhouse-Geisser correction costs power, so plan for more people. The same test is the F test for condition in a mixed model with a random intercept per person (Lesson 14-4). The plan is for the overall F test; follow-up comparisons between pairs of conditions have their own, often lower, power.', }, + { + id: 'factorial', + title: 'Two factors at once, like a 2 × 2 design', + example: 'Does a training raise wellbeing more than a waiting list, measured before and after? Or: do photo and text ads work differently for young and old viewers?', + test: '2 × 2 ANOVA', + lessonId: '13-2', + symbol: 'f', + effectName: "Cohen's f", + effectMeaning: 'the size of one effect relative to the variation left after the other effects.', + benchmarks: [0.1, 0.25, 0.4], + defaultEffect: 0.25, + assumes: + 'Two factors with two levels each, the same number of people in every cell or group, and roughly normal scores with the same spread in every cell. For factors within people, the correlation between conditions is the same for every pair. Each effect in a 2 × 2 design has 1 degree of freedom, so its F test is a t-test on one contrast; for a design with both factors between people this is G*Power\'s "ANOVA: fixed effects, special, main effects and interactions". A factor with three or more levels, or several stimuli per condition, needs a simulation or the Superpower package. If each condition repeats the same trial several times, plan on each person\'s average over those trials.', + }, { id: 'one-sample', title: 'Compare one group with a fixed value', @@ -133,7 +147,7 @@ export const DESIGNS: Design[] = [ { id: 'proportions', title: 'Compare a yes/no outcome between two groups', - example: 'Do fewer mentored employees leave the company?', + example: 'Do fewer mentored employees leave the company? Or an A/B test: do more visitors click with version B than with version A?', test: 'test of two proportions (a chi-square test on a 2 × 2 table)', lessonId: '09-3', symbol: 'p₂', @@ -173,7 +187,7 @@ export type FieldValues = Record; * What to ask for each design and route, and how it becomes the effect size. * `rows` and `cols` (chi-square) come from the earlier table-size question. */ -export function effectInputs(id: DesignId, route: 'matter' | 'research'): { fields: EffectField[]; compute: (v: FieldValues, raw: Record) => number } { +export function effectInputs(id: DesignId, route: 'matter' | 'research', factors?: { layout: Layout; which: Which }): { fields: EffectField[]; compute: (v: FieldValues, raw: Record) => number } { const sd: EffectField = { key: 'sd', label: 'Standard deviation of the outcome', hint: 'How much scores vary within one group. Take it from earlier studies that used the same measure, or from the scale\'s manual. Not sure? Think of the range where almost everyone\'s scores fall and divide it by 4: if almost everyone scores between 30 and 70, the SD is about 10.', placeholder: 'e.g. 10' }; const r2Full: EffectField = { key: 'r2', label: 'Expected R² of the full model', hint: 'What all predictors together explain, controls included. If unsure, take what the controls explain in earlier studies and add the change.', placeholder: 'e.g. .30' }; switch (id) { @@ -220,6 +234,32 @@ export function effectInputs(id: DesignId, route: 'matter' | 'research'): { fiel // λ = n(k − 1)·ηp²/(1 − ηp²), written as G*Power's f for the chosen correlation. compute: (v) => Math.sqrt((v.eta2 / (1 - v.eta2)) * ((v.k - 1) * (1 - v.rho)) / v.k), }; + case 'factorial': { + const { layout, which } = factors ?? { layout: 'between', which: 'interaction' }; + const cellSd: EffectField = { ...sd, label: 'Standard deviation of the scores within one cell', hint: 'A cell is one combination of the two factors. How much people differ from each other there, from earlier studies or a pilot. Not sure? Think of the range where almost everyone\'s scores fall and divide it by 4: if almost everyone scores between 30 and 70, the SD is about 10.' }; + if (route === 'research') { + return { + fields: [{ key: 'eta2', label: 'Partial η² of the same effect in a similar study', hint: 'Only from a study with the same layout (the same factors between or within people): partial η² from another layout is not comparable.', placeholder: 'e.g. .04' }], + compute: (v) => fFromEta2(v.eta2), + }; + } + return which === 'interaction' + ? { + fields: [ + { key: 'eff1', label: 'Effect of one factor at the first level of the other', hint: 'A difference in the outcome\'s own units. In a pre-post design with a control group: the expected change in the training group, for example 5 points.', placeholder: 'e.g. 5' }, + { key: 'eff2', label: 'The same effect at the other level', hint: 'For example 0 points of change in the waiting-list group. Use a minus sign if the effect goes the other way there.', placeholder: 'e.g. 0' }, + cellSd, + ], + compute: (v) => factorialF(layout, which, (v.eff1 - v.eff2) / v.sd, v.rho), + } + : { + fields: [ + { key: 'diff', label: 'Smallest difference between the two levels of the factor that would matter', hint: 'Averaged over the other factor, in the outcome\'s own units, for example 3 points.', placeholder: 'e.g. 3' }, + cellSd, + ], + compute: (v) => factorialF(layout, which, v.diff / v.sd, v.rho), + }; + } case 'anova': return route === 'matter' ? { @@ -267,7 +307,8 @@ export function effectInputs(id: DesignId, route: 'matter' | 'research'): { fiel /** Effect sizes in plain words, for a student with no numbers to go on. */ export type Preset = { value: number; label: string; note: string; recommended?: boolean }; -export function presets(design: Design): Preset[] { +export function presets(design: Design, factors?: { layout: Layout; which: Which; rho?: number }): Preset[] { + if (design.id === 'factorial' && factors) return factorialPresets(factors.layout, factors.which, factors.rho); const [s, m, l] = design.benchmarks ?? [0, 0, 0]; const d = design.symbol === 'd' || design.symbol === 'dz'; const cohen: Preset[] = [ @@ -284,6 +325,24 @@ export function presets(design: Design): Preset[] { return cohen; } +/** 2 × 2 effects in plain words: a main effect as Cohen's d, an interaction as the shape of the two simple effects. */ +function factorialPresets(layout: Layout, which: Which, rho = 0.5): Preset[] { + const f = (delta: number) => factorialF(layout, which, delta, rho); + const show = (delta: number) => `f = ${f(delta).toFixed(2)}`; + if (which === 'interaction') { + return [ + { value: f(0.5), label: `The effect disappears (${show(0.5)})`, note: 'One factor has a medium effect (d = 0.50) at one level of the other factor and none at the other level. For example: the training helps, the waiting list does not.', recommended: true }, + { value: f(0.25), label: `The effect halves (${show(0.25)})`, note: 'A medium effect (d = 0.50) at one level shrinks to a small one (d = 0.25) at the other. Common, and very hard to detect.' }, + { value: f(1), label: `The effect reverses (${show(1)})`, note: 'A medium effect (d = 0.50) one way at one level, and the same effect the other way at the other level. Rare; choose it only with a strong reason.' }, + ]; + } + return [ + { value: f(0.2), label: `Small: subtle (d = 0.20, ${show(0.2)})`, note: 'The two levels differ by a fifth of a standard deviation, averaged over the other factor. Real, but hard to notice.' }, + { value: f(0.5), label: `Medium: noticeable (d = 0.50, ${show(0.5)})`, note: 'Half a standard deviation. Common in student projects, but real effects are often smaller.' }, + { value: f(0.8), label: `Large: obvious (d = 0.80, ${show(0.8)})`, note: 'Most people would notice it without any statistics. Rarely realistic; choose it only with a strong reason.' }, + ]; +} + function fmt(design: Design, value: number): string { return design.id === 'correlation' ? value.toFixed(2).replace(/^0\./, '.') : value.toFixed(2); } diff --git a/src/stats/power.itest.ts b/src/stats/power.itest.ts index 6e3cc21..c5e3842 100644 --- a/src/stats/power.itest.ts +++ b/src/stats/power.itest.ts @@ -30,6 +30,9 @@ const PLANS: Plan[] = [ { design: 'anova', effect: 0.2, alpha: 0.05, sides: 2, k: 4 }, { design: 'repeated', effect: 0.25, alpha: 0.05, sides: 2, k: 4, rho: 0.5 }, { design: 'repeated', effect: 0.15, alpha: 0.01, sides: 2, k: 3, rho: 0.7 }, + { design: 'factorial', effect: 0.25, alpha: 0.05, sides: 2, layout: 'between', which: 'main' }, + { design: 'factorial', effect: 0.15, alpha: 0.05, sides: 2, layout: 'mixed', which: 'interaction', rho: 0.5 }, + { design: 'factorial', effect: 0.2, alpha: 0.01, sides: 2, layout: 'within', which: 'interaction', rho: 0.7 }, { design: 'correlation', effect: 0.25, alpha: 0.05, sides: 2 }, { design: 'correlation', effect: 0.2, alpha: 0.05, sides: 1 }, { design: 'regression', effect: 0.1, alpha: 0.05, sides: 2, predictors: 4 }, @@ -43,14 +46,14 @@ const PLANS: Plan[] = [ async function runAndRead(code: string, plan: Plan, solveFor: 'n' | 'effect'): Promise { const lines = code.split('\n'); // The regression code ends with the number of people; every other call returns a power.htest. - // The repeated-measures code is base R and ends with its answer. - if (plan.design === 'repeated') return webR.evalRNumber(`local({\n${code}\n})`); + // The repeated-measures and 2 × 2 code is base R and ends with its answer. + if (plan.design === 'repeated' || plan.design === 'factorial') return webR.evalRNumber(`local({\n${code}\n})`); if (solveFor === 'n' && (plan.design === 'regression' || plan.design === 'r2-change')) { return webR.evalRNumber(`local({\n${lines.filter((l) => !/^res$|^#/.test(l)).join('\n')}\n})`); } const field = solveFor === 'n' ? (plan.design === 'chi-square' ? 'N' : 'n') - : ({ 'two-groups': 'd', paired: 'd', repeated: 'f', 'one-sample': 'd', anova: 'f', correlation: 'r', regression: 'f2', 'r2-change': 'f2', proportions: 'p2', 'chi-square': 'w' } as const)[plan.design]; + : ({ 'two-groups': 'd', paired: 'd', repeated: 'f', factorial: 'f', 'one-sample': 'd', anova: 'f', correlation: 'r', regression: 'f2', 'r2-change': 'f2', proportions: 'p2', 'chi-square': 'w' } as const)[plan.design]; const call = lines.filter((l) => !l.startsWith('library')).join('\n'); const value = await webR.evalRNumber(`local({ library(pwr); (${call})$${field} })`); return solveFor === 'n' ? Math.ceil(value) : value; diff --git a/src/stats/power.test.ts b/src/stats/power.test.ts index a8be765..eab1f3c 100644 --- a/src/stats/power.test.ts +++ b/src/stats/power.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { dzFromD, f2FromR2, fFromEta2, fFromMeans, powerAt, rCode, solveEffect, solveN, totalN, wFromV, type Plan } from './power'; +import { dzFromD, f2FromR2, factorialF, fFromEta2, fFromMeans, powerAt, rCode, solveEffect, solveN, totalN, wFromV, type Layout, type Plan, type Which } from './power'; // Printed by the pwr package (1.3) and base R's power.prop.test, with n // rounded up: ceiling(pwr.t.test(d = 0.5, power = 0.8)$n) and so on. @@ -138,6 +138,31 @@ describe('repeated measures match G*Power', () => { }); }); +describe('2 × 2 designs', () => { + const plan = (layout: Layout, which: Which, effect: number, rho = 0.5): Plan => ({ design: 'factorial', effect, alpha: 0.05, sides: 2, layout, which, rho }); + test('between: G*Power\'s "fixed effects, special" gives 128 people for f = .25 (32 per cell)', () => { + expect(solveN(plan('between', 'main', 0.25), 0.8)).toBe(32); + expect(totalN(plan('between', 'main', 0.25), 32)).toBe(128); + }); + test('power, from R\'s pf, and within simulation error of a t-test on the contrast (2000 runs)', () => { + // Simple effects of 0.5 and 0 SD, r = .5: simulated power .475 (mixed, 30 per group) and .582 (within, 40 people). + expect(powerAt(plan('mixed', 'interaction', factorialF('mixed', 'interaction', 0.5)), 30)).toBeCloseTo(0.4778965, 6); + expect(powerAt(plan('within', 'interaction', factorialF('within', 'interaction', 0.5)), 40)).toBeCloseTo(0.5874031, 6); + expect(powerAt(plan('mixed', 'main-between', factorialF('mixed', 'main-between', 0.5)), 40)).toBeCloseTo(0.7224485, 6); + }); + test('a within main effect of two conditions is the paired t-test', () => { + // Mixed, within factor: each person's change, dz = d / √(2(1 − r)). + const f = factorialF('mixed', 'main-within', 0.5, 0.5); + expect(f).toBeCloseTo(0.5, 10); + }); + test('an interaction that makes an effect disappear needs about four times the people of that effect alone', () => { + const simple = solveN({ design: 'two-groups', effect: 0.5, alpha: 0.05, sides: 2 }, 0.8)! * 2; + const interaction = totalN(plan('between', 'interaction', 0.125), solveN(plan('between', 'interaction', 0.125), 0.8)!); + expect(interaction / simple).toBeGreaterThan(3.8); + expect(interaction / simple).toBeLessThan(4.2); + }); +}); + describe('planning helpers', () => { test('the solved n reaches the power and one fewer does not', () => { const plan: Plan = { design: 'two-groups', effect: 0.5, alpha: 0.05, sides: 2 }; diff --git a/src/stats/power.ts b/src/stats/power.ts index 5527e1e..03896f3 100644 --- a/src/stats/power.ts +++ b/src/stats/power.ts @@ -11,6 +11,7 @@ export type DesignId = | 'two-groups' | 'paired' | 'repeated' + | 'factorial' | 'one-sample' | 'anova' | 'correlation' @@ -38,13 +39,22 @@ export type Plan = { p1?: number; /** Correlation between a person's scores in different conditions (repeated). */ rho?: number; + /** Factorial: which factors each person does, and the effect the plan is for. */ + layout?: Layout; + which?: Which; }; +/** A 2 × 2 design: both factors between people, both within, or one of each. */ +export type Layout = 'between' | 'within' | 'mixed'; +/** The effect in a 2 × 2 design. In a mixed design, main effects say which factor. */ +export type Which = 'main' | 'main-within' | 'main-between' | 'interaction'; + /** Whether n counts people per group (or pairs), or everyone in the study. */ export const PER_GROUP: Record = { 'two-groups': true, paired: false, repeated: false, + factorial: true, 'one-sample': false, anova: true, correlation: false, @@ -59,6 +69,7 @@ export const HAS_SIDES: Record = { 'two-groups': true, paired: true, repeated: false, + factorial: false, 'one-sample': true, anova: false, correlation: true, @@ -80,10 +91,34 @@ export function minN(plan: Plan): number { } } +/** A 2 × 2 design's n counts people per cell, per group, or everyone. */ +const CELLS: Record = { between: 4, mixed: 2, within: 1 }; +/** Error degrees of freedom are N minus this. */ +const LOST_DF: Record = { between: 4, mixed: 2, within: 1 }; + +/** + * Cohen's f of one effect in a 2 × 2 design, from its size in standard + * deviations of the scores in one cell, and the correlation r between a + * person's scores in different conditions (compound symmetry). For a main + * effect `delta` is the difference between the two levels, averaged over the + * other factor; for the interaction it is the difference between the two + * simple effects. Every effect has 1 df, so its test is a t-test on one + * contrast, and λ = N·f² in every layout. + */ +export function factorialF(layout: Layout, which: Which, delta: number, r = 0.5): number { + const d = Math.abs(delta); + if (layout === 'between') return which === 'interaction' ? d / 4 : d / 2; + if (layout === 'within') return which === 'interaction' ? d / (2 * Math.sqrt(1 - r)) : d / Math.sqrt(1 - r); + if (which === 'interaction') return d / Math.sqrt(8 * (1 - r)); + if (which === 'main-between') return d / Math.sqrt(2 * (1 + r)); + return d / Math.sqrt(2 * (1 - r)); +} + /** Everyone in the study, for a given n. */ export function totalN(plan: Plan, n: number): number { if (plan.design === 'anova') return n * (plan.k ?? 3); if (plan.design === 'two-groups' || plan.design === 'proportions') return 2 * n; + if (plan.design === 'factorial') return n * CELLS[plan.layout ?? 'between']; return n; } @@ -115,6 +150,11 @@ export function powerAt(plan: Plan, n: number): number { const k = plan.k ?? 3; return fPower(k - 1, (n - 1) * (k - 1), (n * k * effect ** 2) / (1 - (plan.rho ?? 0.5)), alpha); } + case 'factorial': { + // One effect of a 2 × 2 ANOVA: G*Power's "fixed effects, special" for a between design. + const N = totalN(plan, n); + return fPower(1, N - LOST_DF[plan.layout ?? 'between'], N * effect ** 2, alpha); + } case 'regression': case 'r2-change': { const p = plan.predictors ?? 1; @@ -239,6 +279,14 @@ export function rCode(plan: Plan, solveFor: 'n' | 'effect', target: number, n?: ? `${head}n <- 2\nwhile (power_at(n, ${num(plan.effect)}) < target) n <- n + 1\nn` : `${head}uniroot(function(f) power_at(${n}, f) - target, c(1e-6, 10), tol = 1e-10)$root`; } + case 'factorial': { + const layout = plan.layout ?? 'between'; + const per = { between: 'people per cell', mixed: 'people per group', within: 'people' }[layout]; + const head = `# One effect in a 2 × 2 ANOVA (${layout === 'mixed' ? 'one factor between, one within people' : `both factors ${layout} people`}): F test with 1 df\nalpha <- ${num(plan.alpha)}; target <- ${num(target)}\npower_at <- function(n, f) { # n = ${per}\n N <- ${CELLS[layout]} * n; df2 <- N - ${LOST_DF[layout]}\n 1 - pf(qf(1 - alpha, 1, df2), 1, df2, ncp = N * f^2)\n}\n`; + return e + ? `${head}n <- 2\nwhile (power_at(n, ${num(plan.effect)}) < target) n <- n + 1\nn # ${per}` + : `${head}uniroot(function(f) power_at(${n}, f) - target, c(1e-6, 10), tol = 1e-10)$root`; + } case 'correlation': { const size = e ? `r = ${num(plan.effect)}` : `n = ${n}`; return `library(pwr)\npwr.r.test(${size}, ${a}, ${pw}, alternative = "${alt}")`;