diff --git a/DESCRIPTION b/DESCRIPTION index 23c1668..02c8cc0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: effectcheck Type: Package Title: Statistical Consistency Checker for Published Research Results -Version: 0.7.13 +Version: 0.7.14 Authors@R: c(person(given = "Gilad", family = "Feldman", role = c("aut", "cre"), email = "giladfel@gmail.com", comment = c(ORCID = "0000-0003-2812-6599"))) Description: A conservative, assumption-aware statistical consistency checker diff --git a/NEWS.md b/NEWS.md index 66f54ee..64dab0b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,117 @@ +# effectcheck 0.7.14 + +Ten open defects, re-tested on 0.7.13 first; the ones that still reproduced are fixed here. Every +new test was watched failing on 0.7.13. **Several published values change** -- listed in full so a +consumer can tell a fix from a regression. + +**A confidence-interval verdict of INCONSISTENT now requires an interval comparable with the +paper's.** `ci_check_status = "INCONSISTENT"` says the paper's interval is wrong; 0.7.9 withdrew +its status escalation after it accused 3 correct papers out of 3, but the column kept publishing +INCONSISTENT on the same false premises. It is now `UNVERIFIABLE`, with the new column +`ci_unverifiable_reason` saying why, when the row itself shows the intervals are not comparable: + +- `referent_not_effect` -- the interval excludes the reported estimate (and its sign-flip), or, + for the d family (whose intervals are near-symmetric), is centred more than a quarter of its + half-width from it. `ci_referent = "not_effect_reported"` can now be set on any row, not only + regression rows. Such a row keeps `estimate_outside_ci = TRUE`, its IMPOSSIBLE VALUE note and + `status = NOTE`: the finding is that at least one of the three printed numbers is wrong, not + that the interval specifically is. Reproduced at 0.7.13: `t(58) = 4.00, d = 1.03; the + unstandardized contrast had a 95% CI [9.99, 30.01]` was INCONSISTENT. +- `no_estimate_parsed` -- no effect size was read beside the interval (none printed, or not + parseable), so it was never established to be a standardized interval (the 0.7.9 notes counted + 11 of 15 such corpus rows as raw mean differences). Correlations are exempt: r is its own + estimate. +- `one_sided_interval` -- stated one-sided/one-tailed in the row's own clause, or a correlation + interval running to 1 or -1. `r(198) = .34, one-sided 95% CI [0.23, 1.00]` was INCONSISTENT; + its true one-sided lower bound is 0.2326. +- `unstated_allocation` -- group sizes unstated, graded at an equal split, and an unequal split + reproduces BOTH the printed d and the interval (the split is named). `t(98) = 0.30, d = 0.075, + 95% CI [-0.415, 0.565]` is exact at n1 = 20, n2 = 80 and was INCONSISTENT. +- `estimand_not_computable` -- an interval on generalized eta-squared (below). +- The pre-existing UNVERIFIABLE causes are now named too: `sample_size_ambiguous`, + `paired_design_independent_approximation`, `one_bound_reported`, + `no_comparable_interval_computed`. + +MATCH and PLAUSIBLE are never altered (except generalized eta-squared). A genuinely wrong +interval of each shape stays INCONSISTENT -- each rule has that control in +`test-v0714-ci-verdict-premise.R`. `ci_match` is NA (not FALSE) on these rows. (d-cf56d6, +d-4efdc1) + +**A Welch test's effect size is no longer "verified" against an N solved from it.** With group +sizes unstated a Welch df only bounds N from below, so the Welch branch back-solved +N = 4t^2/d^2 from the reported d and then recomputed d from that N -- reproducing it by +construction. Measured at 0.7.13: a deliberately wrong d = 0.70 on `Welch's t(223) = 8.11` (true +about 0.99) returned PASS at N = 537. Such a row now checks its p-value only, says the effect +size is not verified, and publishes `N_source = "effect_backsolved"` (it said `global_text` or +`not_found`). When the back-solve falls below the Welch floor the N is clamped to df + 2, the +check there is real, and the label is now `df_inferred`. Stated group sizes now give +N = n1 + n2 (`N_source = "group_sizes"`) instead of being ignored in favour of the back-solve: +`Welch's t(222.87) = 8.11, d = 0.99` with `n1 = 131, n2 = 135` now verifies d at N = 266 as PASS. +An integer Welch df no longer produces the note "Non-integer df (223.00)". (d-48266c; the filed +symptom, N = 225 and a false WARN, was already gone at 0.7.13.) +Two further Welch fixes from the /ship review (2026-09-30): a Welch test compares independent +groups, so paired-design variants (dz, dav, drm) are no longer offered on a Welch row -- a wrong +d = 0.70 with `n1 = 131, n2 = 135` had passed against drm = 0.685 (0.7.13 too); and a tiny, +rounded Welch d that the Welch floor N = df + 2 reproduces within rounding is graded at that N +instead of a scraped study total. Measured on the 27-paper seam (docpluck 2.4.147): 6 rows move, +all in 10.1016/j.jesp.2020.104052 and 10.5334/irsp.571 -- the five jesp rows now grade at +N = 171-203 (`df_inferred`) instead of the document total N = 827 (3 NOTE -> PASS), irsp.571 +stays WARN against the independent-groups d. A row whose effect is explicitly labelled `dz`, +`dav` or `drm` keeps its paired variants even when its df is fractional or its clause mentions +Welch: the tier-2 cross-model check showed a correct `dz = 0.44` on a within-subject +Satterthwaite contrast `t(45.3) = 3.00` falling from PASS to NOTE under the Welch gate. + +**Generalized eta-squared is no longer published as partial eta-squared.** The +`generalized_eta2` column and `all_variants` computed it with the partial formula whenever the +design was unclear -- on collabra.126266 it equalled `partial_eta2` on all 15 F rows beside a +row message saying it cannot be computed. It depends on which factors are measured or +within-subjects (Olejnik & Algina, 2003; Bakeman, 2005), which F and df do not carry, so it is +now always NA. A row reporting it no longer carries two false notes ("unusual for F-test", +"symbol unclear -- may be OCR"), checks its p-value (it was SKIP, "nothing checked"), and its +interval is not graded against partial-eta-squared intervals (collabra.126266: 8 INCONSISTENT, +5 PLAUSIBLE, 1 MATCH, all against the wrong estimand). The 9 bare abstract restatements the +gold counts carry no test statistic; effectcheck extracts no bare effect size of any family, +so that is a scope question, logged (d-368d63). (d-b1c193) + +**A sentence reporting both the average direct effect and the ACME yields both.** Chan & +Feldman (2025, doi 10.1080/02699931.2024.2434156) print "The average direct effect was 0.15, +95% CI [-0.13 to 0.45], p = .3, whereas the ... indirect effect (ACME) was 0.67, 95% CI +[0.47-0.89], p < .001". 0.7.13 emitted one row: the direct effect, labelled +`indirect_effect`; the ACME was absent. Now two rows, `direct_effect` and `indirect_effect`. +A sentence mixing one CI-form and one Sobel-form effect still yields one row (pre-existing, +logged d-564e1b). (d-4fe38e) + +**A statistic quoted twice is one result.** 10.24072/pci.rr.100726 is a review letter quoting +`t(868) = -3.01, p = .006` twice to show comma placement; both copies were scored. v0.6.18 +withdrew three dedup rules because two distinct results can share every printed number, and +named the missing signal: quoted material. Rows now merge only when the printed signature is +identical, they are adjacent, and every copy is inside quotation marks; the kept row says so. +The v0.6.18 guard-rail tests (distinct results must never merge) still pass. The N = 870 +provenance and the 2.2x p-value discrepancy on that paper were already fixed at 0.7.13. +(d-43ed61) + +**An undecidable post-hoc contrast names the contrast reading.** collabra.90203's +`t(998) = 0.097, d = 0.01` sits under a Bonferroni post-hoc announcement with an omnibus +`F(2, 998)`; at a d that small, N = 1000 and the contrast N of about 667 both fit, so the +incumbent N = 1000 stands -- and the row now says the contrast reading exists. The filed +defect (the decisive contrasts binding N = 1000, a false WARN and two false CI flags) was +already fixed at 0.7.13. (d-43c69d) + +**Already fixed at 0.7.13, closed on re-test:** collabra.57785's ambiguous-design t(742) row +publishes N = 743 (gold 743) and names both readings (d-bd5e80); cog_emo Table 9's three false +CI-mismatch flags are gone -- the rows keep N = 794 under the 2026-09-04 ruling to report, not +silently correct, an ambiguous sample, and the one remaining INCONSISTENT is the paper's own +dropped minus sign (d-4ea375). + +**The API returns every column on every row (worker).** JSON endpoints serialized rows with +jsonlite's data-frame default, which drops a key whose value is NA; a row carried 62 keys where +`check_text()` returns 138. Every row now carries every column, NA as `null`. No other response +field changed (measured by diffing the full process-text response). A consumer that detects +schema drift will log each newly visible key once. (d-c7c216) + +**Harness:** the gold lock is promoted to article-finder's 2026-09-25 page-checked correction of +10.1016/j.jesp.2021.104154 (81 -> 81 results); `run_validation.R` refused on origin/main without it. + # effectcheck 0.7.13 **Rows from a table no caption claimed are no longer checked.** docpluck 2.4.145 (released and diff --git a/R/check.R b/R/check.R index 4637b9a..943e4e3 100644 --- a/R/check.R +++ b/R/check.R @@ -427,7 +427,7 @@ VARIANT_METADATA <- list( closest_method = NA_character_, delta_effect_abs = NA_real_, ci_match = as.logical(NA), ci_delta_lower = NA_real_, ci_delta_upper = NA_real_, - ci_check_status = NA_character_, ci_method_match = NA_character_, ci_referent = NA_character_, + ci_check_status = NA_character_, ci_method_match = NA_character_, ci_referent = NA_character_, ci_unverifiable_reason = NA_character_, effect_guard_rejected = isTRUE(g1("effect_guard_rejected", FALSE)), effect_guard_reason = as.character(g1("effect_guard_reason", NA_character_)), SE_guard_rejected = isTRUE(g1("SE_guard_rejected", FALSE)), @@ -644,6 +644,101 @@ VARIANT_METADATA <- list( parsed } +#' Is a parsed statistic inside quotation marks in its own clause? +#' +#' v0.7.14. Looks at the row's own `raw_text` only (never the context window, +#' which bleeds neighbouring sentences). The statistic is located by its printed +#' value after "=", and it is inside a quotation when an odd number of straight +#' quotes, or more opening than closing curly quotes, precede it -- or, when +#' the clause starts inside a quotation that opened in the previous chunk, +#' when nothing opens before it and an unmatched closing quote follows it. +#' +#' @param raw The row's `raw_text`. +#' @param stat The parsed statistic value. +#' @param decs Decimal places it was printed with (may be NA). +#' @return TRUE, FALSE, or NA when the statistic cannot be located. +#' @keywords internal +.stat_in_quotation <- function(raw, stat, decs) { + if (is.na(raw) || !nzchar(raw) || is.na(stat)) return(NA) + d <- if (is.na(decs)) 2L else as.integer(decs) + num <- formatC(abs(stat), format = "f", digits = d) + num_alt <- sub("^0\\.", ".", num) + num_re <- paste0("(?:", gsub(".", "\\.", num, fixed = TRUE), "|", + gsub(".", "\\.", num_alt, fixed = TRUE), ")") + m <- regexpr(paste0("=\\s*[-\u2212\u2013]?\\s*", num_re, "(?![0-9])"), raw, perl = TRUE) + if (m < 1) return(NA) + before <- substr(raw, 1L, m - 1L) + after <- substr(raw, m, nchar(raw)) + cnt <- function(s, ch) lengths(regmatches(s, gregexpr(ch, s, fixed = TRUE))) + sq_before <- cnt(before, "\"") + if (sq_before %% 2L == 1L) return(TRUE) + if (cnt(before, "\u201c") > cnt(before, "\u201d")) return(TRUE) + opened_before <- sq_before > 0L || cnt(before, "\u201c") > 0L + if (!opened_before) { + if (cnt(after, "\"") %% 2L == 1L) return(TRUE) + if (cnt(after, "\u201d") > cnt(after, "\u201c")) return(TRUE) + } + FALSE +} + +#' Collapse a statistic QUOTED more than once into one row +#' +#' v0.7.14 (d-43ed61). 10.24072/pci.rr.100726 is a peer-review letter whose +#' reviewer quotes one t-test twice to illustrate comma placement, and both +#' copies were scored. v0.6.18 withdrew three dedup rules because two distinct +#' results can share every printed number (v0.6.14: `r(797) = .16` twice in one +#' sentence, different variables) -- see test-v0618-prose-restatement-dedup.R, +#' which names the missing signal: quoted material. Distinct results in running +#' prose are not quoted, so the rule requires ALL of: an identical printed +#' signature, adjacent chunks (location gap <= 2), prose rows only, and EVERY +#' copy inside quotation marks. The first copy is kept and carries +#' `quoted_restatement_count` so the row can say so; nothing is dropped silently. +#' +#' @param parsed A parsed-row tibble (prose rows). +#' @return `parsed` with quoted duplicates removed. +#' @keywords internal +.dedup_quoted_restatements <- function(parsed) { + if (is.null(parsed)) return(parsed) + parsed$quoted_restatement_count <- rep(0L, nrow(parsed)) + if (nrow(parsed) < 2L || + !all(c("raw_text", "stat_value", "location") %in% names(parsed))) { + return(parsed) + } + sig_cols <- intersect(c("test_type", "stat_value", "df1", "df2", "p_symbol", + "p_reported", "effect_reported_name", "effect_reported", + "ciL_reported", "ciU_reported"), names(parsed)) + sig <- do.call(paste, c(lapply(sig_cols, function(cn) { + v <- parsed[[cn]] + if (is.numeric(v)) formatC(v, format = "g", digits = 10) else as.character(v) + }), sep = "\r")) + decs <- if ("stat_value_decimals" %in% names(parsed)) parsed$stat_value_decimals else rep(NA_integer_, nrow(parsed)) + quoted <- vapply(seq_len(nrow(parsed)), function(i) { + isTRUE(.stat_in_quotation(as.character(parsed$raw_text[i]), + as.numeric(parsed$stat_value[i]), decs[i])) + }, logical(1)) + is_table <- if ("from_table" %in% names(parsed)) { + vapply(parsed$from_table, isTRUE, logical(1)) + } else rep(FALSE, nrow(parsed)) + drop <- rep(FALSE, nrow(parsed)) + cand <- which(quoted & !is_table & !is.na(parsed$stat_value)) + for (s in unique(sig[cand])) { + grp <- cand[sig[cand] == s] + if (length(grp) < 2L) next + grp <- grp[order(parsed$location[grp])] + keep <- grp[1L] + for (j in grp[-1L]) { + if (!is.na(parsed$location[j]) && !is.na(parsed$location[keep]) && + parsed$location[j] - parsed$location[keep] <= 2L) { + drop[j] <- TRUE + parsed$quoted_restatement_count[keep] <- parsed$quoted_restatement_count[keep] + 1L + } else { + keep <- j + } + } + } + parsed[!drop, , drop = FALSE] +} + #' Drop docpluck table rows that duplicate a text-parsed (prose) row #' #' v0.6.4: a table cell often restates a result already reported inline in the @@ -1229,7 +1324,7 @@ compute_and_compare_one <- function(row, closest_method = NA_character_, delta_effect_abs = NA_real_, ci_match = as.logical(NA), ci_delta_lower = NA_real_, ci_delta_upper = NA_real_, - ci_check_status = NA_character_, ci_method_match = NA_character_, ci_referent = NA_character_, + ci_check_status = NA_character_, ci_method_match = NA_character_, ci_referent = NA_character_, ci_unverifiable_reason = NA_character_, effect_guard_rejected = effect_guard_rejected, effect_guard_reason = effect_guard_reason, SE_guard_rejected = SE_guard_rejected, @@ -1347,7 +1442,7 @@ compute_and_compare_one <- function(row, } else if (tt == "F") { # F-tests/ANOVA: eta\u00b2, partial eta\u00b2, omega\u00b2, Cohen's f, R\u00b2 # When df1=1, F is equivalent to t\u00b2, so d/g/r are also valid - valid_types <- c("eta", "eta2", "etap2", "omega2", "partial_omega2", "epsilon_squared", "cohens_f", "R2", "f2") + valid_types <- c("eta", "eta2", "etap2", "generalized_eta2", "omega2", "partial_omega2", "epsilon_squared", "cohens_f", "R2", "f2") if (!is.na(df1) && df1 == 1) { valid_types <- c(valid_types, "d", "g", "dz", "dav", "drm", "r") } @@ -1405,6 +1500,15 @@ compute_and_compare_one <- function(row, if (effect_guard_rejected) { uncertainty <- c(uncertainty, effect_guard_reason) } + # v0.7.14 (d-43ed61): say when adjacent quoted copies of this statistic + # were collapsed into this row, so the count change is never silent. + if ("quoted_restatement_count" %in% names(row) && + isTRUE(as.integer(row$quoted_restatement_count[1]) > 0L)) { + uncertainty <- c(uncertainty, sprintf(paste0( + "This statistic is quoted %d times in the same passage (inside quotation ", + "marks, identical printed values); it is scored once, as one result."), + as.integer(row$quoted_restatement_count[1]) + 1L)) + } # v0.7.6: same treatment for a refused standard error -- a suppressed value # must never be indistinguishable from an absent one. if (SE_guard_rejected && !is.na(SE_guard_reason)) { @@ -1436,7 +1540,7 @@ compute_and_compare_one <- function(row, # Define which effect sizes are valid for which test types valid_effects_for_test <- list( t = c("d", "g", "dz", "dav", "drm", "r"), # t-tests: d family, g family, r - F = c("eta2", "etap2", "omega2", "cohens_f", "R2", "f2"), # F-tests: ANOVA effects + F = c("eta2", "etap2", "generalized_eta2", "omega2", "cohens_f", "R2", "f2"), # F-tests: ANOVA effects r = c("r", "R2"), # Correlation tests chisq = c("phi", "V", "h"), # Chi-square tests (h: one-proportion / GOF chi-square) # v0.6.20 (downstream O-5) -- RULING: a z-test reporting an odds ratio is NOT @@ -1546,6 +1650,8 @@ compute_and_compare_one <- function(row, # "ambiguous_b_equals_effect" or "unknown". Set in the regression branch; # stays NA for every other test type, where the question does not arise. ci_referent <- NA_character_ + # v0.7.14: why an interval is UNVERIFIABLE, machine-readably (NA otherwise). + ci_unverifiable_reason <- NA_character_ # Note: uncertainty and assumptions already initialized at top of function (line 275-276) # Do NOT reset them here or earlier messages will be lost @@ -1585,6 +1691,11 @@ compute_and_compare_one <- function(row, NA_character_ } + # v0.7.14 (d-48266c): TRUE when N was back-solved from the reported effect + # size (Welch, group sizes unstated). Recomputing that effect from that N + # reproduces it by construction, so the effect-size check is then circular. + N_from_effect_backsolve <- FALSE + if (tt == "t") { # ------ T-TEST COMPUTATIONS ------ @@ -1635,7 +1746,13 @@ compute_and_compare_one <- function(row, if (is_welch) { uncertainty <- c( uncertainty, - sprintf("Non-integer df (%.2f) suggests Welch's t-test (unequal variances)", df1) + if (abs(df1 - round(df1)) > 0.01) { + sprintf("Non-integer df (%.2f) suggests Welch's t-test (unequal variances)", df1) + } else { + # v0.7.14: an integer df reaches here only because the clause names + # Welch; "Non-integer df (223.00)" was false. + sprintf("The clause states a Welch test; df = %.0f is taken as a (possibly rounded) Welch df", df1) + } ) assumptions <- c(assumptions, "Welch's t-test assumed (unequal variances)") } @@ -1706,11 +1823,42 @@ compute_and_compare_one <- function(row, if (N_from_d >= round(0.85 * min_N_welch) && N_from_d < N) { N_original <- N N <- max(N_from_d, min_N_welch) + # v0.7.14: only an N that IS the back-solved value makes the effect + # check circular. When the back-solve dipped under the Welch floor the + # N is clamped to df + 2 -- a df-derived value the effect was not fit + # to -- so the check stays real and the label says df, not global text. + if (N > N_from_d) { + N_from_df_inference <- TRUE + } else { + N_from_effect_backsolve <- TRUE + } assumptions <- c(assumptions, sprintf("Welch: global N=%d likely from different study; used N=%d from reported d", as.integer(N_original), as.integer(N))) uncertainty <- c(uncertainty, "Global sample size overridden by back-computation from reported effect size") + } else if (N_from_d < N && { + # v0.7.14: the band above tests the POINT value of d. A + # tiny d printed to two decimals (d = 0.04 on t = 0.24) + # back-solves anywhere from ~114 to ~188, so a point + # N_from_d of 144 can miss the band although the Welch + # floor df + 2 reproduces the printed d within its own + # rounding. Then the floor is the evidence-backed N, and + # the scraped study total is not (jesp.2020.104052: + # N = 827 turned d = 0.04 into a WARN). + d_dec <- if (!is.na(effect_reported_decimals)) effect_reported_decimals else 2L + d_at_floor <- 2 * abs(stat) / sqrt(min_N_welch) + abs(d_at_floor - abs(eff_as_d)) <= 0.5 * 10^(-d_dec) + 1e-9 + }) { + N_original <- N + N <- min_N_welch + N_from_df_inference <- TRUE + assumptions <- c(assumptions, + sprintf(paste0("Welch: global N=%d likely from different study; the reported d is ", + "reproduced within its rounding at the Welch minimum N=%d (df + 2)"), + as.integer(N_original), as.integer(N))) + uncertainty <- c(uncertainty, + "Global sample size overridden by the Welch minimum N (df + 2), which reproduces the reported d") } } } else { @@ -1841,6 +1989,18 @@ compute_and_compare_one <- function(row, } } + # v0.7.14 (d-48266c): stated group sizes ARE the sample size. Until 0.7.14 + # a row with parsed n1/n2 but no N fell through to the inference below, and + # on a Welch row that back-solved N from the reported d (circular) while + # "n1 = 131, n2 = 135" sat in the same clause. + N_from_group_sizes <- FALSE + if (is.na(N) && !is.na(n1) && !is.na(n2) && n1 > 0 && n2 > 0) { + N <- n1 + n2 + N_from_group_sizes <- TRUE + assumptions <- c(assumptions, + sprintf("N = n1 + n2 = %d from the stated group sizes", as.integer(N))) + } + # Infer N when not reported (or if replaced above) if (is.na(N) && !is.na(df1) && df1 > 0) { if (is_welch) { @@ -1860,6 +2020,7 @@ compute_and_compare_one <- function(row, if (!is.na(N_from_d)) { N <- N_from_d + N_from_effect_backsolve <- TRUE assumptions <- c(assumptions, sprintf("Welch: estimated N=%d from reported d (validated against df+2=%d)", as.integer(N), as.integer(N_from_df))) @@ -2006,6 +2167,11 @@ compute_and_compare_one <- function(row, N <- omni_N N_independent <- omni_N N_source_omnibus_adopted <- TRUE + # The N is no longer the back-solved value, so the circularity flag must + # not survive it (unreachable today by arithmetic -- a back-solve needs + # N >= 0.85 * (df + 2), an omnibus contrast is ~2/3 of it -- cleared so + # the claim is true by construction; Sonnet consult 2026-09-28). + N_from_effect_backsolve <- FALSE assumptions <- c( assumptions, sprintf( @@ -2024,6 +2190,20 @@ compute_and_compare_one <- function(row, omni_N, df1 ) ) + } else if (posthoc_stated) { + # v0.7.14 (d-43c69d): the effect size cannot decide between the two + # readings (a d near 0 fits both), so the incumbent df + 2 stands -- + # but the row sits under a post-hoc announcement, so say that the + # contrast reading exists rather than publishing one reading as fact. + uncertainty <- c( + uncertainty, + sprintf( + paste0("This t may be a post-hoc contrast reprinting the omnibus ANOVA ", + "error df (df=%.0f); if so its N~%.0f (balanced cells), not ", + "N=%.0f. The reported effect size cannot tell the two apart."), + df1, omni_N, N + ) + ) } } @@ -2172,7 +2352,18 @@ compute_and_compare_one <- function(row, assumptions <- c(assumptions, "Using N for paired computation (df unavailable)") } - if (!is.na(n_paired) && n_paired > 0) { + # v0.7.14 (review finding 2026-09-30): a Welch test compares two + # INDEPENDENT groups by definition, so paired-design effect sizes (dz, dav, + # drm and their g forms) cannot describe it. Offering them let a wrong + # d = 0.70 on Welch t(223) = 8.11 with n1 = 131, n2 = 135 (true ~0.99) + # publish PASS against drm = 0.685. + # An explicit paired label (dz / dav / drm) is the author's own design + # evidence and outranks is_welch, which fires on a fractional df or the bare + # word "Welch": a within-subject Satterthwaite contrast reporting dz, or a + # clause saying "Welch's correction not needed", fell from PASS to NOTE + # (tier-2 cross-model finding, 2026-09-30). + paired_label <- !is.na(canonical_type) && canonical_type %in% c("dz", "dav", "drm") + if (!is.na(n_paired) && n_paired > 0 && (!isTRUE(is_welch) || paired_label)) { dz <- tryCatch(dz_from_t(stat, n_paired), error = function(e) NA_real_) if (!is.na(dz)) { computed_variants$dz <- list( @@ -4723,7 +4914,10 @@ compute_and_compare_one <- function(row, # rho as an effect and flag a spurious WARN -- the defect this fixes). # collabra.126266 (Outcome Bias replication+extension): H2 + H5 mediation rows. check_type <- "extraction_only" - reported_type <- "indirect_effect" + # v0.7.14 (d-4fe38e): the CI form also carries the DIRECT effect (ADE); the + # type is whatever parse.R named, never a hard-coded "indirect_effect". + reported_type <- if (!is.na(effect_reported_name)) effect_reported_name else "indirect_effect" + med_has_sobel <- !is.na(stat) matched_value <- NA_real_ matched_variant <- NA_character_ delta_effect_abs <- NA_real_ @@ -4735,13 +4929,19 @@ compute_and_compare_one <- function(row, # Mirrors the interaction_p handler above. collabra.126266 H2/H5 mediation rows. N <- NA_real_ N_source <- NA_character_ + med_label <- if (identical(reported_type, "direct_effect")) { + "mediation direct effect (ADE)" + } else { + "mediation indirect effect" + } uncertainty <- c(uncertainty, - paste("Bootstrapped mediation indirect effect reported with a Sobel Z: the", - "indirect effect and its bootstrapped CI are surfaced, but the indirect", - "effect is not independently recomputable from the reported statistics", - "(it depends on the a/b path coefficients), so this is extraction-only.", - "Any 'ACME robust until rho = ...' value is a sensitivity-analysis bound,", - "not the reported effect size.")) + paste0("Bootstrapped ", med_label, + if (med_has_sobel) " reported with a Sobel Z" else " reported with a CI", + ": the estimate and its CI are surfaced, but it is not independently ", + "recomputable from the reported statistics (it depends on the path ", + "coefficients or resampled draws), so this is extraction-only.", + if (med_has_sobel) paste(" Any 'ACME robust until rho = ...' value is a", + "sensitivity-analysis bound, not the reported effect size.") else "")) } else if (tt == "mcnemar_or") { # ------ McNEMAR TEST REPORTED AS AN ODDS RATIO ------ # v0.6.11 (E-mcnemar-OR): a McNemar test reported only as a discordant-pairs odds @@ -5896,6 +6096,184 @@ compute_and_compare_one <- function(row, } } + # ------------------------------------------------------------------ + # v0.7.14 (d-cf56d6, d-4efdc1): INCONSISTENT NEEDS AN AFFIRMATIVE PREMISE. + # ------------------------------------------------------------------ + # INCONSISTENT says the PAPER's interval is wrong. That follows from "no + # candidate matched" only if the candidates were comparable with what the + # paper printed: the same tail, the same group allocation, the same quantity. + # v0.7.9 withdrew the status escalation after it accused 3 correct papers out + # of 3, but this column kept publishing INCONSISTENT on the same premises. + # Three ways the premise is false are recognisable from the row itself, and + # each is checked here. Each can only move INCONSISTENT to UNVERIFIABLE -- a + # MATCH / PLAUSIBLE is affirmative evidence and is never touched -- and each + # is scoped so that a genuinely wrong interval of the same shape still fails + # (the controls in test-v0714-ci-verdict-premise.R). + # + # 1. REFERENT (widens ci_referent beyond regression rows, as CLAUDE.md says it + # must be). An interval on the reported effect contains it; one that does + # not -- and whose sign-flip does not either, so it is not a dropped minus -- + # cannot be an interval on that effect. For the d family, whose intervals + # are near-symmetric by construction (Wald exactly; noncentral t skews by a + # few percent of the half-width even at d = 3 with n = 5 per group), a + # midpoint far from d says the same thing. Asymmetric families (r, OR, + # eta2) use containment only. The impossible-value finding is NOT + # laundered: it stays in `estimate_outside_ci`, the field that owns it. + if (is.na(ci_referent) && !is.na(ciL_rep) && !is.na(ciU_rep) && + is.finite(ciL_rep) && is.finite(ciU_rep) && ciL_rep < ciU_rep) { + ref_est <- if (!is.na(effect_reported)) effect_reported else NA_real_ + d_family_ref <- !is.na(canonical_type) && + canonical_type %in% c("d", "g", "dz", "dav", "drm") + mid_far <- FALSE + if (d_family_ref && !is.na(ref_est)) { + decs_mid <- c(effect_reported_decimals, ciL_reported_decimals, ciU_reported_decimals) + decs_mid <- decs_mid[!is.na(decs_mid)] + eps_mid <- if (length(decs_mid) > 0) 10^(-min(decs_mid)) else 0.01 + half_w <- (ciU_rep - ciL_rep) / 2 + mid_far <- abs((ciL_rep + ciU_rep) / 2 - ref_est) > 0.25 * half_w + eps_mid + } + if (isTRUE(estimate_outside_ci) || mid_far) { + ci_referent <- "not_effect_reported" + } + } + # 0. ESTIMAND NOT COMPUTABLE (d-b1c193). A generalized eta-squared interval + # was graded against PARTIAL eta-squared intervals -- a different + # estimand -- beside a row message saying generalized eta-squared cannot be + # computed from F and df. Any verdict from that comparison (collabra.126266: + # 8 INCONSISTENT, 5 PLAUSIBLE, 1 MATCH) says nothing about the paper. + if (!is.na(canonical_type) && canonical_type == "generalized_eta2" && + !is.na(ci_check_status) && + ci_check_status %in% c("MATCH", "PLAUSIBLE", "INCONSISTENT")) { + ci_check_status <- "UNVERIFIABLE" + ci_unverifiable_reason <- "estimand_not_computable" + ci_match <- as.logical(NA) + uncertainty <- c(uncertainty, paste0( + "The reported interval is on generalized eta-squared, which cannot be ", + "computed from F and df, so it was not graded (the intervals computed ", + "here are on partial eta-squared, a different quantity).")) + } + if (identical(ci_check_status, "INCONSISTENT")) { + premise_reason <- NA_character_ + premise_note <- NA_character_ + + if (identical(ci_referent, "not_effect_reported")) { + premise_reason <- "referent_not_effect" + premise_note <- paste0( + "The reported interval cannot be an interval on the reported effect ", + "(it is not centred on it, or does not contain it), so it was not ", + "graded as one. It may belong to another quantity, such as an ", + "unstandardized difference.") + } + + # 1b. NO ESTIMATE. With no reported effect size the interval's referent is + # unstated, and grading it as a standardized interval assumes the answer. + # Measured by v0.7.9 (test-v079-ci-severity-tiers.R): 11 of the 15 + # corpus rows of this shape come from one paper whose intervals are raw + # mean differences ("95%CI difference [0.102, 0.776]"). A correlation is + # exempt: its statistic IS its estimate (adopted into effect_reported + # only later in the pipeline, so it is NA here). + if (is.na(premise_reason) && is.na(effect_reported) && + !(!is.na(tt) && tt == "r")) { + premise_reason <- "no_estimate_parsed" + premise_note <- paste0( + "No effect size was read beside this interval (none is printed, or it ", + "could not be parsed), so what it is an interval OF is not established ", + "(often an unstandardized difference); it was not graded as a ", + "standardized one.") + } + + # 2. TAIL. Every candidate computed here is two-sided. A one-sided interval + # either says so or runs to the parameter's limit (r = 1 / -1). + if (is.na(premise_reason)) { + raw_txt <- if ("raw_text" %in% names(row) && length(row$raw_text) > 0) { + as.character(row$raw_text[1]) + } else "" + says_one_sided <- !is.na(raw_txt) && + grepl("one[- ]?(sided|tailed)", raw_txt, ignore.case = TRUE) + is_r_row <- (!is.na(tt) && tt == "r") || + (!is.na(canonical_type) && canonical_type == "r") + at_r_limit <- is_r_row && (isTRUE(ciU_rep >= 1) || isTRUE(ciL_rep <= -1)) + if (says_one_sided || at_r_limit) { + premise_reason <- "one_sided_interval" + premise_note <- paste0( + "The reported interval is one-sided (", if (says_one_sided) + "stated as such" else "it runs to the parameter's limit", + "); every interval computed here is two-sided, so they are not ", + "comparable and the interval was not graded.") + } + } + + # 3. ALLOCATION. With group sizes unstated, the candidates were built at an + # assumed n1 = n2 = N/2. The equal split gives the NARROWEST interval for + # a given t and N, so an unequal split can reproduce a wider one. Search + # every split (normal approximation, the Wald form ci_d_ind() uses as its + # large-sample engine): if one reproduces both bounds within tol_ci, the + # mismatch is the assumption's, not the paper's. If none does -- e.g. an + # interval narrower than the equal split's -- INCONSISTENT stands. + if (is.na(premise_reason) && !is.na(ci_method_match) && + grepl("^(d|g)_ind_equalN", ci_method_match) && + !is.na(effect_reported) && + !is.na(tt) && tt == "t" && !is.na(stat) && + (is.na(n1) || is.na(n2))) { + N_alloc <- if (!is.na(N)) as.integer(round(N)) else + if (!is.na(df1)) as.integer(round(df1)) + 2L else NA_integer_ + if (!is.na(N_alloc) && N_alloc >= 6L && N_alloc <= 100000L) { + n1s <- 2:(N_alloc - 2L) + n2s <- N_alloc - n1s + d_a <- stat * sqrt(1 / n1s + 1 / n2s) + if (startsWith(ci_method_match, "g_ind")) { + J_a <- hedges_J(N_alloc - 2L) + if (!is.na(J_a)) d_a <- d_a * J_a + } + se_a <- sqrt((n1s + n2s) / (n1s * n2s) + d_a^2 / (2 * (n1s + n2s))) + z_a <- stats::qnorm(1 - (1 - ci_level_used) / 2) + # The split must explain the WHOLE row -- the printed d as well as the + # interval. Matching the interval alone searches a one-dimensional + # family of intervals for a coincidence (Sonnet consult 2026-09-28: + # d = 0.20 printed with the n1 = 10 / n2 = 90 interval of d = 0.10 was + # reported as an unstated allocation). + fits <- which(abs(d_a - z_a * se_a - ciL_rep) <= tol_ci & + abs(d_a + z_a * se_a - ciU_rep) <= tol_ci & + abs(d_a - abs(effect_reported)) <= tol_ci) + if (length(fits) > 0) { + best <- fits[which.min(abs(d_a[fits] - z_a * se_a[fits] - ciL_rep) + + abs(d_a[fits] + z_a * se_a[fits] - ciU_rep))] + premise_reason <- "unstated_allocation" + premise_note <- sprintf(paste0( + "Group sizes are not stated and the interval was graded at an ", + "assumed equal split; the reported interval is reproduced at ", + "n1 = %d, n2 = %d (or the mirror split), so the mismatch is the ", + "assumption's, not evidence against the paper."), + as.integer(min(n1s[best], n2s[best])), + as.integer(max(n1s[best], n2s[best]))) + } + } + } + + if (!is.na(premise_reason)) { + ci_check_status <- "UNVERIFIABLE" + ci_unverifiable_reason <- premise_reason + # `ci_match` is a separate contract column; leaving it FALSE would + # republish the withdrawn verdict through another field (same reasoning + # as the sample-size ambiguity cap above). + ci_match <- as.logical(NA) + uncertainty <- c(uncertainty, premise_note) + } + } + if (identical(ci_check_status, "UNVERIFIABLE") && is.na(ci_unverifiable_reason)) { + # Name the pre-existing causes too, so the column is never NA on an + # UNVERIFIABLE row: a consumer filtering on it must not read NA as "none". + ci_unverifiable_reason <- if (is.na(ciL_rep) || is.na(ciU_rep)) { + "one_bound_reported" + } else if (isTRUE(corr_matched_df_ambiguous)) { + "sample_size_ambiguous" + } else if (isTRUE(paired_ci_independent_approx_only)) { + "paired_design_independent_approximation" + } else { + "no_comparable_interval_computed" + } + } + # If no CI was matched but we computed some, extract the "primary" CI for display if (is.na(computed_ciL) || is.na(computed_ciU)) { primary_effects <- if (tt == "t") { @@ -6047,7 +6425,7 @@ compute_and_compare_one <- function(row, # that to `delta <= tol_ci`, so until now PLAUSIBLE and INCONSISTENT produced # the IDENTICAL published status -- a CI wrong by 0.07 and one wrong by 8.0 # both read NOTE, while the effect-size arm grades PASS / WARN (3x) / ERROR - # (5x) on the same row. Downstream, SciMeto maps NOTE to severity "info" by + # (5x) on the same row. Downstream, a consumer maps NOTE to severity "info" by # design, so an interval contradicting every computable method reached the # author as a neutral chip that counted toward no issue. # @@ -6190,6 +6568,14 @@ compute_and_compare_one <- function(row, "the check rather than a finding about the paper.")) } } + # v0.7.14: an estimate outside its own interval is an impossible combination + # of published numbers whatever the interval's referent, so the row can never + # present as a clean PASS. Until 0.7.14 the CI mismatch above supplied the + # NOTE; now that such an interval is UNVERIFIABLE (not graded as the effect's + # own), the invariant supplies it directly. Same opt-out as the CI arm. + if (ci_affects_status && isTRUE(estimate_outside_ci) && status == "PASS") { + status <- "NOTE" + } # Determine check_type (Issue 4): what drove the status classification check_type <- if (has_effect_reported && !is.na(delta_effect_abs)) { @@ -6335,7 +6721,10 @@ compute_and_compare_one <- function(row, # Route to NOTE with explanation rather than false ERROR. if (!is.na(canonical_type) && canonical_type == "generalized_eta2" && check_type == "effect_size") { - check_type <- "extraction_only" + # v0.7.14 (d-b1c193): the EFFECT SIZE is unverifiable, the p-value is not. + # Falling to extraction_only made a row whose p had been recomputed and + # matched publish SKIP ("nothing was checked"), which is false. + check_type <- if (!is.na(p_reported)) "p_value" else "extraction_only" matched_value <- NA_real_ matched_variant <- NA_character_ delta_effect_abs <- NA_real_ @@ -6346,6 +6735,24 @@ compute_and_compare_one <- function(row, "Only partial eta-squared is computable from F and df.")) } + # v0.7.14 (d-48266c): an N back-solved from the reported effect size cannot + # then verify that effect size -- d recomputed from N = 4t^2/d^2 IS d. Until + # 0.7.14 this published PASS for any d at all on a Welch row with unstated + # group sizes (measured: d = 0.70 on t(223) = 8.11, true ~0.99, was PASS at + # N = 537). The p-value does not depend on N and is still checked. + if (isTRUE(N_from_effect_backsolve) && check_type == "effect_size") { + check_type <- if (!is.na(p_reported)) "p_value" else "extraction_only" + matched_value <- NA_real_ + matched_variant <- NA_character_ + delta_effect_abs <- NA_real_ + if (status %in% c("ERROR", "WARN", "PASS")) status <- "NOTE" + uncertainty <- c(uncertainty, paste0( + "The effect size is NOT verified: group sizes are not stated, a Welch df ", + "only bounds N from below, and the N used here was back-solved from the ", + "reported effect size itself, so recomputing the effect from it would ", + "reproduce the reported value by construction.")) + } + # CI mismatch overrides check_type when the CI arm is what moved the status. # # v0.7.9: this used to test `status == "NOTE"`, which was the ONLY status the @@ -8744,6 +9151,15 @@ compute_and_compare_one <- function(row, if (exists("N_source_omnibus_adopted") && isTRUE(N_source_omnibus_adopted)) { N_source_out <- "omnibus_df_contrast" } + # v0.7.14: an N back-solved from the reported effect size is neither observed + # nor a df derivation, and it is what makes that effect unverifiable. + if (isTRUE(N_from_effect_backsolve)) { + N_source_out <- "effect_backsolved" + } + if (exists("N_from_group_sizes") && isTRUE(N_from_group_sizes) && + (is.na(N_source_out[1]) || as.character(N_source_out[1]) == "not_found")) { + N_source_out <- "group_sizes" + } tibble::tibble( location = row$location, @@ -8869,6 +9285,7 @@ compute_and_compare_one <- function(row, # explicitly so a consumer can DISTINGUISH a suppressed effect size from # an absent one, which needs a column to filter on. ci_referent = ci_referent, + ci_unverifiable_reason = ci_unverifiable_reason, effect_guard_rejected = effect_guard_rejected, effect_guard_reason = effect_guard_reason, SE_guard_rejected = SE_guard_rejected, @@ -9442,6 +9859,9 @@ check_text <- function(text, parsed <- parsed[parsed$test_type %in% stats, ] } + # v0.7.14 (d-43ed61): one statistic quoted twice is one result. + parsed <- .dedup_quoted_restatements(parsed) + # v0.6.4: map docpluck structured table rows and append them. Bound AFTER the # whitelist filter so a "table_estimate" row is never dropped, then # deduplicated against any identical prose row (a table cell that restates a @@ -9636,7 +10056,7 @@ check_text <- function(text, closest_method = NA_character_, delta_effect_abs = NA_real_, ci_match = as.logical(NA), ci_delta_lower = NA_real_, ci_delta_upper = NA_real_, - ci_check_status = NA_character_, ci_method_match = NA_character_, ci_referent = NA_character_, + ci_check_status = NA_character_, ci_method_match = NA_character_, ci_referent = NA_character_, ci_unverifiable_reason = NA_character_, effect_guard_rejected = FALSE, effect_guard_reason = NA_character_, SE_guard_rejected = FALSE, SE_guard_reason = NA_character_, df_guard_rejected = FALSE, df_guard_reason = NA_character_, diff --git a/R/compute.R b/R/compute.R index 2c4cee1..a8c44dd 100644 --- a/R/compute.R +++ b/R/compute.R @@ -796,28 +796,17 @@ partial_eta2_from_F <- function(F_val, df1, df2) { (F_val * df1) / (F_val * df1 + df2) } -# Generalized eta-squared (eta_G^2) from F-statistic -# Formula: eta_G^2 = SS_effect / (SS_effect + SS_error + SS_subjects) -# This requires more information than just F, df1, df2 -# Approximation for between-subjects: same as eta^2 -# For within-subjects: requires knowledge of design structure -generalized_eta2_from_F <- function(F_val, df1, df2, design = "between") { - # Ensure scalar inputs - F_val <- as.numeric(F_val[1]) - df1 <- as.numeric(df1[1]) - df2 <- as.numeric(df2[1]) - - if (any(is.na(c(F_val, df1, df2))) || F_val < 0 || df1 <= 0 || df2 <= 0) { - return(NA_real_) - } - # For between-subjects, generalized eta^2 ~= eta^2 - if (design == "between") { - return(eta2_from_F(F_val, df1, df2)) - } - # For within-subjects, we need more information (SS_subjects) - # Return NA with note that more info needed - NA_real_ -} +# Generalized eta-squared (eta_G^2) is deliberately NOT computed from F. +# eta_G^2 = SS_effect / (SS_effect + SS_error + SS of every MEASURED factor and of +# subjects) (Olejnik & Algina, 2003; Bakeman, 2005). Which terms enter the +# denominator is a property of the design -- which factors were manipulated, +# which measured, which within-subjects -- and F, df1, df2 carry none of it. +# v0.7.14 (d-b1c193): this used to fall through to the between-subjects formula +# for an UNCLEAR design, publishing partial eta-squared under the generalized +# name (collabra.126266: identical on all 15 F rows) beside a row message saying +# it could not be computed. Even a detected "between" design does not license +# it: equality with partial eta-squared needs every factor to be manipulated, +# which the text does not establish. # Omega-squared (omega^2) from F-statistic # Formula: omega^2 = (F * df1 - df1) / (F * df1 + df2 + 1) @@ -948,19 +937,9 @@ compute_all_anova_effects <- function(F_val, df1, df2, design = "unclear") { results$omega2 <- omega2_from_F(F_val, df1, df2) results$cohens_f <- cohens_f_from_F(F_val, df1, df2) - # Generalized eta^2 depends on design - if (design == "between") { - results$generalized_eta2 <- generalized_eta2_from_F(F_val, df1, df2, "between") - } else if (design == "within") { - # Would need SS_subjects for accurate computation - results$generalized_eta2 <- NA_real_ - } else if (design == "mixed") { - # Would need SS_subjects and design structure - results$generalized_eta2 <- NA_real_ - } else { - # Design unclear - compute what we can - results$generalized_eta2 <- generalized_eta2_from_F(F_val, df1, df2, "between") - } + # Generalized eta^2 is never computable from F and df alone (see the note + # above omega2_from_F); `design` is kept in the signature for callers. + results$generalized_eta2 <- NA_real_ results } diff --git a/R/parse.R b/R/parse.R index fc24b06..81064d3 100644 --- a/R/parse.R +++ b/R/parse.R @@ -1835,6 +1835,27 @@ parse_text <- function(text, context_window_size = 2) { # source stays pure ASCII (R CMD check warns on non-ASCII in code). chi_tok <- "(?:chi-?square|Chi-?square|chi\\s*\\^?2|\u03c7\\s*\\^?2|\u03c7\u00b2|X\\s*\\^?2|X\u00b2)" + # v0.6.16 (E10 / E-bare-mediation-ci): a BOOTSTRAPPED mediation effect is + # reported with a CI (and usually a p), never a Sobel Z -- so the Sobel-anchored + # pattern above never fired and the result was silently dropped. Cognition & + # Emotion 10.1080/02699931.2024.2434156 reports "The average direct effect was + # 0.15, 95% CI [-0.13 to 0.45], p = .3, whereas the bootstrapped unstandardised + # indirect effect (Average Causal Mediation Effect, ACME) was 0.67, 95% CI + # [0.47-0.89], p < .001." -- two extractable results, zero rows emitted. The + # text WAS delivered by the extractor (verified in the render), so this is an + # ESCImate parse defect, not a docpluck gap. Found by the Sonnet canary audit + # 2026-08-04. + # + # Anchored on an explicit mediation-effect NAME so an ordinary "the effect was + # 0.15, 95% CI [...]" sentence cannot be mistaken for a mediation path. The CI + # separator accepts "to", "-", or "," (all three appear in this paper alone). + pat_mediation_ci <- paste0( + "\\b(?:average\\s+(?:causal\\s+mediation|direct)\\s+effect|ACME|ADE|", + "indirect\\s+effect|direct\\s+effect)\\b", + "[^.]{0,120}?\\bwas\\s+([-+]?\\d*\\.?\\d+)", + "[^.]{0,40}?\\b(\\d{2})\\s*%\\s*CI\\s*[\\[\\(]\\s*", + "([-+]?\\d*\\.?\\d+)\\s*(?:to|,|-)\\s*([-+]?\\d*\\.?\\d+)\\s*[\\]\\)]" + ) # Sub-chunk splitting: when a sentence contains multiple test statistics, # split it so each sub-chunk has exactly one test statistic. # This prevents str_match() from silently dropping 2nd/3rd/4th matches. @@ -1923,6 +1944,26 @@ parse_text <- function(text, context_window_size = 2) { })) chunks <- chunks[nchar(trimws(chunks)) > 0] + # v0.7.14 (d-4fe38e): a sentence reporting TWO named mediation effects with + # their own CIs -- "The average direct effect was 0.15, 95% CI [...], p = .3, + # whereas the ... indirect effect (ACME) was 0.67, 95% CI [...], p < .001" -- + # carries no test statistic, so the splitter above leaves it whole and + # str_match() takes only the FIRST effect (the direct one), which v0.6.16 then + # labelled "indirect_effect" and the ACME was lost. Split at each complete + # mediation-effect-with-CI clause. Requires 2+ complete clauses, so a single + # effect (or the Sobel-Z form) is untouched. + chunks <- unlist(lapply(chunks, function(chunk) { + mm <- gregexpr(pat_mediation_ci, chunk, perl = TRUE, ignore.case = FALSE)[[1]] + if (length(mm) < 2L || mm[1] == -1L) return(chunk) + pos <- as.integer(mm) + out <- character(length(pos)) + for (j in seq_along(pos)) { + out[j] <- substr(chunk, pos[j], if (j < length(pos)) pos[j + 1L] - 1L else nchar(chunk)) + } + if (pos[1] > 1L) out[1] <- paste0(substr(chunk, 1L, pos[1] - 1L), out[1]) + out[nchar(trimws(out)) > 0] + })) + if (length(chunks) == 0) { return(tibble::tibble( location = integer(0), @@ -2185,27 +2226,7 @@ parse_text <- function(text, context_window_size = 2) { "indirect\\s+effect\\b.{0,40}?\\bwas\\s+([-+]?\\d*\\.?\\d+)", ".{0,80}?\\bSobel\\s+[Zz]\\s*=\\s*([-+]?\\d*\\.?\\d+)" ) - # v0.6.16 (E10 / E-bare-mediation-ci): a BOOTSTRAPPED mediation effect is - # reported with a CI (and usually a p), never a Sobel Z -- so the Sobel-anchored - # pattern above never fired and the result was silently dropped. Cognition & - # Emotion 10.1080/02699931.2024.2434156 reports "The average direct effect was - # 0.15, 95% CI [-0.13 to 0.45], p = .3, whereas the bootstrapped unstandardised - # indirect effect (Average Causal Mediation Effect, ACME) was 0.67, 95% CI - # [0.47-0.89], p < .001." -- two extractable results, zero rows emitted. The - # text WAS delivered by the extractor (verified in the render), so this is an - # ESCImate parse defect, not a docpluck gap. Found by the Sonnet canary audit - # 2026-08-04. - # - # Anchored on an explicit mediation-effect NAME so an ordinary "the effect was - # 0.15, 95% CI [...]" sentence cannot be mistaken for a mediation path. The CI - # separator accepts "to", "-", or "," (all three appear in this paper alone). - pat_mediation_ci <- paste0( - "\\b(?:average\\s+(?:causal\\s+mediation|direct)\\s+effect|ACME|ADE|", - "indirect\\s+effect|direct\\s+effect)\\b", - "[^.]{0,120}?\\bwas\\s+([-+]?\\d*\\.?\\d+)", - "[^.]{0,40}?\\b(\\d{2})\\s*%\\s*CI\\s*[\\[\\(]\\s*", - "([-+]?\\d*\\.?\\d+)\\s*(?:to|,|-)\\s*([-+]?\\d*\\.?\\d+)\\s*[\\]\\)]" - ) + # (pat_mediation_ci is defined above the sub-chunk splitter, which uses it.) # v0.5.16: clinical-trial risk ratio with two-proportion slash counts. # Form: "/ (%) versus|and|vs / (%) ... RR ; # 95% CI to ; p[-]?(value)? = ". The two-proportion clause @@ -3852,7 +3873,16 @@ parse_text <- function(text, context_window_size = 2) { effect_reported <- d_ci_nostat_effect effect_reported_decimals <- d_ci_nostat_effect_decimals } else if (isTRUE(is_mediation_indirect) && !is.na(mediation_indirect_effect)) { - effect_name <- "indirect_effect" + # v0.7.14 (d-4fe38e): name the effect the clause names. The CI form's + # anchor also matches the DIRECT effect (ADE), which v0.6.16 published as + # "indirect_effect". + effect_name <- if (!all(is.na(m_mediation_ci)) && all(is.na(m_mediation_indirect)) && + grepl("^(?:average\\s+direct\\s+effect|ADE|direct\\s+effect)\\b", + m_mediation_ci[1], perl = TRUE, ignore.case = TRUE)) { + "direct_effect" + } else { + "indirect_effect" + } effect_reported <- mediation_indirect_effect effect_reported_decimals <- mediation_indirect_effect_decimals } else if (!all(is.na(m_f2))) { @@ -3870,7 +3900,12 @@ parse_text <- function(text, context_window_size = 2) { effect_name <- "generalized_eta2" effect_reported <- numify(m_eta2_corrupted[2]) effect_reported_decimals <- count_decimal_places(m_eta2_corrupted[2]) - effect_fallback <- TRUE # Flag as uncertain extraction + # v0.7.14 (d-b1c193): only a CORRUPTED form ("2G", "n2G") is an uncertain + # symbol. An explicit label ("generalized eta-squared", "etaG2", Greek + # eta with a G subscript) is not, and flagging it told the reader the + # extraction was suspect when it was not. + effect_fallback <- !grepl( + "^(?:[Gg]eta|generalized|\u03b7[Gg]|etaG2)", m_eta2_corrupted[1], perl = TRUE) } else if (!all(is.na(m_eta2))) { effect_name <- "eta2" effect_reported <- numify(m_eta2[2]) diff --git a/cran-comments.md b/cran-comments.md index b1c4100..09b8302 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,7 +1,7 @@ ## Submission This is an update of 'effectcheck' from 0.2.3 (the current CRAN release) to -0.7.13. Development has been active across 0.2.4-0.7.13 -- new test types, +0.7.14. Development has been active across 0.2.4-0.7.14 -- new test types, nonparametric and regression support, confidence-interval computation, and many parser and consistency fixes -- and the most significant change is structural, in 0.4.0 (see "Breaking change" below). Every intervening version @@ -11,10 +11,14 @@ is documented in NEWS.md. * Windows 11, R 4.4.0 (local), `R CMD check --as-cran --no-manual` * win-builder, R Under development (unstable) (2026-08-31 r90457 ucrt), - run on **version 0.7.8** (NOT this 0.7.13 submission candidate) on + run on **version 0.7.8** (NOT this 0.7.14 submission candidate) on 2026-09-02: **Status: OK** (0 errors | 0 warnings | 0 notes; install 26s, check 370s). See the correction immediately below -- this archived run is evidence about 0.7.8, not about this candidate. +* win-builder, R Under development (unstable) (2026-09-25 r90590 ucrt), + run on **version 0.7.13** (NOT this 0.7.14 submission candidate): + **Status: OK** (`.winbuilder-logs/R-devel_2026-09-30_00check.log`, line 12 + names version 0.7.13, `Status: OK` at line 67). Evidence about 0.7.13 only. **CORRECTED 2026-09-11 by reading the log instead of the sentence describing it. The archived win-builder run checked 0.7.8 — not 0.7.9, and not this @@ -28,47 +32,49 @@ with `Status: OK` at line 67. The previous text here claimed the log recorded `'0.7.9'`. It does not, and never did; the wrong number entered in the 0.7.9 release commit and stood through two subsequent edits of this very paragraph — including one that *emphasised* the false version while believing it was -protecting the evidence. **None of 0.7.9, 0.7.10, 0.7.11, 0.7.12 or 0.7.13 has ever been checked by -win-builder.** The log is archived here because win-builder deletes its own +protecting the evidence. **Of 0.7.9 through 0.7.14, only 0.7.13 has been checked by +win-builder (2026-09-30, Status OK, archived); this 0.7.14 candidate has not.** The log is archived here because win-builder deletes its own result directories after roughly 72 hours. **What actually separates the checked version from this candidate is large, not two lines.** Measured `v0.7.8..HEAD` over the package sources: ``` -effectcheck/R/check.R | 1034 ++++++++++++++++++++++++++++++++--- -effectcheck/R/compute.R | 2 +- +effectcheck/R/check.R | 1454 ++++++++++++++++++++++++++++++++--- +effectcheck/R/compute.R | 51 +- effectcheck/R/effectcheck-class.R | 57 +- effectcheck/R/effectcheck-package.R | 2 +- -effectcheck/R/parse.R | 223 +++++++- +effectcheck/R/parse.R | 329 ++++++-- effectcheck/R/report.R | 6 +- -6 files changed, 1214 insertions(+), 110 deletions(-) +6 files changed, 1716 insertions(+), 183 deletions(-) ``` -**A fresh win-builder run on the 0.7.13 tarball is REQUIRED before submission +**A fresh win-builder run on the 0.7.14 tarball is REQUIRED before submission and has NOT been made.** Local `R CMD check --as-cran --no-manual` on this -candidate is the only evidence behind the 0-warning claim. Run on 0.7.13 on 2026-09-27, on the -built tarball `effectcheck_0.7.13.tar.gz` (HEAD c942c9e): `Status: 1 NOTE`, 0 errors, 0 warnings. +candidate is the only evidence behind the 0-warning claim. Run on 0.7.14 on 2026-09-30, on the +built tarball `effectcheck_0.7.14.tar.gz` (HEAD 03af3d6; the log reads "this is package +'effectcheck' version '0.7.14'"): `Status: 1 NOTE`, 0 errors, 0 warnings. The NOTE is `unable to verify current time` (no network time source). (On 0.7.12, 2026-09-25, a second NOTE -- one example at 6.14 s elapsed, CPU 0.81 s, under heavy machine load -- did not recur.) A local check is not a substitute for win-builder on a submission, and an -archived log for a version three releases back is not evidence about this one. +archived log for an earlier version is not evidence about this one. `--no-manual` is used locally because the local machine has no LaTeX, so the PDF-manual step fails with "pdflatex is not available" -- a toolchain gap rather than an Rd defect. That gap does not apply to the archived win-builder -run above, which reports `checking PDF version of manual ... OK` -- but that -run checked 0.7.8, so it is evidence the gap is toolchain-only, not evidence -the manual builds clean for this 0.7.13 candidate. +runs above, which report `checking PDF version of manual ... OK` -- but those +runs checked 0.7.8 and 0.7.13, so they are evidence the gap is toolchain-only, not evidence +the manual builds clean for this 0.7.14 candidate. ## R CMD check results -win-builder R-devel, **version 0.7.8 (archived run, NOT this 0.7.13 -candidate)**: 0 errors | 0 warnings | 0 notes. **This 0.7.13 submission +win-builder R-devel, **version 0.7.8 (archived run, NOT this 0.7.14 +candidate)**: 0 errors | 0 warnings | 0 notes. win-builder R-devel, **version +0.7.13 (archived run, NOT this candidate)**: Status OK. **This 0.7.14 submission candidate has not yet been win-builder-checked** -- see "A fresh win-builder run ... REQUIRED" above. -Local Windows 11 / R 4.4.0 (0.7.13, 2026-09-27): 0 errors | 0 warnings | 1 note: +Local Windows 11 / R 4.4.0 (0.7.14, 2026-09-30): 0 errors | 0 warnings | 1 note: "checking for future file timestamps ... unable to verify current time" -- a transient failure to reach the time server from the local check machine, not a package problem; it does not appear on win-builder. @@ -78,13 +84,13 @@ line; there are no misspelling or URL findings. ## Test suite -1327 test_that blocks across 160 test files; all pass with 0 failures, +1364 test_that blocks across 166 test files; all pass with 0 failures, 0 errors, and 0 warnings under `R CMD check` (the test runner exits non-zero on any failure, error or warning, and `checking tests ... OK`). Both counts were measured in the committed tree that produced this tarball -(0.7.13, HEAD c942c9e). `scripts/verify-executed-test-count.R` on 2026-09-27 -EXECUTED 1327 blocks with 0 failed, 0 errors and 0 skipped, so the blocks +(0.7.14, HEAD 03af3d6). `scripts/verify-executed-test-count.R` on 2026-09-30 +EXECUTED 1364 blocks with 0 failed, 0 errors and 0 skipped, so the blocks EXECUTED rather than merely existing. The archived win-builder run above (`checking tests ... [228s] OK`) is evidence about 0.7.8's suite, not this one. diff --git a/inst/docpluck-contract/boundary_contract_golden.json b/inst/docpluck-contract/boundary_contract_golden.json index 70537aa..aaba24f 100644 --- a/inst/docpluck-contract/boundary_contract_golden.json +++ b/inst/docpluck-contract/boundary_contract_golden.json @@ -2,16 +2,16 @@ "schema": "docpluck-boundary-contract/1", "consumer": "ESCImate worker -- worker/docpluck_client.R", "what_this_pins": "The field set and statistical-symbol surface docpluck returns for two committed fixtures, captured THROUGH THE PRODUCTION CLIENT. Refresh with: Rscript worker/tests/docpluck-contract.R --refresh", - "captured_utc": "2026-09-27T12:38:27Z", + "captured_utc": "2026-09-29T14:05:06Z", "docpluck": { - "docpluck_version": "2.4.145", - "docpluck_identity": "2.4.145/deb69ae/clean", + "docpluck_version": "2.4.147", + "docpluck_identity": "2.4.147/f0237ba/clean", "version_crosscheck": "agreed", "service_version": "1.8.0", "sectioning_version": "1.2.5", - "table_extraction_version": "2.4.17", - "normalization_version": "1.9.69", - "library_repo_sha": "deb69ae", + "table_extraction_version": "2.4.18", + "normalization_version": "1.9.70", + "library_repo_sha": "f0237ba", "service_repo_sha": "a96fff2", "captured_from": "http://127.0.0.1:6117" }, @@ -76,7 +76,7 @@ } }, "provenance_only": { - "normalization_steps_applied": ["C0_furniture_control_strip", "H0_header_banner_strip", "T0_toc_dot_leader_strip", "P0_page_footer_strip", "P0r_recurring_running_header_strip", "P1_frontmatter_metadata_leak_strip", "H0b_letterspaced_label_collapse", "W0_watermark_strip", "W0b_minus_sign_recovery", "W0c_lt_operator_recovery", "W0o_lt_as_b_recovery", "W0d_minus_ci_pairing", "W0j_prose_minus_recovery", "W0q_ci_upper_detached_minus", "W0e_pua_glyph_recovery", "S1_encoding_validation", "S2_accent_recombination", "S3_ligature_expansion", "S4_quote_normalization", "S5_dash_normalization", "S5a_fffd_context_recovery", "S5b_fffd_comparison_recovery", "S6_whitespace_normalization", "S7_hyphenation_repair", "S7a_space_broken_compound_rejoin", "S8_line_break_joining", "A1_stat_linebreak_repair", "P0_standalone_page_numbers", "S9_header_footer_removal", "A3b_stat_bracket_to_paren", "A4_ci_delimiter_harmonization", "A5_math_symbol_normalization", "A6_footnote_removal", "A7_doi_rejoin", "R2_inline_pgnum_scrub", "R3_continuation_join", "LateJoin_line_break_rejoin", "H0r_header_banner_restrip", "P1r_frontmatter_leak_restrip", "P0r_page_footer_restrip", "G5c2_split_numbered_heading_rejoin"], + "normalization_steps_applied": ["C0_furniture_control_strip", "H0_header_banner_strip", "T0_toc_dot_leader_strip", "P0_page_footer_strip", "P0r_recurring_running_header_strip", "P1_frontmatter_metadata_leak_strip", "H0b_letterspaced_label_collapse", "W0_watermark_strip", "W0b_minus_sign_recovery", "W0c_lt_operator_recovery", "W0o_lt_as_b_recovery", "W0d_minus_ci_pairing", "W0j_prose_minus_recovery", "W0q_ci_upper_detached_minus", "W0e_pua_glyph_recovery", "S1_encoding_validation", "S2_accent_recombination", "S3_ligature_expansion", "S4_quote_normalization", "S5_dash_normalization", "S6_whitespace_normalization", "S7_hyphenation_repair", "S7a_space_broken_compound_rejoin", "S8_line_break_joining", "A1_stat_linebreak_repair", "P0_standalone_page_numbers", "S9_header_footer_removal", "A3b_stat_bracket_to_paren", "A4_ci_delimiter_harmonization", "A5_math_symbol_normalization", "A6_footnote_removal", "A7_doi_rejoin", "R2_inline_pgnum_scrub", "R3_continuation_join", "LateJoin_line_break_rejoin", "H0r_header_banner_restrip", "P1r_frontmatter_leak_restrip", "P0r_page_footer_restrip", "G5c2_split_numbered_heading_rejoin"], "pdf_pages": "1" } } diff --git a/man/check_text.Rd b/man/check_text.Rd index 1727b65..add1d0a 100644 --- a/man/check_text.Rd +++ b/man/check_text.Rd @@ -76,7 +76,26 @@ check_text( \item{method_context_action}{Action when method context detected in chunk ("NOTE", "WARN", or "SKIP"; default "NOTE")} -\item{design_ambiguous_action}{Action when design-ambiguous t-test (or F(1,df)) effect size ERROR occurs ("WARN", "NOTE", or "ERROR"; default "WARN")} +\item{design_ambiguous_action}{Action when a design-ambiguous t-test (or +F(1,df), or z with d/g) effect-size ERROR occurs ("WARN", "NOTE", or +"ERROR"; default "WARN"). + +\strong{It has one exception, and it is deliberate.} The downgrade is +applied only where design ambiguity is a candidate EXPLANATION for the +discrepancy -- that is, where the reported effect lies within the range +of the computed independent and paired variants plus a 50\\% margin. A +reported effect matching NEITHER design is not explained by not knowing +which design was used, so it keeps its ERROR and this parameter is inert +on that row. Same principle as the v0.6.18 omnibus-df rule: an effect +matching neither candidate keeps its flag. The row states the reason +("Extreme discrepancy ... likely reflects data extraction error"). + +Documented in v0.7.9 after a downstream consumer probed the parameter and found 2 +rows in 148,984 where a caller asking for "WARN" got "ERROR". The +behaviour was correct and the SILENCE was the defect: a policy knob +whose exception is not documented reads, from outside, exactly like a +policy knob that does not work. Pinned by +\code{test-v079-design-ambiguous-action-scope.R}, both directions.} \item{unknown_groups_action}{Action when d/g ERROR occurs with unknown group sizes n1/n2 ("WARN", "NOTE", or "ERROR"; default "WARN")} @@ -155,6 +174,20 @@ reported with \code{95\% CI [-0.92, -0.30]}). FLAG ONLY: the parsed value is never mutated; the violation is also surfaced in \code{uncertainty_reasons}. NA on error/short-circuit rows, FALSE otherwise.} +\item{\code{estimate_outside_ci}}{Logical (since v0.7.9). TRUE when the +same estimate-in-CI invariant is violated with NO sign explanation: +the reported estimate lies outside its reported CI \emph{and so does +its sign-flip}, so a dropped minus cannot account for it and at least +one of the three published numbers is wrong. Mutually exclusive with +\code{sign_ci_violation}, which keeps its own, more specific +diagnosis. Added because the invariant was evaluated but only ever +REPORTED the dropped-minus shape, so a row whose estimate lay outside +its own interval was indistinguishable from one where it lay inside -- +while the row's message said the invariant had been checked. Real +instance: Chan & Feldman (2025), \emph{Cognition and Emotion} 39(6), +p. 1238, Table 9 row 2a prints \code{r = .70, 95\% CI [0.73, 0.76]}. +FLAG ONLY: the parsed value is never mutated. NA on +error/short-circuit rows, FALSE otherwise.} } Plus all other columns: \code{location}, \code{raw_text}, test identification (\code{test_type}, \code{chisq_subtype}, \code{df1}, diff --git a/man/compute_and_compare_one.Rd b/man/compute_and_compare_one.Rd index 0e684d4..5c84596 100644 --- a/man/compute_and_compare_one.Rd +++ b/man/compute_and_compare_one.Rd @@ -52,7 +52,26 @@ compute_and_compare_one( \item{method_context_action}{Action when method context detected in chunk ("NOTE", "WARN", "SKIP")} -\item{design_ambiguous_action}{Action when design-ambiguous t-test (or F(1,df)) effect size ERROR occurs ("WARN", "NOTE", or "ERROR"; default "WARN")} +\item{design_ambiguous_action}{Action when a design-ambiguous t-test (or +F(1,df), or z with d/g) effect-size ERROR occurs ("WARN", "NOTE", or +"ERROR"; default "WARN"). + +\strong{It has one exception, and it is deliberate.} The downgrade is +applied only where design ambiguity is a candidate EXPLANATION for the +discrepancy -- that is, where the reported effect lies within the range +of the computed independent and paired variants plus a 50\\% margin. A +reported effect matching NEITHER design is not explained by not knowing +which design was used, so it keeps its ERROR and this parameter is inert +on that row. Same principle as the v0.6.18 omnibus-df rule: an effect +matching neither candidate keeps its flag. The row states the reason +("Extreme discrepancy ... likely reflects data extraction error"). + +Documented in v0.7.9 after a downstream consumer probed the parameter and found 2 +rows in 148,984 where a caller asking for "WARN" got "ERROR". The +behaviour was correct and the SILENCE was the defect: a policy knob +whose exception is not documented reads, from outside, exactly like a +policy knob that does not work. Pinned by +\code{test-v079-design-ambiguous-action-scope.R}, both directions.} \item{unknown_groups_action}{Action when d/g ERROR occurs with unknown group sizes n1/n2 ("WARN", "NOTE", or "ERROR"; default "WARN")} } diff --git a/tests/testthat/test-ci-multimethod.R b/tests/testthat/test-ci-multimethod.R index 288bb50..7aac413 100644 --- a/tests/testthat/test-ci-multimethod.R +++ b/tests/testthat/test-ci-multimethod.R @@ -134,12 +134,21 @@ test_that("matching CI still gets ci_match=TRUE", { }) test_that("badly wrong CI still gets ci_match=FALSE", { - # CI [0.90, 1.50] is way off for d=0.71 with n=52 - r <- check_text("t(50) = 2.50, p = .016, d = 0.71, 95% CI [0.90, 1.50]") + # v0.7.14: centred on d = 0.71 but far too narrow for n = 52 -- wrong on the + # effect's own scale. (The old fixture, [0.90, 1.50], excludes its estimate; + # since 0.7.14 such an interval is UNVERIFIABLE -- see the next block.) + r <- check_text("t(50) = 2.50, p = .016, d = 0.71, 95% CI [0.60, 0.82]") expect_false(isTRUE(r$ci_match[1])) expect_equal(r$ci_check_status[1], "INCONSISTENT") }) +test_that("v0.7.14: a CI excluding its own estimate is not graded as its interval", { + r <- check_text("t(50) = 2.50, p = .016, d = 0.71, 95% CI [0.90, 1.50]") + expect_equal(r$ci_check_status[1], "UNVERIFIABLE") + expect_true(is.na(r$ci_match[1])) + expect_true(isTRUE(r$estimate_outside_ci[1])) +}) + # ========================================================================= # Decimal CI level parsing (v0.3.0j fix) # Bug: (\d+)% regex couldn't match "99.9%" — captured "9" → ci_level=0.09 diff --git a/tests/testthat/test-v030f-parser-fixes.R b/tests/testthat/test-v030f-parser-fixes.R index b92eb99..57eab23 100644 --- a/tests/testthat/test-v030f-parser-fixes.R +++ b/tests/testthat/test-v030f-parser-fixes.R @@ -82,11 +82,13 @@ test_that("generalized eta-squared (spelled out) parsed correctly", { expect_equal(r$effect_reported[1], 0.12) }) -test_that("generalized_eta2 gets SKIP (extraction_only) not ERROR in check_text", { +test_that("generalized_eta2 is not ERROR, and a checked p is not SKIP", { # generalized_eta2 is mathematically unverifiable from summary statistics - # (Bakeman 2005) — routed to extraction_only, which becomes SKIP + # (Bakeman 2005). v0.7.14: the p-value still IS checked, so the row is a + # p_value check rather than a "nothing checked" SKIP. r <- check_text("F(1, 100) = 5.0, p = .028, geta-squared = 0.04") - expect_equal(r$status[1], "SKIP") + expect_equal(r$check_type[1], "p_value") + expect_false(r$status[1] %in% c("SKIP", "ERROR", "WARN")) expect_true(any(grepl("Generalized eta-squared cannot be verified", r$uncertainty_reasons))) }) diff --git a/tests/testthat/test-v0613-welch-n-recovery-clamp.R b/tests/testthat/test-v0613-welch-n-recovery-clamp.R index 504b40a..a6db4fe 100644 --- a/tests/testthat/test-v0613-welch-n-recovery-clamp.R +++ b/tests/testthat/test-v0613-welch-n-recovery-clamp.R @@ -31,7 +31,11 @@ test_that("a small-d negative-t Welch clause recovers N clamped to the Welch flo # floor df + 2 = round(520.72) + 2 = 523. expect_true(rr$N[1] < 700) expect_equal(rr$N[1], round(520.72) + 2) - expect_equal(rr$N_source[1], "global_text") + # v0.7.14: the published N is the Welch floor df + 2, a df derivation -- the + # old "global_text" label named a provenance (N = 794) this value no longer + # has. And because it is NOT the back-solved value (516), the effect check at + # 523 is a real one, so the row is not "effect_backsolved". + expect_equal(rr$N_source[1], "df_inferred") # No spurious WARN from the wrong (global) N corrupting the recomputed d. expect_false(identical(rr$status[1], "WARN")) }) diff --git a/tests/testthat/test-v0616-ci-sign-align.R b/tests/testthat/test-v0616-ci-sign-align.R index 695f288..cd0a3ad 100644 --- a/tests/testthat/test-v0616-ci-sign-align.R +++ b/tests/testthat/test-v0616-ci-sign-align.R @@ -50,8 +50,11 @@ test_that("v0.6.16: genuinely wrong CI magnitudes stay INCONSISTENT despite oppo # convert a real inconsistency into a pass. txt <- paste0( "Price sensitivity predicted higher estimates of self-interest in others, ", - "t(1596) = -7.67, p < .0001, d = 0.19 [0.45, 0.60]." + "t(1596) = -7.67, p < .0001, d = 0.19 [0.05, 0.33]." ) + # v0.7.14: centred on the estimate, wrong in magnitude (true dz half-width ~0.05). + # The old fixture [0.45, 0.60] excluded its own estimate, which since 0.7.14 + # makes the interval UNVERIFIABLE (referent) before sign alignment matters. res <- check_text(txt) expect_equal(nrow(res), 1L) expect_equal(res$ci_check_status[1], "INCONSISTENT") diff --git a/tests/testthat/test-v0618-welch-stated-in-text.R b/tests/testthat/test-v0618-welch-stated-in-text.R index a9d43fa..7561f55 100644 --- a/tests/testthat/test-v0618-welch-stated-in-text.R +++ b/tests/testthat/test-v0618-welch-stated-in-text.R @@ -44,10 +44,15 @@ test_that("an author-rounded Welch df stated in the text takes the Welch path", n_bound <- suppressWarnings(as.numeric(row$N[1])) expect_gt(n_bound, 225) - # With the right N the reported d is consistent -- the WARN was a false - # positive produced entirely by the wrong sample size. - expect_lt(suppressWarnings(as.numeric(row$delta_effect[1])), 0.02) + # The false WARN (produced by N = 225) is gone. expect_false(identical(as.character(row$status[1]), "WARN")) + # v0.7.14 (d-48266c): this assertion used to be `delta_effect < 0.02`, i.e. a + # PASS -- but the N here (268) was back-solved FROM d = 0.99, so recomputing d + # from it reproduces 0.99 by construction and the check could not fail (a + # deliberately wrong d = 0.70 also passed, at N = 537). The effect size is + # now reported as not verified; the p-value is still checked. + expect_equal(as.character(row$N_source[1]), "effect_backsolved") + expect_equal(as.character(row$check_type[1]), "p_value") }) test_that("a fractional df still takes the Welch path (existing behaviour intact)", { @@ -55,7 +60,11 @@ test_that("a fractional df still takes the Welch path (existing behaviour intact res <- effectcheck::check_text(txt) rows <- res[!is.na(res$test_type) & res$test_type == "t", ] expect_gt(nrow(rows), 0) - expect_lt(suppressWarnings(as.numeric(rows$delta_effect[1])), 0.02) + # v0.7.14: the Welch path is taken (N back-solved above the df + 2 floor), and + # for that reason the effect size is honestly unverified -- see above. + expect_gt(suppressWarnings(as.numeric(rows$N[1])), 225) + expect_equal(as.character(rows$N_source[1]), "effect_backsolved") + expect_false(identical(as.character(rows$status[1]), "WARN")) }) test_that("a plain integer-df t-test with no Welch wording is unaffected", { diff --git a/tests/testthat/test-v0714-ci-verdict-premise.R b/tests/testthat/test-v0714-ci-verdict-premise.R new file mode 100644 index 0000000..9e75e85 --- /dev/null +++ b/tests/testthat/test-v0714-ci-verdict-premise.R @@ -0,0 +1,116 @@ +# v0.7.14 (d-cf56d6, d-4efdc1): INCONSISTENT is a claim about the PAPER, so it +# needs an affirmative premise: that the intervals we computed are comparable +# with the one the paper printed -- same tail, same allocation, same quantity. +# +# v0.7.9 withdrew the status escalation (0 of 3 precision on the corpus) but +# `ci_check_status` kept publishing INCONSISTENT on the same false premises. +# Three classes, reproduced at 0.7.13 on correctly reported numbers: +# +# one-sided interval r(198) = .34, one-sided 95% CI [0.23, 1.00] +# -> the two-sided candidates cannot reproduce it; the +# true one-sided lower bound is 0.2326. +# unstated allocation t(98) = 0.30, d = 0.075, 95% CI [-0.415, 0.565] +# -> exact at n1 = 20, n2 = 80; graded only at 50/50. +# different referent d = 1.03 with "the unstandardized contrast had a 95% +# CI [9.99, 30.01]" -> not an interval on d at all. +# +# Each is now UNVERIFIABLE with a machine-readable `ci_unverifiable_reason`, +# and `ci_match` is NA (not FALSE) so no other field republishes the verdict. +# The controls matter as much as the fixes: a genuinely wrong interval of each +# shape must STAY INCONSISTENT, or this becomes a way to launder real errors. + +row1 <- function(txt) { + r <- check_text(txt) + expect_equal(nrow(r), 1L, info = txt) + r +} + +test_that("one-sided interval: UNVERIFIABLE, not INCONSISTENT", { + r <- row1("r(198) = .34, p < .001, one-sided 95% CI [0.23, 1.00]") + expect_equal(r$ci_check_status, "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason, "one_sided_interval") + expect_true(is.na(r$ci_match)) + # a bound at the parameter limit alone is also the one-sided signature + r2 <- row1("r(198) = .34, p < .001, 95% CI [0.23, 1.00]") + expect_equal(r2$ci_check_status, "UNVERIFIABLE") + expect_equal(r2$ci_unverifiable_reason, "one_sided_interval") +}) + +test_that("one-sided controls: a correct two-sided r CI still MATCHes, a wrong one stays INCONSISTENT", { + ok <- row1("r(198) = .34, p < .001, 95% CI [0.21, 0.46]") + expect_equal(ok$ci_check_status, "MATCH") + expect_true(is.na(ok$ci_unverifiable_reason)) + bad <- row1("r(198) = .34, p < .001, 95% CI [0.30, 0.38]") + expect_equal(bad$ci_check_status, "INCONSISTENT") + expect_true(is.na(bad$ci_unverifiable_reason)) +}) + +test_that("unstated allocation: an unequal split that reproduces the CI makes it UNVERIFIABLE", { + r <- row1("t(98) = 0.30, p = .765, d = 0.075, 95% CI [-0.415, 0.565]") + expect_equal(r$ci_check_status, "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason, "unstated_allocation") + expect_true(is.na(r$ci_match)) + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_true(grepl("n1 = 20, n2 = 80", u, fixed = TRUE)) +}) + +test_that("allocation controls: 50/50 interval MATCHes; one narrower than ANY split stays INCONSISTENT", { + ok <- row1("t(98) = 0.30, p = .765, d = 0.075, 95% CI [-0.317, 0.467]") + expect_equal(ok$ci_check_status, "MATCH") + # The equal split gives the NARROWEST interval, so a narrower one fits no split. + bad <- row1("t(98) = 0.30, p = .765, d = 0.06, 95% CI [-0.10, 0.22]") + expect_equal(bad$ci_check_status, "INCONSISTENT") + expect_true(is.na(bad$ci_unverifiable_reason)) + # Stated group sizes remove the allocation question entirely. + st <- row1("With n1 = 50 and n2 = 50, t(98) = 0.30, p = .765, d = 0.075, 95% CI [-0.415, 0.565]") + expect_false(identical(st$ci_unverifiable_reason, "unstated_allocation")) +}) + +test_that("different referent: an interval that cannot be on d is UNVERIFIABLE, and says why", { + r <- row1("t(58) = 4.00, p < .001, d = 1.03; the unstandardized contrast had a 95% CI [9.99, 30.01]") + expect_equal(r$ci_check_status, "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason, "referent_not_effect") + expect_equal(r$ci_referent, "not_effect_reported") + # the impossible-value finding is NOT laundered: it stays on its own field + expect_true(isTRUE(r$estimate_outside_ci)) +}) + +test_that("d-family referent by midpoint: a d interval is centred on d (item 9 widening)", { + # Contains d = 1.03 but is centred on 1.75: no d-interval method is that skewed. + r <- row1("t(58) = 4.00, p < .001, d = 1.03, 95% CI [0.50, 3.00]") + expect_equal(r$ci_referent, "not_effect_reported") + expect_equal(r$ci_check_status, "UNVERIFIABLE") + # control: the correct d interval is untouched and MATCHes + ok <- row1("t(58) = 4.00, p < .001, d = 1.03, 95% CI [0.49, 1.57]") + expect_true(is.na(ok$ci_referent)) + expect_true(ok$ci_check_status %in% c("MATCH", "PLAUSIBLE")) +}) + +test_that("ci_unverifiable_reason names the pre-existing UNVERIFIABLE causes too", { + r <- row1("A paired t-test showed t(742) = 12.24, p < .001, d = 0.55, 95% CI [0.47, 0.62]") + if (identical(r$ci_check_status, "UNVERIFIABLE")) { + expect_false(is.na(r$ci_unverifiable_reason)) + } + # MISSING / MATCH rows carry no reason + m <- row1("t(58) = 4.00, p < .001, d = 1.03") + expect_true(is.na(m$ci_unverifiable_reason)) +}) + +test_that("an interval with no reported estimate is not graded as a standardized one", { + # Verbatim corpus shape (10.1038/s41562-024-01961-1): a raw mean difference. + r <- row1("t(181) = 2.571, P = 0.011, 95%CI difference [0.102, 0.776]") + expect_equal(r$ci_check_status, "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason, "no_estimate_parsed") + # a correlation is exempt: its statistic IS its estimate + bad_r <- row1("r(198) = .34, p < .001, 95% CI [0.30, 0.38]") + expect_equal(bad_r$ci_check_status, "INCONSISTENT") +}) + +test_that("an allocation fit must also reproduce the reported d (Sonnet consult 2026-09-28)", { + # A split that reproduces the interval but implies a DIFFERENT d than the one + # printed does not explain the row; it is a coincidence on a 1-D family of + # intervals. d = 0.20 (centred interval, so the referent rule does not fire): + # [-0.553, 0.753] is the n1 = 10, n2 = 90 interval for d = 0.10, not 0.20. + r <- row1("t(98) = 0.30, p = .765, d = 0.20, 95% CI [-0.553, 0.753]") + expect_false(identical(r$ci_unverifiable_reason, "unstated_allocation")) +}) diff --git a/tests/testthat/test-v0714-generalized-eta2.R b/tests/testthat/test-v0714-generalized-eta2.R new file mode 100644 index 0000000..4ce336b --- /dev/null +++ b/tests/testthat/test-v0714-generalized-eta2.R @@ -0,0 +1,76 @@ +# v0.7.14 (d-b1c193): generalized eta-squared is not computable from F and df. +# +# eta_G^2 = SS_effect / (SS_effect + SS_error + SS of every MEASURED factor and +# of subjects) (Olejnik & Algina, 2003; Bakeman, 2005). Which terms enter the +# denominator depends on the design, which F and df do not carry. Until 0.7.14 +# compute_all_anova_effects() fell through to the between-subjects formula for +# an UNCLEAR design, so the published `generalized_eta2` column and +# all_variants$same_type$generalized_eta2 were partial eta-squared under a +# different name -- on the very row whose uncertainty text said generalized +# eta-squared "cannot be verified from summary statistics". Measured on +# collabra.126266: equal to partial_eta2 on all 15 F rows. +# +# The same row also carried two false warnings (generalized eta-squared called +# "unusual for F-test", and its explicit label called an "unclear symbol") and +# collapsed to SKIP although its p-value WAS checked. + +gen_row <- function(txt) check_text(txt) + +test_that("generalized_eta2 is NA when the design is not known, never partial eta2", { + r <- gen_row("F(1, 98) = 5.00, p = .028, generalized eta-squared = .05") + expect_equal(nrow(r), 1L) + expect_true(is.na(r$generalized_eta2)) + av <- jsonlite::fromJSON(r$all_variants[[1]], simplifyVector = FALSE) + expect_null(av$same_type$generalized_eta2) +}) + +test_that("an unreported-type F row does not publish a generalized eta2 either", { + r <- gen_row("F(2, 97) = 5.00, p = .009, partial eta-squared = .09") + expect_true(is.na(r$generalized_eta2)) + # partial eta2 itself is unchanged + expect_equal(r$partial_eta2, 10 / (10 + 97), tolerance = 1e-9) + expect_equal(r$status, "PASS") +}) + +test_that("compute_all_anova_effects: generalized eta2 NA for every design it cannot know", { + for (d in c("unclear", "within", "mixed", "between")) { + expect_true(is.na(compute_all_anova_effects(5, 1, 98, d)$generalized_eta2), + info = d) + } +}) + +test_that("generalized eta2 is not called unusual for an F-test, nor an unclear symbol", { + r <- gen_row("F(1, 98) = 5.00, p = .028, generalized eta-squared = .05") + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_false(grepl("unusual", u, fixed = TRUE)) + expect_false(grepl("unclear - may be OCR", u, fixed = TRUE)) + # the honest limitation stays + expect_true(grepl("Generalized eta-squared cannot be verified", u, fixed = TRUE)) +}) + +test_that("a corrupted generalized-eta2 form IS still flagged as an uncertain symbol", { + r <- gen_row("F(1, 98) = 5.00, p = .028, n2G = .05") + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_true(grepl("unclear - may be OCR", u, fixed = TRUE)) +}) + +test_that("a generalized-eta2 row whose p-value was checked is not SKIP", { + r <- gen_row("F(1, 98) = 5.00, p = .028, generalized eta-squared = .05") + expect_equal(r$check_type, "p_value") + expect_false(r$status == "SKIP") + # and a p that is wrong is still caught + r2 <- gen_row("F(1, 98) = 5.00, p = .60, generalized eta-squared = .05") + expect_true(r2$status %in% c("WARN", "ERROR")) +}) + +test_that("a generalized-eta2 CI is not graded against partial-eta2 intervals", { + # collabra.126266 at 0.7.13: 8 INCONSISTENT / 5 PLAUSIBLE / 1 MATCH, every one + # a comparison against a different estimand. + r <- gen_row("F(1, 265) = 26.51, p < .001, generalized eta-squared = .037, 95% CI [.01, .08]") + expect_equal(r$ci_check_status, "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason, "estimand_not_computable") + expect_true(is.na(r$ci_match)) + # control: a partial-eta2 interval is still graded + p <- gen_row("F(1, 265) = 26.51, p < .001, partial eta-squared = .09, 95% CI [.04, .16]") + expect_true(p$ci_check_status %in% c("MATCH", "PLAUSIBLE", "INCONSISTENT")) +}) diff --git a/tests/testthat/test-v0714-mediation-direct-vs-acme.R b/tests/testthat/test-v0714-mediation-direct-vs-acme.R new file mode 100644 index 0000000..4b95ea7 --- /dev/null +++ b/tests/testthat/test-v0714-mediation-direct-vs-acme.R @@ -0,0 +1,63 @@ +# v0.7.14 (d-4fe38e): a sentence reporting the direct effect AND the ACME yields +# both, each under its own name. +# +# Chan & Feldman (2025), Cognition and Emotion, doi 10.1080/02699931.2024.2434156: +# +# "The average direct effect was 0.15, 95% CI [-0.13 to 0.45], p = .3, whereas +# the bootstrapped unstandardised indirect effect (Average Causal Mediation +# Effect, ACME) was 0.67, 95% CI [0.47-0.89], p < .001." +# +# v0.6.16 added pat_mediation_ci for exactly this sentence and its changelog says +# both effects were recovered. Re-tested at 0.7.13: ONE row, carrying the DIRECT +# effect (0.15, [-0.13, 0.45], p = .3) under effect_reported_name +# "indirect_effect", and the ACME (the paper's actual mediation finding) absent. +# A reader of that row sees a non-significant "indirect effect" the paper never +# reported. + +MED <- paste( + "The average direct effect was 0.15, 95% CI [-0.13 to 0.45], p = .3, whereas", + "the bootstrapped unstandardised indirect effect (Average Causal Mediation", + "Effect, ACME) was 0.67, 95% CI [0.47-0.89], p < .001.") + +med_rows <- function(txt) { + r <- as.data.frame(check_text(txt)) + r[!is.na(r$test_type) & r$test_type == "mediation_indirect", , drop = FALSE] +} + +test_that("both mediation effects are extracted, each under its own name", { + r <- med_rows(MED) + expect_equal(nrow(r), 2L) + ade <- r[r$effect_reported_name == "direct_effect", ] + acme <- r[r$effect_reported_name == "indirect_effect", ] + expect_equal(nrow(ade), 1L) + expect_equal(nrow(acme), 1L) + expect_equal(ade$effect_reported, 0.15) + expect_equal(c(ade$ciL_reported, ade$ciU_reported), c(-0.13, 0.45)) + expect_equal(ade$p_reported, 0.3) + expect_equal(acme$effect_reported, 0.67) + expect_equal(c(acme$ciL_reported, acme$ciU_reported), c(0.47, 0.89)) + expect_equal(acme$p_reported, 0.001) +}) + +test_that("the direct effect is never labelled indirect, and the note says which", { + r <- med_rows(MED) + ade <- r[r$effect_reported == 0.15, ] + expect_equal(ade$effect_reported_name, "direct_effect") + u <- paste(unlist(ade$uncertainty_reasons), collapse = " | ") + expect_true(grepl("direct effect", u, fixed = TRUE)) + expect_false(grepl("Sobel Z", u, fixed = TRUE)) +}) + +test_that("a lone ACME sentence still yields one indirect-effect row (control)", { + r <- med_rows("The indirect effect (ACME) was 0.67, 95% CI [0.47, 0.89], p < .001.") + expect_equal(nrow(r), 1L) + expect_equal(r$effect_reported_name, "indirect_effect") +}) + +test_that("the Sobel-Z form is unchanged (control)", { + r <- med_rows(paste("The bootstrapped indirect effect of X on Y was .05, 95% CI", + "[-.04, .12], Sobel Z = 0.84, p = .40.")) + expect_equal(nrow(r), 1L) + expect_equal(r$effect_reported_name, "indirect_effect") + expect_equal(r$stat_value, 0.84) +}) diff --git a/tests/testthat/test-v0714-omnibus-contrast-undecided.R b/tests/testthat/test-v0714-omnibus-contrast-undecided.R new file mode 100644 index 0000000..99c270d --- /dev/null +++ b/tests/testthat/test-v0714-omnibus-contrast-undecided.R @@ -0,0 +1,49 @@ +# v0.7.14 (d-43c69d): a post-hoc contrast whose effect size cannot decide +# between the two sample-size readings must SAY both. +# +# collabra.90203 reprints the omnibus error df F(2, 998) on its Bonferroni +# post-hoc contrasts. v0.6.18 adopts the balanced contrast N (~667) when the +# row's own d decisively favours it -- t(998) = 2.46, d = 0.19 does, and was +# re-tested MATCH at 0.7.13. But t(998) = 0.097, d = 0.01 cannot decide: at a d +# that small both N = 1000 and N ~ 667 reproduce it. The row kept N = 1000 and +# said nothing about the contrast reading, although the announcement it sits +# under says it is a post-hoc contrast (the paper's cells are 335 + 335). +# Publishing the df + 2 reading is the package's convention; publishing it as +# the only reading is not. (Owner ruling 2026-09-04: report, never silently +# correct.) + +OMNI <- paste( + "There was a main effect of condition, F(2, 998) = 3.91, p = .02.", + "Post-hoc comparisons with Bonferroni correction showed no difference between", + "statistical and identifiable victims, t(998) = 0.097, p = 1.00, d = 0.01, 95% CI [-0.16, 0.14],", + "and a difference between statistical and joint, t(998) = 2.46, p = .041, d = 0.19 [0.04, 0.34].") + +t_by_stat <- function(res, v) { + res <- as.data.frame(res) + res[!is.na(res$test_type) & res$test_type == "t" & abs(res$stat_value - v) < 1e-9, , drop = FALSE] +} + +test_that("an undecidable post-hoc contrast names the contrast reading", { + r <- t_by_stat(check_text(OMNI), 0.097) + expect_equal(nrow(r), 1L) + expect_equal(r$N, 1000) # the incumbent reading is still published + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_true(grepl("post-hoc contrast", u, fixed = TRUE)) + expect_true(grepl("N~667", u, fixed = TRUE)) +}) + +test_that("the decisive contrast is unchanged: adopted, labelled, MATCH", { + r <- t_by_stat(check_text(OMNI), 2.46) + expect_equal(r$N, 667) + expect_equal(r$N_source, "omnibus_df_contrast") + expect_equal(r$status, "PASS") +}) + +test_that("no post-hoc announcement -> no contrast note (control)", { + txt <- paste( + "There was a main effect of condition, F(2, 998) = 3.91, p = .02.", + "Separately, the two sexes did not differ, t(998) = 0.097, p = .92, d = 0.01.") + r <- t_by_stat(check_text(txt), 0.097) + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_false(grepl("post-hoc contrast", u, fixed = TRUE)) +}) diff --git a/tests/testthat/test-v0714-quoted-restatement.R b/tests/testthat/test-v0714-quoted-restatement.R new file mode 100644 index 0000000..b6d1cb7 --- /dev/null +++ b/tests/testthat/test-v0714-quoted-restatement.R @@ -0,0 +1,70 @@ +# v0.7.14 (d-43ed61): a statistic QUOTED twice is one result, not two. +# +# 10.24072/pci.rr.100726 is a peer-review letter. Reviewer #2 quotes one test +# twice to illustrate APA comma placement: +# +# (e.g. "... and t(868) = -3.01, p = .006 for the gender..." should be +# "... and, t(868) = -3.01, p = .006, for the gender...") +# +# and effectcheck emitted two fully scored rows. v0.6.18 built three dedup rules +# and withdrew all three (test-v0618-prose-restatement-dedup.R), because two +# genuinely distinct results can share every printed number -- v0.6.14's +# `r(797) = .16` twice in one sentence, different variables. That file named +# the missing signal: "a span marked as quoted material". The text carries it: +# both copies sit inside quotation marks. Distinct results in running prose are +# not quoted, so the collabra.23443 case is untouched by construction. +# +# The rule: identical printed signature + adjacent + BOTH inside quotation +# marks -> one row, and that row SAYS the statistic was quoted more than once. + +t_rows <- function(txt) { + r <- check_text(txt) + r[!is.na(r$test_type) & r$test_type == "t", ] +} + +PCI <- paste0( + "(1) I believe that according to the APA style commas should be included ", + "before and after statistics (e.g. \"... and t(868) = -3.01, p = .006 for the ", + "gender...\" should be \"... and, t(868) = -3.01, p = .006, for the gender...\"). ", + "Thank you.") + +test_that("a statistic quoted twice in one passage is scored once, and says so", { + r <- t_rows(PCI) + expect_equal(nrow(r), 1L) + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_true(grepl("quoted 2 times", u, fixed = TRUE)) +}) + +test_that("curly quotation marks count as quotation too", { + txt <- paste0( + "The reviewer wrote “the effect, t(40) = 2.10, p = .042 was small” and ", + "later “the effect, t(40) = 2.10, p = .042, was small”.") + expect_equal(nrow(t_rows(txt)), 1L) +}) + +test_that("the same numbers twice in UNQUOTED prose are both kept (v0.6.14 invariant)", { + txt <- paste0( + "Paid participants showed r(797) = .16, p < .001, and unpaid participants ", + "also showed r(797) = .16, p < .001.") + r <- check_text(txt) + expect_equal(sum(!is.na(r$test_type) & r$test_type == "r"), 2L) + txt2 <- "The first test was t(40) = 2.10, p = .042, and the second was t(40) = 2.10, p = .042." + expect_equal(nrow(t_rows(txt2)), 2L) +}) + +test_that("one quoted copy beside one unquoted result keeps both", { + txt <- paste0( + "We found t(40) = 2.10, p = .042. The reviewer asked us to write ", + "\"t(40) = 2.10, p = .042,\" with a comma.") + expect_equal(nrow(t_rows(txt)), 2L) +}) + +test_that("two DIFFERENT quoted statistics are both kept", { + # Separate sentences: a second quoted statistic in the SAME sentence after a + # closing quote is currently lost by the parser (TODO d-6ac077), which is a + # different defect from the one pinned here. + txt <- paste0( + "The reviewer wrote \"t(868) = -3.01, p = .006 for the gender\". ", + "Another reviewer wrote \"t(868) = -2.01, p = .045 for the age\".") + expect_equal(nrow(t_rows(txt)), 2L) +}) diff --git a/tests/testthat/test-v0714-welch-backsolved-n.R b/tests/testthat/test-v0714-welch-backsolved-n.R new file mode 100644 index 0000000..d884a24 --- /dev/null +++ b/tests/testthat/test-v0714-welch-backsolved-n.R @@ -0,0 +1,109 @@ +# v0.7.14 (d-48266c): a Welch N back-solved FROM the reported d cannot verify d. +# +# collabra.77859 prints "Welch's t(223) = 8.11, p < .001, (d = 0.99, 95% CI: +# [0.72, 1.25])". v0.6.18 fixed the filed symptom (the rounded Welch df bound +# N = df + 2 = 225 and raised a false WARN). Re-tested at 0.7.13 the row was +# PASS -- but for a circular reason. With group sizes unstated, a Welch df only +# bounds N from below, so the Welch branch back-solves N = 4 t^2 / d^2 from the +# REPORTED d and then recomputes d from that N, which reproduces the reported d +# by construction. Measured at 0.7.13: a deliberately wrong d = 0.70 on the same +# t(223) = 8.11 (true d ~ 0.99) also returned PASS, at N = 537. A check that +# cannot fail is not a check. It also labelled that N `global_text` / +# `not_found`, naming a provenance it did not have, and said "Non-integer df +# (223.00)" of an integer df. +# +# Now: the effect size is reported as not verifiable (the p-value still is), +# and N_source says what the N is: "effect_backsolved". + +welch_row <- function(txt) { + r <- as.data.frame(check_text(txt)) + r[!is.na(r$test_type) & r$test_type == "t", , drop = FALSE] +} + +test_that("a WRONG d on a Welch test with no group sizes is not PASS", { + r <- welch_row("The scarf was rated higher than the coat, Welch's t(223) = 8.11, p < .001, d = 0.70.") + expect_equal(nrow(r), 1L) + expect_false(r$status == "PASS") + expect_equal(r$N_source, "effect_backsolved") + expect_true(is.na(r$matched_value)) +}) + +test_that("the paper's own (correct) row is honest too: no circular PASS, p still checked", { + txt <- paste("We recruited N = 403 participants. Many analyses followed.", + "The scarf was rated higher than the coat, Welch's t(223) = 8.11,", + "p < .001, d = 0.99, 95% CI [0.72, 1.25].") + r <- welch_row(txt) + expect_equal(nrow(r), 1L) + expect_equal(r$check_type, "p_value") + expect_equal(r$N_source, "effect_backsolved") + u <- paste(unlist(r$uncertainty_reasons), collapse = " | ") + expect_true(grepl("back-solved from the reported effect size", u, fixed = TRUE)) + expect_false(grepl("Non-integer df (223.00)", u, fixed = TRUE)) + expect_true(grepl("Welch", u, fixed = TRUE)) +}) + +test_that("stated group sizes still verify a Welch d (control)", { + r <- welch_row("With n1 = 131 and n2 = 135, Welch's t(222.87) = 8.11, p < .001, d = 0.99.") + expect_equal(nrow(r), 1L) + expect_false(identical(r$N_source, "effect_backsolved")) + expect_equal(r$check_type, "effect_size") +}) + +test_that("a non-Welch t with an equal-n df is unaffected (control)", { + r <- welch_row("The groups differed, t(98) = 2.50, p = .014, d = 0.50.") + expect_equal(r$check_type, "effect_size") + expect_equal(r$status, "PASS") +}) + +test_that("a Welch row never matches a paired-design variant (review finding 2026-09-30)", { + # A Welch test compares two independent groups by definition, so dz / dav / + # drm cannot describe it. At 0.7.13 and in the 0.7.14 candidate a WRONG + # d = 0.70 on Welch t(223) = 8.11 with n1 = 131, n2 = 135 (true d ~ 0.99) + # published PASS by matching drm = 0.685. + r <- welch_row("Welch t(223.00) = 8.11, p < .001, d = 0.70, n1 = 131, n2 = 135.") + expect_false(r$status == "PASS") + expect_false(isTRUE(r$matched_variant %in% c("dz", "dav", "drm", "gz", "gav", "grm"))) + # control: the correct d still passes against the independent-groups value + ok <- welch_row("Welch t(223.00) = 8.11, p < .001, d = 0.99, n1 = 131, n2 = 135.") + expect_equal(ok$status, "PASS") + # control: a paired t-test still matches dz + p <- welch_row("A paired t-test showed t(49) = 3.00, p = .004, dz = 0.42.") + expect_equal(p$matched_variant, "dz") +}) + +test_that("an explicit paired label (dz) keeps its paired variants on a Welch-looking row", { + # Tier-2 cross-model finding (2026-09-30): the Welch gate keyed on the word + # "Welch" or a fractional df alone, so a correct dz on a within-subject + # Satterthwaite contrast, or in a clause that merely mentions Welch, lost dz + # and fell from PASS to NOTE. The author's own paired label is design evidence. + sat <- welch_row("In the within-subject contrast, t(45.3) = 3.00, p = .004, dz = 0.44 (Satterthwaite df).") + expect_equal(sat$matched_variant, "dz") + expect_equal(sat$status, "PASS") + neg <- welch_row("A paired t-test (Welch's correction not needed) showed t(49) = 3.00, p = .004, dz = 0.42.") + expect_equal(neg$matched_variant, "dz") + expect_equal(neg$status, "PASS") + # control: a generic d on a Welch row is still never matched to a paired variant + w <- welch_row("A Welch t-test showed t(223) = 8.11, p < .001, d = 0.70, n1 = 131, n2 = 135.") + expect_false(isTRUE(w$matched_variant %in% c("dz", "dav", "drm"))) +}) + +test_that("a tiny rounded Welch d is graded at the Welch floor, not a scraped study total", { + # 10.1016/j.jesp.2020.104052 (0.7.14 + docpluck 2.4.147): Welch t(169.45) = + # 0.24, d = 0.04 bound the document's global N = 827. The back-solve + # N = 4t^2/d^2 = 144 fell just outside the 0.85 * floor band because d is + # rounded to two decimals, so the scraped 827 stood and the recomputed + # d = 0.0167 raised a WARN. At the floor N = df + 2 = 171, d = 0.0367, which + # the printed 0.04 reproduces within its rounding -- a real (not fitted) check. + filler <- paste(rep("This is unrelated filler prose about methodology and procedures.", 12), + collapse = " ") + txt <- paste0("We recruited N = 827 participants. ", filler, " ", filler, + " The conditions did not differ, Welch t(169.45) = 0.24, p = .81, d = 0.04.") + r <- welch_row(txt) + expect_equal(nrow(r), 1L) + expect_equal(r$N, 171) + expect_equal(r$N_source, "df_inferred") + expect_false(r$status %in% c("WARN", "ERROR")) + # control: a d the floor N cannot reproduce is still flagged (0.20 vs 0.037) + bad <- welch_row(sub("d = 0.04", "d = 0.20", txt, fixed = TRUE)) + expect_false(bad$status %in% c("PASS")) +}) diff --git a/tests/testthat/test-v079-ci-severity-tiers.R b/tests/testthat/test-v079-ci-severity-tiers.R index 67e8b62..214302d 100644 --- a/tests/testthat/test-v079-ci-severity-tiers.R +++ b/tests/testthat/test-v079-ci-severity-tiers.R @@ -30,9 +30,19 @@ # [0.54, 0.56] INCONSISTENT NOTE # [8.00, 9.00] INCONSISTENT NOTE # +# v0.7.14: the blocks below that used [8.00, 9.00] as their "absurd" interval now +# use [0.40, 0.70] -- centred on the estimate, containing it, and far too narrow, +# so it is wrong ON THE EFFECT'S OWN SCALE. [8.00, 9.00] excludes its own +# estimate, and an interval that excludes its estimate cannot be graded as that +# estimate's interval: since 0.7.14 it is UNVERIFIABLE with ci_referent = +# "not_effect_reported", while `estimate_outside_ci` (and status NOTE) carry the +# impossible-value finding. That case is pinned in test-v0714-ci-verdict-premise.R +# and at the end of this file. The rows that still use [8.00, 9.00] assert +# something else (bounded r range, a stated level of 263.95%, cross-family). +# # A confidence interval wrong by 0.07 and one wrong by 8.0 published the IDENTICAL # verdict, while the effect-size arm grades PASS / WARN (3x) / ERROR (5x) on the -# same input. Downstream, SciMeto maps NOTE -> severity "info" by explicit design, +# same input. Downstream, a consumer maps NOTE -> severity "info" by explicit design, # so an interval that contradicts every computable method reached the author as a # neutral chip that turned no badge and counted toward no issue. # @@ -87,7 +97,7 @@ test_that("an interval matching NO computed candidate does NOT escalate status", # measured at precision 0 of 3 on the corpus -- see the block comment at the # end of this file -- and now leaves `status` alone. The TIER is still # computed and still published; it is `ci_check_status` that carries it. - df <- ci_row("95% CI [8.00, 9.00]") + df <- ci_row("95% CI [0.40, 0.70]") expect_equal(nrow(df), 1L) expect_equal(df$ci_check_status[1], "INCONSISTENT") expect_equal(df$status[1], "NOTE") @@ -130,7 +140,7 @@ test_that("the three tiers are genuinely distinguishable IN THE PUBLISHED ROW", # parse, and without any way to accuse a correct paper. exact <- ci_row("95% CI [0.03, 1.07]", tol_ci = 0.05) near <- ci_row("95% CI [0.10, 1.00]", tol_ci = 0.05) - absurd <- ci_row("95% CI [8.00, 9.00]", tol_ci = 0.05) + absurd <- ci_row("95% CI [0.40, 0.70]", tol_ci = 0.05) tiers <- c(exact$ci_check_status[1], near$ci_check_status[1], absurd$ci_check_status[1]) expect_equal(tiers, c("MATCH", "PLAUSIBLE", "INCONSISTENT")) @@ -152,7 +162,7 @@ test_that("the three tiers are genuinely distinguishable IN THE PUBLISHED ROW", test_that("ci_affects_status = FALSE still suppresses the CI's effect entirely", { # The escalation must honour the existing opt-out, or a consumer that has # deliberately turned the CI arm off would start seeing new WARNs. - df <- ci_row("95% CI [8.00, 9.00]", ci_affects_status = FALSE) + df <- ci_row("95% CI [0.40, 0.70]", ci_affects_status = FALSE) expect_equal(df$ci_check_status[1], "INCONSISTENT") expect_equal(df$status[1], "PASS") }) @@ -179,7 +189,7 @@ test_that("an INCONSISTENT CI never DOWNGRADES an existing ERROR", { ) txt <- paste( "Groups were compared with an independent-samples t test.", - "The effect was reliable, t(58) = 2.10, p = .040, d = 1.10, 95% CI [8.00, 9.00]." + "The effect was reliable, t(58) = 2.10, p = .040, d = 1.10, 95% CI [1.00, 1.20]." ) base <- as.data.frame(check_text(base_txt)) df <- as.data.frame(check_text(txt)) @@ -212,7 +222,7 @@ test_that("an ASSUMED confidence level is not escalated -- the universe was not # # An INCONSISTENT verdict on an assumed level is therefore partly a statement # about OUR inference, not purely about the paper. It stays NOTE. - df <- ci_row("CI [8.00, 9.00]") # no stated level -> ci_level_source assumed_95 + df <- ci_row("CI [0.40, 0.70]") # no stated level -> ci_level_source assumed_95 expect_equal(nrow(df), 1L) expect_equal(as.character(df$ci_level_source[1]), "assumed_95") expect_equal(df$ci_check_status[1], "INCONSISTENT") @@ -225,7 +235,7 @@ test_that("a STATED level reaches the same status -- level no longer changes sta # NOTE. The DISTINCTION still exists and is still published -- `ci_level_source` # tells a consumer whether the level was the paper's or ours -- it simply no # longer moves `status`, which is a contract read by three projects. - df <- ci_row("95% CI [8.00, 9.00]") + df <- ci_row("95% CI [0.40, 0.70]") expect_equal(nrow(df), 1L) expect_equal(as.character(df$ci_level_source[1]), "explicit_with_bounds") expect_equal(df$ci_check_status[1], "INCONSISTENT") @@ -248,7 +258,8 @@ test_that("a CORRELATION with an impossible interval does not publish PASS", { df <- as.data.frame(check_text( "The correlation was significant, r(198) = .34, p < .001, 95% CI [8.00, 9.00].")) expect_equal(nrow(df), 1L) - expect_equal(df$ci_check_status[1], "INCONSISTENT") + expect_equal(df$ci_check_status[1], "UNVERIFIABLE") + expect_equal(df$ci_unverifiable_reason[1], "referent_not_effect") # v0.7.14, see header expect_false(identical(df$status[1], "PASS")) expect_equal(df$status[1], "WARN") }) @@ -273,7 +284,8 @@ test_that("an IMPLAUSIBLE stated level is treated like an assumed one", { "263.95% CI [8.00, 9.00]."))) expect_equal(nrow(df), 1L) expect_equal(as.character(df$ci_level_source[1]), "implausible_level") - expect_equal(df$ci_check_status[1], "INCONSISTENT") + expect_equal(df$ci_check_status[1], "UNVERIFIABLE") + expect_equal(df$ci_unverifiable_reason[1], "referent_not_effect") # v0.7.14, see header expect_equal(df$status[1], "NOTE") }) @@ -291,7 +303,8 @@ test_that("a row whose EXTRACTION is suspect is not escalated by the CI arm", { df <- as.data.frame(check_text(txt)) expect_equal(nrow(df), 1L) expect_true(isTRUE(df$extraction_suspect[1])) - expect_equal(df$ci_check_status[1], "INCONSISTENT") + expect_equal(df$ci_check_status[1], "UNVERIFIABLE") + expect_equal(df$ci_unverifiable_reason[1], "referent_not_effect") # v0.7.14, see header expect_equal(df$status[1], "NOTE") }) @@ -354,7 +367,8 @@ test_that("a cross-family row with an ABSURD interval also stays NOTE", { r <- crossfam_row("95% CI [8.00, 9.00]") expect_equal(nrow(r), 1L) expect_equal(as.character(r$ambiguity_level[1]), "highly_ambiguous") - expect_equal(r$ci_check_status[1], "INCONSISTENT") + expect_equal(r$ci_check_status[1], "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason[1], "referent_not_effect") # v0.7.14, see header expect_equal(as.character(r$status[1]), "NOTE") }) @@ -370,7 +384,8 @@ test_that("the SAME-family row reaches the same status -- the exemption was subs # together justified the withdrawal. r <- ci_row("95% CI [8.00, 9.00]") expect_equal(as.character(r$ambiguity_level[1]), "clear") - expect_equal(r$ci_check_status[1], "INCONSISTENT") + expect_equal(r$ci_check_status[1], "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason[1], "referent_not_effect") # v0.7.14, see header expect_equal(as.character(r$status[1]), "NOTE") }) @@ -418,7 +433,8 @@ test_that("inferred_from_context is NOT escalated -- 15/15 corpus rows say it ne r <- as.data.frame(check_text(txt)) expect_equal(nrow(r), 1L) expect_equal(as.character(r$ci_level_source[1]), "inferred_from_context") - expect_equal(r$ci_check_status[1], "INCONSISTENT") + expect_equal(r$ci_check_status[1], "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason[1], "referent_not_effect") # v0.7.14, see header expect_false(is.na(r$matched_value[1])) # exclusion 1 is NOT doing the work here expect_equal(as.character(r$status[1]), "NOTE") }) @@ -431,7 +447,8 @@ test_that("the real corpus shape -- a raw-difference interval -- is not escalate r <- as.data.frame(check_text(txt)) expect_equal(nrow(r), 1L) expect_equal(as.character(r$ci_level_source[1]), "inferred_from_context") - expect_equal(r$ci_check_status[1], "INCONSISTENT") + expect_equal(r$ci_check_status[1], "UNVERIFIABLE") + expect_equal(r$ci_unverifiable_reason[1], "no_estimate_parsed") # v0.7.14, see header expect_true(as.character(r$status[1]) %in% c("OK", "NOTE")) expect_false(identical(as.character(r$status[1]), "WARN")) }) @@ -554,7 +571,10 @@ test_that("a WRONG BUT POSSIBLE correlation interval is NOT flagged", { df <- as.data.frame(check_text( "The correlation was significant, r(198) = .34, p < .001, 95% CI [0.90, 0.99].")) expect_equal(nrow(df), 1L) - expect_equal(df$ci_check_status[1], "INCONSISTENT") + # v0.7.14: the interval excludes its own estimate, so it is not graded as the + # estimate's interval (see header); `estimate_outside_ci` carries the finding. + expect_equal(df$ci_check_status[1], "UNVERIFIABLE") + expect_equal(df$ci_unverifiable_reason[1], "referent_not_effect") expect_false(impossible_ci(df)) expect_equal(as.character(df$status[1]), "PASS") }) @@ -577,3 +597,15 @@ test_that("an UNBOUNDED family is never touched by the bounded-CI check", { expect_equal(nrow(df), 1L) expect_false(impossible_ci(df)) }) + +test_that("v0.7.14: an interval excluding its own estimate is UNVERIFIABLE, never PASS", { + # The old "absurd" fixture, pinned under the 0.7.14 semantics (see header). + df <- ci_row("95% CI [8.00, 9.00]") + expect_equal(df$ci_check_status[1], "UNVERIFIABLE") + expect_equal(df$ci_referent[1], "not_effect_reported") + expect_true(isTRUE(df$estimate_outside_ci[1])) + expect_equal(df$status[1], "NOTE") + # and the opt-out still suppresses the CI arm's effect on status entirely + off <- ci_row("95% CI [8.00, 9.00]", ci_affects_status = FALSE) + expect_equal(off$status[1], "PASS") +})