diff --git a/DESCRIPTION b/DESCRIPTION index 920704f8..292203c7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,7 +14,7 @@ Imports: Matrix, pbapply, ggplot2, - DRDID (>= 1.3.0), + DRDID (>= 1.4.0), generics, methods, tidyr, @@ -31,7 +31,7 @@ Suggests: covr, backports, broom, - testthat (>= 3.0.0), + testthat (>= 3.1.8), withr, remotes, callr diff --git a/NEWS.md b/NEWS.md index 1307456c..92d25f20 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,42 @@ +# did (development version) + + * Requires `DRDID (>= 1.4.0)` so repeated-cross-section estimation and + inference use the corrected upstream implementation. + * Correct weighted-estimation feasibility checks: observations with zero + normalized sampling weight no longer cause false overlap failures, and + regression conditioning uses the sampling-weighted control design used by + DRDID. Unweighted checks retain their fast path without weight scans or + unnecessary covariate copies. + * `aggte(type = "dynamic", balance_e = ...)` now retains only cohorts with + non-missing estimates at every observed post-treatment event time through + the requested horizon, including on irregular grids. Balancing precedes + display limits, and an error explains when no complete cohort remains. + * Standard errors, Wald tests, and simultaneous bands no longer discard + finite positive variances based on absolute outcome units. Exact-zero and + non-finite scales remain unavailable. `att_gt()` reuses the multiplier + bootstrap's critical value instead of repeating its quantiles and scaling. + * Fast preprocessing accepts used data columns named `args`, `tlist`, or + `cutoff_t` without confusing them with local variables. + * `conditional_did_pretest()` resolves omitted covariates and observation IDs + and uses the actual panel structure after preprocessing. Balanced panels + with `allow_unbalanced_panel = TRUE` and repeated cross sections with that + ignored flag no longer fail as if they were unbalanced panels. + * The legacy conditional pre-test now supports only its justified `"ipw"` + method. Regression, doubly robust, and custom methods error clearly: the + original outcome model need not remain correct after indicator weighting, + so those alternatives did not inherit ordinary outcome-model robustness. + `att_gt()` continues to support those estimation methods. + * Restore the anticipation correction from PR #276, which had merged into an + intermediate branch without reaching master. Anticipation-window cells use + the clean base period under either base-period setting and are excluded + from the Wald pre-test. `anticipation = 0` and post-treatment estimates are + unchanged. + * Include the small-group protection and restored tests from PR #278. Built-in + `dr` and `reg` repeated-cross-section cells, unbalanced panels, and balanced + panels with `fix_weights = "varying"` check normalized positive-weight + support in every required outcome regression before native fitting. + Undersized designs warn and return an NA cell while healthy cells continue. + # did 2.5.1.902 * Bug fix (`att_gt()`, balanced panel): the never-treated group is now re-checked after the panel is balanced. Balancing drops units whole, so a covariate missing in one period for every never-treated unit removed the whole group after the check had passed; every ATT(g,t) was then `NA` — silently under `faster_mode = TRUE`, and reported as "overlap condition violated" under `faster_mode = FALSE`. The last treated cohort is now used as the comparison group (as when the data contain no never-treated group), with a warning, and the estimates equal those from the balanced sample. Under `control_group = "notyettreated"` the last cohort's pre-treatment cells are no longer computed in this case (it is a comparison group only). diff --git a/R/MP.R b/R/MP.R index 251d39be..f0d27d06 100644 --- a/R/MP.R +++ b/R/MP.R @@ -20,6 +20,7 @@ #' align rows by rowname, never by position. #' @param n the number of unique cross-sectional units (unique values of idname) #' @param W the Wald statistic for pre-testing the common trends assumption +#' (from the pre-treatment cells, t < g - anticipation) #' @param Wpval the p-value of the Wald statistic for pre-testing the #' common trends assumption #' @param aggte an aggregate treatment effects object diff --git a/R/aggte.R b/R/aggte.R index 2a700fea..aa37e57a 100644 --- a/R/aggte.R +++ b/R/aggte.R @@ -23,8 +23,12 @@ #' the sample with respect to event time. For example, if `balance_e=2`, #' `aggte` will drop groups that are not exposed to treatment for #' at least three periods. (the initial period when `e=0` as well as the -#' next two periods when `e=1` and the `e=2`). This ensures that -#' the composition of groups does not change when event time changes. +#' next two periods when `e=1` and the `e=2`). Each retained group must +#' have a non-missing ATT estimate at every observed post-treatment event +#' time through `balance_e`, so the composition of groups does not change +#' across those event times. On irregular time grids, only event times +#' observed among the eligible groups are required. Balancing is done before +#' applying `min_e` and `max_e`; an error is returned if no group remains. #' @param min_e The smallest event time (length of exposure, `e = t - g`, #' measured in the same units as `tname`) to include. By default, #' `min_e = -Inf` so that effects at all lengths of exposure are included. diff --git a/R/att_gt.R b/R/att_gt.R index 35130bfe..484bd2d3 100644 --- a/R/att_gt.R +++ b/R/att_gt.R @@ -26,7 +26,7 @@ #' \describe{ #' \item{Balanced panel}{Each 2x2 DiD comparison uses the weight from the #' earlier of the two time periods involved. For post-treatment cells, -#' this is the base period (g-1). For pre-treatment cells with +#' this is the base period (g-anticipation-1). For pre-treatment cells with #' \code{base_period="varying"}, this is the pre-treatment period itself. #' The panel DRDID estimators are used.} #' \item{Repeated cross sections and unbalanced panels}{Both periods' @@ -44,7 +44,7 @@ #' \describe{ #' \item{\code{NULL} (default)}{For balanced panel: uses the weight from #' the earlier of the two time periods in each 2x2 comparison. For -#' post-treatment cells, this is the base period (g-1). For +#' post-treatment cells, this is the base period (g-anticipation-1). For #' pre-treatment cells, this depends on the \code{base_period} setting. #' For RC/unbalanced panel: uses per-observation weights from both #' periods.} @@ -192,7 +192,10 @@ #' date (net of `anticipation`) onward are dropped, with a warning. #' @param anticipation The number of time periods before participating #' in the treatment where units can anticipate participating in the -#' treatment and therefore it can affect their untreated potential outcomes +#' treatment and therefore it can affect their untreated potential outcomes. +#' Periods `g - anticipation`, ..., `g - 1` use the base period +#' `g - anticipation - 1` (under either `base_period`) and are not used +#' in the pre-test of parallel trends. #' @param faster_mode This option enables a faster version of `did`, optimizing #' computation time for large datasets by improving data management within the package. #' The default is set to `TRUE`. Both modes produce identical results up to @@ -201,7 +204,7 @@ #' @param base_period Whether to use a "varying" base period or a #' "universal" base period. Either choice results in the same #' post-treatment estimates of ATT(g,t)'s. In pre-treatment -#' periods, using a varying base period amounts to computing a +#' periods (t < g - anticipation), using a varying base period amounts to computing a #' pseudo-ATT in each treatment period by comparing the change #' in outcomes for a particular group relative to its comparison #' group in the pre-treatment periods (i.e., in pre-treatment @@ -596,8 +599,11 @@ att_gt <- function(yname, } se <- sqrt(Matrix::diag(V) / n) - # Zero standard error replaced by NA - se[se <= sqrt(.Machine$double.eps) * 10] <- NA + # Only unavailable or nonpositive standard errors are degenerate. An + # absolute tolerance would change inference when outcome units change. + # Magnitude alone cannot distinguish a small valid IF from nuisance-fit + # roundoff; numerical singularity is checked using rcond(preV) below. + se[!is.finite(se) | se <= 0] <- NA_real_ # If a cluster variable beyond idname is requested without the bootstrap but the cluster-robust variance # could not be formed (e.g. cluster identifiers unavailable), fall back to i.i.d. and let the user know. @@ -611,7 +617,6 @@ att_gt <- function(yname, # bootstrap variance matrix if (bstrap) { bout <- mboot(inffunc, DIDparams = dp, pl = pl, cores = cores, return_V = FALSE) - bres <- bout$bres if (length(zero_na_sd_entry) > 0) { se[-zero_na_sd_entry] <- bout$se[-zero_na_sd_entry] @@ -619,8 +624,7 @@ att_gt <- function(yname, se <- bout$se } } - # Zero standard error replaced by NA - se[se <= sqrt(.Machine$double.eps) * 10] <- NA + se[!is.finite(se) | se <= 0] <- NA_real_ #----------------------------------------------------------------------------- @@ -652,12 +656,20 @@ att_gt <- function(yname, } if (is.null(wald_invalid)) { - # select which periods are pre-treatment - pre <- which(group > tt) + # select which periods are pre-treatment (anticipation periods are not) + pre <- which(group - anticipation > tt) # number of pre-treatment cells before the variance filter below, so the # no-pre-periods warning can distinguish "no pre-treatment cells exist" # from "all pre-treatment cells were dropped for NA/zero variance" - n_pre_cells <- length(pre) + # (the universal base cell is normalized to 0 and does not count) + if (base_period == "universal" && length(pre) > 0L) { + time_grid <- if (faster_mode) dp$time_periods else dp$tlist + base_times <- time_grid[findInterval(group[pre] - anticipation, time_grid, + left.open = TRUE)] + n_pre_cells <- sum(tt[pre] != base_times) + } else { + n_pre_cells <- length(pre) + } # Drop group-periods that have variance equal to zero (singularity problems) if (length(zero_na_sd_entry) > 0) { @@ -722,35 +734,14 @@ att_gt <- function(yname, # HAVE to use the bootstrap if (bstrap) { if (cband) { - # for uniform confidence band - # compute new critical value - # see paper for details - # Vectorized like mboot(): both quantiles of each bootstrap column are - # taken in a single type = 1 call (one sort per column instead of two), - # with the qnorm normalization hoisted out of the per-column loop. - iqr_norm <- qnorm(.75) - qnorm(.25) - bSigma <- apply(bres, 2, function(b) { - qs <- quantile(b, c(.25, .75), type = 1, na.rm = TRUE) - (qs[2L] - qs[1L]) / iqr_norm - }) - - bSigma[bSigma <= sqrt(.Machine$double.eps) * 10] <- NA - - # sup-t confidence band: per bootstrap draw, the max over columns of - # |b / bSigma|, as a pmax column-fold (equivalent to the previous row-wise - # apply(max, na.rm = TRUE): NA entries are ignored via -Inf, and an all-NA - # row yields -Inf, exactly as before). - if (ncol(bres) == 0L) { - # every bootstrap dimension was degenerate (mboot dropped all columns): - # the previous row-wise apply(max, na.rm = TRUE) returned -Inf per row - bT <- rep(-Inf, nrow(bres)) - } else { - scaled <- abs(bres / rep(bSigma, each = nrow(bres))) - scaled[is.na(scaled)] <- -Inf - bT <- scaled[, 1L] - for (j in seq_len(ncol(scaled))[-1L]) bT <- pmax(bT, scaled[, j]) + # mboot already computed the studentized sup-t critical value from + # these draws. Reuse it rather than sorting and scaling a second time. + cval <- bout$crit.val + if (!is.finite(cval)) { + warning("Simultaneous critical value is not finite, likely because the standard errors could not be computed. Falling back to pointwise confidence intervals.") + cval <- qnorm(1 - alp / 2) + dp$cband <- FALSE } - cval <- quantile(bT, 1 - alp, type = 1, na.rm = TRUE) if (cval >= 7) { warning("Simultaneous critical value is arguably `too large' to be reliable. This usually happens when the number of observations per group is small and/or there is not much variation in outcomes.") } diff --git a/R/compute.aggte.R b/R/compute.aggte.R index 44e6240d..6687ae71 100644 --- a/R/compute.aggte.R +++ b/R/compute.aggte.R @@ -155,6 +155,12 @@ compute.aggte <- function(MP, stop("No post-treatment ATT(g,t) estimates are available to aggregate (all cells have t < g).") } + # Keep the observed cell support before na.rm removes estimates. An event + # time with all-NA estimates is still required for balancing; an event time + # absent from the original grid is not. + full_group <- group + full_t <- t + if (na.rm) { notna <- !is.na(att) if (!any(notna)) { @@ -264,7 +270,6 @@ compute.aggte <- function(MP, group <- orig2t_vec(originalgroup) glist <- orig2t_vec(originalglist) tlist <- unique(t) - maxT <- max(t) # Set the weights if (".w" %in% names(dta)) { @@ -342,9 +347,6 @@ compute.aggte <- function(MP, # get standard errors from overall influence function simple.se <- getSE(simple.if, dp) - if (!is.na(simple.se)) { - if (simple.se <= sqrt(.Machine$double.eps) * 10) simple.se <- NA - } return(AGGTEobj( @@ -408,7 +410,6 @@ compute.aggte <- function(MP, # recover standard errors separately by group selective.se.g <- unlist(BMisc::get_list_element(selective.se.inner, "se")) - selective.se.g[selective.se.g <= sqrt(.Machine$double.eps) * 10] <- NA # recover influence function separately by group selective.inf.func.g <- simplify2array(BMisc::get_list_element(selective.se.inner, "inf.func")) @@ -465,9 +466,6 @@ compute.aggte <- function(MP, selective.inf.func <- as.numeric(selective.inf.func) # get overall standard error selective.se <- getSE(selective.inf.func, dp) - if (!is.na(selective.se)) { - if ((selective.se <= sqrt(.Machine$double.eps) * 10)) selective.se <- NA - } return(AGGTEobj( overall.att = selective.att, @@ -508,12 +506,44 @@ compute.aggte <- function(MP, # if we balance the sample with resepect to event time if (!is.null(balance_e)) { - include.balanced.gt <- (t2orig(maxT) - originalgroup >= balance_e) + # Calendar reach alone does not guarantee that a cohort contributes at + # every event time: irregularly spaced periods can leave holes, and + # na.rm can remove individual ATT(g,t) cells. Require common observed + # post-treatment support through balance_e before applying min_e/max_e. + # Use the event times actually represented by the eligible cohorts so + # globally unobserved times do not rule out every cohort on sparse grids. + event_time <- originalt - originalgroup + full_event_time <- full_t - full_group + balance_max_t <- max(full_t) + eligible_full <- balance_max_t - full_group >= balance_e + eligible_g <- unique(full_group[eligible_full]) + required_e <- unique(full_event_time[eligible_full & + full_event_time >= 0 & full_event_time <= balance_e]) + n_required_e <- length(required_e) + complete <- logical(length(eligible_g)) + if (n_required_e > 0L) { + # Count distinct (group, event-time) pairs in one pass rather than + # rescanning every cell for each group. Integer indices encode pairs + # without depending on the magnitude or spacing of the original times. + group_index <- match(originalgroup, eligible_g, nomatch = 0L) + event_index <- match(event_time, required_e, nomatch = 0L) + present <- group_index > 0L & event_index > 0L + present_g <- group_index[present] + pair_key <- present_g + length(eligible_g) * (as.double(event_index[present]) - 1) + counts <- tabulate(present_g[!duplicated(pair_key)], nbins = length(eligible_g)) + complete <- counts == n_required_e + } + if (!any(complete)) { + stop("No groups have non-missing ATT(g,t) estimates at every observed ", + "post-treatment event time through 'balance_e'. Reduce 'balance_e' ", + "or inspect the missing group-time estimates.") + } + include.balanced.gt <- c(FALSE, complete)[group_index + 1L] eseq <- unique(originalt[include.balanced.gt] - originalgroup[include.balanced.gt]) eseq <- eseq[order(eseq)] - eseq <- eseq[(eseq <= balance_e) & (eseq >= balance_e - t2orig(maxT) + t2orig(1))] + eseq <- eseq[(eseq <= balance_e) & (eseq >= balance_e - balance_max_t + t2orig(1))] } # only looks at some event times @@ -555,7 +585,6 @@ compute.aggte <- function(MP, }) dynamic.se.e <- unlist(BMisc::get_list_element(dynamic.se.inner, "se")) - dynamic.se.e[dynamic.se.e <= sqrt(.Machine$double.eps) * 10] <- NA dynamic.inf.func.e <- simplify2array(BMisc::get_list_element(dynamic.se.inner, "inf.func")) @@ -597,9 +626,6 @@ compute.aggte <- function(MP, dynamic.inf.func <- as.numeric(dynamic.inf.func) dynamic.se <- getSE(dynamic.inf.func, dp) - if (!is.na(dynamic.se)) { - if (dynamic.se <= sqrt(.Machine$double.eps) * 10) dynamic.se <- NA - } return(AGGTEobj( overall.att = dynamic.att, @@ -686,7 +712,6 @@ compute.aggte <- function(MP, # recover standard errors separately by time calendar.se.t <- unlist(BMisc::get_list_element(calendar.se.inner, "se")) - calendar.se.t[calendar.se.t <= sqrt(.Machine$double.eps) * 10] <- NA # recover influence function separately by time calendar.inf.func.t <- simplify2array(BMisc::get_list_element(calendar.se.inner, "inf.func")) @@ -731,9 +756,6 @@ compute.aggte <- function(MP, calendar.inf.func <- as.numeric(calendar.inf.func) # get overall standard error calendar.se <- getSE(calendar.inf.func, dp) - if (!is.na(calendar.se)) { - if (calendar.se <= sqrt(.Machine$double.eps) * 10) calendar.se <- NA - } return(AGGTEobj( overall.att = calendar.att, overall.se = calendar.se, @@ -873,7 +895,7 @@ getSE <- function(thisinffunc, DIDparams = NULL) { if (bstrap) { bout <- mboot(thisinffunc, DIDparams, return_V = FALSE) - return(bout$se) + se <- bout$se } else { # Analytical cluster-robust SE (no bootstrap): cluster sums of the aggregated influence function, # matching the cluster-sum aggregation of the multiplier bootstrap (CS 2021, Remark 10). Falls back to @@ -883,8 +905,12 @@ getSE <- function(thisinffunc, DIDparams = NULL) { extra_cl <- if (!is.null(DIDparams)) DIDparams$clustervars[!(DIDparams$clustervars %in% c(DIDparams$idname, ""))] else character(0) if (length(extra_cl) > 0 && !is.null(cv) && length(cv) == length(thisinffunc)) { S <- rowsum(thisinffunc, cv) - return(sqrt(sum(S^2)) / n) + se <- sqrt(sum(S^2)) / n + } else { + se <- sqrt(mean((thisinffunc)^2) / n) } - return(sqrt(mean((thisinffunc)^2) / n)) } + # No absolute cutoff: the same positive variance must remain available in + # every outcome unit. Zero and non-finite scales cannot support inference. + if (!is.finite(se) || se <= 0) NA_real_ else se } diff --git a/R/compute.att_gt.R b/R/compute.att_gt.R index 481ee7d1..e4c67c71 100644 --- a/R/compute.att_gt.R +++ b/R/compute.att_gt.R @@ -331,8 +331,9 @@ compute.att_gt <- function(dp) { pret <- pret_g } - # check if in post-treatment period - if ((glist[g] <= tlist[(t + tfac)])) { + # check if in post-treatment period (anticipation periods, t >= g - anticipation, + # also use the base period g - anticipation - 1) + if ((glist[g] - anticipation <= tlist[(t + tfac)])) { # update pre-period if in post-treatment period to # be period (g-delta-1) pret <- pret_g @@ -347,7 +348,7 @@ compute.att_gt <- function(dp) { # pretest-only: skip post-treatment cells entirely (see # pretreatment_only above; conditional_did_pretest discards these # cells unread, so none of the per-cell work below is needed) - if (pretreatment_only && glist[g] <= tlist[t + tfac]) next + if (pretreatment_only && glist[g] - anticipation <= tlist[t + tfac]) next # use "not yet treated as control" # that is, never treated + units that are eventually treated, @@ -541,14 +542,22 @@ compute.att_gt <- function(dp) { # Run overlap/rank checks on RC data (not wide panel data) if (!is.function(est_method)) { + if (est_method %in% c("dr", "reg") && + rc_outcome_has_too_few_obs(G_rc, post_rc, w_rc, + ncol(covariates_rc), + include_treated = est_method == "dr")) { + warning(paste0("Outcome regression design is singular (fewer positive-weight observations than covariates) for group ", glist[g], " in time period ", tlist[t + tfac], "; the ATT for this cell will be set to NA.")) + stop("singular") + } + if (est_method %in% c("dr", "ipw")) { - if (overlap_check_fail(covariates_rc, G_rc, intercept_only)) { + if (overlap_check_fail(covariates_rc, G_rc, intercept_only, if (is.null(weightsname)) NULL else w_rc)) { warning(paste0("overlap condition violated for group ", glist[g], " in time period ", tlist[t + tfac])) stop("overlap") } } if (est_method %in% c("dr", "reg")) { - if (rcond_check_fail(covariates_rc, G_rc, intercept_only)) { + if (rcond_check_fail(covariates_rc, G_rc, intercept_only, if (is.null(weightsname)) NULL else w_rc)) { warning(paste0("Covariate matrix for control units is singular or numerically ill-conditioned for group ", glist[g], " in time period ", tlist[t + tfac], "; consider centering/rescaling covariates or removing collinear terms")) stop("singular") } @@ -594,9 +603,9 @@ compute.att_gt <- function(dp) { # check for outcome-regression methods, run only if overlap # passed (matching the stop("overlap") short-circuit below) overlap_fail <- (est_method %in% c("dr", "ipw")) && - overlap_check_fail(covariates, G, intercept_only) + overlap_check_fail(covariates, G, intercept_only, if (is.null(weightsname)) NULL else w) rcond_fail <- !overlap_fail && (est_method %in% c("dr", "reg")) && - rcond_check_fail(covariates, G, intercept_only) + rcond_check_fail(covariates, G, intercept_only, if (is.null(weightsname)) NULL else w) guard <- list(overlap_fail = overlap_fail, rcond_fail = rcond_fail) if (use_check_cache) check_cache[[check_key]] <- guard } @@ -866,10 +875,17 @@ compute.att_gt <- function(dp) { # as the panel branch) so that a failing preliminary fit degrades to # an NA cell with a warning instead of aborting the entire att_gt() if (!is.function(est_method)) { + if (est_method %in% c("dr", "reg") && + rc_outcome_has_too_few_obs(G, post, w, ncol(covariates), + include_treated = est_method == "dr")) { + warning(paste0("Outcome regression design is singular (fewer positive-weight observations than covariates) for group ", glist[g], " in time period ", tlist[t + tfac], "; the ATT for this cell will be set to NA.")) + stop("singular") + } + # checks for pscore based methods (no caching on the RC branch: the # cohort index varies cell by cell here) if (est_method %in% c("dr", "ipw")) { - if (overlap_check_fail(covariates, G, intercept_only)) { + if (overlap_check_fail(covariates, G, intercept_only, if (is.null(weightsname)) NULL else w)) { warning(paste0("overlap condition violated for group ", glist[g], " in time period ", tlist[t + tfac])) stop("overlap") } @@ -877,7 +893,7 @@ compute.att_gt <- function(dp) { # check if can run regression using control units if (est_method %in% c("dr", "reg")) { - if (rcond_check_fail(covariates, G, intercept_only)) { + if (rcond_check_fail(covariates, G, intercept_only, if (is.null(weightsname)) NULL else w)) { warning(paste0("Covariate matrix for control units is singular or numerically ill-conditioned for group ", glist[g], " in time period ", tlist[t + tfac], "; consider centering/rescaling covariates or removing collinear terms")) stop("singular") } diff --git a/R/compute.att_gt2.R b/R/compute.att_gt2.R index 4d02b970..efb591db 100644 --- a/R/compute.att_gt2.R +++ b/R/compute.att_gt2.R @@ -153,9 +153,9 @@ run_DRDID <- function(cohort_data, covariates, dp2, g_val = NULL, t_val = NULL, # for outcome-regression methods, run only if overlap passed (matching # the early return on overlap failure below) overlap_fail <- (dp2$est_method %in% c("dr", "ipw")) && - overlap_check_fail(covariates, D_vec, intercept_only) + overlap_check_fail(covariates, D_vec, intercept_only, if (is.null(dp2$weightsname)) NULL else i.weights) rcond_fail <- !overlap_fail && (dp2$est_method %in% c("dr", "reg")) && - rcond_check_fail(covariates, D_vec, intercept_only) + rcond_check_fail(covariates, D_vec, intercept_only, if (is.null(dp2$weightsname)) NULL else i.weights) guard <- list(overlap_fail = overlap_fail, rcond_fail = rcond_fail) if (!is.null(check_cache_key)) dp2$.check_cache[[check_cache_key]] <- guard } @@ -296,10 +296,18 @@ run_DRDID <- function(cohort_data, covariates, dp2, g_val = NULL, t_val = NULL, if (!custom_est_method) { D_vec <- D + if (dp2$est_method %in% c("dr", "reg") && + rc_outcome_has_too_few_obs(D, post, i.weights, ncol(covariates), + include_treated = dp2$est_method == "dr")) { + warning(paste0("Outcome regression design is singular (fewer positive-weight observations than covariates)", gt_label, "; the ATT for this cell will be set to NA.")) + return(list(att = NA, if_i = if (do_inf) seq_len(n_inf) else NULL, + if_x = if (do_inf) rep(NA_real_, n_inf) else NULL)) + } + # checks for pscore based methods (no caching on the RC branch: the # cohort index varies cell by cell here) if (dp2$est_method %in% c("dr", "ipw")) { - if (overlap_check_fail(covariates, D_vec, intercept_only)) { + if (overlap_check_fail(covariates, D_vec, intercept_only, if (is.null(dp2$weightsname)) NULL else i.weights)) { warning(paste0("overlap condition violated", gt_label)) return(list(att = NA, if_i = if (do_inf) seq_len(n_inf) else NULL, if_x = if (do_inf) rep(NA_real_, n_inf) else NULL)) @@ -308,7 +316,7 @@ run_DRDID <- function(cohort_data, covariates, dp2, g_val = NULL, t_val = NULL, # check if can run regression using control units if (dp2$est_method %in% c("dr", "reg")) { - if (rcond_check_fail(covariates, D_vec, intercept_only)) { + if (rcond_check_fail(covariates, D_vec, intercept_only, if (is.null(dp2$weightsname)) NULL else i.weights)) { warning(paste0("Covariate matrix for control units is singular or numerically ill-conditioned", gt_label, "; consider centering/rescaling covariates or removing collinear terms")) return(list(att = NA, if_i = if (do_inf) seq_len(n_inf) else NULL, if_x = if (do_inf) rep(NA_real_, n_inf) else NULL)) @@ -432,8 +440,9 @@ run_att_gt_estimation <- function(g, t, dp2){ pret <- pret_g } - # check if in post-treatment period - if ((dp2$treated_groups[g] <= dp2$time_periods[(t+tfac)])) { + # check if in post-treatment period (anticipation periods, t >= g - anticipation, + # also use the base period g - anticipation - 1) + if ((dp2$treated_groups[g] - dp2$anticipation <= dp2$time_periods[(t+tfac)])) { # update pre-period if in post-treatment period to # be period (g-delta-1) diff --git a/R/conditional_did_pretest.R b/R/conditional_did_pretest.R index 65c8c5f2..82c48865 100644 --- a/R/conditional_did_pretest.R +++ b/R/conditional_did_pretest.R @@ -6,11 +6,25 @@ #' @inheritParams att_gt #' @param weightsname The name of the column containing the sampling weights. #' If not set, all observations have same weight. -#' @param est_method the method to compute group-time average treatment -#' effects. The default for `conditional_did_pretest` is "ipw" for -#' inverse probability weighting. Other built-in methods include "dr" -#' which uses the doubly robust approach in the `DRDID` package and "reg" -#' for first step regression estimators. +#' @param xformla A formula such as `~ X1 + X2` specifying covariates for the +#' propensity score model and the indicator weighting functions. The default +#' `NULL` is equivalent to `~1`. For panel data, use conditioning variables +#' that are fixed within each unit across periods. If raw covariates vary +#' over time, create explicit unit-level baseline copies and include those +#' copies in `xformla`; the test then conditions on those fixed baseline +#' variables. For repeated cross sections (`panel = FALSE`), each observation's +#' observed covariates are used. +#' @param allow_unbalanced_panel Passed to panel preprocessing. The default +#' `FALSE` balances the panel by dropping units observed in only some periods. +#' With `TRUE`, an already balanced panel is accepted, but a genuinely +#' unbalanced panel is not supported by this conditional pre-test. Ignored +#' when `panel = FALSE`. +#' @param est_method The estimation method for the conditional moments. +#' Only "ipw" (the default), for inverse probability weighting, is supported. +#' This test requires a correctly specified propensity score model. +#' Regression ("reg"), doubly robust ("dr"), and custom estimation methods +#' are not supported by this legacy test. For pre-treatment tests using +#' those estimation methods, see the tests reported by [att_gt()]. #' @param bstrap Not used by the pre-test. Critical values and p-values #' for the Cramer von Mises and Kolmogorov-Smirnov test statistics are #' always computed with the multiplier bootstrap (using `biters` @@ -34,6 +48,24 @@ #' parallelizes Step 1 (computing the test statistic); Step 2's multiplier #' bootstrap is vectorized and runs in a single process. #' +#' @details +#' The conditional moments use outcomes multiplied by covariate indicator +#' functions. A correctly specified outcome regression for the original +#' outcome need not remain correctly specified for every indicator-weighted +#' outcome. Consequently, refitting the original regression model to those +#' outcomes does not provide a valid regression or doubly robust conditional +#' test under correctness of the original outcome model alone. This function +#' implements the inverse probability weighted test only. +#' +#' For panel data, the indicator weighting functions use the covariate vector +#' from the first observed period. The conditional null is formulated using +#' a fixed vector of conditioning variables throughout the pre-treatment +#' comparisons. The legacy test is not justified as a test of separate +#' conditional nulls whose conditioning variables change with the baseline +#' period of each group-time comparison. Supplying time-varying raw panel +#' covariates does not establish that different interpretation; choose and +#' supply fixed baseline variables explicitly when that is the intended null. +#' #' @examples #' \dontrun{ #' data(mpdta) @@ -70,6 +102,11 @@ conditional_did_pretest <- function(yname, pl=FALSE, cores=1) { + validate_choice_scalar( + est_method, "est_method", "ipw", + "conditional_did_pretest() only supports est_method = \"ipw\". The legacy conditional-moment test is not implemented for regression (\"reg\"), doubly robust (\"dr\"), or custom estimation methods. For pre-treatment tests using those methods, see att_gt()." + ) + message("We are no longer updating this function. It should continue to work, but most users find the pre-tests already reported by the `att_gt` function to be sufficient for most empirical applications.") if (missing(control_group)) control_group <- "nevertreated" @@ -101,6 +138,14 @@ conditional_did_pretest <- function(yname, tlist <- dp$tlist glist <- dp$glist n <- dp$n + # Use the pre-processor's resolved data structure, formula, and observation id. + # A balanced panel resets allow_unbalanced_panel to FALSE; genuine repeated + # cross sections ignore that flag. The legacy DIDparams records unbalanced + # panels as panel = FALSE and true_repeated_cross_sections = FALSE. + xformla <- dp$xformla + idname <- dp$idname + panel <- dp$panel + allow_unbalanced_panel <- !panel && !dp$true_repeated_cross_sections # check if possible to do test # note: tlist[2] contains the 2nd time period diff --git a/R/mboot.R b/R/mboot.R index 15eda490..c4fc7eaa 100644 --- a/R/mboot.R +++ b/R/mboot.R @@ -140,12 +140,14 @@ mboot <- function(inf.func, DIDparams, pl = FALSE, cores = 1, return_V = TRUE) { if (isTRUE(class(bres) == "numeric")) bres <- as.matrix(bres) - # Non-degenerate dimensions - # ndg.dim <- (base::colSums(bres) != 0) - ndg.dim <- (!is.na(colSums(bres))) & (base::colSums(bres^2) > sqrt(.Machine$double.eps)*10) + # Drop only non-finite or identically zero bootstrap dimensions. An absolute + # cutoff on their sum of squares incorrectly drops valid dimensions when the + # outcome is expressed in smaller units (and squaring can underflow). + ndg.dim <- colSums(!is.finite(bres)) == 0L & + colSums(bres != 0, na.rm = TRUE) > 0L # If NA, set it to false #ndg.dim[is.na(ndg.dim)] <- FALSE - bres <- as.matrix(bres[ , ndg.dim]) + bres <- as.matrix(bres[ , ndg.dim, drop = FALSE]) # bootstrap variance matrix (this matrix can be defective because of degenerate cases). # Computed only on request: no internal caller consumes it, and it is the only @@ -163,6 +165,9 @@ mboot <- function(inf.func, DIDparams, pl = FALSE, cores = 1, return_V = TRUE) { nb <- length(b) (b[ceiling(.75 * nb)] - b[ceiling(.25 * nb)]) / iqr_norm }) + # A zero/non-finite IQR scale cannot studentize a bootstrap dimension. + # Retain arbitrarily small positive scales regardless of outcome units. + bSigma[!is.finite(bSigma) | bSigma <= 0] <- NA_real_ # critical value for uniform confidence band: per bootstrap draw, the max over # columns of |b / bSigma|. Vectorized as a column-fold of pmax (equivalent to the diff --git a/R/pre_process_did2.R b/R/pre_process_did2.R index 33fe64d1..9c667890 100644 --- a/R/pre_process_did2.R +++ b/R/pre_process_did2.R @@ -40,17 +40,19 @@ validate_args <- function(args, data){ checkvar_message <- "__ARG__ must be NULL or a character scalar that is a name of a column from the dataset." dreamerr::check_set_arg(args$weightsname, args$clustervars, "NULL | match", .choices = data_names, .message = checkvar_message, .up = 1) + # Access argument-named columns outside data.table expressions. Otherwise a + # user column named "args" shadows this function's argument list. # check if time periods are numeric - if(!data[, is.numeric(get(args$tname))]){stop("The time variable '", args$tname, "' must be numeric. Please convert it.")} + if(!is.numeric(data[[args$tname]])){stop("The time variable '", args$tname, "' must be numeric. Please convert it.")} # Check if gname is numeric - if(!data[, is.numeric(get(args$gname))]){stop("The group variable '", args$gname, "' must be numeric. Please convert it.")} + if(!is.numeric(data[[args$gname]])){stop("The group variable '", args$gname, "' must be numeric. Please convert it.")} # gname must be 0 (never-treated) or a positive treatment-timing value; # negative codes are not supported (0 is reserved for never-treated). Reject # them up front so the fast and slow paths behave identically (the fast path # previously accepted negative codes silently while the slow path errored). - if (data[, any(get(args$gname) < 0, na.rm = TRUE)]) { + if (any(data[[args$gname]] < 0, na.rm = TRUE)) { stop("The group variable '", args$gname, "' must be 0 (never-treated) or a ", "positive treatment-timing value; negative values are not supported. ", "If your time periods are non-positive, shift them so the earliest ", @@ -58,7 +60,7 @@ validate_args <- function(args, data){ } # Check if yname is numeric (logical 0/1 outcomes are also allowed) - if(!data[, is.numeric(get(args$yname)) || is.logical(get(args$yname))]){stop("The outcome variable '", args$yname, "' must be numeric. Please convert it.")} + if(!(is.numeric(data[[args$yname]]) || is.logical(data[[args$yname]]))){stop("The outcome variable '", args$yname, "' must be numeric. Please convert it.")} # Flag for idname if(!is.null(args$idname)){ @@ -67,7 +69,7 @@ validate_args <- function(args, data){ dreamerr::check_set_arg(args$idname, "match", .choices = data_names, .message = name_message, .up = 1) # check if idname is numeric - if(!data[, is.numeric(get(args$idname))]){stop("The id variable '", args$idname, "' must be numeric. Please convert it.")} + if(!is.numeric(data[[args$idname]])){stop("The id variable '", args$idname, "' must be numeric. Please convert it.")} # With user-level panel = FALSE the data are genuine repeated cross sections: # every observation is its own sampling unit, so a supplied idname must not @@ -117,7 +119,7 @@ validate_args <- function(args, data){ # no within-unit time variation to check and idname is not yet available here -- skip the check then. if (length(args$clustervars) > 0 && !is.null(args$idname)) { # Efficiently check for time-varying cluster variables - clust_tv <- data[, lapply(.SD, function(col) length(unique(col)) == 1), by = get(args$idname), .SDcols = args$clustervars] + clust_tv <- data[, lapply(.SD, function(col) length(unique(col)) == 1), by = c(args$idname), .SDcols = args$clustervars] # If any cluster variable varies over time within any unit, stop execution # (same wording as the slow path and mboot()) if (!all(unlist(clust_tv[, -1, with = FALSE]))) { @@ -159,22 +161,23 @@ did_standardization <- function(data, args){ data <- data[, ..cols_to_keep] # Check if any covariates were missing - n_orig <- data[, .N] + n_orig <- nrow(data) # gname is excluded from the finite check because Inf is a valid never-treated # code there (see complete_finite_cases); missing/NaN gname is still dropped. - data <- data[complete_finite_cases(data, finite_exclude = args$gname)] + finite_rows <- complete_finite_cases(data, finite_exclude = args$gname) + data <- data[finite_rows] # also drop rows whose EVALUATED design is missing/non-finite (e.g. log of a non-positive # covariate); safe now that raw-covariate NAs are removed (so poly()/ns()/... will # not error on NA input). Use model.frame (NOT model.matrix) with na.action = # na.pass: model.frame keeps every row -- including NA/NaN-valued terms -- so the # complete.cases() mask stays aligned with `data` (model.matrix would silently drop # NaN rows, shortening the mask and letting the offending rows survive). - if (length(xvars) > 0L && data[, .N] > 0L) { + if (length(xvars) > 0L && nrow(data) > 0L) { mf_check <- suppressWarnings(model.frame(args$xformla, data = data, na.action = na.pass)) finite_rows <- complete_finite_cases(mf_check) if (!all(finite_rows)) data <- data[finite_rows] } - n_new <- data[, .N] + n_new <- nrow(data) n_diff <- n_orig - n_new if (n_diff != 0) { warning(paste0("dropped ", n_diff, " rows from original data due to missing or non-finite data")) @@ -214,15 +217,15 @@ did_standardization <- function(data, args){ } # get a list of dates from min to max - tlist <- data[, sort(unique(get(args$tname)))] + tlist <- sort(unique(data[[args$tname]])) # Ensure gname column is numeric (double) so it can hold Inf if (is.integer(data[[args$gname]])) { - data[, (args$gname) := as.numeric(get(args$gname))] + set(data, j = args$gname, value = as.numeric(data[[args$gname]])) } # Coerce control group identified with zero as Inf - data[get(args$gname) == 0, (args$gname) := Inf] + set(data, i = which(data[[args$gname]] == 0), j = args$gname, value = Inf) # Working out the dates # Identify groups with treatment time bigger than the maximum treatment time + anticipation. @@ -276,14 +279,16 @@ did_standardization <- function(data, args){ ) # Drop all periods ≥ (latest_g - anticipation) - data <- data[get(args$tname) < cutoff_t] + keep_periods <- data[[args$tname]] < cutoff_t + data <- data[keep_periods] # For any row where gname == latest_g, set gname := Inf - data[, (args$gname) := as.numeric(get(args$gname))] # Convert the column to numeric so Inf can be stored - data[get(args$gname) == latest_g, (args$gname) := Inf] + set(data, j = args$gname, value = as.numeric(data[[args$gname]])) # Convert the column to numeric so Inf can be stored + set(data, i = which(data[[args$gname]] == latest_g), j = args$gname, value = Inf) } else { # 3. If not "nevertreated", we simply drop those periods and leave gnames alone - data <- data[get(args$tname) < cutoff_t] + keep_periods <- data[[args$tname]] < cutoff_t + data <- data[keep_periods] } if (nrow(data) == 0) { stop("No periods before the last treated cohort's treatment date (net of anticipation) remain. Check that 'gname' is measured on the same scale as 'tname'.") @@ -331,8 +336,8 @@ did_standardization <- function(data, args){ data <- data[!treated_first_period] # update tlist and glist - tlist <- data[, sort(unique(get(args$tname)))] - glist <- data[, sort(unique(get(args$gname)))] + tlist <- sort(unique(data[[args$tname]])) + glist <- sort(unique(data[[args$gname]])) # Drop groups treated in the first period or before first_period <- tlist[1] @@ -404,7 +409,8 @@ did_standardization <- function(data, args){ unit_count <- data[, .(count = .N), by = c(args$idname)] mis_unit_ids <- unit_count[[1L]][unit_count[[2L]] < raw_time_size] warning(length(mis_unit_ids), " units are missing in some periods. Converting to balanced panel by dropping them.") - data <- data[!(data[[args$idname]] %in% mis_unit_ids)] + keep_units <- !(data[[args$idname]] %in% mis_unit_ids) + data <- data[keep_units] } # If all data is dropped, stop execution @@ -424,10 +430,11 @@ did_standardization <- function(data, args){ if (!any(is.infinite(data[[args$gname]])) && (is.na(latest_g) || max(data[[args$gname]]) < latest_g)) { latest_g <- max(data[[args$gname]]) cutoff_t <- latest_g - args$anticipation - data <- data[get(args$tname) < cutoff_t] + keep_periods <- data[[args$tname]] < cutoff_t + data <- data[keep_periods] if (args$control_group == "nevertreated") { warning("No never-treated group is available after converting to balanced panel (set allow_unbalanced_panel = TRUE to keep the never-treated units). The last treated cohort is being coerced as 'never-treated' units, and data from periods after that is being filtered out (no available comparison groups).") - data[get(args$gname) == latest_g, (args$gname) := Inf] + set(data, i = which(data[[args$gname]] == latest_g), j = args$gname, value = Inf) } else { warning("No never-treated group is available after converting to balanced panel (set allow_unbalanced_panel = TRUE to keep the never-treated units). The last treated cohort is used only as a comparison group, and data from periods after that is being filtered out (no available comparison groups).") } @@ -460,7 +467,7 @@ did_standardization <- function(data, args){ args$idname <- ".rowid" } else { # Set rowid to idname for repeated cross section/unbalanced - data[, .rowid := get(args$idname)] + set(data, j = ".rowid", value = data[[args$idname]]) } # Count unique number of cross section observations @@ -493,13 +500,16 @@ did_standardization <- function(data, args){ # Check against very small groups # Calculate group sizes, dividing the count of each group by the length of tlist - gsize <- data[, .N / length(tlist), by = get(args$gname)] + gsize <- data[, .N, by = c(args$gname)] + setnames(gsize, c("group", "V1")) + set(gsize, j = "V1", value = gsize[[2L]] / length(tlist)) # How many in each group before giving a warning reqsize <- length(BMisc::rhs_vars(args$xformla)) + 5 # Filter groups smaller than reqsize - gsize <- gsize[V1 < reqsize] + small_group_rows <- gsize[[2L]] < reqsize + gsize <- gsize[small_group_rows] # Warn if some groups are small if (nrow(gsize) > 0) { @@ -596,11 +606,13 @@ get_did_tensors <- function(data, args){ #if(!args$allow_unbalanced_panel){ if(args$panel){ # We can do this filtering because the data is already sorted appropriately - invariant_data <- data[1:args$id_count] + first_period_rows <- seq_len(args$id_count) + invariant_data <- data[first_period_rows] } else { if (!args$allow_unbalanced_panel) { # get the first observation for each unit - invariant_data <- data[data[, .I[1], by = get(args$idname)]$V1] + first_unit_rows <- data[, .I[1], by = c(args$idname)][[2L]] + invariant_data <- data[first_unit_rows] # # order by idname # setorderv(invariant_data, c(args$idname), c(1)) } else { @@ -616,7 +628,7 @@ get_did_tensors <- function(data, args){ cohorts <- c(args$treated_groups, Inf) if (args$panel) { # Get cohort counts - cohort_counts <- invariant_data[, .(cohort_size = .N) , by = get(args$gname)] + cohort_counts <- invariant_data[, .(cohort_size = .N) , by = c(args$gname)] names(cohort_counts)[1] <- "cohort" # changing the name # Get period counts: all invariant_data rows share the first time period @@ -629,7 +641,7 @@ get_did_tensors <- function(data, args){ N = cohort_counts$cohort_size) } else { # Get crosstable counts (long form): count occurrences by tname and gname - crosstable_counts <- invariant_data[, .N, by = .(get(args$tname), get(args$gname))] + crosstable_counts <- invariant_data[, .N, by = c(args$tname, args$gname)] names(crosstable_counts)[1] <- "period" # changing the name names(crosstable_counts)[2] <- "cohort" # changing the name diff --git a/R/utility_functions.R b/R/utility_functions.R index 16fa1fdf..3777b339 100644 --- a/R/utility_functions.R +++ b/R/utility_functions.R @@ -82,9 +82,23 @@ overlap_logit_fit <- function(x, y) { #' @param x covariate matrix #' @param y treatment indicator #' @param intercept_only whether `x` is a single column of ones +#' @param i.weights sampling weights passed to the 2x2 estimator; observations +#' whose normalized weight is zero cannot restrict the estimator's support #' @return TRUE if the overlap condition is violated #' @noRd -overlap_check_fail <- function(x, y, intercept_only = FALSE) { +overlap_check_fail <- function(x, y, intercept_only = FALSE, i.weights = NULL) { + if (!is.null(i.weights)) { + weight_mean <- mean(i.weights) + # Positive normalized support is the usual case. Test its lower bound + # without allocating a normalized vector or copying the whole design. + if (!isTRUE(min(i.weights) / weight_mean > 0)) { + supported <- (i.weights / weight_mean) > 0 + if (anyNA(supported)) supported[is.na(supported)] <- FALSE + x <- x[supported, , drop = FALSE] + y <- y[supported] + } + } + if (length(y) == 0L) return(TRUE) if (intercept_only) { pbar <- mean(y) if (abs(pbar - 0.999) > 1e-6) return(pbar >= 0.999) @@ -94,22 +108,73 @@ overlap_check_fail <- function(x, y, intercept_only = FALSE) { } #' @title rcond_check_fail -#' @description Evaluate the per-(g,t) regression-feasibility guard, i.e. -#' whether `rcond(crossprod(x[controls, ]))` is below machine epsilon. For an -#' intercept-only design the control-unit Gram matrix is the scalar count of -#' control units (n0): `rcond` is exactly 1 when n0 > 0 and exactly 0 when -#' n0 == 0, so the test reduces to "no control units" with no Gram matrix. +#' @description Evaluate the per-(g,t) regression-feasibility guard using the +#' normalized sampling-weighted control Gram matrix, as in DRDID's outcome +#' regression. Zero-weight observations cannot supply rank or spoil conditioning. +#' For an intercept-only design the weighted Gram matrix is a scalar: +#' `rcond` is exactly 1 when controls have positive weight and exactly 0 +#' otherwise, so the test reduces to "no supported control units". #' @param x covariate matrix #' @param y treatment indicator (controls are `y == 0`) #' @param intercept_only whether `x` is a single column of ones +#' @param i.weights sampling weights passed to the 2x2 estimator #' @return TRUE if the control-unit design is singular or ill-conditioned #' @noRd -rcond_check_fail <- function(x, y, intercept_only = FALSE) { - if (intercept_only) { - return(!any(y == 0)) +rcond_check_fail <- function(x, y, intercept_only = FALSE, i.weights = NULL) { + if (is.null(i.weights)) { + # With no user weights, retain the original unweighted rank guard. + if (intercept_only) return(!any(y == 0)) + control_covs <- x[y == 0, , drop = FALSE] + return(rcond(crossprod(control_covs)) < .Machine$double.eps) + } + n <- length(y) + controls <- y == 0 + if (!any(controls)) return(TRUE) + # Constant positive weights normalize to one. Avoid normalizing and copying + # the full design, while retaining the same normalized control Gram matrix. + # Keep the two-matrix crossprod call even at unit weights: its roundoff can + # differ from the symmetric crossprod shortcut near the rank cutoff. + weight_min <- min(i.weights) + constant_weights <- isTRUE(weight_min > 0 && weight_min == max(i.weights)) + if (constant_weights) { + if (intercept_only) return(FALSE) + control_covs <- x[controls, , drop = FALSE] + return(rcond(crossprod(control_covs * 1, control_covs) / n) < .Machine$double.eps) + } + i.weights <- i.weights / mean(i.weights) + controls <- controls & i.weights > 0 + if (anyNA(controls)) controls[is.na(controls)] <- FALSE + if (!any(controls)) return(TRUE) + if (intercept_only) return(FALSE) + control_covs <- x[controls, , drop = FALSE] + rcond(crossprod(control_covs * i.weights[controls], control_covs) / n) < + .Machine$double.eps +} + +# A repeated-cross-section outcome regression needs at least as many +# positive-weight observations as design columns in EACH fitted group/period. +# This necessary rank condition must be checked before entering native fitting: +# tryCatch cannot recover a native crash on an undersized design. The default +# DR estimator fits all four regressions; regression-only fits just controls. +# Mirror DRDID's local mean normalization: a finite positive input can round to +# zero there. Count the actual normalized support without changing passed weights. +# Passing this check does not establish full rank (the estimator checks that). +rc_outcome_has_too_few_obs <- function(D, post, i.weights, n_covariates, + include_treated = TRUE) { + scale <- mean(i.weights) + if (!is.finite(scale) || scale <= 0) return(TRUE) + # Ordinary positive weights retain every row after this local normalization. + # Avoid allocating the normalized vector, support mask, and two row subsets. + # Underflow and zero weights still take the exact normalized-support route. + if (isTRUE(min(i.weights) / scale > 0)) { + counts <- tabulate(1L + post + 2L * D, nbins = 4L) + } else { + normalized <- i.weights / scale + positive <- is.finite(normalized) & normalized > 0 + counts <- tabulate(1L + post[positive] + 2L * D[positive], nbins = 4L) } - control_covs <- x[y == 0, , drop = FALSE] - rcond(crossprod(control_covs)) < .Machine$double.eps + required <- if (include_treated) 1:4 else 1:2 + any(counts[required] < n_covariates) } #' @title Check User-Supplied Names Against Internal Names diff --git a/dev/benchmarks/stress-fixes/README.md b/dev/benchmarks/stress-fixes/README.md new file mode 100644 index 00000000..fb3afe41 --- /dev/null +++ b/dev/benchmarks/stress-fixes/README.md @@ -0,0 +1,116 @@ +# Performance checks for the stress-test fixes + +The main benchmark compared clean `origin/master` commit `74b88eb07f1faa644df1271055a0f77848f6550c` with the combined fixes at `1fbfd29`, using the same installed DRDID 1.4.0. It detected no material end-to-end slowdown in the tested workloads. Three end-to-end medians increased by 0.4–0.9% (0.5–1 ms), with overlapping interquartile ranges. Across the 28 end-to-end cases, the geometric mean final/baseline time ratio was 0.950. + +The isolated weighted rank guard costs more: 100 calls took 4 ms on master and 9 ms with the correction (about 50 microseconds extra per call). The corrected guard evaluates the sampling-weighted design; master checked the wrong unweighted matrix. The corresponding weighted panel workloads showed no measured end-to-end regression. This does not mean every internal check is faster. + +## Reproduce + +The examples below run from the repository root, with R and all package dependencies installed. The script may also be invoked by absolute path from another working directory: it finds the repository and worker from its own location. Copied source trees stay outside the tracked benchmark directory. Both workers assert DRDID 1.4.0. + +```sh +Rscript dev/benchmarks/stress-fixes/performance.R 7 2 /tmp/did-main-performance main +Rscript dev/benchmarks/stress-fixes/performance.R 7 2 /tmp/did-supplemental-performance supplemental +``` + +The driver archives the fixed baseline and snapshots the current working source. Its metadata records the source commit, dirty-file summary, and R-source hashes. Re-running after further code changes measures those changes. Output includes copied source trees, complete worker sessions, raw timings, medians, interquartile ranges, and final/baseline ratios. Do not commit the output source trees. + +## Measurement design + +- R 4.6.0 on aarch64 macOS Tahoe 26.5.1, Accelerate BLAS; DRDID 1.4.0. Full versions are in `session-info.txt`. +- Data seed: 20261028. T = 6 with n = 1,000 and 5,000 panel units; repeated cross sections have one independent observation ID per row. The supplement adds n = 3,000, T = 12. +- Separate sequential R processes in baseline/final/final/baseline order. Each path is warmed twice in each process, then measured seven times: 14 measurements per source/case. +- Cases are randomly ordered within each timed round. Loading, data generation, and explicit garbage collection occur outside timings. Automatic allocation/collection costs during the function call remain included. +- Other agents and the parent process stopped heavy R work during timing. +- Bootstrap cases use 499 iterations. The small conditional pre-test uses n = 60 and 99 iterations. +- Millisecond elapsed-time resolution and local machine noise limit interpretation of small differences. IQR overlap is descriptive and is not a formal equivalence test. These workloads do not establish a universal performance bound. + +## Main measurements + +All values below are elapsed milliseconds. Guard microbenchmarks batch 100 calls, or 1,000 calls for intercept-only checks. + +| Case | Master median [Q25, Q75] | Fixed median [Q25, Q75] | Fixed / master | +|---|---:|---:|---:| +| `aggte_1k_dynamic` | 4.00 [4.00, 5.00] | 4.00 [4.00, 4.75] | 1.000 | +| `aggte_1k_dynamic_boot` | 18.00 [17.00, 18.75] | 17.00 [17.00, 18.00] | 0.944 | +| `aggte_5k_dynamic` | 6.00 [6.00, 6.00] | 6.00 [6.00, 6.00] | 1.000 | +| `aggte_5k_simple` | 2.00 [1.25, 2.00] | 1.50 [1.00, 2.00] | 0.750 | +| `aggte_unbalanced_dynamic` | 30.50 [30.00, 32.75] | 29.50 [29.00, 30.00] | 0.967 | +| `conditional_pretest_60` | 88.00 [82.25, 98.50] | 81.50 [81.00, 82.00] | 0.926 | +| `guard_overlap_5k` | 75.00 [72.50, 77.00] | 73.00 [72.00, 73.00] | 0.973 | +| `guard_overlap_intercept_5k` | 12.50 [12.00, 13.00] | 12.00 [12.00, 13.00] | 0.960 | +| `guard_rank_5k` | 4.00 [4.00, 4.00] | 4.00 [3.00, 4.00] | 1.000 | +| `guard_rank_intercept_5k` | 7.00 [7.00, 7.00] | 7.00 [7.00, 7.00] | 1.000 | +| `guard_rank_weighted_5k` | 4.00 [4.00, 4.00] | 9.00 [9.00, 9.75] | 2.250 | +| `panel_1k_fast_dr_boot` | 30.00 [29.00, 30.75] | 26.00 [26.00, 26.75] | 0.867 | +| `panel_1k_fast_dr_nox` | 19.50 [19.00, 20.00] | 17.50 [17.00, 18.00] | 0.897 | +| `panel_1k_fast_dr_x` | 25.00 [24.00, 26.00] | 22.00 [22.00, 22.00] | 0.880 | +| `panel_1k_fast_reg_x` | 18.00 [17.00, 18.00] | 16.00 [16.00, 16.00] | 0.889 | +| `panel_1k_slow_dr_x` | 16.00 [16.00, 17.00] | 16.00 [15.25, 16.00] | 1.000 | +| `panel_5k_fast_dr_boot` | 81.50 [81.00, 84.50] | 78.00 [77.00, 78.00] | 0.957 | +| `panel_5k_fast_dr_nox` | 40.00 [39.25, 42.50] | 37.00 [37.00, 38.00] | 0.925 | +| `panel_5k_fast_dr_weighted` | 71.50 [71.00, 74.50] | 67.50 [67.00, 68.00] | 0.944 | +| `panel_5k_fast_dr_x` | 65.00 [64.00, 68.25] | 63.00 [62.00, 64.00] | 0.969 | +| `panel_5k_fast_reg_x` | 44.50 [43.00, 45.75] | 41.00 [41.00, 41.75] | 0.921 | +| `panel_5k_slow_dr_nox` | 29.50 [29.00, 30.75] | 29.00 [28.25, 29.00] | 0.983 | +| `panel_5k_slow_dr_weighted` | 50.00 [49.25, 52.75] | 50.00 [49.00, 50.75] | 1.000 | +| `panel_5k_slow_dr_x` | 48.00 [46.00, 49.00] | 46.50 [45.25, 47.00] | 0.969 | +| `panel_5k_slow_reg_x` | 24.00 [23.00, 25.75] | 24.00 [23.00, 24.00] | 1.000 | +| `rc_1k_fast_dr_boot` | 69.00 [68.00, 69.75] | 66.50 [64.25, 68.00] | 0.964 | +| `rc_1k_fast_dr_x` | 49.00 [49.00, 49.00] | 48.00 [47.25, 49.75] | 0.980 | +| `rc_5k_fast_dr_x` | 148.00 [147.00, 149.00] | 149.00 [148.00, 153.00] | 1.007 | +| `rc_5k_slow_dr_x` | 135.50 [134.00, 139.00] | 136.00 [136.00, 137.75] | 1.004 | +| `unbalanced_1k_fast_dr_boot` | 54.50 [53.25, 56.00] | 52.00 [50.25, 52.75] | 0.954 | +| `unbalanced_1k_fast_dr_x` | 48.50 [47.00, 49.75] | 46.00 [46.00, 48.00] | 0.948 | +| `unbalanced_5k_fast_dr_x` | 137.50 [135.25, 144.00] | 137.00 [136.00, 138.00] | 0.996 | +| `unbalanced_5k_slow_dr_x` | 113.50 [112.00, 118.00] | 114.50 [113.25, 118.25] | 1.009 | + +Raw observations are in `main-raw-timings.csv`; the complete summary, including CPU medians, is in `main-comparison.csv`. + +## Supplemental measurements + +The first supplemental run at `1a16c46` detected two regressions: n = 3,000, T = 12 dynamic aggregation with `balance_e = 2` increased from 8 to 9 ms; n = 5,000 slow varying-weight DR increased from 169 to 175.5 ms. Both directions repeated across the two process rounds with non-overlapping IQRs. Those observations remain in `supplemental-before-optimization-{comparison,raw-timings}.csv`. + +The code was then changed to replace repeated cohort/event scans with vectorized support counting and to avoid allocating normalized RC weights/support subsets when all weights remain positive after normalization. Independent differential checks preserved selector and support decisions. The same seeded supplemental design was measured again in a new quiet ABBA run. + +In the final supplemental run, no material end-to-end slowdown was detected. Slow varying-weight DR measured 169 versus 170 ms (0.6%, overlapping IQRs); the T = 12 balance_e = 2 case measured 9 versus 9 ms. The baseline balance measurement itself moved from 8 to 9 ms between runs, so the full 1 ms change should not be attributed solely to optimization. The geometric mean final/baseline time ratio across the 12 supplemental cases was 0.968. + +The frozen optimized source was a working tree above `1a16c46`; exact R-file hashes and the dirty-file summary are in `optimized-source-manifest.txt`. The main 33-case measurements preceded these last two targeted optimizations; the final run remeasured all 12 supplemental cases. + +| Case | Master median [Q25, Q75] | Fixed median [Q25, Q75] | Fixed / master | +|---|---:|---:|---:| +| `aggte_1k_dynamic_balance_e2` | 3.00 [3.00, 3.00] | 3.00 [3.00, 3.00] | 1.000 | +| `aggte_3k_T12_dynamic_balance_e2` | 9.00 [8.25, 9.00] | 9.00 [8.00, 9.00] | 1.000 | +| `aggte_3k_T12_dynamic_balance_e5` | 9.00 [9.00, 9.00] | 9.00 [9.00, 9.00] | 1.000 | +| `aggte_5k_dynamic_balance_e2` | 4.00 [4.00, 4.00] | 4.00 [4.00, 4.00] | 1.000 | +| `panel_1k_fast_dr_varying` | 49.50 [49.00, 50.75] | 47.00 [47.00, 48.00] | 0.949 | +| `panel_1k_fast_reg_weighted` | 19.00 [18.00, 19.00] | 17.00 [17.00, 17.00] | 0.895 | +| `panel_5k_fast_dr_varying` | 142.00 [140.50, 142.75] | 140.00 [138.50, 141.00] | 0.986 | +| `panel_5k_fast_reg_weighted` | 48.00 [48.00, 48.00] | 45.00 [45.00, 45.00] | 0.937 | +| `panel_5k_slow_dr_varying` | 169.00 [168.00, 170.75] | 170.00 [169.25, 171.00] | 1.006 | +| `panel_5k_slow_reg_weighted` | 26.50 [26.00, 27.00] | 26.00 [26.00, 26.00] | 0.981 | +| `rc_1k_fast_reg_weighted` | 29.00 [28.25, 30.00] | 27.00 [27.00, 27.75] | 0.931 | +| `unbalanced_1k_fast_reg_weighted` | 30.00 [30.00, 30.75] | 28.00 [28.00, 28.00] | 0.933 | + +Final raw observations are in `supplemental-raw-timings.csv`; the complete final summary is in `supplemental-comparison.csv`. + +### RC preflight microbenchmark + +`support-performance.R` compares the original correct RC support helper from `1a16c46` with the optimized helper: 500 calls per timed batch, 20 batches per source/case, alternating before/after order. Positive support measured 80 → 46 microseconds per call. The zero-weight and normalization-underflow fallbacks measured 78 → 80 and 70 → 74 microseconds, respectively; the additional minimum scan has a small cost when the fast path is unavailable. All support semantics remain unchanged. These are internal-check measurements, separate from end-to-end runtime. + +| Support | Before median [Q25, Q75], μs/call | After median [Q25, Q75], μs/call | +|---|---:|---:| +| positive | 80.0 [80.0, 80.0] | 46.0 [44.0, 46.0] | +| zero | 78.0 [76.0, 78.0] | 80.0 [79.5, 82.0] | +| underflow | 70.0 [68.0, 70.0] | 74.0 [74.0, 74.0] | + +To reproduce the helper checks and microbenchmark: + +```sh +Rscript dev/benchmarks/stress-fixes/guard-support-differential.R +Rscript dev/benchmarks/stress-fixes/support-performance.R /tmp/did-rc-support-performance.csv +``` + +## Guard optimization check + +A fixed-seed 2,000-case comparison against the first audit implementation preserved all supplied-weight overlap/rank guard decisions, including zero weights, normalization underflow, and designs close to the rank cutoff. The RC support optimization additionally passed 4,000 decisions against an independent four-subgroup summation reference; its script is included. A proposed symmetric-Gram shortcut was rejected because its floating-point accumulation changed a near-cutoff rank decision. The final weighted path retains the original two-matrix weighted Gram calculation; the no-user-weights path retains the released unweighted guard. diff --git a/dev/benchmarks/stress-fixes/guard-support-differential-result.txt b/dev/benchmarks/stress-fixes/guard-support-differential-result.txt new file mode 100644 index 00000000..6beaf80c --- /dev/null +++ b/dev/benchmarks/stress-fixes/guard-support-differential-result.txt @@ -0,0 +1,3 @@ +4000 support decisions match the independent reference +utility_functions.R SHA256 ad9f23a11305257374851ac767430125b2b6f398159f31e6706153a2f6f86b60 +seed 20261028 diff --git a/dev/benchmarks/stress-fixes/guard-support-differential.R b/dev/benchmarks/stress-fixes/guard-support-differential.R new file mode 100644 index 00000000..bfb64f2e --- /dev/null +++ b/dev/benchmarks/stress-fixes/guard-support-differential.R @@ -0,0 +1,33 @@ +# Independent four-cell support-count reference, including local normalization. +script <- grep("^--file=", commandArgs(), value = TRUE)[[1L]] +root <- normalizePath(file.path(dirname(sub("^--file=", "", script)), "..", "..", "..")) +pkgload::load_all(root, quiet = TRUE) +reference <- function(D, post, w, k, include_treated) { + mean_w <- mean(w) + if (!is.finite(mean_w) || mean_w <= 0) return(TRUE) + normalized <- w / mean_w + supported <- is.finite(normalized) & normalized > 0 + groups <- if (include_treated) 0:1 else 0L + any(vapply(groups, function(g) any(vapply(0:1, function(t) { + sum(supported & D == g & post == t) < k + }, logical(1))), logical(1))) +} +set.seed(20261028) +checks <- logical() +for (i in seq_len(2000L)) { + n <- sample(0:400, 1L) + D <- sample(0:1, n, TRUE); post <- sample(0:1, n, TRUE) + w <- switch(1L + i %% 8L, + rep(1, n), runif(n), exp(rnorm(n, 0, 150)), + sample(c(0, 1), n, TRUE), rep(2^-1074, n), + sample(c(2^1000, 2^-76, 2^-75, 0), n, TRUE), + rep(Inf, n), rep(NA_real_, n)) + k <- sample(1:6, 1L) + for (include_treated in c(TRUE, FALSE)) { + actual <- did:::rc_outcome_has_too_few_obs(D, post, w, k, include_treated) + expected <- reference(D, post, w, k, include_treated) + checks <- c(checks, identical(actual, expected)) + } +} +stopifnot(all(checks)) +cat(length(checks), "support decisions match the independent reference\n") diff --git a/dev/benchmarks/stress-fixes/main-comparison.csv b/dev/benchmarks/stress-fixes/main-comparison.csv new file mode 100644 index 00000000..5fdabb0e --- /dev/null +++ b/dev/benchmarks/stress-fixes/main-comparison.csv @@ -0,0 +1,34 @@ +"case","source_baseline","n_baseline","median_baseline","q25_baseline","q75_baseline","median_cpu_baseline","source_final","n_final","median_final","q25_final","q75_final","median_cpu_final","ratio","delta_ms","iqr_overlap" +"aggte_1k_dynamic","baseline",14,0.00400000000000311,0.004,0.00500000000000167,0.00400000000000489,"final",14,0.00400000000000311,0.00399999999999823,0.00475000000000314,0.00400000000000134,1,0,TRUE +"aggte_1k_dynamic_boot","baseline",14,0.0180000000000007,0.0169999999999972,0.0187499999999989,0.0179999999999974,"final",14,0.0170000000000012,0.0169999999999995,0.0180000000000007,0.0170000000000003,0.944444444444477,-0.999999999999446,TRUE +"aggte_5k_dynamic","baseline",14,0.00600000000000023,0.00600000000000023,0.00600000000000023,0.00600000000000023,"final",14,0.00600000000000023,0.00600000000000023,0.00600000000000023,0.00600000000000023,1,0,TRUE +"aggte_5k_simple","baseline",14,0.00199999999999889,0.00124999999999975,0.00200000000000244,0.00199999999999889,"final",14,0.00149999999999828,0.000999999999997669,0.00200000000000156,0.00199999999999712,0.749999999999556,-0.500000000000611,TRUE +"aggte_unbalanced_dynamic","baseline",14,0.0305,0.0299999999999985,0.0327499999999983,0.0309999999999964,"final",14,0.0294999999999987,0.0289999999999973,0.0300000000000002,0.0300000000000011,0.967213114754058,-1.00000000000122,TRUE +"conditional_pretest_60","baseline",14,0.087999999999997,0.0822500000000002,0.0984999999999996,0.087499999999999,"final",14,0.0815000000000001,0.0809999999999997,0.0820000000000007,0.0819999999999995,0.926136363636397,-6.49999999999684,FALSE +"guard_overlap_5k","baseline",14,0.0749999999999957,0.0725000000000016,0.0769999999999995,0.0744999999999992,"final",14,0.0729999999999986,0.0720000000000027,0.0730000000000004,0.0724999999999995,0.97333333333337,-1.99999999999712,TRUE +"guard_overlap_intercept_5k","baseline",14,0.0124999999999993,0.0120000000000005,0.0130000000000039,0.0120000000000005,"final",14,0.0120000000000005,0.0120000000000005,0.0129999999999981,0.0120000000000005,0.960000000000091,-0.499999999998835,TRUE +"guard_rank_5k","baseline",14,0.00399999999999778,0.00399999999999778,0.00400000000000356,0.00399999999999867,"final",14,0.00399999999999778,0.00300000000000011,0.00400000000000134,0.00399999999999867,1,0,TRUE +"guard_rank_intercept_5k","baseline",14,0.00699999999999878,0.0069999999999979,0.007000000000001,0.00699999999999967,"final",14,0.00700000000000056,0.0069999999999979,0.00700000000000145,0.00700000000000145,1.00000000000025,1.77635683940025e-12,TRUE +"guard_rank_weighted_5k","baseline",14,0.00400000000000134,0.00399999999999778,0.004000000000004,0.00349999999999895,"final",14,0.00900000000000034,0.00900000000000034,0.00974999999999859,0.00900000000000034,2.24999999999933,4.999999999999,FALSE +"panel_1k_fast_dr_boot","baseline",14,0.0299999999999967,0.0289999999999973,0.0307499999999994,0.0299999999999964,"final",14,0.0259999999999998,0.0259999999999962,0.0267500000000016,0.0260000000000039,0.866666666666756,-3.9999999999969,FALSE +"panel_1k_fast_dr_nox","baseline",14,0.0195000000000007,0.0189999999999988,0.0200000000000031,0.0199999999999984,"final",14,0.0174999999999983,0.017000000000003,0.0180000000000007,0.018000000000003,0.897435897435776,-2.00000000000244,FALSE +"panel_1k_fast_dr_x","baseline",14,0.0249999999999986,0.0240000000000009,0.0259999999999998,0.0254999999999998,"final",14,0.0219999999999985,0.0219999999999985,0.0220000000000016,0.0220000000000008,0.879999999999989,-3.00000000000011,FALSE +"panel_1k_fast_reg_x","baseline",14,0.0180000000000007,0.0170000000000003,0.0180000000000007,0.0180000000000018,"final",14,0.0159999999999982,0.0159999999999982,0.016,0.0160000000000006,0.888888888888757,-2.00000000000244,FALSE +"panel_1k_slow_dr_x","baseline",14,0.0159999999999991,0.0159999999999982,0.0170000000000012,0.0169999999999982,"final",14,0.0159999999999982,0.01525,0.0160000000000053,0.0159999999999985,0.999999999999944,-8.88178419700125e-13,TRUE +"panel_5k_fast_dr_boot","baseline",14,0.0814999999999984,0.0810000000000031,0.0844999999999954,0.0824999999999995,"final",14,0.0780000000000003,0.0770000000000004,0.078000000000003,0.0789999999999991,0.957055214723949,-3.49999999999806,FALSE +"panel_5k_fast_dr_nox","baseline",14,0.0399999999999991,0.0392500000000009,0.0424999999999986,0.0414999999999985,"final",14,0.0370000000000026,0.036999999999999,0.0380000000000029,0.0380000000000001,0.925000000000084,-2.99999999999656,FALSE +"panel_5k_fast_dr_weighted","baseline",14,0.0715000000000003,0.0709999999999984,0.0744999999999969,0.0729999999999997,"final",14,0.067499999999999,0.0670000000000002,0.0680000000000005,0.0690000000000002,0.944055944055926,-4.00000000000134,FALSE +"panel_5k_fast_dr_x","baseline",14,0.0649999999999977,0.0640000000000001,0.0682499999999995,0.0659999999999989,"final",14,0.0629999999999988,0.0619999999999985,0.0640000000000001,0.0644999999999982,0.969230769230785,-1.99999999999889,TRUE +"panel_5k_fast_reg_x","baseline",14,0.0444999999999993,0.0429999999999997,0.04575,0.0454999999999999,"final",14,0.0410000000000039,0.0410000000000004,0.0417500000000022,0.043000000000001,0.921348314606844,-3.4999999999954,FALSE +"panel_5k_slow_dr_nox","baseline",14,0.0295000000000014,0.0289999999999973,0.0307500000000021,0.0295000000000017,"final",14,0.0289999999999999,0.0282499999999994,0.0290000000000026,0.0290000000000035,0.983050847457577,-0.500000000001499,TRUE +"panel_5k_slow_dr_weighted","baseline",14,0.0499999999999989,0.0492499999999989,0.0527499999999992,0.0509999999999972,"final",14,0.0499999999999972,0.0489999999999995,0.050749999999999,0.0505000000000001,0.999999999999964,-1.77635683940025e-12,TRUE +"panel_5k_slow_dr_x","baseline",14,0.0479999999999947,0.0459999999999994,0.0490000000000048,0.0470000000000018,"final",14,0.0465000000000018,0.0452500000000011,0.0470000000000006,0.0469999999999994,0.968750000000143,-1.49999999999295,TRUE +"panel_5k_slow_reg_x","baseline",14,0.0240000000000009,0.0229999999999997,0.0257499999999973,0.0240000000000021,"final",14,0.0239999999999974,0.0230000000000006,0.0240000000000009,0.0234999999999985,0.999999999999852,-3.5527136788005e-12,TRUE +"rc_1k_fast_dr_boot","baseline",14,0.0690000000000004,0.0680000000000023,0.0697500000000009,0.0690000000000002,"final",14,0.0664999999999996,0.0642499999999995,0.0679999999999998,0.0669999999999992,0.963768115942017,-2.50000000000083,FALSE +"rc_1k_fast_dr_x","baseline",14,0.0489999999999995,0.0489999999999995,0.0490000000000048,0.0500000000000018,"final",14,0.0480000000000018,0.0472500000000027,0.0497499999999991,0.0489999999999982,0.979591836734741,-0.999999999997669,TRUE +"rc_5k_fast_dr_x","baseline",14,0.148000000000001,0.146999999999998,0.149000000000001,0.1495,"final",14,0.149000000000001,0.148,0.153000000000001,0.151000000000002,1.00675675675675,0.999999999999446,TRUE +"rc_5k_slow_dr_x","baseline",14,0.135500000000003,0.134,0.138999999999999,0.135,"final",14,0.136000000000003,0.135999999999999,0.137749999999999,0.136999999999997,1.00369003690037,0.499999999999723,TRUE +"unbalanced_1k_fast_dr_boot","baseline",14,0.0545000000000009,0.0532499999999994,0.0560000000000036,0.0539999999999984,"final",14,0.0519999999999996,0.0502500000000037,0.052750000000001,0.0515000000000043,0.95412844036695,-2.50000000000128,FALSE +"unbalanced_1k_fast_dr_x","baseline",14,0.0485000000000007,0.0470000000000006,0.0497500000000022,0.0485000000000006,"final",14,0.0459999999999994,0.0459999999999994,0.0480000000000018,0.0470000000000005,0.948453608247397,-2.50000000000128,TRUE +"unbalanced_5k_fast_dr_x","baseline",14,0.137499999999999,0.135249999999999,0.144,0.137999999999996,"final",14,0.137,0.136,0.137999999999998,0.137999999999999,0.996363636363645,-0.499999999998835,TRUE +"unbalanced_5k_slow_dr_x","baseline",14,0.1135,0.111999999999997,0.118,0.112500000000002,"final",14,0.114500000000003,0.113249999999999,0.11825,0.114999999999998,1.00881057268725,1.000000000003,TRUE diff --git a/dev/benchmarks/stress-fixes/main-raw-timings.csv b/dev/benchmarks/stress-fixes/main-raw-timings.csv new file mode 100644 index 00000000..7e0bb506 --- /dev/null +++ b/dev/benchmarks/stress-fixes/main-raw-timings.csv @@ -0,0 +1,925 @@ +"source","round","iteration","case","elapsed","user","system" +"baseline",1,1,"panel_1k_fast_dr_nox",0.0250000000000004,0.0209999999999999,0.001 +"baseline",1,1,"conditional_pretest_60",0.109,0.0949999999999998,0.00799999999999995 +"baseline",1,1,"panel_1k_fast_reg_x",0.0240000000000009,0.0199999999999996,0.002 +"baseline",1,1,"panel_1k_fast_dr_boot",0.0339999999999989,0.0309999999999997,0.001 +"baseline",1,1,"rc_5k_slow_dr_x",0.147,0.137,0.00700000000000001 +"baseline",1,1,"panel_5k_fast_dr_nox",0.0429999999999993,0.0410000000000004,0.003 +"baseline",1,1,"guard_overlap_intercept_5k",0.0149999999999988,0.0129999999999999,0 +"baseline",1,1,"aggte_1k_dynamic",0.00400000000000134,0.00500000000000078,0 +"baseline",1,1,"panel_5k_slow_dr_nox",0.0329999999999995,0.0310000000000006,0.001 +"baseline",1,1,"panel_5k_slow_reg_x",0.0280000000000005,0.0259999999999998,0.001 +"baseline",1,1,"panel_5k_slow_dr_x",0.0519999999999996,0.0490000000000013,0.001 +"baseline",1,1,"rc_5k_fast_dr_x",0.167,0.156000000000001,0.005 +"baseline",1,1,"aggte_1k_dynamic_boot",0.0190000000000001,0.0190000000000001,0 +"baseline",1,1,"panel_5k_fast_reg_x",0.0459999999999994,0.0449999999999999,0.002 +"baseline",1,1,"unbalanced_1k_fast_dr_x",0.0559999999999992,0.0510000000000002,0.004 +"baseline",1,1,"aggte_5k_dynamic",0.00599999999999845,0.00600000000000023,0 +"baseline",1,1,"panel_5k_fast_dr_boot",0.093,0.0869999999999997,0.004 +"baseline",1,1,"panel_5k_fast_dr_weighted",0.0759999999999987,0.0740000000000016,0.002 +"baseline",1,1,"guard_rank_weighted_5k",0.00399999999999956,0.00300000000000011,0 +"baseline",1,1,"rc_1k_fast_dr_boot",0.081999999999999,0.0719999999999992,0.006 +"baseline",1,1,"guard_overlap_5k",0.0820000000000007,0.077,0.001 +"baseline",1,1,"unbalanced_5k_fast_dr_x",0.144,0.141,0.003 +"baseline",1,1,"aggte_5k_simple",0.00200000000000067,0.00200000000000067,0 +"baseline",1,1,"rc_1k_fast_dr_x",0.0549999999999997,0.0520000000000014,0.003 +"baseline",1,1,"aggte_unbalanced_dynamic",0.0330000000000013,0.0309999999999988,0 +"baseline",1,1,"unbalanced_5k_slow_dr_x",0.115,0.112,0.001 +"baseline",1,1,"panel_5k_fast_dr_x",0.0690000000000008,0.0670000000000002,0.002 +"baseline",1,1,"panel_1k_fast_dr_x",0.027000000000001,0.0259999999999998,0.002 +"baseline",1,1,"guard_rank_5k",0.00399999999999956,0.00399999999999956,0 +"baseline",1,1,"panel_1k_slow_dr_x",0.0180000000000007,0.0179999999999989,0.001 +"baseline",1,1,"guard_rank_intercept_5k",0.00699999999999967,0.00699999999999967,0 +"baseline",1,1,"panel_5k_slow_dr_weighted",0.052999999999999,0.0519999999999996,0.001 +"baseline",1,1,"unbalanced_1k_fast_dr_boot",0.0579999999999998,0.0549999999999997,0.002 +"baseline",1,2,"guard_overlap_5k",0.077,0.0739999999999998,0.001 +"baseline",1,2,"unbalanced_1k_fast_dr_x",0.0470000000000006,0.0449999999999999,0.001 +"baseline",1,2,"rc_1k_fast_dr_x",0.0489999999999995,0.0489999999999995,0.002 +"baseline",1,2,"panel_1k_fast_dr_x",0.0249999999999986,0.0240000000000009,0.001 +"baseline",1,2,"panel_1k_slow_dr_x",0.0159999999999982,0.016,0.001 +"baseline",1,2,"rc_5k_fast_dr_x",0.148,0.147,0.00199999999999995 +"baseline",1,2,"panel_1k_fast_dr_nox",0.0190000000000019,0.0180000000000007,0.001 +"baseline",1,2,"panel_1k_fast_reg_x",0.0169999999999995,0.0160000000000018,0.001 +"baseline",1,2,"conditional_pretest_60",0.100999999999999,0.0910000000000011,0.003 +"baseline",1,2,"panel_5k_slow_dr_weighted",0.0539999999999985,0.0519999999999996,0.001 +"baseline",1,2,"panel_5k_fast_dr_weighted",0.0789999999999971,0.0760000000000005,0.003 +"baseline",1,2,"panel_5k_fast_dr_nox",0.0449999999999982,0.0440000000000005,0.002 +"baseline",1,2,"aggte_unbalanced_dynamic",0.0329999999999977,0.032,0 +"baseline",1,2,"panel_1k_fast_dr_boot",0.0330000000000013,0.032,0.001 +"baseline",1,2,"aggte_1k_dynamic",0.00600000000000023,0.004999999999999,0.001 +"baseline",1,2,"panel_5k_slow_dr_x",0.149000000000001,0.143000000000001,0.002 +"baseline",1,2,"panel_5k_fast_dr_x",0.0700000000000003,0.0680000000000014,0.002 +"baseline",1,2,"guard_overlap_intercept_5k",0.0139999999999993,0.0139999999999993,0.001 +"baseline",1,2,"panel_5k_fast_reg_x",0.0459999999999994,0.0459999999999994,0.001 +"baseline",1,2,"aggte_5k_dynamic",0.00700000000000145,0.00600000000000023,0 +"baseline",1,2,"panel_5k_fast_dr_boot",0.108000000000001,0.088000000000001,0.004 +"baseline",1,2,"unbalanced_5k_fast_dr_x",0.151,0.142999999999997,0.00399999999999995 +"baseline",1,2,"guard_rank_weighted_5k",0.00400000000000134,0.00300000000000011,0 +"baseline",1,2,"unbalanced_5k_slow_dr_x",0.140000000000001,0.117000000000001,0.00700000000000001 +"baseline",1,2,"guard_rank_intercept_5k",0.00700000000000145,0.00799999999999912,0 +"baseline",1,2,"panel_5k_slow_dr_nox",0.0289999999999999,0.0279999999999987,0.001 +"baseline",1,2,"aggte_5k_simple",0.00100000000000122,0.00199999999999889,0 +"baseline",1,2,"rc_1k_fast_dr_boot",0.0730000000000004,0.0689999999999991,0.002 +"baseline",1,2,"aggte_1k_dynamic_boot",0.0280000000000022,0.0199999999999996,0 +"baseline",1,2,"panel_5k_slow_reg_x",0.0229999999999997,0.0229999999999997,0 +"baseline",1,2,"unbalanced_1k_fast_dr_boot",0.0689999999999991,0.0579999999999998,0.003 +"baseline",1,2,"guard_rank_5k",0.004999999999999,0.00400000000000134,0 +"baseline",1,2,"rc_5k_slow_dr_x",0.133000000000003,0.132000000000001,0 +"baseline",1,3,"aggte_5k_simple",0.00199999999999889,0.00100000000000122,0 +"baseline",1,3,"aggte_1k_dynamic",0.00399999999999778,0.00399999999999778,0 +"baseline",1,3,"panel_5k_fast_dr_nox",0.0390000000000015,0.0390000000000015,0.002 +"baseline",1,3,"panel_5k_fast_dr_weighted",0.0710000000000015,0.0710000000000015,0.001 +"baseline",1,3,"panel_1k_fast_dr_x",0.0259999999999998,0.0249999999999986,0.001 +"baseline",1,3,"guard_rank_5k",0.00399999999999778,0.00300000000000011,0 +"baseline",1,3,"rc_5k_fast_dr_x",0.153000000000002,0.151,0.003 +"baseline",1,3,"aggte_unbalanced_dynamic",0.032,0.032,0 +"baseline",1,3,"guard_overlap_intercept_5k",0.0309999999999988,0.0140000000000029,0.002 +"baseline",1,3,"rc_5k_slow_dr_x",0.138999999999999,0.134,0.001 +"baseline",1,3,"guard_overlap_5k",0.0790000000000006,0.0769999999999982,0 +"baseline",1,3,"unbalanced_1k_fast_dr_boot",0.0560000000000009,0.0549999999999997,0.002 +"baseline",1,3,"panel_5k_fast_dr_boot",0.0850000000000009,0.0820000000000007,0.002 +"baseline",1,3,"panel_1k_fast_reg_x",0.0180000000000007,0.0169999999999995,0 +"baseline",1,3,"unbalanced_1k_fast_dr_x",0.0500000000000007,0.0479999999999983,0.002 +"baseline",1,3,"panel_5k_slow_dr_x",0.0459999999999994,0.0450000000000017,0.001 +"baseline",1,3,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0 +"baseline",1,3,"panel_5k_fast_dr_x",0.0700000000000003,0.0670000000000002,0.002 +"baseline",1,3,"rc_1k_fast_dr_boot",0.070999999999998,0.0689999999999991,0.002 +"baseline",1,3,"rc_1k_fast_dr_x",0.0489999999999995,0.0480000000000018,0.002 +"baseline",1,3,"panel_1k_fast_dr_boot",0.0289999999999999,0.0280000000000022,0 +"baseline",1,3,"conditional_pretest_60",0.160999999999998,0.109000000000002,0.006 +"baseline",1,3,"guard_rank_weighted_5k",0.00400000000000134,0.00400000000000134,0 +"baseline",1,3,"aggte_5k_dynamic",0.00600000000000023,0.00599999999999667,0 +"baseline",1,3,"aggte_1k_dynamic_boot",0.0180000000000007,0.0169999999999995,0 +"baseline",1,3,"panel_5k_slow_dr_weighted",0.0869999999999997,0.059000000000001,0.003 +"baseline",1,3,"unbalanced_5k_fast_dr_x",0.327000000000002,0.164999999999999,0.016 +"baseline",1,3,"panel_5k_slow_reg_x",0.0339999999999989,0.0280000000000022,0.001 +"baseline",1,3,"unbalanced_5k_slow_dr_x",0.272999999999996,0.135000000000002,0.01 +"baseline",1,3,"panel_5k_slow_dr_nox",0.0380000000000038,0.0330000000000013,0.003 +"baseline",1,3,"panel_1k_slow_dr_x",0.0179999999999936,0.017000000000003,0 +"baseline",1,3,"panel_5k_fast_reg_x",0.0480000000000018,0.0459999999999994,0.002 +"baseline",1,3,"panel_1k_fast_dr_nox",0.0189999999999984,0.0189999999999984,0.001 +"baseline",1,4,"panel_1k_fast_dr_boot",0.0330000000000013,0.0300000000000011,0.00099999999999989 +"baseline",1,4,"panel_1k_slow_dr_x",0.0249999999999986,0.0199999999999996,0.001 +"baseline",1,4,"panel_5k_slow_dr_nox",0.0300000000000011,0.0289999999999999,0.001 +"baseline",1,4,"panel_5k_slow_dr_weighted",0.0519999999999996,0.0509999999999984,0.002 +"baseline",1,4,"aggte_5k_dynamic",0.00600000000000023,0.00500000000000256,0 +"baseline",1,4,"unbalanced_1k_fast_dr_boot",0.0629999999999953,0.0579999999999998,0.003 +"baseline",1,4,"unbalanced_1k_fast_dr_x",0.0520000000000067,0.0489999999999995,0.002 +"baseline",1,4,"guard_rank_5k",0.00399999999999778,0.00400000000000489,0 +"baseline",1,4,"panel_1k_fast_dr_x",0.0279999999999987,0.0259999999999962,0.001 +"baseline",1,4,"guard_rank_weighted_5k",0.00399999999999778,0.00399999999999778,0.001 +"baseline",1,4,"aggte_5k_simple",0.000999999999997669,0.00100000000000477,0 +"baseline",1,4,"panel_5k_fast_reg_x",0.0450000000000017,0.0450000000000017,0.001 +"baseline",1,4,"panel_5k_fast_dr_weighted",0.0850000000000009,0.0750000000000028,0.003 +"baseline",1,4,"aggte_1k_dynamic_boot",0.0189999999999984,0.0180000000000007,0 +"baseline",1,4,"unbalanced_5k_fast_dr_x",0.137999999999998,0.135999999999996,0.002 +"baseline",1,4,"panel_5k_slow_dr_x",0.0489999999999995,0.0470000000000041,0.001 +"baseline",1,4,"rc_5k_fast_dr_x",0.153999999999996,0.150999999999996,0.005 +"baseline",1,4,"guard_overlap_5k",0.0749999999999957,0.0739999999999981,0 +"baseline",1,4,"panel_1k_fast_dr_nox",0.0200000000000031,0.0200000000000031,0.001 +"baseline",1,4,"unbalanced_5k_slow_dr_x",0.119,0.114000000000004,0.002 +"baseline",1,4,"panel_5k_fast_dr_nox",0.0409999999999968,0.0409999999999968,0.002 +"baseline",1,4,"panel_1k_fast_reg_x",0.0180000000000007,0.0180000000000007,0.001 +"baseline",1,4,"panel_5k_fast_dr_boot",0.0819999999999936,0.080999999999996,0.002 +"baseline",1,4,"rc_5k_slow_dr_x",0.148000000000003,0.137,0.003 +"baseline",1,4,"panel_5k_slow_reg_x",0.0259999999999962,0.0250000000000057,0.001 +"baseline",1,4,"aggte_1k_dynamic",0.00500000000000256,0.00399999999999778,0 +"baseline",1,4,"aggte_unbalanced_dynamic",0.0309999999999988,0.029999999999994,0.001 +"baseline",1,4,"rc_1k_fast_dr_x",0.0529999999999973,0.0500000000000043,0.003 +"baseline",1,4,"rc_1k_fast_dr_boot",0.0690000000000026,0.0670000000000002,0.002 +"baseline",1,4,"panel_5k_fast_dr_x",0.0659999999999954,0.0649999999999977,0.001 +"baseline",1,4,"guard_rank_intercept_5k",0.00799999999999557,0.0069999999999979,0 +"baseline",1,4,"conditional_pretest_60",0.0910000000000011,0.088000000000001,0.001 +"baseline",1,4,"guard_overlap_intercept_5k",0.0129999999999981,0.0120000000000005,0 +"baseline",1,5,"panel_1k_fast_reg_x",0.0180000000000007,0.0180000000000007,0.001 +"baseline",1,5,"unbalanced_5k_slow_dr_x",0.120999999999995,0.113999999999997,0.003 +"baseline",1,5,"unbalanced_1k_fast_dr_boot",0.054000000000002,0.0519999999999996,0.001 +"baseline",1,5,"panel_5k_slow_reg_x",0.0240000000000009,0.0230000000000032,0.001 +"baseline",1,5,"guard_rank_intercept_5k",0.0069999999999979,0.007000000000005,0 +"baseline",1,5,"guard_rank_5k",0.00399999999999778,0.00399999999999778,0 +"baseline",1,5,"unbalanced_5k_fast_dr_x",0.143999999999998,0.143000000000001,0.002 +"baseline",1,5,"guard_overlap_5k",0.0940000000000012,0.0790000000000006,0.001 +"baseline",1,5,"panel_5k_fast_dr_boot",0.0849999999999937,0.0849999999999937,0.002 +"baseline",1,5,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"baseline",1,5,"panel_1k_fast_dr_boot",0.0289999999999964,0.0279999999999987,0.001 +"baseline",1,5,"panel_5k_fast_dr_x",0.0649999999999977,0.0649999999999977,0.002 +"baseline",1,5,"guard_rank_weighted_5k",0.00399999999999778,0.00300000000000011,0 +"baseline",1,5,"rc_5k_slow_dr_x",0.136000000000003,0.134,0.001 +"baseline",1,5,"panel_1k_slow_dr_x",0.017000000000003,0.0160000000000053,0 +"baseline",1,5,"panel_5k_fast_dr_nox",0.046999999999997,0.0429999999999993,0.004 +"baseline",1,5,"aggte_unbalanced_dynamic",0.0330000000000013,0.0310000000000059,0 +"baseline",1,5,"panel_5k_slow_dr_nox",0.0310000000000059,0.029999999999994,0.001 +"baseline",1,5,"panel_5k_fast_reg_x",0.0449999999999946,0.0439999999999969,0.001 +"baseline",1,5,"guard_overlap_intercept_5k",0.0130000000000052,0.0129999999999981,0 +"baseline",1,5,"aggte_1k_dynamic_boot",0.0180000000000007,0.0189999999999984,0 +"baseline",1,5,"panel_5k_fast_dr_weighted",0.0730000000000004,0.070999999999998,0.002 +"baseline",1,5,"panel_1k_fast_dr_x",0.0249999999999986,0.0250000000000057,0.001 +"baseline",1,5,"rc_1k_fast_dr_x",0.0509999999999948,0.0499999999999972,0.003 +"baseline",1,5,"panel_5k_slow_dr_x",0.0479999999999947,0.0470000000000041,0.001 +"baseline",1,5,"aggte_1k_dynamic",0.00500000000000256,0.00500000000000256,0 +"baseline",1,5,"panel_1k_fast_dr_nox",0.019999999999996,0.0189999999999984,0.001 +"baseline",1,5,"aggte_5k_simple",0.00200000000000244,0.00199999999999534,0 +"baseline",1,5,"panel_5k_slow_dr_weighted",0.0519999999999996,0.0509999999999948,0.001 +"baseline",1,5,"rc_1k_fast_dr_boot",0.0700000000000003,0.0679999999999978,0.002 +"baseline",1,5,"unbalanced_1k_fast_dr_x",0.0489999999999995,0.0470000000000041,0.001 +"baseline",1,5,"conditional_pretest_60",0.0900000000000034,0.088000000000001,0.001 +"baseline",1,5,"rc_5k_fast_dr_x",0.148000000000003,0.145000000000003,0.003 +"baseline",1,6,"panel_1k_fast_reg_x",0.0180000000000007,0.0180000000000007,0.001 +"baseline",1,6,"panel_5k_fast_dr_boot",0.0810000000000031,0.0799999999999983,0.002 +"baseline",1,6,"guard_rank_intercept_5k",0.00800000000000267,0.0069999999999979,0 +"baseline",1,6,"panel_1k_fast_dr_x",0.027000000000001,0.0260000000000034,0.001 +"baseline",1,6,"unbalanced_5k_fast_dr_x",0.161999999999999,0.140999999999998,0.004 +"baseline",1,6,"guard_rank_weighted_5k",0.00900000000000034,0.00500000000000256,0.001 +"baseline",1,6,"conditional_pretest_60",0.107999999999997,0.0999999999999943,0.003 +"baseline",1,6,"panel_5k_fast_dr_x",0.070999999999998,0.0670000000000002,0.002 +"baseline",1,6,"panel_1k_slow_dr_x",0.0169999999999959,0.017000000000003,0 +"baseline",1,6,"aggte_1k_dynamic",0.00399999999999778,0.00499999999999545,0 +"baseline",1,6,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"baseline",1,6,"aggte_5k_simple",0.00199999999999534,0.00200000000000244,0 +"baseline",1,6,"aggte_1k_dynamic_boot",0.0169999999999959,0.0169999999999959,0.001 +"baseline",1,6,"panel_5k_fast_reg_x",0.0459999999999994,0.0449999999999946,0.002 +"baseline",1,6,"panel_5k_slow_reg_x",0.0239999999999938,0.0229999999999961,0 +"baseline",1,6,"panel_1k_fast_dr_nox",0.0200000000000031,0.0189999999999984,0 +"baseline",1,6,"panel_1k_fast_dr_boot",0.0309999999999988,0.0289999999999964,0.001 +"baseline",1,6,"aggte_unbalanced_dynamic",0.0300000000000011,0.0300000000000011,0 +"baseline",1,6,"panel_5k_fast_dr_weighted",0.070999999999998,0.0710000000000051,0.001 +"baseline",1,6,"panel_5k_slow_dr_weighted",0.0499999999999972,0.0489999999999995,0.001 +"baseline",1,6,"guard_overlap_5k",0.0719999999999956,0.0720000000000027,0 +"baseline",1,6,"guard_rank_5k",0.00399999999999778,0.00300000000000011,0 +"baseline",1,6,"rc_1k_fast_dr_x",0.0489999999999995,0.0480000000000018,0.002 +"baseline",1,6,"rc_1k_fast_dr_boot",0.0679999999999978,0.0670000000000002,0.001 +"baseline",1,6,"panel_5k_slow_dr_nox",0.0289999999999964,0.0289999999999964,0 +"baseline",1,6,"panel_5k_fast_dr_nox",0.0510000000000019,0.0419999999999945,0.002 +"baseline",1,6,"unbalanced_5k_slow_dr_x",0.109000000000002,0.107999999999997,0.001 +"baseline",1,6,"rc_5k_fast_dr_x",0.146000000000001,0.146000000000001,0.002 +"baseline",1,6,"rc_5k_slow_dr_x",0.133999999999993,0.131999999999998,0 +"baseline",1,6,"unbalanced_1k_fast_dr_boot",0.0519999999999996,0.0510000000000019,0.001 +"baseline",1,6,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"baseline",1,6,"panel_5k_slow_dr_x",0.0459999999999994,0.0450000000000017,0.001 +"baseline",1,6,"unbalanced_1k_fast_dr_x",0.046999999999997,0.0459999999999994,0.001 +"baseline",1,7,"aggte_1k_dynamic_boot",0.017000000000003,0.017000000000003,0 +"baseline",1,7,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0459999999999994,0.001 +"baseline",1,7,"unbalanced_5k_fast_dr_x",0.134999999999998,0.135000000000005,0.002 +"baseline",1,7,"panel_5k_fast_dr_boot",0.0810000000000031,0.0810000000000031,0.001 +"baseline",1,7,"panel_5k_slow_dr_weighted",0.0499999999999972,0.0499999999999972,0.001 +"baseline",1,7,"panel_5k_fast_dr_x",0.0659999999999954,0.0649999999999977,0.001 +"baseline",1,7,"panel_5k_slow_dr_x",0.0459999999999994,0.0450000000000017,0 +"baseline",1,7,"guard_rank_weighted_5k",0.00300000000000011,0.00400000000000489,0 +"baseline",1,7,"conditional_pretest_60",0.0850000000000009,0.0839999999999961,0 +"baseline",1,7,"rc_1k_fast_dr_boot",0.0659999999999954,0.0649999999999977,0.002 +"baseline",1,7,"panel_1k_slow_dr_x",0.0150000000000006,0.0150000000000006,0 +"baseline",1,7,"panel_5k_fast_dr_nox",0.0380000000000038,0.0380000000000038,0.00100000000000011 +"baseline",1,7,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0 +"baseline",1,7,"aggte_5k_simple",0.00200000000000244,0.00200000000000244,0 +"baseline",1,7,"aggte_1k_dynamic",0.00499999999999545,0.00400000000000489,0 +"baseline",1,7,"panel_5k_slow_dr_nox",0.0290000000000035,0.0289999999999964,0 +"baseline",1,7,"aggte_unbalanced_dynamic",0.0289999999999964,0.0300000000000011,0 +"baseline",1,7,"panel_5k_slow_reg_x",0.0229999999999961,0.0230000000000032,0.00100000000000011 +"baseline",1,7,"rc_1k_fast_dr_x",0.0490000000000066,0.0480000000000018,0.002 +"baseline",1,7,"rc_5k_fast_dr_x",0.146000000000001,0.144999999999996,0.002 +"baseline",1,7,"panel_1k_fast_dr_nox",0.0189999999999984,0.0189999999999984,0.00100000000000011 +"baseline",1,7,"panel_1k_fast_reg_x",0.0180000000000007,0.017000000000003,0.00099999999999989 +"baseline",1,7,"panel_5k_fast_reg_x",0.0429999999999993,0.0429999999999993,0.002 +"baseline",1,7,"panel_1k_fast_dr_boot",0.0300000000000011,0.0290000000000035,0.00099999999999989 +"baseline",1,7,"unbalanced_5k_slow_dr_x",0.111999999999995,0.110999999999997,0.00099999999999989 +"baseline",1,7,"unbalanced_1k_fast_dr_boot",0.0529999999999973,0.0519999999999996,0.00099999999999989 +"baseline",1,7,"aggte_5k_dynamic",0.00600000000000023,0.00599999999999312,0 +"baseline",1,7,"panel_1k_fast_dr_x",0.0240000000000009,0.0240000000000009,0.00100000000000011 +"baseline",1,7,"panel_5k_fast_dr_weighted",0.0719999999999956,0.0720000000000027,0.002 +"baseline",1,7,"guard_overlap_intercept_5k",0.0120000000000005,0.0129999999999981,0 +"baseline",1,7,"guard_overlap_5k",0.0769999999999982,0.0720000000000027,0.004 +"baseline",1,7,"guard_rank_5k",0.00400000000000489,0.00400000000000489,0.00099999999999989 +"baseline",1,7,"rc_5k_slow_dr_x",0.135000000000005,0.134,0.00100000000000011 +"final",1,1,"panel_1k_fast_dr_nox",0.0209999999999999,0.0180000000000007,0.001 +"final",1,1,"conditional_pretest_60",0.0809999999999995,0.0800000000000001,0.001 +"final",1,1,"panel_1k_fast_reg_x",0.016,0.0149999999999997,0.001 +"final",1,1,"panel_1k_fast_dr_boot",0.0279999999999987,0.0259999999999998,0.002 +"final",1,1,"rc_5k_slow_dr_x",0.140000000000001,0.137,0.002 +"final",1,1,"panel_5k_fast_dr_nox",0.0389999999999997,0.0389999999999997,0.002 +"final",1,1,"guard_overlap_intercept_5k",0.0120000000000005,0.0119999999999987,0 +"final",1,1,"aggte_1k_dynamic",0.00600000000000023,0.004999999999999,0 +"final",1,1,"panel_5k_slow_dr_nox",0.0410000000000004,0.0329999999999995,0.001 +"final",1,1,"panel_5k_slow_reg_x",0.0259999999999998,0.0240000000000009,0.001 +"final",1,1,"panel_5k_slow_dr_x",0.0500000000000007,0.048,0.001 +"final",1,1,"rc_5k_fast_dr_x",0.159000000000001,0.151,0.006 +"final",1,1,"aggte_1k_dynamic_boot",0.0180000000000007,0.0170000000000012,0 +"final",1,1,"panel_5k_fast_reg_x",0.0440000000000005,0.0429999999999993,0.002 +"final",1,1,"unbalanced_1k_fast_dr_x",0.0490000000000013,0.0469999999999988,0.00299999999999995 +"final",1,1,"aggte_5k_dynamic",0.00699999999999967,0.00600000000000023,0 +"final",1,1,"panel_5k_fast_dr_boot",0.077,0.077,0.002 +"final",1,1,"panel_5k_fast_dr_weighted",0.065999999999999,0.0670000000000002,0.002 +"final",1,1,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",1,1,"rc_1k_fast_dr_boot",0.0670000000000002,0.0629999999999988,0.004 +"final",1,1,"guard_overlap_5k",0.072000000000001,0.0709999999999997,0.001 +"final",1,1,"unbalanced_5k_fast_dr_x",0.200000000000001,0.146000000000001,0.00900000000000001 +"final",1,1,"aggte_5k_simple",0.000999999999999446,0.00100000000000122,0 +"final",1,1,"rc_1k_fast_dr_x",0.0499999999999989,0.048,0.003 +"final",1,1,"aggte_unbalanced_dynamic",0.0300000000000011,0.0299999999999994,0.001 +"final",1,1,"unbalanced_5k_slow_dr_x",0.116000000000001,0.115,0.001 +"final",1,1,"panel_5k_fast_dr_x",0.0629999999999988,0.0630000000000006,0.002 +"final",1,1,"panel_1k_fast_dr_x",0.0229999999999997,0.0220000000000002,0.001 +"final",1,1,"guard_rank_5k",0.00400000000000134,0.00400000000000134,0 +"final",1,1,"panel_1k_slow_dr_x",0.0180000000000007,0.0169999999999995,0.001 +"final",1,1,"guard_rank_intercept_5k",0.00699999999999967,0.0080000000000009,0 +"final",1,1,"panel_5k_slow_dr_weighted",0.052999999999999,0.0520000000000014,0.001 +"final",1,1,"unbalanced_1k_fast_dr_boot",0.052999999999999,0.0510000000000002,0.003 +"final",1,2,"guard_overlap_5k",0.0750000000000011,0.0750000000000011,0 +"final",1,2,"unbalanced_1k_fast_dr_x",0.0469999999999988,0.0459999999999994,0.002 +"final",1,2,"rc_1k_fast_dr_x",0.0760000000000005,0.0510000000000002,0.002 +"final",1,2,"panel_1k_fast_dr_x",0.0229999999999997,0.0220000000000002,0.001 +"final",1,2,"panel_1k_slow_dr_x",0.0159999999999982,0.0149999999999988,0.001 +"final",1,2,"rc_5k_fast_dr_x",0.147000000000002,0.147,0.003 +"final",1,2,"panel_1k_fast_dr_nox",0.0169999999999995,0.016,0.001 +"final",1,2,"panel_1k_fast_reg_x",0.0159999999999982,0.0150000000000006,0 +"final",1,2,"conditional_pretest_60",0.0819999999999972,0.0800000000000018,0.001 +"final",1,2,"panel_5k_slow_dr_weighted",0.0579999999999998,0.0500000000000007,0.001 +"final",1,2,"panel_5k_fast_dr_weighted",0.0680000000000014,0.0670000000000002,0.002 +"final",1,2,"panel_5k_fast_dr_nox",0.036999999999999,0.036999999999999,0.001 +"final",1,2,"aggte_unbalanced_dynamic",0.0299999999999976,0.0309999999999988,0 +"final",1,2,"panel_1k_fast_dr_boot",0.0259999999999998,0.0250000000000021,0.001 +"final",1,2,"aggte_1k_dynamic",0.00500000000000256,0.00400000000000134,0 +"final",1,2,"panel_5k_slow_dr_x",0.0449999999999982,0.0460000000000029,0.001 +"final",1,2,"panel_5k_fast_dr_x",0.0620000000000012,0.0620000000000012,0.002 +"final",1,2,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"final",1,2,"panel_5k_fast_reg_x",0.0430000000000028,0.0429999999999993,0.002 +"final",1,2,"aggte_5k_dynamic",0.004999999999999,0.00600000000000023,0 +"final",1,2,"panel_5k_fast_dr_boot",0.0770000000000017,0.0769999999999982,0.001 +"final",1,2,"unbalanced_5k_fast_dr_x",0.222000000000001,0.220999999999997,0.002 +"final",1,2,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",1,2,"unbalanced_5k_slow_dr_x",0.114000000000001,0.113,0.001 +"final",1,2,"guard_rank_intercept_5k",0.00700000000000145,0.0069999999999979,0 +"final",1,2,"panel_5k_slow_dr_nox",0.0279999999999987,0.0279999999999987,0.001 +"final",1,2,"aggte_5k_simple",0.00100000000000122,0.00199999999999889,0 +"final",1,2,"rc_1k_fast_dr_boot",0.0640000000000001,0.0629999999999988,0.002 +"final",1,2,"aggte_1k_dynamic_boot",0.0169999999999995,0.0169999999999995,0 +"final",1,2,"panel_5k_slow_reg_x",0.0229999999999997,0.0229999999999997,0 +"final",1,2,"unbalanced_1k_fast_dr_boot",0.0500000000000007,0.0489999999999995,0.001 +"final",1,2,"guard_rank_5k",0.00400000000000134,0.00399999999999778,0 +"final",1,2,"rc_5k_slow_dr_x",0.135999999999999,0.135000000000002,0.001 +"final",1,3,"aggte_5k_simple",0.000999999999997669,0.000999999999997669,0 +"final",1,3,"aggte_1k_dynamic",0.00400000000000134,0.00399999999999778,0 +"final",1,3,"panel_5k_fast_dr_nox",0.036999999999999,0.0379999999999967,0.002 +"final",1,3,"panel_5k_fast_dr_weighted",0.0689999999999991,0.0679999999999978,0.002 +"final",1,3,"panel_1k_fast_dr_x",0.0219999999999985,0.0210000000000008,0.001 +"final",1,3,"guard_rank_5k",0.00300000000000011,0.00399999999999778,0 +"final",1,3,"rc_5k_fast_dr_x",0.148,0.148,0.002 +"final",1,3,"aggte_unbalanced_dynamic",0.0299999999999976,0.0289999999999999,0 +"final",1,3,"guard_overlap_intercept_5k",0.0120000000000005,0.0119999999999969,0 +"final",1,3,"rc_5k_slow_dr_x",0.134999999999998,0.134,0 +"final",1,3,"guard_overlap_5k",0.0729999999999968,0.070999999999998,0.001 +"final",1,3,"unbalanced_1k_fast_dr_boot",0.0519999999999996,0.0500000000000007,0.001 +"final",1,3,"panel_5k_fast_dr_boot",0.0779999999999994,0.0779999999999994,0.002 +"final",1,3,"panel_1k_fast_reg_x",0.0160000000000018,0.0160000000000018,0.001 +"final",1,3,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0459999999999994,0.001 +"final",1,3,"panel_5k_slow_dr_x",0.0470000000000006,0.0470000000000006,0.001 +"final",1,3,"guard_rank_intercept_5k",0.00700000000000145,0.00700000000000145,0 +"final",1,3,"panel_5k_fast_dr_x",0.0619999999999976,0.0620000000000012,0.002 +"final",1,3,"rc_1k_fast_dr_boot",0.0689999999999991,0.0649999999999977,0.006 +"final",1,3,"rc_1k_fast_dr_x",0.0479999999999983,0.0470000000000006,0.002 +"final",1,3,"panel_1k_fast_dr_boot",0.0280000000000022,0.027000000000001,0.001 +"final",1,3,"conditional_pretest_60",0.131999999999998,0.0960000000000001,0.001 +"final",1,3,"guard_rank_weighted_5k",0.0100000000000016,0.00899999999999679,0.001 +"final",1,3,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"final",1,3,"aggte_1k_dynamic_boot",0.0210000000000008,0.0199999999999996,0 +"final",1,3,"panel_5k_slow_dr_weighted",0.0509999999999984,0.0500000000000007,0.001 +"final",1,3,"unbalanced_5k_fast_dr_x",0.136000000000003,0.135000000000002,0.00299999999999989 +"final",1,3,"panel_5k_slow_reg_x",0.0230000000000032,0.0229999999999997,0 +"final",1,3,"unbalanced_5k_slow_dr_x",0.113,0.113,0.001 +"final",1,3,"panel_5k_slow_dr_nox",0.0289999999999999,0.0280000000000022,0.001 +"final",1,3,"panel_1k_slow_dr_x",0.0159999999999982,0.0159999999999982,0.001 +"final",1,3,"panel_5k_fast_reg_x",0.0410000000000004,0.0409999999999968,0.002 +"final",1,3,"panel_1k_fast_dr_nox",0.0169999999999995,0.0169999999999995,0 +"final",1,4,"panel_1k_fast_dr_boot",0.027000000000001,0.0260000000000034,0.001 +"final",1,4,"panel_1k_slow_dr_x",0.0150000000000006,0.0150000000000006,0 +"final",1,4,"panel_5k_slow_dr_nox",0.0289999999999999,0.0289999999999999,0.001 +"final",1,4,"panel_5k_slow_dr_weighted",0.0499999999999972,0.0500000000000007,0.001 +"final",1,4,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"final",1,4,"unbalanced_1k_fast_dr_boot",0.0599999999999987,0.0509999999999984,0.002 +"final",1,4,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0450000000000017,0.001 +"final",1,4,"guard_rank_5k",0.00300000000000011,0.00399999999999778,0 +"final",1,4,"panel_1k_fast_dr_x",0.0210000000000008,0.0210000000000008,0.001 +"final",1,4,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",1,4,"aggte_5k_simple",0.000999999999997669,0.00100000000000122,0 +"final",1,4,"panel_5k_fast_reg_x",0.0409999999999968,0.041999999999998,0.001 +"final",1,4,"panel_5k_fast_dr_weighted",0.0679999999999978,0.0679999999999978,0.001 +"final",1,4,"aggte_1k_dynamic_boot",0.0169999999999959,0.0159999999999982,0 +"final",1,4,"unbalanced_5k_fast_dr_x",0.137,0.135000000000002,0.002 +"final",1,4,"panel_5k_slow_dr_x",0.0459999999999994,0.0459999999999994,0 +"final",1,4,"rc_5k_fast_dr_x",0.149999999999999,0.150000000000006,0.002 +"final",1,4,"guard_overlap_5k",0.0720000000000027,0.070999999999998,0 +"final",1,4,"panel_1k_fast_dr_nox",0.0169999999999959,0.017000000000003,0.001 +"final",1,4,"unbalanced_5k_slow_dr_x",0.113,0.113,0.001 +"final",1,4,"panel_5k_fast_dr_nox",0.036999999999999,0.036999999999999,0.001 +"final",1,4,"panel_1k_fast_reg_x",0.0159999999999982,0.0150000000000006,0.001 +"final",1,4,"panel_5k_fast_dr_boot",0.0790000000000006,0.0769999999999982,0.002 +"final",1,4,"rc_5k_slow_dr_x",0.136000000000003,0.135999999999996,0.001 +"final",1,4,"panel_5k_slow_reg_x",0.0240000000000009,0.0239999999999938,0 +"final",1,4,"aggte_1k_dynamic",0.00500000000000256,0.00399999999999778,0 +"final",1,4,"aggte_unbalanced_dynamic",0.0289999999999964,0.0300000000000011,0 +"final",1,4,"rc_1k_fast_dr_x",0.0470000000000041,0.0460000000000065,0.002 +"final",1,4,"rc_1k_fast_dr_boot",0.0619999999999976,0.0620000000000047,0.002 +"final",1,4,"panel_5k_fast_dr_x",0.0660000000000025,0.0620000000000047,0.006 +"final",1,4,"guard_rank_intercept_5k",0.00799999999999557,0.007000000000005,0.001 +"final",1,4,"conditional_pretest_60",0.0810000000000031,0.080999999999996,0 +"final",1,4,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"final",1,5,"panel_1k_fast_reg_x",0.0159999999999982,0.0159999999999982,0.002 +"final",1,5,"unbalanced_5k_slow_dr_x",0.115000000000002,0.111999999999995,0.004 +"final",1,5,"unbalanced_1k_fast_dr_boot",0.054000000000002,0.0519999999999996,0.003 +"final",1,5,"panel_5k_slow_reg_x",0.0240000000000009,0.0229999999999961,0 +"final",1,5,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0 +"final",1,5,"guard_rank_5k",0.00399999999999778,0.00399999999999778,0 +"final",1,5,"unbalanced_5k_fast_dr_x",0.135999999999996,0.134999999999998,0.003 +"final",1,5,"guard_overlap_5k",0.0720000000000027,0.0719999999999956,0 +"final",1,5,"panel_5k_fast_dr_boot",0.0750000000000028,0.0760000000000005,0.001 +"final",1,5,"aggte_5k_dynamic",0.00500000000000256,0.00600000000000023,0 +"final",1,5,"panel_1k_fast_dr_boot",0.0259999999999962,0.0249999999999986,0.001 +"final",1,5,"panel_5k_fast_dr_x",0.0640000000000001,0.0640000000000001,0.002 +"final",1,5,"guard_rank_weighted_5k",0.00900000000000034,0.00899999999999324,0 +"final",1,5,"rc_5k_slow_dr_x",0.135999999999996,0.136000000000003,0 +"final",1,5,"panel_1k_slow_dr_x",0.0149999999999935,0.0150000000000006,0 +"final",1,5,"panel_5k_fast_dr_nox",0.0380000000000038,0.036999999999999,0.001 +"final",1,5,"aggte_unbalanced_dynamic",0.029999999999994,0.0300000000000011,0 +"final",1,5,"panel_5k_slow_dr_nox",0.0279999999999987,0.0279999999999987,0 +"final",1,5,"panel_5k_fast_reg_x",0.0410000000000039,0.0410000000000039,0.002 +"final",1,5,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"final",1,5,"aggte_1k_dynamic_boot",0.0180000000000007,0.017000000000003,0 +"final",1,5,"panel_5k_fast_dr_weighted",0.0679999999999978,0.0670000000000002,0.002 +"final",1,5,"panel_1k_fast_dr_x",0.0219999999999985,0.0220000000000056,0.001 +"final",1,5,"rc_1k_fast_dr_x",0.0480000000000018,0.0459999999999994,0.003 +"final",1,5,"panel_5k_slow_dr_x",0.0460000000000065,0.0459999999999994,0.001 +"final",1,5,"aggte_1k_dynamic",0.00400000000000489,0.00399999999999778,0 +"final",1,5,"panel_1k_fast_dr_nox",0.0179999999999936,0.017000000000003,0.001 +"final",1,5,"aggte_5k_simple",0.00199999999999534,0.00199999999999534,0 +"final",1,5,"panel_5k_slow_dr_weighted",0.0499999999999972,0.0500000000000043,0.001 +"final",1,5,"rc_1k_fast_dr_boot",0.0640000000000001,0.0629999999999953,0.001 +"final",1,5,"unbalanced_1k_fast_dr_x",0.0450000000000017,0.0450000000000017,0.002 +"final",1,5,"conditional_pretest_60",0.0820000000000007,0.080999999999996,0.001 +"final",1,5,"rc_5k_fast_dr_x",0.149000000000001,0.149000000000001,0.003 +"final",1,6,"panel_1k_fast_reg_x",0.0150000000000006,0.0140000000000029,0.001 +"final",1,6,"panel_5k_fast_dr_boot",0.0769999999999982,0.0769999999999982,0.001 +"final",1,6,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0 +"final",1,6,"panel_1k_fast_dr_x",0.0219999999999985,0.0210000000000008,0.001 +"final",1,6,"unbalanced_5k_fast_dr_x",0.137999999999998,0.137,0.00199999999999989 +"final",1,6,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",1,6,"conditional_pretest_60",0.0820000000000007,0.0820000000000007,0 +"final",1,6,"panel_5k_fast_dr_x",0.0609999999999999,0.0609999999999999,0.002 +"final",1,6,"panel_1k_slow_dr_x",0.0159999999999982,0.0150000000000006,0 +"final",1,6,"aggte_1k_dynamic",0.00399999999999778,0.00399999999999778,0 +"final",1,6,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"final",1,6,"aggte_5k_simple",0.000999999999997669,0.000999999999997669,0 +"final",1,6,"aggte_1k_dynamic_boot",0.0169999999999959,0.017000000000003,0 +"final",1,6,"panel_5k_fast_reg_x",0.0410000000000039,0.0420000000000016,0.002 +"final",1,6,"panel_5k_slow_reg_x",0.0230000000000032,0.0240000000000009,0.001 +"final",1,6,"panel_1k_fast_dr_nox",0.017000000000003,0.0169999999999959,0.001 +"final",1,6,"panel_1k_fast_dr_boot",0.0259999999999962,0.0249999999999986,0.001 +"final",1,6,"aggte_unbalanced_dynamic",0.0290000000000035,0.0290000000000035,0 +"final",1,6,"panel_5k_fast_dr_weighted",0.0660000000000025,0.0660000000000025,0.002 +"final",1,6,"panel_5k_slow_dr_weighted",0.0499999999999972,0.0500000000000043,0.001 +"final",1,6,"guard_overlap_5k",0.0730000000000004,0.0720000000000027,0.001 +"final",1,6,"guard_rank_5k",0.00300000000000011,0.00300000000000011,0 +"final",1,6,"rc_1k_fast_dr_x",0.0700000000000003,0.0510000000000019,0.004 +"final",1,6,"rc_1k_fast_dr_boot",0.0649999999999977,0.0640000000000001,0.002 +"final",1,6,"panel_5k_slow_dr_nox",0.0289999999999964,0.0290000000000035,0 +"final",1,6,"panel_5k_fast_dr_nox",0.036999999999999,0.036999999999999,0.001 +"final",1,6,"unbalanced_5k_slow_dr_x",0.113999999999997,0.112000000000002,0.002 +"final",1,6,"rc_5k_fast_dr_x",0.149999999999999,0.149999999999999,0.002 +"final",1,6,"rc_5k_slow_dr_x",0.136000000000003,0.134999999999998,0.001 +"final",1,6,"unbalanced_1k_fast_dr_boot",0.0510000000000019,0.0490000000000066,0.002 +"final",1,6,"guard_overlap_intercept_5k",0.0129999999999981,0.0120000000000005,0 +"final",1,6,"panel_5k_slow_dr_x",0.0459999999999994,0.0459999999999994,0.001 +"final",1,6,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0449999999999946,0.001 +"final",1,7,"aggte_1k_dynamic_boot",0.0180000000000007,0.017000000000003,0 +"final",1,7,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0450000000000017,0.002 +"final",1,7,"unbalanced_5k_fast_dr_x",0.136000000000003,0.135000000000005,0.002 +"final",1,7,"panel_5k_fast_dr_boot",0.078000000000003,0.0770000000000053,0.002 +"final",1,7,"panel_5k_slow_dr_weighted",0.0489999999999995,0.0480000000000018,0 +"final",1,7,"panel_5k_fast_dr_x",0.0640000000000001,0.0640000000000001,0.001 +"final",1,7,"panel_5k_slow_dr_x",0.0450000000000017,0.0459999999999994,0 +"final",1,7,"guard_rank_weighted_5k",0.00999999999999801,0.00900000000000034,0 +"final",1,7,"conditional_pretest_60",0.080999999999996,0.0810000000000031,0.001 +"final",1,7,"rc_1k_fast_dr_boot",0.0659999999999954,0.0640000000000001,0.002 +"final",1,7,"panel_1k_slow_dr_x",0.0160000000000053,0.0159999999999982,0 +"final",1,7,"panel_5k_fast_dr_nox",0.0370000000000061,0.036999999999999,0.001 +"final",1,7,"guard_rank_intercept_5k",0.0069999999999979,0.007000000000005,0 +"final",1,7,"aggte_5k_simple",0.00200000000000244,0.00200000000000244,0 +"final",1,7,"aggte_1k_dynamic",0.00400000000000489,0.00400000000000489,0 +"final",1,7,"panel_5k_slow_dr_nox",0.0290000000000035,0.0290000000000035,0.001 +"final",1,7,"aggte_unbalanced_dynamic",0.0300000000000011,0.0300000000000011,0 +"final",1,7,"panel_5k_slow_reg_x",0.0239999999999938,0.0230000000000032,0 +"final",1,7,"rc_1k_fast_dr_x",0.0459999999999994,0.0459999999999994,0.002 +"final",1,7,"rc_5k_fast_dr_x",0.156999999999996,0.152000000000001,0.006 +"final",1,7,"panel_1k_fast_dr_nox",0.017000000000003,0.017000000000003,0.001 +"final",1,7,"panel_1k_fast_reg_x",0.0159999999999982,0.0159999999999982,0.002 +"final",1,7,"panel_5k_fast_reg_x",0.0410000000000039,0.0420000000000016,0.002 +"final",1,7,"panel_1k_fast_dr_boot",0.0260000000000034,0.0249999999999986,0.001 +"final",1,7,"unbalanced_5k_slow_dr_x",0.121000000000002,0.113,0.004 +"final",1,7,"unbalanced_1k_fast_dr_boot",0.054000000000002,0.0509999999999948,0.003 +"final",1,7,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"final",1,7,"panel_1k_fast_dr_x",0.0220000000000056,0.0210000000000008,0.001 +"final",1,7,"panel_5k_fast_dr_weighted",0.0690000000000026,0.0679999999999978,0.002 +"final",1,7,"guard_overlap_intercept_5k",0.0129999999999981,0.0120000000000005,0 +"final",1,7,"guard_overlap_5k",0.0730000000000004,0.0720000000000027,0.001 +"final",1,7,"guard_rank_5k",0.00399999999999778,0.00399999999999778,0 +"final",1,7,"rc_5k_slow_dr_x",0.137,0.136000000000003,0.001 +"final",2,1,"conditional_pretest_60",0.0810000000000004,0.0809999999999995,0.001 +"final",2,1,"aggte_5k_dynamic",0.00499999999999989,0.00499999999999989,0 +"final",2,1,"panel_5k_slow_dr_x",0.0469999999999997,0.0460000000000003,0.002 +"final",2,1,"rc_1k_fast_dr_boot",0.0680000000000005,0.0649999999999995,0.004 +"final",2,1,"unbalanced_1k_fast_dr_x",0.0490000000000004,0.0470000000000006,0.003 +"final",2,1,"panel_1k_fast_reg_x",0.016,0.0149999999999997,0.001 +"final",2,1,"panel_5k_slow_dr_nox",0.0280000000000005,0.0280000000000005,0.001 +"final",2,1,"guard_overlap_5k",0.0730000000000004,0.0719999999999992,0.001 +"final",2,1,"aggte_unbalanced_dynamic",0.0289999999999999,0.0289999999999999,0 +"final",2,1,"unbalanced_1k_fast_dr_boot",0.0520000000000014,0.0510000000000002,0.003 +"final",2,1,"guard_rank_weighted_5k",0.00999999999999979,0.00999999999999979,0 +"final",2,1,"rc_5k_slow_dr_x",0.143000000000001,0.139000000000001,0.003 +"final",2,1,"guard_overlap_intercept_5k",0.0129999999999999,0.0129999999999999,0 +"final",2,1,"panel_5k_fast_dr_boot",0.0780000000000012,0.077,0.002 +"final",2,1,"aggte_1k_dynamic",0.00400000000000134,0.00500000000000078,0 +"final",2,1,"guard_rank_5k",0.00399999999999956,0.00399999999999956,0 +"final",2,1,"panel_5k_slow_reg_x",0.0229999999999997,0.0239999999999991,0.001 +"final",2,1,"guard_rank_intercept_5k",0.00699999999999967,0.00700000000000145,0 +"final",2,1,"panel_1k_fast_dr_nox",0.0179999999999989,0.0170000000000012,0.001 +"final",2,1,"aggte_5k_simple",0.00199999999999889,0.00200000000000067,0 +"final",2,1,"rc_1k_fast_dr_x",0.0499999999999989,0.0489999999999995,0.003 +"final",2,1,"panel_5k_fast_dr_nox",0.0389999999999997,0.0390000000000015,0.002 +"final",2,1,"panel_1k_fast_dr_boot",0.0299999999999994,0.0280000000000005,0.002 +"final",2,1,"aggte_1k_dynamic_boot",0.0169999999999995,0.0169999999999995,0 +"final",2,1,"rc_5k_fast_dr_x",0.154000000000002,0.153,0.003 +"final",2,1,"panel_5k_fast_dr_weighted",0.0670000000000002,0.0670000000000002,0.002 +"final",2,1,"panel_5k_fast_dr_x",0.0640000000000001,0.0640000000000001,0.002 +"final",2,1,"unbalanced_5k_slow_dr_x",0.116,0.113,0.005 +"final",2,1,"panel_1k_slow_dr_x",0.0170000000000012,0.016,0.001 +"final",2,1,"panel_5k_fast_reg_x",0.043000000000001,0.0420000000000016,0.002 +"final",2,1,"panel_1k_fast_dr_x",0.0220000000000002,0.020999999999999,0.001 +"final",2,1,"unbalanced_5k_fast_dr_x",0.137,0.135999999999999,0.002 +"final",2,1,"panel_5k_slow_dr_weighted",0.0500000000000007,0.0489999999999995,0.001 +"final",2,2,"panel_5k_slow_dr_weighted",0.0490000000000013,0.0489999999999995,0.001 +"final",2,2,"aggte_1k_dynamic_boot",0.0169999999999995,0.0179999999999989,0 +"final",2,2,"guard_rank_intercept_5k",0.00700000000000145,0.00699999999999967,0 +"final",2,2,"unbalanced_1k_fast_dr_boot",0.0519999999999996,0.0500000000000007,0.001 +"final",2,2,"rc_1k_fast_dr_boot",0.065999999999999,0.0649999999999995,0.002 +"final",2,2,"aggte_1k_dynamic",0.00399999999999956,0.00400000000000134,0 +"final",2,2,"unbalanced_5k_fast_dr_x",0.137,0.133999999999999,0.003 +"final",2,2,"panel_1k_fast_dr_boot",0.0259999999999998,0.0249999999999986,0.002 +"final",2,2,"conditional_pretest_60",0.0820000000000007,0.0820000000000007,0.001 +"final",2,2,"panel_5k_slow_dr_x",0.0450000000000017,0.0440000000000005,0.001 +"final",2,2,"panel_5k_fast_dr_weighted",0.0679999999999978,0.0670000000000002,0.002 +"final",2,2,"rc_1k_fast_dr_x",0.0489999999999995,0.0470000000000006,0.001 +"final",2,2,"panel_5k_fast_dr_nox",0.036999999999999,0.036999999999999,0.001 +"final",2,2,"aggte_5k_dynamic",0.00600000000000023,0.004999999999999,0 +"final",2,2,"panel_1k_slow_dr_x",0.0169999999999995,0.0169999999999995,0 +"final",2,2,"panel_1k_fast_dr_nox",0.0180000000000007,0.0180000000000007,0.001 +"final",2,2,"panel_5k_fast_dr_boot",0.078000000000003,0.0770000000000017,0.001 +"final",2,2,"guard_rank_weighted_5k",0.0100000000000016,0.00900000000000034,0 +"final",2,2,"aggte_unbalanced_dynamic",0.0300000000000011,0.0300000000000011,0 +"final",2,2,"rc_5k_slow_dr_x",0.135999999999999,0.135999999999999,0.001 +"final",2,2,"aggte_5k_simple",0.00199999999999889,0.00199999999999889,0 +"final",2,2,"unbalanced_5k_slow_dr_x",0.202000000000002,0.201,0.001 +"final",2,2,"rc_5k_fast_dr_x",0.145,0.145999999999997,0.002 +"final",2,2,"panel_5k_slow_reg_x",0.0229999999999997,0.0230000000000032,0 +"final",2,2,"panel_5k_fast_reg_x",0.0410000000000004,0.0410000000000004,0.001 +"final",2,2,"guard_rank_5k",0.00300000000000011,0.00400000000000134,0 +"final",2,2,"panel_5k_fast_dr_x",0.0620000000000012,0.0620000000000012,0.002 +"final",2,2,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0449999999999982,0.001 +"final",2,2,"guard_overlap_5k",0.070999999999998,0.0719999999999992,0 +"final",2,2,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"final",2,2,"panel_1k_fast_reg_x",0.0160000000000018,0.0150000000000006,0.001 +"final",2,2,"panel_1k_fast_dr_x",0.0210000000000008,0.0210000000000008,0.001 +"final",2,2,"panel_5k_slow_dr_nox",0.0289999999999999,0.0289999999999999,0 +"final",2,3,"aggte_unbalanced_dynamic",0.032,0.0309999999999988,0 +"final",2,3,"panel_5k_slow_dr_x",0.0470000000000006,0.0459999999999994,0.001 +"final",2,3,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"final",2,3,"panel_1k_fast_dr_boot",0.0259999999999998,0.0259999999999998,0.002 +"final",2,3,"panel_5k_slow_reg_x",0.0240000000000009,0.0229999999999997,0 +"final",2,3,"unbalanced_5k_slow_dr_x",0.112000000000002,0.111999999999998,0.001 +"final",2,3,"aggte_5k_simple",0.00200000000000244,0.00199999999999889,0 +"final",2,3,"panel_5k_fast_dr_nox",0.0380000000000003,0.0380000000000003,0.001 +"final",2,3,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0 +"final",2,3,"panel_1k_fast_reg_x",0.0160000000000018,0.0150000000000006,0.001 +"final",2,3,"aggte_1k_dynamic",0.00399999999999778,0.00400000000000134,0 +"final",2,3,"panel_5k_slow_dr_weighted",0.0479999999999983,0.0480000000000018,0.001 +"final",2,3,"panel_1k_fast_dr_x",0.022000000000002,0.0210000000000008,0.001 +"final",2,3,"panel_5k_fast_dr_weighted",0.0670000000000002,0.0670000000000002,0.002 +"final",2,3,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",2,3,"guard_overlap_5k",0.0710000000000015,0.070999999999998,0 +"final",2,3,"rc_5k_slow_dr_x",0.137999999999998,0.137,0 +"final",2,3,"panel_1k_slow_dr_x",0.0160000000000018,0.0160000000000018,0.001 +"final",2,3,"rc_5k_fast_dr_x",0.148,0.146999999999998,0.002 +"final",2,3,"panel_5k_fast_dr_x",0.0629999999999988,0.0620000000000012,0.001 +"final",2,3,"panel_5k_fast_dr_boot",0.083000000000002,0.0790000000000006,0.005 +"final",2,3,"unbalanced_1k_fast_dr_x",0.0480000000000018,0.0449999999999982,0.002 +"final",2,3,"panel_1k_fast_dr_nox",0.0180000000000007,0.0159999999999982,0.001 +"final",2,3,"panel_5k_slow_dr_nox",0.0329999999999977,0.0309999999999988,0.001 +"final",2,3,"unbalanced_1k_fast_dr_boot",0.0500000000000007,0.0489999999999995,0.001 +"final",2,3,"aggte_1k_dynamic_boot",0.0169999999999995,0.0169999999999995,0 +"final",2,3,"panel_5k_fast_reg_x",0.0410000000000004,0.0410000000000004,0.001 +"final",2,3,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"final",2,3,"guard_rank_5k",0.00400000000000134,0.00400000000000134,0 +"final",2,3,"unbalanced_5k_fast_dr_x",0.135999999999999,0.134,0.003 +"final",2,3,"conditional_pretest_60",0.0829999999999984,0.0809999999999995,0.001 +"final",2,3,"rc_1k_fast_dr_x",0.0489999999999995,0.046999999999997,0.002 +"final",2,3,"rc_1k_fast_dr_boot",0.0630000000000024,0.0620000000000012,0.002 +"final",2,4,"panel_1k_fast_dr_x",0.0219999999999985,0.0210000000000008,0.001 +"final",2,4,"aggte_1k_dynamic_boot",0.0169999999999995,0.0169999999999995,0 +"final",2,4,"panel_5k_slow_dr_x",0.0470000000000006,0.0459999999999994,0.001 +"final",2,4,"aggte_5k_dynamic",0.00600000000000023,0.004999999999999,0 +"final",2,4,"guard_rank_intercept_5k",0.00700000000000145,0.00700000000000145,0.001 +"final",2,4,"rc_5k_slow_dr_x",0.140000000000001,0.137,0.002 +"final",2,4,"panel_1k_slow_dr_x",0.0150000000000006,0.0160000000000018,0 +"final",2,4,"panel_5k_fast_dr_nox",0.0380000000000003,0.0380000000000003,0.001 +"final",2,4,"aggte_1k_dynamic",0.00399999999999778,0.004999999999999,0 +"final",2,4,"panel_5k_slow_reg_x",0.0229999999999997,0.0229999999999997,0 +"final",2,4,"rc_1k_fast_dr_boot",0.0690000000000026,0.065999999999999,0.003 +"final",2,4,"rc_1k_fast_dr_x",0.0470000000000006,0.0460000000000029,0.001 +"final",2,4,"panel_5k_slow_dr_nox",0.0289999999999999,0.0289999999999999,0.001 +"final",2,4,"guard_rank_5k",0.00400000000000489,0.00400000000000134,0 +"final",2,4,"unbalanced_5k_fast_dr_x",0.134,0.134,0.002 +"final",2,4,"panel_1k_fast_reg_x",0.0159999999999982,0.0159999999999982,0.001 +"final",2,4,"panel_1k_fast_dr_boot",0.0259999999999962,0.0250000000000057,0.001 +"final",2,4,"panel_5k_fast_dr_boot",0.0800000000000054,0.0779999999999959,0.002 +"final",2,4,"unbalanced_1k_fast_dr_boot",0.0500000000000043,0.0489999999999995,0.002 +"final",2,4,"conditional_pretest_60",0.0800000000000054,0.0799999999999983,0 +"final",2,4,"rc_5k_fast_dr_x",0.145999999999994,0.146000000000001,0.002 +"final",2,4,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",2,4,"panel_1k_fast_dr_nox",0.0180000000000007,0.0180000000000007,0.001 +"final",2,4,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0450000000000017,0.001 +"final",2,4,"unbalanced_5k_slow_dr_x",0.113,0.111999999999995,0.000999999999999945 +"final",2,4,"aggte_unbalanced_dynamic",0.0290000000000035,0.0290000000000035,0 +"final",2,4,"panel_5k_fast_dr_x",0.0609999999999999,0.0609999999999999,0.002 +"final",2,4,"guard_overlap_5k",0.0720000000000027,0.0710000000000051,0 +"final",2,4,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"final",2,4,"panel_5k_fast_reg_x",0.0409999999999968,0.0409999999999968,0.002 +"final",2,4,"aggte_5k_simple",0.00200000000000244,0.000999999999997669,0 +"final",2,4,"panel_5k_fast_dr_weighted",0.068000000000005,0.068000000000005,0.001 +"final",2,4,"panel_5k_slow_dr_weighted",0.0510000000000019,0.0489999999999995,0.003 +"final",2,5,"guard_rank_intercept_5k",0.00800000000000267,0.007000000000005,0.001 +"final",2,5,"aggte_unbalanced_dynamic",0.0289999999999964,0.0300000000000011,0 +"final",2,5,"rc_5k_fast_dr_x",0.149000000000001,0.148000000000003,0.003 +"final",2,5,"aggte_5k_simple",0.000999999999997669,0.00200000000000244,0 +"final",2,5,"aggte_1k_dynamic_boot",0.0180000000000007,0.0169999999999959,0 +"final",2,5,"panel_5k_fast_dr_x",0.0650000000000048,0.0629999999999953,0.002 +"final",2,5,"panel_5k_fast_dr_boot",0.0770000000000053,0.0760000000000005,0.002 +"final",2,5,"rc_1k_fast_dr_boot",0.0670000000000002,0.0640000000000001,0.004 +"final",2,5,"panel_5k_fast_reg_x",0.0410000000000039,0.0409999999999968,0.002 +"final",2,5,"guard_rank_5k",0.00300000000000011,0.00400000000000489,0 +"final",2,5,"rc_5k_slow_dr_x",0.137,0.135000000000005,0 +"final",2,5,"panel_1k_slow_dr_x",0.0150000000000006,0.0150000000000006,0 +"final",2,5,"guard_overlap_intercept_5k",0.0130000000000052,0.0120000000000005,0 +"final",2,5,"guard_overlap_5k",0.0760000000000005,0.0739999999999981,0 +"final",2,5,"unbalanced_1k_fast_dr_x",0.0480000000000018,0.0470000000000041,0.002 +"final",2,5,"unbalanced_5k_slow_dr_x",0.120999999999995,0.119,0.001 +"final",2,5,"panel_5k_slow_reg_x",0.0240000000000009,0.0240000000000009,0 +"final",2,5,"panel_1k_fast_dr_nox",0.0180000000000007,0.017000000000003,0.001 +"final",2,5,"unbalanced_1k_fast_dr_boot",0.0499999999999972,0.0489999999999995,0.002 +"final",2,5,"aggte_1k_dynamic",0.00500000000000256,0.00399999999999778,0 +"final",2,5,"panel_1k_fast_reg_x",0.0149999999999935,0.0150000000000006,0.001 +"final",2,5,"unbalanced_5k_fast_dr_x",0.133999999999993,0.134,0.002 +"final",2,5,"rc_1k_fast_dr_x",0.0480000000000018,0.046999999999997,0.002 +"final",2,5,"guard_rank_weighted_5k",0.00800000000000267,0.00900000000000034,0 +"final",2,5,"panel_5k_fast_dr_nox",0.0390000000000015,0.0379999999999967,0.002 +"final",2,5,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"final",2,5,"panel_1k_fast_dr_x",0.0219999999999985,0.0210000000000008,0.001 +"final",2,5,"panel_5k_slow_dr_x",0.0450000000000017,0.0459999999999994,0 +"final",2,5,"panel_5k_slow_dr_weighted",0.0489999999999995,0.0489999999999995,0.001 +"final",2,5,"panel_1k_fast_dr_boot",0.0249999999999986,0.0249999999999986,0.001 +"final",2,5,"conditional_pretest_60",0.0810000000000031,0.0810000000000031,0.001 +"final",2,5,"panel_5k_slow_dr_nox",0.0290000000000035,0.0289999999999964,0 +"final",2,5,"panel_5k_fast_dr_weighted",0.0670000000000002,0.0670000000000002,0.002 +"final",2,6,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0 +"final",2,6,"panel_1k_fast_dr_nox",0.017000000000003,0.017000000000003,0 +"final",2,6,"panel_5k_slow_dr_weighted",0.0479999999999947,0.0489999999999995,0.001 +"final",2,6,"panel_1k_fast_reg_x",0.0159999999999982,0.0150000000000006,0.001 +"final",2,6,"panel_5k_slow_reg_x",0.0240000000000009,0.0229999999999961,0.001 +"final",2,6,"aggte_1k_dynamic",0.00400000000000489,0.00500000000000256,0.001 +"final",2,6,"panel_5k_fast_dr_x",0.0630000000000024,0.0630000000000024,0.002 +"final",2,6,"panel_1k_slow_dr_x",0.0160000000000053,0.0149999999999935,0 +"final",2,6,"guard_overlap_5k",0.0730000000000004,0.0730000000000004,0.001 +"final",2,6,"aggte_1k_dynamic_boot",0.017000000000003,0.0169999999999959,0 +"final",2,6,"rc_1k_fast_dr_boot",0.0700000000000003,0.0659999999999954,0.003 +"final",2,6,"panel_5k_slow_dr_x",0.0479999999999947,0.0450000000000017,0.002 +"final",2,6,"panel_5k_fast_dr_nox",0.036999999999999,0.0360000000000014,0.00199999999999989 +"final",2,6,"unbalanced_5k_slow_dr_x",0.114000000000004,0.111999999999995,0.002 +"final",2,6,"panel_5k_fast_dr_boot",0.078000000000003,0.0779999999999959,0.002 +"final",2,6,"aggte_5k_dynamic",0.00600000000000023,0.00500000000000256,0 +"final",2,6,"panel_1k_fast_dr_x",0.0219999999999985,0.0210000000000008,0.001 +"final",2,6,"panel_1k_fast_dr_boot",0.0249999999999986,0.0249999999999986,0.001 +"final",2,6,"aggte_unbalanced_dynamic",0.0289999999999964,0.0290000000000035,0 +"final",2,6,"panel_5k_slow_dr_nox",0.0279999999999987,0.0290000000000035,0 +"final",2,6,"panel_5k_fast_dr_weighted",0.0670000000000002,0.0670000000000002,0.002 +"final",2,6,"panel_5k_fast_reg_x",0.0420000000000016,0.0420000000000016,0.002 +"final",2,6,"guard_rank_5k",0.00399999999999778,0.00400000000000489,0 +"final",2,6,"conditional_pretest_60",0.0869999999999962,0.0850000000000009,0.001 +"final",2,6,"unbalanced_1k_fast_dr_x",0.0480000000000018,0.0459999999999994,0.002 +"final",2,6,"aggte_5k_simple",0.00200000000000244,0.000999999999997669,0 +"final",2,6,"unbalanced_5k_fast_dr_x",0.140999999999998,0.139000000000003,0.003 +"final",2,6,"rc_5k_fast_dr_x",0.183999999999997,0.146999999999998,0.003 +"final",2,6,"guard_rank_intercept_5k",0.0069999999999979,0.00600000000000023,0 +"final",2,6,"rc_1k_fast_dr_x",0.0470000000000041,0.046999999999997,0.001 +"final",2,6,"guard_overlap_intercept_5k",0.0200000000000031,0.0120000000000005,0 +"final",2,6,"unbalanced_1k_fast_dr_boot",0.0520000000000067,0.0510000000000019,0.001 +"final",2,6,"rc_5k_slow_dr_x",0.135999999999996,0.134999999999998,0.001 +"final",2,7,"guard_overlap_intercept_5k",0.0119999999999933,0.0120000000000005,0 +"final",2,7,"panel_5k_fast_reg_x",0.0409999999999968,0.0410000000000039,0.001 +"final",2,7,"panel_5k_fast_dr_weighted",0.0660000000000025,0.0660000000000025,0.001 +"final",2,7,"aggte_1k_dynamic_boot",0.0180000000000007,0.0180000000000007,0 +"final",2,7,"panel_5k_slow_dr_nox",0.0289999999999964,0.0290000000000035,0 +"final",2,7,"panel_5k_slow_dr_weighted",0.0480000000000018,0.0489999999999995,0.001 +"final",2,7,"rc_5k_slow_dr_x",0.136000000000003,0.136000000000003,0.001 +"final",2,7,"aggte_5k_simple",0.000999999999997669,0.00200000000000244,0 +"final",2,7,"rc_5k_fast_dr_x",0.149000000000001,0.149000000000001,0.002 +"final",2,7,"panel_5k_slow_dr_x",0.046999999999997,0.0449999999999946,0 +"final",2,7,"guard_rank_weighted_5k",0.00900000000000034,0.00900000000000034,0.001 +"final",2,7,"panel_1k_fast_dr_nox",0.017000000000003,0.017000000000003,0.001 +"final",2,7,"panel_5k_fast_dr_x",0.0619999999999976,0.0619999999999976,0.001 +"final",2,7,"rc_1k_fast_dr_x",0.0480000000000018,0.0480000000000018,0.002 +"final",2,7,"panel_5k_fast_dr_boot",0.0769999999999982,0.0770000000000053,0.001 +"final",2,7,"aggte_unbalanced_dynamic",0.0289999999999964,0.029999999999994,0 +"final",2,7,"panel_5k_slow_reg_x",0.0240000000000009,0.0240000000000009,0 +"final",2,7,"panel_1k_slow_dr_x",0.0159999999999982,0.0150000000000006,0 +"final",2,7,"panel_5k_fast_dr_nox",0.0359999999999943,0.036999999999999,0.001 +"final",2,7,"panel_1k_fast_dr_x",0.0219999999999985,0.0210000000000008,0.001 +"final",2,7,"panel_1k_fast_dr_boot",0.0249999999999986,0.0240000000000009,0.001 +"final",2,7,"rc_1k_fast_dr_boot",0.0679999999999978,0.0649999999999977,0.004 +"final",2,7,"unbalanced_1k_fast_dr_boot",0.0510000000000019,0.0499999999999972,0.003 +"final",2,7,"guard_rank_intercept_5k",0.00800000000000267,0.0069999999999979,0.001 +"final",2,7,"aggte_1k_dynamic",0.00399999999999778,0.00500000000000256,0 +"final",2,7,"conditional_pretest_60",0.080999999999996,0.080999999999996,0 +"final",2,7,"panel_1k_fast_reg_x",0.0159999999999982,0.0150000000000006,0.001 +"final",2,7,"aggte_5k_dynamic",0.00600000000000023,0.00499999999999545,0 +"final",2,7,"guard_overlap_5k",0.0790000000000006,0.0749999999999957,0.002 +"final",2,7,"unbalanced_5k_slow_dr_x",0.119,0.115000000000002,0.003 +"final",2,7,"guard_rank_5k",0.00400000000000489,0.00399999999999778,0 +"final",2,7,"unbalanced_5k_fast_dr_x",0.137999999999998,0.135999999999996,0.003 +"final",2,7,"unbalanced_1k_fast_dr_x",0.0459999999999994,0.0439999999999969,0.002 +"baseline",2,1,"conditional_pretest_60",0.0880000000000001,0.0809999999999995,0.00700000000000001 +"baseline",2,1,"aggte_5k_dynamic",0.00599999999999934,0.00600000000000023,0 +"baseline",2,1,"panel_5k_slow_dr_x",0.0579999999999998,0.0489999999999995,0.005 +"baseline",2,1,"rc_1k_fast_dr_boot",0.069,0.0659999999999998,0.004 +"baseline",2,1,"unbalanced_1k_fast_dr_x",0.0490000000000004,0.0469999999999997,0.003 +"baseline",2,1,"panel_1k_fast_reg_x",0.0169999999999995,0.0170000000000003,0.001 +"baseline",2,1,"panel_5k_slow_dr_nox",0.0299999999999994,0.0289999999999999,0.001 +"baseline",2,1,"guard_overlap_5k",0.0749999999999993,0.0730000000000004,0.00299999999999997 +"baseline",2,1,"aggte_unbalanced_dynamic",0.0300000000000011,0.0299999999999994,0.001 +"baseline",2,1,"unbalanced_1k_fast_dr_boot",0.0530000000000008,0.0519999999999996,0.001 +"baseline",2,1,"guard_rank_weighted_5k",0.00400000000000134,0.00299999999999834,0 +"baseline",2,1,"rc_5k_slow_dr_x",0.136000000000001,0.134,0.001 +"baseline",2,1,"guard_overlap_intercept_5k",0.0129999999999999,0.0120000000000005,0 +"baseline",2,1,"panel_5k_fast_dr_boot",0.0830000000000002,0.0820000000000007,0.002 +"baseline",2,1,"aggte_1k_dynamic",0.00399999999999956,0.00399999999999956,0 +"baseline",2,1,"guard_rank_5k",0.00500000000000078,0.00400000000000134,0.001 +"baseline",2,1,"panel_5k_slow_reg_x",0.0250000000000004,0.0239999999999991,0.001 +"baseline",2,1,"guard_rank_intercept_5k",0.00699999999999967,0.00699999999999967,0 +"baseline",2,1,"panel_1k_fast_dr_nox",0.0190000000000001,0.0190000000000001,0 +"baseline",2,1,"aggte_5k_simple",0.000999999999999446,0.000999999999999446,0 +"baseline",2,1,"rc_1k_fast_dr_x",0.048,0.0470000000000006,0.002 +"baseline",2,1,"panel_5k_fast_dr_nox",0.0399999999999991,0.0389999999999997,0.001 +"baseline",2,1,"panel_1k_fast_dr_boot",0.0289999999999999,0.0280000000000005,0.001 +"baseline",2,1,"aggte_1k_dynamic_boot",0.0180000000000007,0.0179999999999989,0 +"baseline",2,1,"rc_5k_fast_dr_x",0.145999999999999,0.145,0.002 +"baseline",2,1,"panel_5k_fast_dr_weighted",0.0709999999999997,0.0700000000000003,0.002 +"baseline",2,1,"panel_5k_fast_dr_x",0.0640000000000001,0.0649999999999995,0.002 +"baseline",2,1,"unbalanced_5k_slow_dr_x",0.115,0.114000000000001,0.001 +"baseline",2,1,"panel_1k_slow_dr_x",0.016,0.016,0.001 +"baseline",2,1,"panel_5k_fast_reg_x",0.043000000000001,0.0429999999999993,0.001 +"baseline",2,1,"panel_1k_fast_dr_x",0.0240000000000009,0.0240000000000009,0 +"baseline",2,1,"unbalanced_5k_fast_dr_x",0.136000000000001,0.135,0.002 +"baseline",2,1,"panel_5k_slow_dr_weighted",0.0489999999999995,0.048,0.001 +"baseline",2,2,"panel_5k_slow_dr_weighted",0.0549999999999997,0.052999999999999,0.002 +"baseline",2,2,"aggte_1k_dynamic_boot",0.0170000000000012,0.0170000000000012,0 +"baseline",2,2,"guard_rank_intercept_5k",0.00699999999999967,0.00600000000000023,0 +"baseline",2,2,"unbalanced_1k_fast_dr_boot",0.0540000000000003,0.0519999999999996,0.002 +"baseline",2,2,"rc_1k_fast_dr_boot",0.0690000000000008,0.0670000000000002,0.002 +"baseline",2,2,"aggte_1k_dynamic",0.00400000000000134,0.004999999999999,0 +"baseline",2,2,"unbalanced_5k_fast_dr_x",0.137,0.135,0.002 +"baseline",2,2,"panel_1k_fast_dr_boot",0.0299999999999994,0.0289999999999999,0.001 +"baseline",2,2,"conditional_pretest_60",0.0809999999999995,0.0800000000000001,0.001 +"baseline",2,2,"panel_5k_slow_dr_x",0.0459999999999994,0.0449999999999999,0 +"baseline",2,2,"panel_5k_fast_dr_weighted",0.070999999999998,0.0710000000000015,0.002 +"baseline",2,2,"rc_1k_fast_dr_x",0.0479999999999983,0.0470000000000006,0.002 +"baseline",2,2,"panel_5k_fast_dr_nox",0.0390000000000015,0.0390000000000015,0.001 +"baseline",2,2,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"baseline",2,2,"panel_1k_slow_dr_x",0.0160000000000018,0.0159999999999982,0.001 +"baseline",2,2,"panel_1k_fast_dr_nox",0.0450000000000017,0.0219999999999985,0.001 +"baseline",2,2,"panel_5k_fast_dr_boot",0.0809999999999995,0.0799999999999983,0.001 +"baseline",2,2,"guard_rank_weighted_5k",0.00399999999999778,0.00399999999999778,0 +"baseline",2,2,"aggte_unbalanced_dynamic",0.0299999999999976,0.0289999999999999,0.001 +"baseline",2,2,"rc_5k_slow_dr_x",0.221,0.221,0 +"baseline",2,2,"aggte_5k_simple",0.00200000000000244,0.00199999999999889,0 +"baseline",2,2,"unbalanced_5k_slow_dr_x",0.112000000000002,0.109999999999999,0.001 +"baseline",2,2,"rc_5k_fast_dr_x",0.148,0.146999999999998,0.003 +"baseline",2,2,"panel_5k_slow_reg_x",0.0229999999999997,0.0229999999999997,0 +"baseline",2,2,"panel_5k_fast_reg_x",0.0429999999999993,0.0429999999999993,0.001 +"baseline",2,2,"guard_rank_5k",0.00399999999999778,0.00400000000000134,0 +"baseline",2,2,"panel_5k_fast_dr_x",0.0640000000000001,0.0640000000000001,0.002 +"baseline",2,2,"unbalanced_1k_fast_dr_x",0.0470000000000006,0.0459999999999994,0.001 +"baseline",2,2,"guard_overlap_5k",0.0720000000000027,0.0720000000000027,0 +"baseline",2,2,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"baseline",2,2,"panel_1k_fast_reg_x",0.0169999999999995,0.0180000000000007,0 +"baseline",2,2,"panel_1k_fast_dr_x",0.0229999999999997,0.0229999999999997,0.001 +"baseline",2,2,"panel_5k_slow_dr_nox",0.0289999999999999,0.0289999999999999,0 +"baseline",2,3,"aggte_unbalanced_dynamic",0.0300000000000011,0.0300000000000011,0 +"baseline",2,3,"panel_5k_slow_dr_x",0.0459999999999994,0.0459999999999994,0 +"baseline",2,3,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"baseline",2,3,"panel_1k_fast_dr_boot",0.0279999999999987,0.0280000000000022,0 +"baseline",2,3,"panel_5k_slow_reg_x",0.0229999999999997,0.0229999999999997,0.001 +"baseline",2,3,"unbalanced_5k_slow_dr_x",0.114999999999998,0.111000000000001,0.004 +"baseline",2,3,"aggte_5k_simple",0.00199999999999889,0.00199999999999889,0 +"baseline",2,3,"panel_5k_fast_dr_nox",0.0389999999999979,0.0399999999999991,0.001 +"baseline",2,3,"guard_rank_intercept_5k",0.0069999999999979,0.00700000000000145,0 +"baseline",2,3,"panel_1k_fast_reg_x",0.0179999999999971,0.0180000000000007,0 +"baseline",2,3,"aggte_1k_dynamic",0.004999999999999,0.00399999999999778,0 +"baseline",2,3,"panel_5k_slow_dr_weighted",0.0500000000000007,0.0489999999999995,0.001 +"baseline",2,3,"panel_1k_fast_dr_x",0.0229999999999997,0.0229999999999997,0 +"baseline",2,3,"panel_5k_fast_dr_weighted",0.0719999999999992,0.0710000000000015,0.002 +"baseline",2,3,"guard_rank_weighted_5k",0.00400000000000134,0.00300000000000011,0 +"baseline",2,3,"guard_overlap_5k",0.0719999999999992,0.070999999999998,0.001 +"baseline",2,3,"rc_5k_slow_dr_x",0.129999999999999,0.129000000000001,0.001 +"baseline",2,3,"panel_1k_slow_dr_x",0.0150000000000006,0.0160000000000018,0 +"baseline",2,3,"rc_5k_fast_dr_x",0.149000000000001,0.148,0.002 +"baseline",2,3,"panel_5k_fast_dr_x",0.0630000000000024,0.0629999999999988,0.002 +"baseline",2,3,"panel_5k_fast_dr_boot",0.0799999999999983,0.0790000000000006,0.001 +"baseline",2,3,"unbalanced_1k_fast_dr_x",0.0470000000000006,0.0459999999999994,0.001 +"baseline",2,3,"panel_1k_fast_dr_nox",0.0210000000000008,0.0180000000000007,0.003 +"baseline",2,3,"panel_5k_slow_dr_nox",0.0310000000000024,0.0299999999999976,0.001 +"baseline",2,3,"unbalanced_1k_fast_dr_boot",0.0559999999999974,0.0539999999999985,0.002 +"baseline",2,3,"aggte_1k_dynamic_boot",0.0180000000000007,0.0180000000000007,0 +"baseline",2,3,"panel_5k_fast_reg_x",0.0449999999999982,0.0440000000000005,0.002 +"baseline",2,3,"aggte_5k_dynamic",0.00500000000000256,0.00600000000000023,0 +"baseline",2,3,"guard_rank_5k",0.00399999999999778,0.00399999999999778,0 +"baseline",2,3,"unbalanced_5k_fast_dr_x",0.138000000000002,0.134999999999998,0.003 +"baseline",2,3,"conditional_pretest_60",0.0820000000000007,0.0809999999999995,0 +"baseline",2,3,"rc_1k_fast_dr_x",0.0489999999999995,0.046999999999997,0.002 +"baseline",2,3,"rc_1k_fast_dr_boot",0.0690000000000026,0.065999999999999,0.002 +"baseline",2,4,"panel_1k_fast_dr_x",0.0259999999999998,0.0250000000000021,0.001 +"baseline",2,4,"aggte_1k_dynamic_boot",0.0190000000000019,0.0189999999999984,0 +"baseline",2,4,"panel_5k_slow_dr_x",0.0479999999999983,0.0480000000000018,0 +"baseline",2,4,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"baseline",2,4,"guard_rank_intercept_5k",0.00700000000000145,0.00700000000000145,0 +"baseline",2,4,"rc_5k_slow_dr_x",0.138999999999999,0.134999999999998,0.001 +"baseline",2,4,"panel_1k_slow_dr_x",0.0159999999999982,0.0159999999999982,0 +"baseline",2,4,"panel_5k_fast_dr_nox",0.0399999999999991,0.0400000000000027,0.001 +"baseline",2,4,"aggte_1k_dynamic",0.00400000000000134,0.00400000000000134,0 +"baseline",2,4,"panel_5k_slow_reg_x",0.0229999999999997,0.0229999999999997,0 +"baseline",2,4,"rc_1k_fast_dr_boot",0.0680000000000014,0.065999999999999,0.002 +"baseline",2,4,"rc_1k_fast_dr_x",0.0479999999999983,0.0470000000000006,0.001 +"baseline",2,4,"panel_5k_slow_dr_nox",0.0289999999999964,0.0289999999999999,0 +"baseline",2,4,"guard_rank_5k",0.00300000000000011,0.00400000000000134,0 +"baseline",2,4,"unbalanced_5k_fast_dr_x",0.134999999999998,0.135000000000002,0.00199999999999995 +"baseline",2,4,"panel_1k_fast_reg_x",0.017000000000003,0.017000000000003,0 +"baseline",2,4,"panel_1k_fast_dr_boot",0.029999999999994,0.0280000000000058,0.001 +"baseline",2,4,"panel_5k_fast_dr_boot",0.0800000000000054,0.0799999999999983,0.002 +"baseline",2,4,"unbalanced_1k_fast_dr_boot",0.0529999999999973,0.0519999999999996,0.001 +"baseline",2,4,"conditional_pretest_60",0.0829999999999984,0.0829999999999984,0 +"baseline",2,4,"rc_5k_fast_dr_x",0.146999999999998,0.146999999999998,0.002 +"baseline",2,4,"guard_rank_weighted_5k",0.00399999999999778,0.00400000000000489,0 +"baseline",2,4,"panel_1k_fast_dr_nox",0.0190000000000055,0.0189999999999984,0.001 +"baseline",2,4,"unbalanced_1k_fast_dr_x",0.0480000000000018,0.046999999999997,0.002 +"baseline",2,4,"unbalanced_5k_slow_dr_x",0.111000000000004,0.111000000000004,0.001 +"baseline",2,4,"aggte_unbalanced_dynamic",0.0289999999999964,0.0290000000000035,0 +"baseline",2,4,"panel_5k_fast_dr_x",0.0640000000000001,0.0630000000000024,0.001 +"baseline",2,4,"guard_overlap_5k",0.070999999999998,0.070999999999998,0 +"baseline",2,4,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"baseline",2,4,"panel_5k_fast_reg_x",0.0439999999999969,0.0429999999999993,0.002 +"baseline",2,4,"aggte_5k_simple",0.00200000000000244,0.00200000000000244,0 +"baseline",2,4,"panel_5k_fast_dr_weighted",0.0710000000000051,0.070999999999998,0.002 +"baseline",2,4,"panel_5k_slow_dr_weighted",0.0480000000000018,0.0489999999999995,0 +"baseline",2,5,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0.001 +"baseline",2,5,"aggte_unbalanced_dynamic",0.0309999999999988,0.0300000000000011,0.001 +"baseline",2,5,"rc_5k_fast_dr_x",0.148000000000003,0.146000000000001,0.003 +"baseline",2,5,"aggte_5k_simple",0.00199999999999534,0.00199999999999534,0 +"baseline",2,5,"aggte_1k_dynamic_boot",0.0169999999999959,0.0169999999999959,0 +"baseline",2,5,"panel_5k_fast_dr_x",0.0649999999999977,0.0630000000000024,0.002 +"baseline",2,5,"panel_5k_fast_dr_boot",0.0810000000000031,0.0810000000000031,0.001 +"baseline",2,5,"rc_1k_fast_dr_boot",0.0689999999999955,0.0659999999999954,0.003 +"baseline",2,5,"panel_5k_fast_reg_x",0.0420000000000016,0.0429999999999993,0.001 +"baseline",2,5,"guard_rank_5k",0.00300000000000011,0.00399999999999778,0 +"baseline",2,5,"rc_5k_slow_dr_x",0.134999999999998,0.134,0.001 +"baseline",2,5,"panel_1k_slow_dr_x",0.0159999999999982,0.0160000000000053,0 +"baseline",2,5,"guard_overlap_intercept_5k",0.0129999999999981,0.0120000000000005,0 +"baseline",2,5,"guard_overlap_5k",0.0749999999999957,0.0719999999999956,0.003 +"baseline",2,5,"unbalanced_1k_fast_dr_x",0.0510000000000019,0.0479999999999947,0.003 +"baseline",2,5,"unbalanced_5k_slow_dr_x",0.111000000000004,0.109999999999999,0.001 +"baseline",2,5,"panel_5k_slow_reg_x",0.0249999999999986,0.0229999999999961,0.000999999999999945 +"baseline",2,5,"panel_1k_fast_dr_nox",0.0180000000000007,0.0179999999999936,0 +"baseline",2,5,"unbalanced_1k_fast_dr_boot",0.0539999999999949,0.0529999999999973,0.001 +"baseline",2,5,"aggte_1k_dynamic",0.00399999999999778,0.00400000000000489,0 +"baseline",2,5,"panel_1k_fast_reg_x",0.0180000000000007,0.017000000000003,0.001 +"baseline",2,5,"unbalanced_5k_fast_dr_x",0.134999999999998,0.134,0.002 +"baseline",2,5,"rc_1k_fast_dr_x",0.0489999999999995,0.046999999999997,0.001 +"baseline",2,5,"guard_rank_weighted_5k",0.00400000000000489,0.00399999999999778,0 +"baseline",2,5,"panel_5k_fast_dr_nox",0.0409999999999968,0.0410000000000039,0.001 +"baseline",2,5,"aggte_5k_dynamic",0.00600000000000023,0.0069999999999979,0 +"baseline",2,5,"panel_1k_fast_dr_x",0.0249999999999986,0.0249999999999986,0.001 +"baseline",2,5,"panel_5k_slow_dr_x",0.0479999999999947,0.0470000000000041,0 +"baseline",2,5,"panel_5k_slow_dr_weighted",0.0499999999999972,0.0499999999999972,0.001 +"baseline",2,5,"panel_1k_fast_dr_boot",0.0280000000000058,0.0279999999999987,0.001 +"baseline",2,5,"conditional_pretest_60",0.0820000000000007,0.080999999999996,0 +"baseline",2,5,"panel_5k_slow_dr_nox",0.0300000000000011,0.0300000000000011,0 +"baseline",2,5,"panel_5k_fast_dr_weighted",0.0700000000000003,0.0700000000000003,0.002 +"baseline",2,6,"guard_rank_weighted_5k",0.00400000000000489,0.00300000000000011,0 +"baseline",2,6,"panel_1k_fast_dr_nox",0.0200000000000031,0.0190000000000055,0.001 +"baseline",2,6,"panel_5k_slow_dr_weighted",0.0489999999999995,0.0489999999999995,0.001 +"baseline",2,6,"panel_1k_fast_reg_x",0.0169999999999959,0.0169999999999959,0.001 +"baseline",2,6,"panel_5k_slow_reg_x",0.0240000000000009,0.0240000000000009,0 +"baseline",2,6,"aggte_1k_dynamic",0.00500000000000256,0.00399999999999778,0 +"baseline",2,6,"panel_5k_fast_dr_x",0.0640000000000001,0.0630000000000024,0.001 +"baseline",2,6,"panel_1k_slow_dr_x",0.0159999999999982,0.0159999999999982,0 +"baseline",2,6,"guard_overlap_5k",0.0739999999999981,0.0720000000000027,0 +"baseline",2,6,"aggte_1k_dynamic_boot",0.0169999999999959,0.0169999999999959,0 +"baseline",2,6,"rc_1k_fast_dr_boot",0.068000000000005,0.0670000000000002,0.002 +"baseline",2,6,"panel_5k_slow_dr_x",0.0490000000000066,0.046999999999997,0 +"baseline",2,6,"panel_5k_fast_dr_nox",0.0399999999999991,0.0389999999999944,0.003 +"baseline",2,6,"unbalanced_5k_slow_dr_x",0.112000000000002,0.109999999999999,0.002 +"baseline",2,6,"panel_5k_fast_dr_boot",0.0829999999999984,0.0820000000000007,0.002 +"baseline",2,6,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"baseline",2,6,"panel_1k_fast_dr_x",0.0240000000000009,0.0230000000000032,0.001 +"baseline",2,6,"panel_1k_fast_dr_boot",0.0289999999999964,0.0289999999999964,0.001 +"baseline",2,6,"aggte_unbalanced_dynamic",0.029999999999994,0.0289999999999964,0 +"baseline",2,6,"panel_5k_slow_dr_nox",0.0289999999999964,0.0289999999999964,0 +"baseline",2,6,"panel_5k_fast_dr_weighted",0.0749999999999957,0.0750000000000028,0.001 +"baseline",2,6,"panel_5k_fast_reg_x",0.0420000000000016,0.0420000000000016,0.001 +"baseline",2,6,"guard_rank_5k",0.00400000000000489,0.00399999999999778,0 +"baseline",2,6,"conditional_pretest_60",0.0879999999999939,0.0859999999999985,0.001 +"baseline",2,6,"unbalanced_1k_fast_dr_x",0.0490000000000066,0.0480000000000018,0.002 +"baseline",2,6,"aggte_5k_simple",0.00200000000000244,0.000999999999997669,0 +"baseline",2,6,"unbalanced_5k_fast_dr_x",0.134999999999998,0.134999999999998,0.002 +"baseline",2,6,"rc_5k_fast_dr_x",0.149000000000001,0.149000000000001,0.002 +"baseline",2,6,"guard_rank_intercept_5k",0.0069999999999979,0.0069999999999979,0 +"baseline",2,6,"rc_1k_fast_dr_x",0.0489999999999995,0.0479999999999947,0.003 +"baseline",2,6,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"baseline",2,6,"unbalanced_1k_fast_dr_boot",0.0549999999999997,0.0529999999999973,0.001 +"baseline",2,6,"rc_5k_slow_dr_x",0.134,0.134,0.001 +"baseline",2,7,"guard_overlap_intercept_5k",0.0120000000000005,0.0120000000000005,0 +"baseline",2,7,"panel_5k_fast_reg_x",0.044000000000004,0.044000000000004,0.002 +"baseline",2,7,"panel_5k_fast_dr_weighted",0.070999999999998,0.0700000000000003,0.001 +"baseline",2,7,"aggte_1k_dynamic_boot",0.0169999999999959,0.0169999999999959,0 +"baseline",2,7,"panel_5k_slow_dr_nox",0.0289999999999964,0.0290000000000035,0 +"baseline",2,7,"panel_5k_slow_dr_weighted",0.0480000000000018,0.0480000000000018,0.001 +"baseline",2,7,"rc_5k_slow_dr_x",0.134,0.132999999999996,0.001 +"baseline",2,7,"aggte_5k_simple",0.000999999999997669,0.00200000000000244,0 +"baseline",2,7,"rc_5k_fast_dr_x",0.146999999999998,0.148000000000003,0.002 +"baseline",2,7,"panel_5k_slow_dr_x",0.0449999999999946,0.0459999999999994,0.001 +"baseline",2,7,"guard_rank_weighted_5k",0.00400000000000489,0.00300000000000011,0 +"baseline",2,7,"panel_1k_fast_dr_nox",0.0189999999999984,0.0180000000000007,0.001 +"baseline",2,7,"panel_5k_fast_dr_x",0.0640000000000001,0.0640000000000001,0.001 +"baseline",2,7,"rc_1k_fast_dr_x",0.0489999999999995,0.0480000000000018,0.002 +"baseline",2,7,"panel_5k_fast_dr_boot",0.0810000000000031,0.080999999999996,0.001 +"baseline",2,7,"aggte_unbalanced_dynamic",0.0429999999999993,0.0349999999999966,0.001 +"baseline",2,7,"panel_5k_slow_reg_x",0.0300000000000011,0.0260000000000034,0.001 +"baseline",2,7,"panel_1k_slow_dr_x",0.0159999999999982,0.0159999999999982,0.001 +"baseline",2,7,"panel_5k_fast_dr_nox",0.0399999999999991,0.0399999999999991,0.001 +"baseline",2,7,"panel_1k_fast_dr_x",0.0230000000000032,0.0230000000000032,0.001 +"baseline",2,7,"panel_1k_fast_dr_boot",0.0300000000000011,0.0289999999999964,0.001 +"baseline",2,7,"rc_1k_fast_dr_boot",0.0670000000000002,0.0670000000000002,0.001 +"baseline",2,7,"unbalanced_1k_fast_dr_boot",0.0560000000000045,0.0529999999999973,0.004 +"baseline",2,7,"guard_rank_intercept_5k",0.0069999999999979,0.007000000000005,0 +"baseline",2,7,"aggte_1k_dynamic",0.00400000000000489,0.00400000000000489,0 +"baseline",2,7,"conditional_pretest_60",0.0820000000000007,0.0800000000000054,0.001 +"baseline",2,7,"panel_1k_fast_reg_x",0.0180000000000007,0.0180000000000007,0.001 +"baseline",2,7,"aggte_5k_dynamic",0.00600000000000023,0.00600000000000023,0 +"baseline",2,7,"guard_overlap_5k",0.0739999999999981,0.0720000000000027,0.002 +"baseline",2,7,"unbalanced_5k_slow_dr_x",0.112000000000002,0.111000000000004,0.001 +"baseline",2,7,"guard_rank_5k",0.00399999999999778,0.00399999999999778,0 +"baseline",2,7,"unbalanced_5k_fast_dr_x",0.137,0.135999999999996,0.002 +"baseline",2,7,"unbalanced_1k_fast_dr_x",0.046999999999997,0.0459999999999994,0.002 diff --git a/dev/benchmarks/stress-fixes/optimized-source-manifest.txt b/dev/benchmarks/stress-fixes/optimized-source-manifest.txt new file mode 100644 index 00000000..98e0a045 --- /dev/null +++ b/dev/benchmarks/stress-fixes/optimized-source-manifest.txt @@ -0,0 +1,37 @@ +$source_head +[1] "1a16c46cac7d184b486da556db7c855995d78422" + +$source_diff +[1] " R/att_gt.R | 9 ++++++++-" +[2] " R/compute.aggte.R | 26 +++++++++++++++++++-------" +[3] " R/mboot.R | 2 +-" +[4] " R/utility_functions.R | 13 ++++++++++---" +[5] " tests/testthat/test-anticipation-window.R | 16 ++++++++++++++++" +[6] " tests/testthat/test-inference-outcome-units.R | 27 +++++++++++++++++++++++++++" +[7] " 6 files changed, 81 insertions(+), 12 deletions(-)" + +$source_hashes + R/aggte.R R/AGGTEobj.R +"2d062b18289126f8dba5a67392839b73" "47e4652e98e07951213dbde1768eb87d" + R/att_gt.R R/compute.aggte.R +"bb904b188e78220fd22cf27624d0bef2" "0d956959d1261461640c1dd1b1fde133" + R/compute.att_gt.R R/compute.att_gt2.R +"46411357aea90ce0eee2e2e985a67ac0" "ce78ad65c44926b4bcb68509f151fa3e" + R/conditional_did_pretest.R R/data.R +"29768bf9a07d8dec9b76183c5dae4a28" "0301b6b87092194b31d66260eb39c97b" + R/DIDparams.R R/DIDparams2.R +"1d3ce273c192fc7a0b77042699c16b97" "d7596012220776ff81e9772ae68a21a2" + R/ggdid.R R/gplot.R +"55f186443457b0bb9f022bd3191c5a95" "429d320b12cdc603f689ae2a715b51dd" + R/honest_did/honest_did.R R/imports.R +"efc136cc0805a180cd101d5e799a071b" "c4385e20eca846d0e0b3c35c97ce4243" + R/mboot.R R/MP.R +"55656e2f6cdd3d1b40b6f140c304ea14" "85adea7bd7eea34d70597349526359a8" + R/MP.TEST.R R/pre_process_did.R +"aafbe2d731fd354e4da57bfc05bed9bd" "581a7a33acccb559566f028e816b0404" + R/pre_process_did2.R R/process_attgt.R +"90f155f7497bc59c631b51ac4ede3a9a" "735b44be0c59c0c6b0fae2c0de33db48" + R/simulate_data.R R/tidy.R +"cde999a7898c9eb130346fa21ee14f08" "4b5b2395b86cf6bb7ffe6128aed05bc8" + R/utility_functions.R R/zzz.R +"359a2ac452f8836a0b618eea0f966b35" "78ea69d652140b604b79f12fe0b42614" diff --git a/dev/benchmarks/stress-fixes/performance-worker.R b/dev/benchmarks/stress-fixes/performance-worker.R new file mode 100644 index 00000000..daf2c3c7 --- /dev/null +++ b/dev/benchmarks/stress-fixes/performance-worker.R @@ -0,0 +1,143 @@ +# Invoked by performance.R. Keep package loading/data creation outside timings. +args <- commandArgs(trailingOnly = TRUE) +package_path <- normalizePath(args[[1L]]) +label <- args[[2L]] +output_path <- args[[3L]] +round <- as.integer(args[[4L]]) +repetitions <- as.integer(args[[5L]]) +suite <- if (length(args) > 5L) args[[6L]] else "main" +options(warn = 1) +pkgload::load_all(package_path, quiet = TRUE, export_all = TRUE) +stopifnot(utils::packageVersion("DRDID") == package_version("1.4.0")) +set.seed(20261028) +make_panel <- function(n, periods = 6L) { + X <- rnorm(n); X2 <- rnorm(n); a <- rnorm(n) + groups <- if (periods <= 6L) c(0, 3, 4, 5, 6) else c(0, seq.int(3L, periods)) + probabilities <- if (periods <= 6L) c(.35, .16, .16, .16, .17) else + c(.35, rep(.65 / (length(groups) - 1L), length(groups) - 1L)) + G <- sample(groups, n, TRUE, probabilities) + d <- expand.grid(id = seq_len(n), t = seq_len(periods)) + d$G <- G[d$id]; d$X <- X[d$id]; d$X2 <- X2[d$id] + d$w <- .5 + (d$id %% 7) / 3 + d$Y <- a[d$id] + .2 * d$t + (.3 + .15 * d$t) * d$X + + .2 * d$X2 + (d$G > 0 & d$t >= d$G) + rnorm(nrow(d)) + d +} +d1 <- make_panel(1000L); d5 <- make_panel(5000L) +u1 <- d1[-sample(nrow(d1), 800L), ] +u5 <- d5[-sample(nrow(d5), 4000L), ] +r1 <- d1; r1$id <- seq_len(nrow(r1)) +r5 <- d5; r5$id <- seq_len(nrow(r5)) +pd <- make_panel(60L, 4L) +fit <- function(d, faster = TRUE, covariates = TRUE, method = "dr", + structure = "panel", bootstrap = FALSE, weighted = FALSE, + fix_weights = NULL) { + suppressMessages(suppressWarnings(att_gt( + "Y", "t", "id", "G", data = d, + xformla = if (covariates) ~X + X2 else NULL, + weightsname = if (weighted) "w" else NULL, fix_weights = fix_weights, + est_method = method, faster_mode = faster, + panel = structure != "rc", allow_unbalanced_panel = structure == "unbalanced", + bstrap = bootstrap, cband = bootstrap, biters = 499L, + anticipation = 0 + ))) +} +# Each function exercises the same public entry point on both sources. +cases <- list( + panel_1k_fast_dr_x = function() fit(d1), + panel_5k_fast_dr_x = function() fit(d5), + panel_1k_slow_dr_x = function() fit(d1, faster = FALSE), + panel_5k_slow_dr_x = function() fit(d5, faster = FALSE), + panel_1k_fast_dr_nox = function() fit(d1, covariates = FALSE), + panel_5k_fast_dr_nox = function() fit(d5, covariates = FALSE), + panel_5k_slow_dr_nox = function() fit(d5, faster = FALSE, covariates = FALSE), + panel_1k_fast_reg_x = function() fit(d1, method = "reg"), + panel_5k_fast_reg_x = function() fit(d5, method = "reg"), + panel_5k_slow_reg_x = function() fit(d5, method = "reg", faster = FALSE), + panel_5k_fast_dr_weighted = function() fit(d5, weighted = TRUE), + panel_5k_slow_dr_weighted = function() fit(d5, weighted = TRUE, faster = FALSE), + unbalanced_1k_fast_dr_x = function() fit(u1, structure = "unbalanced"), + unbalanced_5k_fast_dr_x = function() fit(u5, structure = "unbalanced"), + unbalanced_5k_slow_dr_x = function() fit(u5, structure = "unbalanced", faster = FALSE), + rc_1k_fast_dr_x = function() fit(r1, structure = "rc"), + rc_5k_fast_dr_x = function() fit(r5, structure = "rc"), + rc_5k_slow_dr_x = function() fit(r5, structure = "rc", faster = FALSE), + panel_1k_fast_dr_boot = function() fit(d1, bootstrap = TRUE), + panel_5k_fast_dr_boot = function() fit(d5, bootstrap = TRUE), + unbalanced_1k_fast_dr_boot = function() fit(u1, structure = "unbalanced", bootstrap = TRUE), + rc_1k_fast_dr_boot = function() fit(r1, structure = "rc", bootstrap = TRUE) +) +# Reuse pre-fit objects to time aggregation independently of att_gt(). +mp1 <- fit(d1); mp5 <- fit(d5); mpu <- fit(u1, structure = "unbalanced") +cases$aggte_1k_dynamic <- function() suppressWarnings(aggte(mp1, type = "dynamic")) +cases$aggte_5k_dynamic <- function() suppressWarnings(aggte(mp5, type = "dynamic")) +cases$aggte_5k_simple <- function() suppressWarnings(aggte(mp5, type = "simple")) +cases$aggte_1k_dynamic_boot <- function() suppressWarnings(aggte(mp1, type = "dynamic", bstrap = TRUE, cband = TRUE, biters = 499L)) +cases$aggte_unbalanced_dynamic <- function() suppressWarnings(aggte(mpu, type = "dynamic")) +cases$conditional_pretest_60 <- function() { + invisible(capture.output(suppressMessages(suppressWarnings( + conditional_did_pretest("Y", "t", "id", "G", xformla = ~X, + data = pd, biters = 99L) + )))) +} +# Low-level guard probes isolate the added support/weighted-Gram work. +x_guard <- cbind(1, rnorm(5000L), rnorm(5000L)) +y_guard <- rep(c(0, 1), length.out = 5000L) +w_guard <- rep(1, 5000L); w_vary <- rep(c(.5, 1, 2), length.out = 5000L) +overlap_fn <- getFromNamespace("overlap_check_fail", "did") +rank_fn <- getFromNamespace("rcond_check_fail", "did") +supports_weights <- "i.weights" %in% names(formals(overlap_fn)) +call_guard <- function(fun, x, y, intercept = FALSE, w = NULL) { + if (supports_weights) fun(x, y, intercept_only = intercept, i.weights = w) + else fun(x, y, intercept_only = intercept) +} +cases$guard_overlap_5k <- function() for (i in seq_len(100L)) call_guard(overlap_fn, x_guard, y_guard) +cases$guard_rank_5k <- function() for (i in seq_len(100L)) call_guard(rank_fn, x_guard, y_guard) +cases$guard_overlap_intercept_5k <- function() for (i in seq_len(1000L)) call_guard(overlap_fn, x_guard[, 1L, drop = FALSE], y_guard, TRUE) +cases$guard_rank_intercept_5k <- function() for (i in seq_len(1000L)) call_guard(rank_fn, x_guard[, 1L, drop = FALSE], y_guard, TRUE) +cases$guard_rank_weighted_5k <- function() for (i in seq_len(100L)) call_guard(rank_fn, x_guard, y_guard, w = w_vary) +# Supplemental cases cover the changed support-balancing and forced-RC branches. +if (suite == "supplemental") { + d12 <- make_panel(3000L, 12L) + mp12 <- fit(d12) + vary <- d1; vary$w <- vary$w * exp(.1 * vary$t * sin(vary$id)) + vary5 <- d5; vary5$w <- vary5$w * exp(.1 * vary5$t * sin(vary5$id)) + cases <- list( + aggte_1k_dynamic_balance_e2 = function() suppressWarnings(aggte(mp1, type = "dynamic", balance_e = 2L)), + aggte_5k_dynamic_balance_e2 = function() suppressWarnings(aggte(mp5, type = "dynamic", balance_e = 2L)), + aggte_3k_T12_dynamic_balance_e2 = function() suppressWarnings(aggte(mp12, type = "dynamic", balance_e = 2L)), + aggte_3k_T12_dynamic_balance_e5 = function() suppressWarnings(aggte(mp12, type = "dynamic", balance_e = 5L)), + panel_1k_fast_dr_varying = function() fit(vary, weighted = TRUE, fix_weights = "varying"), + panel_5k_fast_dr_varying = function() fit(vary5, weighted = TRUE, fix_weights = "varying"), + panel_5k_slow_dr_varying = function() fit(vary5, weighted = TRUE, fix_weights = "varying", faster = FALSE), + panel_1k_fast_reg_weighted = function() fit(d1, weighted = TRUE, method = "reg"), + panel_5k_fast_reg_weighted = function() fit(d5, weighted = TRUE, method = "reg"), + panel_5k_slow_reg_weighted = function() fit(d5, weighted = TRUE, method = "reg", faster = FALSE), + unbalanced_1k_fast_reg_weighted = function() fit(u1, weighted = TRUE, method = "reg", structure = "unbalanced"), + rc_1k_fast_reg_weighted = function() fit(r1, weighted = TRUE, method = "reg", structure = "rc") + ) +} +# Warm each path twice. Rotate case order across processes to balance drift. +for (name in names(cases)) for (warm in 1:2) { + set.seed(20261028 + warm) + invisible(cases[[name]]()) +} +results <- list(); k <- 0L +for (iteration in seq_len(repetitions)) { + set.seed(20261028 + round * 100L + iteration) + order <- sample(names(cases)) + for (name in order) { + gc(verbose = FALSE) + set.seed(20261028 + iteration) + timing <- system.time(invisible(cases[[name]]())) + k <- k + 1L + results[[k]] <- data.frame(source = label, round = round, + iteration = iteration, case = name, elapsed = unname(timing[["elapsed"]]), + user = unname(timing[["user.self"]]), system = unname(timing[["sys.self"]])) + } +} +out <- list(timings = do.call(rbind, results), session = sessionInfo(), + source_path = package_path, process_id = Sys.getpid(), + cases = names(cases), data_seed = 20261028L) +saveRDS(out, output_path) +cat(label, "round", round, "completed", nrow(out$timings), "timings\n") diff --git a/dev/benchmarks/stress-fixes/performance.R b/dev/benchmarks/stress-fixes/performance.R new file mode 100644 index 00000000..3c1bdbc1 --- /dev/null +++ b/dev/benchmarks/stress-fixes/performance.R @@ -0,0 +1,76 @@ +# Usage after source freeze: +# Rscript dev/benchmarks/stress-fixes/performance.R [repetitions=7] [rounds=2] [output_directory] [suite=main] +# Creates an exact clean-master baseline and a frozen snapshot of the working +# source, then interleaves independent warmed R processes (ABBA across rounds). +# Package loading, input construction, and explicit GC run outside each timing. +args <- commandArgs(trailingOnly = TRUE) +repetitions <- if (length(args)) as.integer(args[[1L]]) else 7L +rounds <- if (length(args) > 1L) as.integer(args[[2L]]) else 2L +script_arg <- grep("^--file=", commandArgs(), value = TRUE)[[1L]] +script_dir <- dirname(normalizePath(sub("^--file=", "", script_arg))) +root <- normalizePath(file.path(script_dir, "..", "..", "..")) +setwd(root) +suite <- if (length(args) > 3L) args[[4L]] else "main" +stopifnot(suite %in% c("main", "supplemental")) +artifact_dir <- if (length(args) > 2L) args[[3L]] else + file.path(tempdir(), paste0("did-stress-performance-", suite)) +dir.create(artifact_dir, recursive = TRUE, showWarnings = FALSE) +baseline <- file.path(artifact_dir, "baseline-source") +final <- file.path(artifact_dir, "final-source") +dir.create(baseline, showWarnings = FALSE); dir.create(final, showWarnings = FALSE) +archive <- file.path(artifact_dir, "baseline.tar") +status <- system2("git", c("archive", "--format=tar", "--output", shQuote(archive), + "74b88eb", "DESCRIPTION", "NAMESPACE", "R", "data")) +stopifnot(status == 0L) +untar(archive, exdir = baseline) +for (path in c("DESCRIPTION", "NAMESPACE", "R", "data")) { + destination <- file.path(final, basename(path)) + if (dir.exists(destination)) unlink(destination, recursive = TRUE) + stopifnot(file.copy(file.path(root, path), final, recursive = TRUE)) +} +metadata <- list(baseline = "74b88eb07f1faa644df1271055a0f77848f6550c", + source_head = system2("git", c("rev-parse", "HEAD"), stdout = TRUE), + source_diff = system2("git", c("diff", "--stat"), stdout = TRUE), + source_hashes = tools::md5sum(list.files(file.path(final, "R"), pattern = "\\.R$", + full.names = TRUE, recursive = TRUE)), + suite = suite, + timestamp = Sys.time(), session = sessionInfo(), + design = "sequential warmed processes; baseline/final then final/baseline; randomized case order; GC excluded") +saveRDS(metadata, file.path(artifact_dir, "metadata.rds")) +worker <- file.path(script_dir, "performance-worker.R") +files <- character() +for (round in seq_len(rounds)) { + labels <- if (round %% 2L) c("baseline", "final") else c("final", "baseline") + for (label in labels) { + output <- file.path(artifact_dir, paste0(label, "-round-", round, ".rds")) + log <- file.path(artifact_dir, paste0(label, "-round-", round, ".log")) + path <- if (label == "baseline") baseline else final + status <- system2(file.path(R.home("bin"), "Rscript"), + c(shQuote(worker), shQuote(path), label, shQuote(output), round, repetitions, suite), + stdout = log, stderr = log) + if (status != 0L) stop("Benchmark worker failed: ", log) + files <- c(files, output) + cat(label, "round", round, "finished\n"); flush.console() + } +} +raw <- do.call(rbind, lapply(files, function(path) readRDS(path)$timings)) +write.csv(raw, file.path(artifact_dir, "raw-timings.csv"), row.names = FALSE) +summary <- do.call(rbind, lapply(split(raw, list(raw$case, raw$source), drop = TRUE), function(z) { + data.frame(case = z$case[[1L]], source = z$source[[1L]], n = nrow(z), + median = median(z$elapsed), q25 = unname(quantile(z$elapsed, .25)), + q75 = unname(quantile(z$elapsed, .75)), + median_cpu = median(z$user + z$system)) +})) +comparison <- merge(summary[summary$source == "baseline", ], + summary[summary$source == "final", ], by = "case", + suffixes = c("_baseline", "_final")) +comparison$ratio <- comparison$median_final / comparison$median_baseline +comparison$delta_ms <- 1000 * (comparison$median_final - comparison$median_baseline) +comparison$iqr_overlap <- comparison$q25_final <= comparison$q75_baseline & + comparison$q25_baseline <= comparison$q75_final +write.csv(comparison, file.path(artifact_dir, "comparison.csv"), row.names = FALSE) +saveRDS(list(metadata = metadata, raw = raw, summary = summary, comparison = comparison), + file.path(artifact_dir, "results.rds")) +cat("Artifacts:", normalizePath(artifact_dir), "\n") +print(comparison[c("case", "median_baseline", "q25_baseline", "q75_baseline", + "median_final", "q25_final", "q75_final", "ratio", "delta_ms", "iqr_overlap")], row.names = FALSE) diff --git a/dev/benchmarks/stress-fixes/session-info.txt b/dev/benchmarks/stress-fixes/session-info.txt new file mode 100644 index 00000000..4f9541b8 --- /dev/null +++ b/dev/benchmarks/stress-fixes/session-info.txt @@ -0,0 +1,48 @@ +R version 4.6.0 (2026-04-24) +Platform: aarch64-apple-darwin23 +Running under: macOS Tahoe 26.5.1 + +Matrix products: default +BLAS: /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib +LAPACK: /Library/Frameworks/R.framework/Versions/4.6/Resources/lib/libRlapack.dylib; LAPACK version 3.12.1 + +locale: +[1] C.UTF-8/C.UTF-8/C.UTF-8/C/C.UTF-8/C.UTF-8 + +time zone: America/New_York +tzcode source: internal + +attached base packages: +[1] stats graphics grDevices utils datasets methods base + +other attached packages: +[1] did_2.5.1.902 + +loaded via a namespace (and not attached): + [1] gtable_0.3.6 ggplot2_4.0.3 recipes_1.3.3 + [4] lattice_0.22-9 vctrs_0.7.3 tools_4.6.0 + [7] generics_0.1.4 sandwich_3.1-1 stats4_4.6.0 +[10] parallel_4.6.0 tibble_3.3.1 pkgconfig_2.0.3 +[13] ModelMetrics_1.2.2.2 DRDID_1.4.0 Matrix_1.7-5 +[16] data.table_1.18.4 RColorBrewer_1.1-3 stringmagic_1.2.0 +[19] S7_0.2.2 desc_1.4.3 uuid_1.2-2 +[22] lifecycle_1.0.5 compiler_4.6.0 farver_2.1.2 +[25] stringr_1.6.0 codetools_0.2-20 class_7.3-23 +[28] Formula_1.2-5 prodlim_2026.03.11 pillar_1.11.1 +[31] tidyr_1.3.2 MASS_7.3-65 gower_1.0.2 +[34] iterators_1.0.14 trust_0.1-9 rpart_4.1.27 +[37] foreach_1.5.2 nlme_3.1-169 parallelly_1.47.0 +[40] lava_1.9.1 tidyselect_1.2.1 digest_0.6.39 +[43] stringi_1.8.7 future_1.70.0 dplyr_1.2.1 +[46] reshape2_1.4.5 purrr_1.2.2 listenv_0.10.1 +[49] splines_4.6.0 rprojroot_2.1.1 grid_4.6.0 +[52] cli_3.6.6 magrittr_2.0.5 survival_3.8-6 +[55] pkgbuild_1.4.8 bigmemory.sri_0.1.8 bigmemory_4.6.6 +[58] future.apply_1.20.2 dreamerr_1.5.0 withr_3.0.2 +[61] scales_1.4.0 lubridate_1.9.5 timechange_0.4.0 +[64] globals_0.19.1 nnet_7.3-20 timeDate_4052.112 +[67] zoo_1.8-15 fastglm_0.1.1 pbapply_1.7-4 +[70] hardhat_1.4.3 caret_7.0-1 rlang_1.2.0 +[73] Rcpp_1.1.1-1.1 glue_1.8.1 BMisc_1.4.8 +[76] pROC_1.19.0.1 ipred_0.9-15 pkgload_1.5.2 +[79] R6_2.6.1 plyr_1.8.9 diff --git a/dev/benchmarks/stress-fixes/supplemental-before-optimization-comparison.csv b/dev/benchmarks/stress-fixes/supplemental-before-optimization-comparison.csv new file mode 100644 index 00000000..9a77b2ae --- /dev/null +++ b/dev/benchmarks/stress-fixes/supplemental-before-optimization-comparison.csv @@ -0,0 +1,13 @@ +"case","source_baseline","n_baseline","median_baseline","q25_baseline","q75_baseline","median_cpu_baseline","source_final","n_final","median_final","q25_final","q75_final","median_cpu_final","ratio","delta_ms","iqr_overlap" +"aggte_1k_dynamic_balance_e2","baseline",14,0.00300000000000011,0.00300000000000011,0.00300000000000011,0.00300000000000011,"final",14,0.00300000000000011,0.00300000000000011,0.00300000000000011,0.00300000000000011,1,0,TRUE +"aggte_3k_T12_dynamic_balance_e2","baseline",14,0.0080000000000009,0.00800000000000001,0.00800000000000223,0.00900000000000034,"final",14,0.00900000000000034,0.00899999999999945,0.00900000000000034,0.00800000000000045,1.12499999999992,0.999999999999446,FALSE +"aggte_3k_T12_dynamic_balance_e5","baseline",14,0.00900000000000034,0.00900000000000034,0.00900000000000034,0.00899999999999768,"final",14,0.00900000000000034,0.00800000000000023,0.00900000000000034,0.00900000000000034,1,0,TRUE +"aggte_5k_dynamic_balance_e2","baseline",14,0.004,0.00399999999999956,0.00400000000000134,0.004,"final",14,0.00399999999999956,0.00399999999999823,0.00400000000000134,0.00400000000000089,0.999999999999889,-4.44089209850063e-13,TRUE +"panel_1k_fast_dr_varying","baseline",14,0.0490000000000013,0.048,0.050749999999999,0.0490000000000018,"final",14,0.0479999999999992,0.0470000000000006,0.0495000000000003,0.0480000000000006,0.979591836734651,-1.00000000000211,TRUE +"panel_1k_fast_reg_weighted","baseline",14,0.0190000000000001,0.0182500000000001,0.0190000000000015,0.0190000000000007,"final",14,0.0170000000000012,0.0169999999999995,0.0179999999999996,0.0179999999999995,0.894736842105322,-1.99999999999889,FALSE +"panel_5k_fast_dr_varying","baseline",14,0.142000000000003,0.141,0.14625,0.1435,"final",14,0.142,0.141000000000002,0.14475,0.143500000000001,0.999999999999981,-2.66453525910038e-12,TRUE +"panel_5k_fast_reg_weighted","baseline",14,0.0485000000000007,0.048,0.0489999999999995,0.0490000000000009,"final",14,0.0459999999999994,0.0449999999999999,0.0467500000000012,0.0474999999999997,0.948453608247397,-2.50000000000128,FALSE +"panel_5k_slow_dr_varying","baseline",14,0.169,0.16725,0.17,0.178000000000001,"final",14,0.1755,0.17425,0.178000000000001,0.186000000000001,1.03846153846154,6.49999999999995,FALSE +"panel_5k_slow_reg_weighted","baseline",14,0.0259999999999998,0.0250000000000008,0.0260000000000014,0.026500000000001,"final",14,0.0264999999999986,0.0259999999999998,0.0269999999999999,0.0269999999999998,1.01923076923072,0.499999999998835,TRUE +"rc_1k_fast_reg_weighted","baseline",14,0.0294999999999996,0.0282500000000017,0.03,0.0295000000000007,"final",14,0.028,0.0279999999999987,0.0280000000000022,0.0290000000000007,0.949152542372894,-1.49999999999961,FALSE +"unbalanced_1k_fast_reg_weighted","baseline",14,0.0299999999999985,0.0289999999999999,0.0300000000000011,0.0309999999999996,"final",14,0.0289999999999999,0.0280000000000009,0.0307499999999989,0.0299999999999991,0.966666666666713,-0.999999999998558,TRUE diff --git a/dev/benchmarks/stress-fixes/supplemental-before-optimization-raw-timings.csv b/dev/benchmarks/stress-fixes/supplemental-before-optimization-raw-timings.csv new file mode 100644 index 00000000..203d7db3 --- /dev/null +++ b/dev/benchmarks/stress-fixes/supplemental-before-optimization-raw-timings.csv @@ -0,0 +1,337 @@ +"source","round","iteration","case","elapsed","user","system" +"baseline",1,1,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,1,"panel_1k_fast_dr_varying",0.0510000000000002,0.0489999999999995,0.002 +"baseline",1,1,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0190000000000001,0.001 +"baseline",1,1,"aggte_3k_T12_dynamic_balance_e2",0.00800000000000001,0.00800000000000001,0.000999999999999973 +"baseline",1,1,"aggte_5k_dynamic_balance_e2",0.00400000000000045,0.00400000000000045,0 +"baseline",1,1,"panel_5k_fast_dr_varying",0.141,0.14,0.003 +"baseline",1,1,"panel_5k_slow_reg_weighted",0.0260000000000007,0.0259999999999998,0.001 +"baseline",1,1,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00800000000000001,0 +"baseline",1,1,"rc_1k_fast_reg_weighted",0.0300000000000002,0.0290000000000008,0.002 +"baseline",1,1,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0289999999999999,0.001 +"baseline",1,1,"panel_5k_slow_dr_varying",0.165999999999999,0.154,0.021 +"baseline",1,1,"panel_5k_fast_reg_weighted",0.048,0.0469999999999997,0.001 +"baseline",1,2,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",1,2,"rc_1k_fast_reg_weighted",0.0360000000000005,0.032,0.003 +"baseline",1,2,"aggte_1k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0.001 +"baseline",1,2,"aggte_3k_T12_dynamic_balance_e2",0.00800000000000001,0.00900000000000034,0 +"baseline",1,2,"panel_5k_slow_reg_weighted",0.0269999999999992,0.0259999999999998,0.001 +"baseline",1,2,"panel_1k_fast_dr_varying",0.048,0.0470000000000006,0.001 +"baseline",1,2,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0180000000000007,0.001 +"baseline",1,2,"unbalanced_1k_fast_reg_weighted",0.0349999999999984,0.032,0.002 +"baseline",1,2,"panel_5k_fast_reg_weighted",0.0560000000000009,0.0530000000000008,0.002 +"baseline",1,2,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"baseline",1,2,"panel_5k_fast_dr_varying",0.167999999999999,0.151,0.003 +"baseline",1,2,"panel_5k_slow_dr_varying",0.191000000000001,0.174,0.026 +"baseline",1,3,"panel_5k_fast_dr_varying",0.168000000000001,0.153,0.005 +"baseline",1,3,"rc_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.002 +"baseline",1,3,"panel_5k_fast_reg_weighted",0.0470000000000006,0.0470000000000006,0.001 +"baseline",1,3,"panel_5k_slow_dr_varying",0.167999999999999,0.156000000000001,0.022 +"baseline",1,3,"panel_1k_fast_reg_weighted",0.0210000000000008,0.0179999999999989,0.001 +"baseline",1,3,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0289999999999999,0.002 +"baseline",1,3,"aggte_3k_T12_dynamic_balance_e2",0.0080000000000009,0.00900000000000034,0 +"baseline",1,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,3,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"baseline",1,3,"panel_1k_fast_dr_varying",0.0479999999999983,0.048,0.002 +"baseline",1,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999856,0 +"baseline",1,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"baseline",1,4,"aggte_3k_T12_dynamic_balance_e2",0.0080000000000009,0.00900000000000034,0 +"baseline",1,4,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0280000000000005,0.001 +"baseline",1,4,"panel_5k_slow_dr_varying",0.17,0.157,0.022 +"baseline",1,4,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0190000000000001,0.001 +"baseline",1,4,"panel_1k_fast_dr_varying",0.048,0.0470000000000006,0.001 +"baseline",1,4,"panel_5k_fast_reg_weighted",0.048,0.048,0.001 +"baseline",1,4,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",1,4,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,4,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0289999999999999,0.002 +"baseline",1,4,"panel_5k_slow_reg_weighted",0.0260000000000016,0.0259999999999998,0.001 +"baseline",1,4,"panel_5k_fast_dr_varying",0.140000000000001,0.139000000000001,0.002 +"baseline",1,4,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.004999999999999,0 +"baseline",1,5,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0179999999999989,0.001 +"baseline",1,5,"panel_1k_fast_dr_varying",0.0470000000000006,0.0469999999999988,0.001 +"baseline",1,5,"panel_5k_slow_reg_weighted",0.0250000000000004,0.0259999999999998,0.001 +"baseline",1,5,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00799999999999912,0 +"baseline",1,5,"rc_1k_fast_reg_weighted",0.0280000000000022,0.0280000000000005,0.001 +"baseline",1,5,"aggte_3k_T12_dynamic_balance_e2",0.0100000000000016,0.00900000000000034,0 +"baseline",1,5,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999778,0 +"baseline",1,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,5,"panel_5k_fast_dr_varying",0.140000000000001,0.138999999999999,0.002 +"baseline",1,5,"panel_5k_slow_dr_varying",0.166,0.155000000000001,0.021 +"baseline",1,5,"panel_5k_fast_reg_weighted",0.0489999999999995,0.0480000000000018,0.001 +"baseline",1,5,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999987,0.001 +"baseline",1,6,"unbalanced_1k_fast_reg_weighted",0.0299999999999976,0.0289999999999999,0.002 +"baseline",1,6,"panel_1k_fast_dr_varying",0.049000000000003,0.0480000000000018,0.001 +"baseline",1,6,"panel_1k_fast_reg_weighted",0.0190000000000019,0.0180000000000007,0.001 +"baseline",1,6,"aggte_3k_T12_dynamic_balance_e5",0.00799999999999912,0.00799999999999912,0 +"baseline",1,6,"panel_5k_fast_reg_weighted",0.0480000000000018,0.046999999999997,0.001 +"baseline",1,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,6,"panel_5k_fast_dr_varying",0.141999999999999,0.140999999999998,0.002 +"baseline",1,6,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0 +"baseline",1,6,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00900000000000034,0 +"baseline",1,6,"panel_5k_slow_reg_weighted",0.0260000000000034,0.0250000000000021,0.001 +"baseline",1,6,"panel_5k_slow_dr_varying",0.167999999999999,0.157,0.021 +"baseline",1,6,"rc_1k_fast_reg_weighted",0.0279999999999987,0.027000000000001,0.002 +"baseline",1,7,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0.001 +"baseline",1,7,"panel_5k_slow_reg_weighted",0.0260000000000034,0.0259999999999998,0.001 +"baseline",1,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999679,0 +"baseline",1,7,"rc_1k_fast_reg_weighted",0.0300000000000011,0.0289999999999999,0.002 +"baseline",1,7,"unbalanced_1k_fast_reg_weighted",0.0309999999999988,0.0289999999999999,0.002 +"baseline",1,7,"panel_1k_fast_dr_varying",0.0509999999999984,0.0500000000000007,0.002 +"baseline",1,7,"panel_1k_fast_reg_weighted",0.0199999999999996,0.0189999999999984,0.001 +"baseline",1,7,"panel_5k_fast_reg_weighted",0.0489999999999995,0.0480000000000018,0.001 +"baseline",1,7,"panel_5k_slow_dr_varying",0.245000000000001,0.163,0.023 +"baseline",1,7,"panel_5k_fast_dr_varying",0.147000000000002,0.143000000000001,0.005 +"baseline",1,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,7,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"final",1,1,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,1,"panel_1k_fast_dr_varying",0.0499999999999998,0.0470000000000006,0.00299999999999997 +"final",1,1,"panel_1k_fast_reg_weighted",0.0170000000000003,0.0170000000000003,0.001 +"final",1,1,"aggte_3k_T12_dynamic_balance_e2",0.00899999999999945,0.00900000000000034,0 +"final",1,1,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"final",1,1,"panel_5k_fast_dr_varying",0.141,0.140000000000001,0.003 +"final",1,1,"panel_5k_slow_reg_weighted",0.0270000000000001,0.0259999999999998,0.001 +"final",1,1,"aggte_3k_T12_dynamic_balance_e5",0.00800000000000001,0.00800000000000001,0 +"final",1,1,"rc_1k_fast_reg_weighted",0.0300000000000002,0.0279999999999996,0.002 +"final",1,1,"unbalanced_1k_fast_reg_weighted",0.0309999999999997,0.0289999999999999,0.002 +"final",1,1,"panel_5k_slow_dr_varying",0.176,0.167000000000001,0.024 +"final",1,1,"panel_5k_fast_reg_weighted",0.0490000000000004,0.0489999999999995,0.002 +"final",1,2,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999945,0 +"final",1,2,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0270000000000001,0.002 +"final",1,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,2,"aggte_3k_T12_dynamic_balance_e2",0.00899999999999945,0.00800000000000001,0 +"final",1,2,"panel_5k_slow_reg_weighted",0.0280000000000005,0.0269999999999992,0.001 +"final",1,2,"panel_1k_fast_dr_varying",0.0470000000000006,0.0469999999999997,0.001 +"final",1,2,"panel_1k_fast_reg_weighted",0.0179999999999989,0.016,0.001 +"final",1,2,"unbalanced_1k_fast_reg_weighted",0.0520000000000014,0.032,0.003 +"final",1,2,"panel_5k_fast_reg_weighted",0.0470000000000006,0.0459999999999994,0.002 +"final",1,2,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00400000000000134,0 +"final",1,2,"panel_5k_fast_dr_varying",0.145,0.144,0.002 +"final",1,2,"panel_5k_slow_dr_varying",0.183,0.170999999999999,0.024 +"final",1,3,"panel_5k_fast_dr_varying",0.145999999999999,0.145999999999999,0.002 +"final",1,3,"rc_1k_fast_reg_weighted",0.0269999999999992,0.0269999999999992,0.002 +"final",1,3,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.002 +"final",1,3,"panel_5k_slow_dr_varying",0.173,0.161,0.02 +"final",1,3,"panel_1k_fast_reg_weighted",0.016,0.016,0.001 +"final",1,3,"unbalanced_1k_fast_reg_weighted",0.0279999999999987,0.0259999999999998,0.001 +"final",1,3,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.0080000000000009,0 +"final",1,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,3,"panel_5k_slow_reg_weighted",0.0269999999999992,0.0259999999999998,0 +"final",1,3,"panel_1k_fast_dr_varying",0.0470000000000006,0.0459999999999994,0.001 +"final",1,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",1,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"final",1,4,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.0080000000000009,0.001 +"final",1,4,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0280000000000005,0.003 +"final",1,4,"panel_5k_slow_dr_varying",0.178000000000001,0.164000000000001,0.022 +"final",1,4,"panel_1k_fast_reg_weighted",0.0170000000000012,0.0169999999999995,0.001 +"final",1,4,"panel_1k_fast_dr_varying",0.0850000000000009,0.0540000000000003,0.002 +"final",1,4,"panel_5k_fast_reg_weighted",0.0459999999999994,0.0459999999999994,0.002 +"final",1,4,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",1,4,"aggte_1k_dynamic_balance_e2",0.00399999999999956,0.00300000000000011,0 +"final",1,4,"unbalanced_1k_fast_reg_weighted",0.0280000000000005,0.0269999999999992,0.001 +"final",1,4,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,4,"panel_5k_fast_dr_varying",0.143000000000001,0.141999999999999,0.002 +"final",1,4,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"final",1,5,"panel_1k_fast_reg_weighted",0.0169999999999995,0.016,0.001 +"final",1,5,"panel_1k_fast_dr_varying",0.0470000000000006,0.0469999999999988,0.002 +"final",1,5,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,5,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.0080000000000009,0 +"final",1,5,"rc_1k_fast_reg_weighted",0.0279999999999987,0.0280000000000005,0.001 +"final",1,5,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00799999999999912,0 +"final",1,5,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00300000000000011,0 +"final",1,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,5,"panel_5k_fast_dr_varying",0.141000000000002,0.141000000000002,0.002 +"final",1,5,"panel_5k_slow_dr_varying",0.174999999999997,0.163,0.021 +"final",1,5,"panel_5k_fast_reg_weighted",0.0449999999999982,0.0449999999999982,0.002 +"final",1,5,"unbalanced_1k_fast_reg_weighted",0.0329999999999977,0.0289999999999999,0.001 +"final",1,6,"unbalanced_1k_fast_reg_weighted",0.0279999999999987,0.0279999999999987,0.002 +"final",1,6,"panel_1k_fast_dr_varying",0.0470000000000006,0.0470000000000006,0.001 +"final",1,6,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0160000000000018,0.001 +"final",1,6,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00799999999999912,0 +"final",1,6,"panel_5k_fast_reg_weighted",0.0459999999999994,0.0459999999999994,0.001 +"final",1,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00299999999999656,0 +"final",1,6,"panel_5k_fast_dr_varying",0.141000000000002,0.140000000000001,0.002 +"final",1,6,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999778,0 +"final",1,6,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00800000000000267,0 +"final",1,6,"panel_5k_slow_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"final",1,6,"panel_5k_slow_dr_varying",0.169999999999998,0.157999999999998,0.021 +"final",1,6,"rc_1k_fast_reg_weighted",0.0280000000000022,0.0259999999999998,0.002 +"final",1,7,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00300000000000011,0 +"final",1,7,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",1,7,"rc_1k_fast_reg_weighted",0.0279999999999987,0.027000000000001,0.002 +"final",1,7,"unbalanced_1k_fast_reg_weighted",0.0280000000000022,0.0269999999999975,0.001 +"final",1,7,"panel_1k_fast_dr_varying",0.0480000000000018,0.0470000000000006,0.001 +"final",1,7,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0169999999999995,0.001 +"final",1,7,"panel_5k_fast_reg_weighted",0.0440000000000005,0.0440000000000005,0.002 +"final",1,7,"panel_5k_slow_dr_varying",0.178000000000001,0.164999999999999,0.022 +"final",1,7,"panel_5k_fast_dr_varying",0.146000000000001,0.141999999999999,0.005 +"final",1,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,7,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",2,1,"rc_1k_fast_reg_weighted",0.0279999999999996,0.0279999999999996,0.00199999999999997 +"final",2,1,"panel_1k_fast_reg_weighted",0.0179999999999998,0.0170000000000003,0.001 +"final",2,1,"aggte_3k_T12_dynamic_balance_e5",0.00800000000000001,0.00900000000000034,0 +"final",2,1,"panel_5k_fast_dr_varying",0.146999999999999,0.146,0.003 +"final",2,1,"panel_1k_fast_dr_varying",0.0529999999999999,0.0499999999999998,0.004 +"final",2,1,"panel_5k_slow_dr_varying",0.175000000000001,0.163,0.024 +"final",2,1,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999996,0.001 +"final",2,1,"panel_5k_fast_reg_weighted",0.0460000000000003,0.0460000000000003,0.002 +"final",2,1,"aggte_3k_T12_dynamic_balance_e2",0.00899999999999945,0.00900000000000034,0 +"final",2,1,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,1,"panel_5k_slow_reg_weighted",0.0269999999999992,0.0270000000000001,0.001 +"final",2,1,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000045,0 +"final",2,2,"rc_1k_fast_reg_weighted",0.0270000000000001,0.0259999999999998,0.002 +"final",2,2,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0 +"final",2,2,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999996,0.002 +"final",2,2,"panel_1k_fast_dr_varying",0.0469999999999997,0.0460000000000003,0.002 +"final",2,2,"panel_5k_fast_dr_varying",0.141,0.14,0.003 +"final",2,2,"panel_5k_slow_dr_varying",0.174000000000001,0.162999999999998,0.022 +"final",2,2,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0169999999999995,0.001 +"final",2,2,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"final",2,2,"panel_5k_fast_reg_weighted",0.048,0.0470000000000006,0.002 +"final",2,2,"aggte_3k_T12_dynamic_balance_e5",0.0080000000000009,0.0080000000000009,0 +"final",2,2,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00400000000000134,0 +"final",2,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,3,"unbalanced_1k_fast_reg_weighted",0.0269999999999992,0.027000000000001,0.002 +"final",2,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.0080000000000009,0 +"final",2,3,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",2,3,"panel_5k_fast_reg_weighted",0.048,0.0470000000000006,0.002 +"final",2,3,"panel_5k_fast_dr_varying",0.142000000000001,0.141,0.002 +"final",2,3,"rc_1k_fast_reg_weighted",0.0280000000000005,0.0259999999999998,0.002 +"final",2,3,"panel_5k_slow_dr_varying",0.176,0.164000000000001,0.022 +"final",2,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,3,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0 +"final",2,3,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",2,3,"panel_1k_fast_dr_varying",0.048,0.0469999999999988,0.001 +"final",2,3,"panel_1k_fast_reg_weighted",0.0179999999999989,0.0169999999999995,0.001 +"final",2,4,"aggte_1k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"final",2,4,"panel_5k_slow_reg_weighted",0.0259999999999998,0.027000000000001,0.001 +"final",2,4,"aggte_3k_T12_dynamic_balance_e5",0.00799999999999912,0.00900000000000034,0 +"final",2,4,"panel_1k_fast_reg_weighted",0.0170000000000012,0.016,0.001 +"final",2,4,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"final",2,4,"panel_5k_slow_dr_varying",0.178000000000001,0.164,0.022 +"final",2,4,"panel_1k_fast_dr_varying",0.0470000000000006,0.0459999999999994,0.001 +"final",2,4,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.002 +"final",2,4,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00899999999999856,0 +"final",2,4,"panel_5k_fast_dr_varying",0.144,0.144,0.002 +"final",2,4,"rc_1k_fast_reg_weighted",0.0269999999999992,0.0259999999999998,0.002 +"final",2,4,"unbalanced_1k_fast_reg_weighted",0.0309999999999988,0.0289999999999999,0.002 +"final",2,5,"unbalanced_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.001 +"final",2,5,"aggte_1k_dynamic_balance_e2",0.00399999999999956,0.00300000000000011,0.001 +"final",2,5,"panel_5k_fast_reg_weighted",0.0439999999999987,0.0449999999999999,0.002 +"final",2,5,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00399999999999956,0 +"final",2,5,"aggte_3k_T12_dynamic_balance_e5",0.00799999999999912,0.00900000000000034,0 +"final",2,5,"panel_5k_fast_dr_varying",0.141999999999999,0.141999999999999,0.002 +"final",2,5,"rc_1k_fast_reg_weighted",0.0280000000000022,0.0279999999999987,0.002 +"final",2,5,"panel_5k_slow_dr_varying",0.175000000000001,0.161999999999999,0.021 +"final",2,5,"panel_1k_fast_reg_weighted",0.0350000000000001,0.0199999999999996,0.000999999999999945 +"final",2,5,"panel_5k_slow_reg_weighted",0.0249999999999986,0.0250000000000021,0.001 +"final",2,5,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",2,5,"panel_1k_fast_dr_varying",0.0479999999999983,0.046999999999997,0.001 +"final",2,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,6,"panel_1k_fast_dr_varying",0.0480000000000018,0.0470000000000006,0.002 +"final",2,6,"panel_1k_fast_reg_weighted",0.0289999999999999,0.022000000000002,0.003 +"final",2,6,"panel_5k_slow_dr_varying",0.192,0.170999999999999,0.025 +"final",2,6,"aggte_3k_T12_dynamic_balance_e5",0.00800000000000267,0.00900000000000034,0.001 +"final",2,6,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0 +"final",2,6,"panel_5k_slow_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"final",2,6,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00799999999999912,0 +"final",2,6,"panel_5k_fast_reg_weighted",0.0459999999999994,0.0459999999999994,0.002 +"final",2,6,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0280000000000022,0.002 +"final",2,6,"panel_5k_fast_dr_varying",0.141000000000002,0.140000000000001,0.002 +"final",2,6,"rc_1k_fast_reg_weighted",0.0279999999999987,0.027000000000001,0.002 +"final",2,7,"panel_5k_fast_reg_weighted",0.0460000000000029,0.0449999999999982,0.001 +"final",2,7,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999987,0.002 +"final",2,7,"panel_5k_slow_reg_weighted",0.0269999999999975,0.027000000000001,0.001 +"final",2,7,"panel_5k_slow_dr_varying",0.173000000000002,0.162000000000003,0.021 +"final",2,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",2,7,"aggte_5k_dynamic_balance_e2",0.00399999999999778,0.00400000000000134,0 +"final",2,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,7,"panel_5k_fast_dr_varying",0.141999999999999,0.140999999999998,0.002 +"final",2,7,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0159999999999982,0.001 +"final",2,7,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",2,7,"panel_1k_fast_dr_varying",0.0519999999999996,0.0479999999999983,0.005 +"final",2,7,"rc_1k_fast_reg_weighted",0.0280000000000022,0.027000000000001,0.002 +"baseline",2,1,"rc_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.002 +"baseline",2,1,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0190000000000001,0 +"baseline",2,1,"aggte_3k_T12_dynamic_balance_e5",0.00999999999999979,0.00899999999999945,0.001 +"baseline",2,1,"panel_5k_fast_dr_varying",0.147,0.143,0.004 +"baseline",2,1,"panel_1k_fast_dr_varying",0.0540000000000003,0.0499999999999998,0.005 +"baseline",2,1,"panel_5k_slow_dr_varying",0.167,0.155,0.022 +"baseline",2,1,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999996,0.001 +"baseline",2,1,"panel_5k_fast_reg_weighted",0.0469999999999997,0.0470000000000006,0.001 +"baseline",2,1,"aggte_3k_T12_dynamic_balance_e2",0.00800000000000001,0.00900000000000034,0 +"baseline",2,1,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,1,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0250000000000004,0.001 +"baseline",2,1,"aggte_5k_dynamic_balance_e2",0.00400000000000045,0.00400000000000045,0 +"baseline",2,2,"rc_1k_fast_reg_weighted",0.0280000000000005,0.0270000000000001,0.001 +"baseline",2,2,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0250000000000004,0.001 +"baseline",2,2,"unbalanced_1k_fast_reg_weighted",0.0300000000000002,0.0299999999999994,0.001 +"baseline",2,2,"panel_1k_fast_dr_varying",0.048,0.0469999999999997,0.001 +"baseline",2,2,"panel_5k_fast_dr_varying",0.144,0.143,0.002 +"baseline",2,2,"panel_5k_slow_dr_varying",0.169,0.156000000000001,0.022 +"baseline",2,2,"panel_1k_fast_reg_weighted",0.0189999999999984,0.0190000000000001,0.001 +"baseline",2,2,"aggte_3k_T12_dynamic_balance_e2",0.0080000000000009,0.00799999999999912,0 +"baseline",2,2,"panel_5k_fast_reg_weighted",0.0489999999999995,0.048,0.002 +"baseline",2,2,"aggte_3k_T12_dynamic_balance_e5",0.00999999999999979,0.00999999999999979,0 +"baseline",2,2,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,3,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0289999999999999,0.002 +"baseline",2,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",2,3,"aggte_3k_T12_dynamic_balance_e2",0.0080000000000009,0.00900000000000034,0 +"baseline",2,3,"panel_5k_fast_reg_weighted",0.048,0.048,0.002 +"baseline",2,3,"panel_5k_fast_dr_varying",0.141,0.140000000000001,0.002 +"baseline",2,3,"rc_1k_fast_reg_weighted",0.0299999999999994,0.0280000000000005,0.002 +"baseline",2,3,"panel_5k_slow_dr_varying",0.167,0.154999999999999,0.022 +"baseline",2,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00300000000000011,0 +"baseline",2,3,"panel_5k_slow_reg_weighted",0.0250000000000004,0.0259999999999998,0 +"baseline",2,3,"panel_1k_fast_dr_varying",0.0500000000000007,0.048,0.001 +"baseline",2,3,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0179999999999989,0.001 +"baseline",2,4,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,4,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0260000000000016,0.001 +"baseline",2,4,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.0080000000000009,0 +"baseline",2,4,"panel_1k_fast_reg_weighted",0.0179999999999989,0.0180000000000007,0.001 +"baseline",2,4,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"baseline",2,4,"panel_5k_slow_dr_varying",0.169,0.156000000000001,0.022 +"baseline",2,4,"panel_1k_fast_dr_varying",0.0489999999999995,0.048,0.001 +"baseline",2,4,"panel_5k_fast_reg_weighted",0.0489999999999995,0.048,0.001 +"baseline",2,4,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00900000000000034,0 +"baseline",2,4,"panel_5k_fast_dr_varying",0.141,0.139999999999999,0.002 +"baseline",2,4,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0269999999999992,0.002 +"baseline",2,4,"unbalanced_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.001 +"baseline",2,5,"unbalanced_1k_fast_reg_weighted",0.0300000000000011,0.0279999999999987,0.001 +"baseline",2,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,5,"panel_5k_fast_reg_weighted",0.0479999999999983,0.048,0.001 +"baseline",2,5,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"baseline",2,5,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.0080000000000009,0 +"baseline",2,5,"panel_5k_fast_dr_varying",0.142000000000003,0.140000000000001,0.002 +"baseline",2,5,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999987,0.001 +"baseline",2,5,"panel_5k_slow_dr_varying",0.169999999999998,0.157,0.022 +"baseline",2,5,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0190000000000019,0.001 +"baseline",2,5,"panel_5k_slow_reg_weighted",0.0249999999999986,0.0249999999999986,0.001 +"baseline",2,5,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"baseline",2,5,"panel_1k_fast_dr_varying",0.0480000000000018,0.0480000000000018,0.001 +"baseline",2,6,"aggte_1k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0 +"baseline",2,6,"panel_1k_fast_dr_varying",0.0500000000000007,0.0489999999999995,0.001 +"baseline",2,6,"panel_1k_fast_reg_weighted",0.0190000000000019,0.0180000000000007,0.001 +"baseline",2,6,"panel_5k_slow_dr_varying",0.182000000000002,0.164000000000001,0.026 +"baseline",2,6,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00799999999999912,0 +"baseline",2,6,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00400000000000134,0 +"baseline",2,6,"panel_5k_slow_reg_weighted",0.0249999999999986,0.0250000000000021,0.001 +"baseline",2,6,"aggte_3k_T12_dynamic_balance_e2",0.00800000000000267,0.00799999999999912,0 +"baseline",2,6,"panel_5k_fast_reg_weighted",0.0489999999999995,0.049000000000003,0.002 +"baseline",2,6,"unbalanced_1k_fast_reg_weighted",0.0300000000000011,0.0289999999999999,0.002 +"baseline",2,6,"panel_5k_fast_dr_varying",0.142000000000003,0.141000000000002,0.003 +"baseline",2,6,"rc_1k_fast_reg_weighted",0.0309999999999988,0.0289999999999999,0.002 +"baseline",2,7,"panel_5k_fast_reg_weighted",0.049000000000003,0.0480000000000018,0.002 +"baseline",2,7,"unbalanced_1k_fast_reg_weighted",0.0300000000000011,0.0289999999999999,0.001 +"baseline",2,7,"panel_5k_slow_reg_weighted",0.0250000000000021,0.0249999999999986,0 +"baseline",2,7,"panel_5k_slow_dr_varying",0.169,0.157999999999998,0.021 +"baseline",2,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.0100000000000016,0 +"baseline",2,7,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00300000000000011,0 +"baseline",2,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,7,"panel_5k_fast_dr_varying",0.143999999999998,0.141999999999999,0.002 +"baseline",2,7,"panel_1k_fast_reg_weighted",0.0189999999999984,0.0190000000000019,0.001 +"baseline",2,7,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00800000000000267,0 +"baseline",2,7,"panel_1k_fast_dr_varying",0.0510000000000019,0.0500000000000007,0.002 +"baseline",2,7,"rc_1k_fast_reg_weighted",0.0279999999999987,0.0269999999999975,0.002 diff --git a/dev/benchmarks/stress-fixes/supplemental-comparison.csv b/dev/benchmarks/stress-fixes/supplemental-comparison.csv new file mode 100644 index 00000000..3b9c4de0 --- /dev/null +++ b/dev/benchmarks/stress-fixes/supplemental-comparison.csv @@ -0,0 +1,13 @@ +"case","source_baseline","n_baseline","median_baseline","q25_baseline","q75_baseline","median_cpu_baseline","source_final","n_final","median_final","q25_final","q75_final","median_cpu_final","ratio","delta_ms","iqr_overlap" +"aggte_1k_dynamic_balance_e2","baseline",14,0.00300000000000011,0.00300000000000011,0.00300000000000011,0.00300000000000011,"final",14,0.00300000000000011,0.00300000000000011,0.00300000000000011,0.00300000000000011,1,0,TRUE +"aggte_3k_T12_dynamic_balance_e2","baseline",14,0.00900000000000034,0.00824999999999987,0.00900000000000034,0.00900000000000017,"final",14,0.00900000000000034,0.00800000000000023,0.00900000000000034,0.00800000000000045,1,0,TRUE +"aggte_3k_T12_dynamic_balance_e5","baseline",14,0.00900000000000034,0.00899999999999879,0.00900000000000034,0.0089999999999999,"final",14,0.00900000000000034,0.00899999999999967,0.00900000000000034,0.00900000000000034,1,0,TRUE +"aggte_5k_dynamic_balance_e2","baseline",14,0.004,0.00399999999999956,0.00400000000000134,0.00399999999999956,"final",14,0.00399999999999956,0.00399999999999823,0.00400000000000023,0.00399999999999956,0.999999999999889,-4.44089209850063e-13,TRUE +"panel_1k_fast_dr_varying","baseline",14,0.049500000000001,0.0489999999999995,0.0507499999999987,0.0499999999999995,"final",14,0.0470000000000006,0.0470000000000006,0.0480000000000014,0.0475000000000009,0.949494949494943,-2.50000000000039,FALSE +"panel_1k_fast_reg_weighted","baseline",14,0.0189999999999997,0.0180000000000007,0.0190000000000001,0.0190000000000007,"final",14,0.0169999999999995,0.0169999999999995,0.0169999999999995,0.017,0.89473684210525,-2.00000000000022,FALSE +"panel_5k_fast_dr_varying","baseline",14,0.141999999999999,0.1405,0.142750000000001,0.143,"final",14,0.140000000000001,0.1385,0.141,0.142000000000001,0.985915492957754,-1.99999999999889,TRUE +"panel_5k_fast_reg_weighted","baseline",14,0.048,0.048,0.0480000000000018,0.0494999999999992,"final",14,0.0449999999999999,0.0449999999999999,0.0449999999999999,0.0469999999999999,0.937499999999998,-3.00000000000011,FALSE +"panel_5k_slow_dr_varying","baseline",14,0.169,0.167999999999999,0.170750000000001,0.178,"final",14,0.17,0.16925,0.170999999999999,0.179000000000001,1.00591715976331,0.999999999999446,TRUE +"panel_5k_slow_reg_weighted","baseline",14,0.0264999999999986,0.0259999999999998,0.0270000000000008,0.0269999999999998,"final",14,0.0259999999999998,0.0259999999999998,0.0260000000000005,0.0269999999999998,0.981132075471741,-0.499999999998835,TRUE +"rc_1k_fast_reg_weighted","baseline",14,0.0289999999999999,0.0282500000000003,0.0299999999999994,0.0294999999999996,"final",14,0.027000000000001,0.0270000000000001,0.0277499999999993,0.0280000000000002,0.931034482758659,-1.99999999999889,FALSE +"unbalanced_1k_fast_reg_weighted","baseline",14,0.0299999999999998,0.0299999999999994,0.0307499999999994,0.0304999999999996,"final",14,0.028,0.0279999999999987,0.0280000000000005,0.028000000000001,0.93333333333334,-1.99999999999978,FALSE diff --git a/dev/benchmarks/stress-fixes/supplemental-raw-timings.csv b/dev/benchmarks/stress-fixes/supplemental-raw-timings.csv new file mode 100644 index 00000000..d6d872d4 --- /dev/null +++ b/dev/benchmarks/stress-fixes/supplemental-raw-timings.csv @@ -0,0 +1,337 @@ +"source","round","iteration","case","elapsed","user","system" +"baseline",1,1,"aggte_1k_dynamic_balance_e2",0.00400000000000045,0.00300000000000011,0 +"baseline",1,1,"panel_1k_fast_dr_varying",0.0499999999999998,0.0489999999999995,0.00199999999999997 +"baseline",1,1,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0190000000000001,0.002 +"baseline",1,1,"aggte_3k_T12_dynamic_balance_e2",0.00899999999999945,0.00800000000000001,0.001 +"baseline",1,1,"aggte_5k_dynamic_balance_e2",0.00400000000000045,0.00400000000000045,0.001 +"baseline",1,1,"panel_5k_fast_dr_varying",0.143000000000001,0.141,0.002 +"baseline",1,1,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0260000000000007,0.001 +"baseline",1,1,"aggte_3k_T12_dynamic_balance_e5",0.00899999999999945,0.00899999999999945,0 +"baseline",1,1,"rc_1k_fast_reg_weighted",0.0309999999999997,0.0300000000000002,0.003 +"baseline",1,1,"unbalanced_1k_fast_reg_weighted",0.0300000000000002,0.0289999999999999,0.002 +"baseline",1,1,"panel_5k_slow_dr_varying",0.174,0.16,0.022 +"baseline",1,1,"panel_5k_fast_reg_weighted",0.048,0.048,0.002 +"baseline",1,2,"aggte_3k_T12_dynamic_balance_e5",0.00899999999999945,0.00900000000000034,0 +"baseline",1,2,"rc_1k_fast_reg_weighted",0.0280000000000005,0.0270000000000001,0.002 +"baseline",1,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,2,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00800000000000001,0 +"baseline",1,2,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0250000000000004,0.001 +"baseline",1,2,"panel_1k_fast_dr_varying",0.0499999999999989,0.0490000000000004,0.002 +"baseline",1,2,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0180000000000007,0.001 +"baseline",1,2,"unbalanced_1k_fast_reg_weighted",0.0310000000000006,0.0289999999999999,0.001 +"baseline",1,2,"panel_5k_fast_reg_weighted",0.048,0.048,0.001 +"baseline",1,2,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999956,0 +"baseline",1,2,"panel_5k_fast_dr_varying",0.140000000000001,0.139000000000001,0.002 +"baseline",1,2,"panel_5k_slow_dr_varying",0.167999999999999,0.157,0.021 +"baseline",1,3,"panel_5k_fast_dr_varying",0.140000000000001,0.141,0.002 +"baseline",1,3,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0269999999999992,0.002 +"baseline",1,3,"panel_5k_fast_reg_weighted",0.048,0.048,0.001 +"baseline",1,3,"panel_5k_slow_dr_varying",0.167000000000002,0.155000000000001,0.021 +"baseline",1,3,"panel_1k_fast_reg_weighted",0.0179999999999989,0.0180000000000007,0.001 +"baseline",1,3,"unbalanced_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.001 +"baseline",1,3,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00799999999999912,0 +"baseline",1,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,3,"panel_5k_slow_reg_weighted",0.0249999999999986,0.0259999999999998,0.001 +"baseline",1,3,"panel_1k_fast_dr_varying",0.0490000000000013,0.048,0.001 +"baseline",1,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",1,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00300000000000011,0 +"baseline",1,4,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0.001 +"baseline",1,4,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0269999999999992,0.002 +"baseline",1,4,"panel_5k_slow_dr_varying",0.167999999999999,0.156000000000001,0.021 +"baseline",1,4,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0180000000000007,0.000999999999999945 +"baseline",1,4,"panel_1k_fast_dr_varying",0.048,0.0470000000000006,0.001 +"baseline",1,4,"panel_5k_fast_reg_weighted",0.0489999999999995,0.048,0.001 +"baseline",1,4,"aggte_3k_T12_dynamic_balance_e5",0.00799999999999912,0.00899999999999856,0 +"baseline",1,4,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,4,"unbalanced_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.002 +"baseline",1,4,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0250000000000004,0 +"baseline",1,4,"panel_5k_fast_dr_varying",0.142000000000001,0.141,0.002 +"baseline",1,4,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999956,0 +"baseline",1,5,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0179999999999989,0.001 +"baseline",1,5,"panel_1k_fast_dr_varying",0.0519999999999996,0.0510000000000002,0.001 +"baseline",1,5,"panel_5k_slow_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"baseline",1,5,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999856,0 +"baseline",1,5,"rc_1k_fast_reg_weighted",0.0299999999999994,0.0280000000000005,0.002 +"baseline",1,5,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"baseline",1,5,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00399999999999956,0 +"baseline",1,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,5,"panel_5k_fast_dr_varying",0.141999999999999,0.141000000000002,0.002 +"baseline",1,5,"panel_5k_slow_dr_varying",0.167000000000002,0.155999999999999,0.022 +"baseline",1,5,"panel_5k_fast_reg_weighted",0.0479999999999983,0.0479999999999983,0.002 +"baseline",1,5,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0289999999999999,0.001 +"baseline",1,6,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0289999999999999,0.002 +"baseline",1,6,"panel_1k_fast_dr_varying",0.0509999999999984,0.0500000000000007,0.002 +"baseline",1,6,"panel_1k_fast_reg_weighted",0.0199999999999996,0.0199999999999996,0.001 +"baseline",1,6,"aggte_3k_T12_dynamic_balance_e5",0.00799999999999912,0.00799999999999912,0 +"baseline",1,6,"panel_5k_fast_reg_weighted",0.0480000000000018,0.0479999999999983,0.000999999999999945 +"baseline",1,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,6,"panel_5k_fast_dr_varying",0.141999999999999,0.140999999999998,0.002 +"baseline",1,6,"aggte_5k_dynamic_balance_e2",0.00399999999999778,0.00400000000000134,0 +"baseline",1,6,"aggte_3k_T12_dynamic_balance_e2",0.00899999999999679,0.00900000000000034,0 +"baseline",1,6,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0249999999999986,0 +"baseline",1,6,"panel_5k_slow_dr_varying",0.169,0.157,0.021 +"baseline",1,6,"rc_1k_fast_reg_weighted",0.0279999999999987,0.0280000000000022,0.002 +"baseline",1,7,"aggte_5k_dynamic_balance_e2",0.00399999999999778,0.00399999999999778,0 +"baseline",1,7,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0260000000000034,0 +"baseline",1,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",1,7,"rc_1k_fast_reg_weighted",0.0279999999999987,0.0279999999999987,0.002 +"baseline",1,7,"unbalanced_1k_fast_reg_weighted",0.0300000000000011,0.0279999999999987,0.001 +"baseline",1,7,"panel_1k_fast_dr_varying",0.0509999999999984,0.0489999999999995,0.001 +"baseline",1,7,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0180000000000007,0.001 +"baseline",1,7,"panel_5k_fast_reg_weighted",0.0480000000000018,0.0480000000000018,0.002 +"baseline",1,7,"panel_5k_slow_dr_varying",0.173000000000002,0.156000000000002,0.026 +"baseline",1,7,"panel_5k_fast_dr_varying",0.148,0.141999999999999,0.002 +"baseline",1,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",1,7,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"final",1,1,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,1,"panel_1k_fast_dr_varying",0.0499999999999998,0.048,0.003 +"final",1,1,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0169999999999995,0.001 +"final",1,1,"aggte_3k_T12_dynamic_balance_e2",0.00800000000000001,0.00800000000000001,0.001 +"final",1,1,"aggte_5k_dynamic_balance_e2",0.00400000000000045,0.00300000000000011,0 +"final",1,1,"panel_5k_fast_dr_varying",0.141,0.140000000000001,0.003 +"final",1,1,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,1,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00800000000000001,0.001 +"final",1,1,"rc_1k_fast_reg_weighted",0.0270000000000001,0.0260000000000007,0.002 +"final",1,1,"unbalanced_1k_fast_reg_weighted",0.0280000000000005,0.0270000000000001,0.001 +"final",1,1,"panel_5k_slow_dr_varying",0.169,0.157999999999999,0.02 +"final",1,1,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.001 +"final",1,2,"aggte_3k_T12_dynamic_balance_e5",0.00899999999999945,0.00899999999999945,0 +"final",1,2,"rc_1k_fast_reg_weighted",0.0270000000000001,0.0259999999999998,0.002 +"final",1,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,2,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00800000000000001,0 +"final",1,2,"panel_5k_slow_reg_weighted",0.0270000000000001,0.0270000000000001,0 +"final",1,2,"panel_1k_fast_dr_varying",0.0470000000000006,0.0470000000000006,0.002 +"final",1,2,"panel_1k_fast_reg_weighted",0.0169999999999995,0.016,0.001 +"final",1,2,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0269999999999992,0.001 +"final",1,2,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.001 +"final",1,2,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999956,0 +"final",1,2,"panel_5k_fast_dr_varying",0.140000000000001,0.140000000000001,0.003 +"final",1,2,"panel_5k_slow_dr_varying",0.17,0.158000000000001,0.021 +"final",1,3,"panel_5k_fast_dr_varying",0.138,0.137,0.00199999999999995 +"final",1,3,"rc_1k_fast_reg_weighted",0.027000000000001,0.0259999999999998,0.002 +"final",1,3,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.002 +"final",1,3,"panel_5k_slow_dr_varying",0.17,0.158000000000001,0.021 +"final",1,3,"panel_1k_fast_reg_weighted",0.0169999999999995,0.016,0.001 +"final",1,3,"unbalanced_1k_fast_reg_weighted",0.0279999999999987,0.0269999999999992,0.001 +"final",1,3,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",1,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,3,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,3,"panel_1k_fast_dr_varying",0.048,0.0470000000000006,0.002 +"final",1,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",1,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0.001 +"final",1,4,"aggte_3k_T12_dynamic_balance_e2",0.00899999999999856,0.0080000000000009,0 +"final",1,4,"rc_1k_fast_reg_weighted",0.0269999999999992,0.0269999999999992,0.001 +"final",1,4,"panel_5k_slow_dr_varying",0.170999999999999,0.158999999999999,0.022 +"final",1,4,"panel_1k_fast_reg_weighted",0.0169999999999995,0.016,0.000999999999999945 +"final",1,4,"panel_1k_fast_dr_varying",0.0469999999999988,0.0459999999999994,0.001 +"final",1,4,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.001 +"final",1,4,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",1,4,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00399999999999956,0 +"final",1,4,"unbalanced_1k_fast_reg_weighted",0.0280000000000005,0.027000000000001,0.001 +"final",1,4,"panel_5k_slow_reg_weighted",0.0250000000000004,0.0260000000000016,0.001 +"final",1,4,"panel_5k_fast_dr_varying",0.141,0.140000000000001,0.002 +"final",1,4,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"final",1,5,"panel_1k_fast_reg_weighted",0.0169999999999995,0.016,0.001 +"final",1,5,"panel_1k_fast_dr_varying",0.0470000000000006,0.0460000000000012,0.001 +"final",1,5,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,5,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999856,0 +"final",1,5,"rc_1k_fast_reg_weighted",0.027000000000001,0.0259999999999998,0.003 +"final",1,5,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.0080000000000009,0.001 +"final",1,5,"aggte_5k_dynamic_balance_e2",0.00399999999999778,0.00399999999999956,0 +"final",1,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00199999999999889,0 +"final",1,5,"panel_5k_fast_dr_varying",0.140000000000001,0.140000000000001,0.002 +"final",1,5,"panel_5k_slow_dr_varying",0.169,0.157999999999998,0.021 +"final",1,5,"panel_5k_fast_reg_weighted",0.0460000000000029,0.0459999999999994,0.002 +"final",1,5,"unbalanced_1k_fast_reg_weighted",0.0279999999999987,0.0269999999999975,0.002 +"final",1,6,"unbalanced_1k_fast_reg_weighted",0.0279999999999987,0.027000000000001,0.001 +"final",1,6,"panel_1k_fast_dr_varying",0.0470000000000006,0.0459999999999994,0.001 +"final",1,6,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0159999999999982,0.001 +"final",1,6,"aggte_3k_T12_dynamic_balance_e5",0.00800000000000267,0.00900000000000034,0 +"final",1,6,"panel_5k_fast_reg_weighted",0.0440000000000005,0.0449999999999982,0.002 +"final",1,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00299999999999656,0 +"final",1,6,"panel_5k_fast_dr_varying",0.140000000000001,0.139999999999997,0.002 +"final",1,6,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0 +"final",1,6,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",1,6,"panel_5k_slow_reg_weighted",0.0269999999999975,0.0259999999999998,0.001 +"final",1,6,"panel_5k_slow_dr_varying",0.169,0.157,0.022 +"final",1,6,"rc_1k_fast_reg_weighted",0.0279999999999987,0.0259999999999998,0.002 +"final",1,7,"aggte_5k_dynamic_balance_e2",0.00399999999999778,0.00400000000000134,0 +"final",1,7,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",1,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999679,0 +"final",1,7,"rc_1k_fast_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"final",1,7,"unbalanced_1k_fast_reg_weighted",0.0280000000000022,0.0259999999999998,0.001 +"final",1,7,"panel_1k_fast_dr_varying",0.046999999999997,0.0459999999999994,0.001 +"final",1,7,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0159999999999982,0.001 +"final",1,7,"panel_5k_fast_reg_weighted",0.0450000000000017,0.0450000000000017,0.001 +"final",1,7,"panel_5k_slow_dr_varying",0.169,0.158000000000001,0.02 +"final",1,7,"panel_5k_fast_dr_varying",0.141000000000002,0.140000000000001,0.002 +"final",1,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",1,7,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00799999999999912,0 +"final",2,1,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0270000000000001,0.002 +"final",2,1,"panel_1k_fast_reg_weighted",0.0179999999999998,0.0179999999999998,0.000999999999999973 +"final",2,1,"aggte_3k_T12_dynamic_balance_e5",0.00999999999999979,0.00999999999999979,0 +"final",2,1,"panel_5k_fast_dr_varying",0.148000000000001,0.146999999999999,0.003 +"final",2,1,"panel_1k_fast_dr_varying",0.0499999999999998,0.0469999999999997,0.004 +"final",2,1,"panel_5k_slow_dr_varying",0.172,0.159,0.023 +"final",2,1,"unbalanced_1k_fast_reg_weighted",0.0279999999999996,0.0270000000000001,0.002 +"final",2,1,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0440000000000005,0.001 +"final",2,1,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00800000000000001,0 +"final",2,1,"aggte_1k_dynamic_balance_e2",0.00299999999999923,0.00299999999999923,0 +"final",2,1,"panel_5k_slow_reg_weighted",0.0260000000000007,0.0260000000000007,0.001 +"final",2,1,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00300000000000011,0 +"final",2,2,"rc_1k_fast_reg_weighted",0.0270000000000001,0.0270000000000001,0.002 +"final",2,2,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",2,2,"unbalanced_1k_fast_reg_weighted",0.0279999999999996,0.0270000000000001,0.001 +"final",2,2,"panel_1k_fast_dr_varying",0.0469999999999997,0.0459999999999994,0.001 +"final",2,2,"panel_5k_fast_dr_varying",0.14,0.14,0.002 +"final",2,2,"panel_5k_slow_dr_varying",0.170000000000002,0.159000000000001,0.021 +"final",2,2,"panel_1k_fast_reg_weighted",0.0169999999999995,0.016,0.001 +"final",2,2,"aggte_3k_T12_dynamic_balance_e2",0.0080000000000009,0.00799999999999912,0 +"final",2,2,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.002 +"final",2,2,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00799999999999912,0 +"final",2,2,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"final",2,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,3,"unbalanced_1k_fast_reg_weighted",0.0280000000000005,0.0269999999999992,0.002 +"final",2,3,"aggte_3k_T12_dynamic_balance_e5",0.0080000000000009,0.00900000000000034,0 +"final",2,3,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"final",2,3,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0450000000000017,0.002 +"final",2,3,"panel_5k_fast_dr_varying",0.138,0.138,0.00199999999999995 +"final",2,3,"rc_1k_fast_reg_weighted",0.0269999999999992,0.0269999999999992,0.001 +"final",2,3,"panel_5k_slow_dr_varying",0.17,0.158000000000001,0.021 +"final",2,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00299999999999834,0 +"final",2,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"final",2,3,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",2,3,"panel_1k_fast_dr_varying",0.0470000000000006,0.0459999999999994,0.001 +"final",2,3,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0169999999999995,0.001 +"final",2,4,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,4,"panel_5k_slow_reg_weighted",0.027000000000001,0.0269999999999992,0 +"final",2,4,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",2,4,"panel_1k_fast_reg_weighted",0.016,0.0169999999999995,0.001 +"final",2,4,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"final",2,4,"panel_5k_slow_dr_varying",0.171000000000001,0.16,0.021 +"final",2,4,"panel_1k_fast_dr_varying",0.048,0.0459999999999994,0.001 +"final",2,4,"panel_5k_fast_reg_weighted",0.0440000000000005,0.0450000000000017,0.002 +"final",2,4,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00900000000000034,0 +"final",2,4,"panel_5k_fast_dr_varying",0.138,0.138,0.00199999999999995 +"final",2,4,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0280000000000005,0.002 +"final",2,4,"unbalanced_1k_fast_reg_weighted",0.0280000000000005,0.027000000000001,0.001 +"final",2,5,"unbalanced_1k_fast_reg_weighted",0.0269999999999992,0.027000000000001,0.002 +"final",2,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,5,"panel_5k_fast_reg_weighted",0.0449999999999999,0.0449999999999999,0.002 +"final",2,5,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999956,0 +"final",2,5,"aggte_3k_T12_dynamic_balance_e5",0.00899999999999856,0.00900000000000034,0 +"final",2,5,"panel_5k_fast_dr_varying",0.140999999999998,0.140000000000001,0.002 +"final",2,5,"rc_1k_fast_reg_weighted",0.027000000000001,0.0260000000000016,0.002 +"final",2,5,"panel_5k_slow_dr_varying",0.175000000000001,0.160999999999998,0.022 +"final",2,5,"panel_1k_fast_reg_weighted",0.0159999999999982,0.0160000000000018,0.001 +"final",2,5,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0 +"final",2,5,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00900000000000034,0 +"final",2,5,"panel_1k_fast_dr_varying",0.0480000000000018,0.0470000000000006,0.002 +"final",2,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,6,"panel_1k_fast_dr_varying",0.0470000000000006,0.0470000000000006,0.001 +"final",2,6,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0169999999999995,0.001 +"final",2,6,"panel_5k_slow_dr_varying",0.169999999999998,0.157999999999998,0.021 +"final",2,6,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00799999999999912,0 +"final",2,6,"aggte_5k_dynamic_balance_e2",0.00399999999999778,0.00400000000000134,0 +"final",2,6,"panel_5k_slow_reg_weighted",0.0250000000000021,0.0259999999999998,0 +"final",2,6,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"final",2,6,"panel_5k_fast_reg_weighted",0.0450000000000017,0.0450000000000017,0.002 +"final",2,6,"unbalanced_1k_fast_reg_weighted",0.0279999999999987,0.0279999999999987,0.001 +"final",2,6,"panel_5k_fast_dr_varying",0.137999999999998,0.136999999999997,0.001 +"final",2,6,"rc_1k_fast_reg_weighted",0.0280000000000022,0.0269999999999975,0.002 +"final",2,7,"panel_5k_fast_reg_weighted",0.0440000000000005,0.0450000000000017,0.002 +"final",2,7,"unbalanced_1k_fast_reg_weighted",0.0280000000000022,0.027000000000001,0.001 +"final",2,7,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0259999999999998,0.001 +"final",2,7,"panel_5k_slow_dr_varying",0.170999999999999,0.158999999999999,0.021 +"final",2,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"final",2,7,"aggte_5k_dynamic_balance_e2",0.00300000000000011,0.00399999999999778,0 +"final",2,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"final",2,7,"panel_5k_fast_dr_varying",0.140999999999998,0.140999999999998,0.002 +"final",2,7,"panel_1k_fast_reg_weighted",0.0169999999999995,0.0159999999999982,0.001 +"final",2,7,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"final",2,7,"panel_1k_fast_dr_varying",0.0480000000000018,0.0470000000000006,0.001 +"final",2,7,"rc_1k_fast_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"baseline",2,1,"rc_1k_fast_reg_weighted",0.0300000000000002,0.0289999999999999,0.002 +"baseline",2,1,"panel_1k_fast_reg_weighted",0.0189999999999992,0.0190000000000001,0.001 +"baseline",2,1,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00899999999999945,0 +"baseline",2,1,"panel_5k_fast_dr_varying",0.145,0.144,0.002 +"baseline",2,1,"panel_1k_fast_dr_varying",0.0520000000000005,0.0499999999999998,0.003 +"baseline",2,1,"panel_5k_slow_dr_varying",0.172,0.161,0.024 +"baseline",2,1,"unbalanced_1k_fast_reg_weighted",0.0310000000000006,0.0299999999999994,0.001 +"baseline",2,1,"panel_5k_fast_reg_weighted",0.0490000000000004,0.0490000000000004,0.002 +"baseline",2,1,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"baseline",2,1,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,1,"panel_5k_slow_reg_weighted",0.0279999999999996,0.0269999999999992,0.001 +"baseline",2,1,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"baseline",2,2,"rc_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.002 +"baseline",2,2,"panel_5k_slow_reg_weighted",0.0270000000000001,0.0270000000000001,0.001 +"baseline",2,2,"unbalanced_1k_fast_reg_weighted",0.0309999999999997,0.0309999999999997,0.002 +"baseline",2,2,"panel_1k_fast_dr_varying",0.0499999999999998,0.0489999999999995,0.002 +"baseline",2,2,"panel_5k_fast_dr_varying",0.140000000000001,0.139,0.002 +"baseline",2,2,"panel_5k_slow_dr_varying",0.17,0.157999999999999,0.021 +"baseline",2,2,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0190000000000001,0.001 +"baseline",2,2,"aggte_3k_T12_dynamic_balance_e2",0.0080000000000009,0.00799999999999912,0 +"baseline",2,2,"panel_5k_fast_reg_weighted",0.048,0.0469999999999988,0.001 +"baseline",2,2,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",2,2,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00399999999999956,0 +"baseline",2,2,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,3,"unbalanced_1k_fast_reg_weighted",0.0309999999999988,0.0299999999999994,0.002 +"baseline",2,3,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.0080000000000009,0 +"baseline",2,3,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00900000000000034,0 +"baseline",2,3,"panel_5k_fast_reg_weighted",0.0510000000000002,0.0490000000000013,0.002 +"baseline",2,3,"panel_5k_fast_dr_varying",0.145999999999999,0.142000000000001,0.006 +"baseline",2,3,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0280000000000005,0.001 +"baseline",2,3,"panel_5k_slow_dr_varying",0.165999999999999,0.154999999999999,0.021 +"baseline",2,3,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,3,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00400000000000134,0 +"baseline",2,3,"panel_5k_slow_reg_weighted",0.0259999999999998,0.0269999999999992,0 +"baseline",2,3,"panel_1k_fast_dr_varying",0.0489999999999995,0.048,0.001 +"baseline",2,3,"panel_1k_fast_reg_weighted",0.0190000000000001,0.0179999999999989,0.001 +"baseline",2,4,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0.001 +"baseline",2,4,"panel_5k_slow_reg_weighted",0.0269999999999992,0.027000000000001,0.001 +"baseline",2,4,"aggte_3k_T12_dynamic_balance_e5",0.00799999999999912,0.00900000000000034,0 +"baseline",2,4,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0169999999999995,0.001 +"baseline",2,4,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00300000000000011,0 +"baseline",2,4,"panel_5k_slow_dr_varying",0.171000000000001,0.159000000000001,0.021 +"baseline",2,4,"panel_1k_fast_dr_varying",0.0489999999999995,0.0489999999999995,0.001 +"baseline",2,4,"panel_5k_fast_reg_weighted",0.048,0.048,0.001 +"baseline",2,4,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00799999999999912,0 +"baseline",2,4,"panel_5k_fast_dr_varying",0.142000000000001,0.142000000000001,0.002 +"baseline",2,4,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0269999999999992,0.001 +"baseline",2,4,"unbalanced_1k_fast_reg_weighted",0.0299999999999994,0.0289999999999999,0.001 +"baseline",2,5,"unbalanced_1k_fast_reg_weighted",0.0299999999999994,0.0279999999999987,0.001 +"baseline",2,5,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,5,"panel_5k_fast_reg_weighted",0.048,0.048,0.002 +"baseline",2,5,"aggte_5k_dynamic_balance_e2",0.00399999999999956,0.00399999999999956,0 +"baseline",2,5,"aggte_3k_T12_dynamic_balance_e5",0.00899999999999856,0.00900000000000034,0 +"baseline",2,5,"panel_5k_fast_dr_varying",0.140000000000001,0.141,0.002 +"baseline",2,5,"rc_1k_fast_reg_weighted",0.0289999999999999,0.0269999999999975,0.001 +"baseline",2,5,"panel_5k_slow_dr_varying",0.168999999999997,0.157,0.021 +"baseline",2,5,"panel_1k_fast_reg_weighted",0.0180000000000007,0.0179999999999971,0.001 +"baseline",2,5,"panel_5k_slow_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"baseline",2,5,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00800000000000267,0 +"baseline",2,5,"panel_1k_fast_dr_varying",0.049000000000003,0.0489999999999995,0.001 +"baseline",2,6,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,6,"panel_1k_fast_dr_varying",0.0480000000000018,0.0470000000000006,0.001 +"baseline",2,6,"panel_1k_fast_reg_weighted",0.0199999999999996,0.0190000000000019,0.000999999999999945 +"baseline",2,6,"panel_5k_slow_dr_varying",0.169,0.157,0.021 +"baseline",2,6,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00799999999999912,0 +"baseline",2,6,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00400000000000134,0 +"baseline",2,6,"panel_5k_slow_reg_weighted",0.027000000000001,0.0259999999999998,0.001 +"baseline",2,6,"aggte_3k_T12_dynamic_balance_e2",0.00799999999999912,0.00800000000000267,0 +"baseline",2,6,"panel_5k_fast_reg_weighted",0.0480000000000018,0.0480000000000018,0.002 +"baseline",2,6,"unbalanced_1k_fast_reg_weighted",0.0300000000000011,0.0299999999999976,0.002 +"baseline",2,6,"panel_5k_fast_dr_varying",0.141999999999999,0.140999999999998,0.002 +"baseline",2,6,"rc_1k_fast_reg_weighted",0.0279999999999987,0.027000000000001,0.001 +"baseline",2,7,"panel_5k_fast_reg_weighted",0.0470000000000006,0.0470000000000006,0.001 +"baseline",2,7,"unbalanced_1k_fast_reg_weighted",0.0289999999999999,0.0279999999999987,0.001 +"baseline",2,7,"panel_5k_slow_reg_weighted",0.0269999999999975,0.0259999999999998,0.001 +"baseline",2,7,"panel_5k_slow_dr_varying",0.169,0.157,0.021 +"baseline",2,7,"aggte_3k_T12_dynamic_balance_e5",0.00900000000000034,0.00900000000000034,0 +"baseline",2,7,"aggte_5k_dynamic_balance_e2",0.00400000000000134,0.00300000000000011,0 +"baseline",2,7,"aggte_1k_dynamic_balance_e2",0.00300000000000011,0.00300000000000011,0 +"baseline",2,7,"panel_5k_fast_dr_varying",0.141999999999999,0.142000000000003,0.00199999999999995 +"baseline",2,7,"panel_1k_fast_reg_weighted",0.0189999999999984,0.0180000000000007,0.001 +"baseline",2,7,"aggte_3k_T12_dynamic_balance_e2",0.00900000000000034,0.00900000000000034,0 +"baseline",2,7,"panel_1k_fast_dr_varying",0.0489999999999995,0.0480000000000018,0.001 +"baseline",2,7,"rc_1k_fast_reg_weighted",0.0300000000000011,0.027000000000001,0.004 diff --git a/dev/benchmarks/stress-fixes/support-comparison.csv b/dev/benchmarks/stress-fixes/support-comparison.csv new file mode 100644 index 00000000..e614628b --- /dev/null +++ b/dev/benchmarks/stress-fixes/support-comparison.csv @@ -0,0 +1,7 @@ +"case","source","median_us_per_call","q25_us_per_call","q75_us_per_call" +"positive","after",45.9999999999994,44.0000000000031,45.9999999999994 +"underflow","after",73.9999999999998,73.9999999999981,74.0000000000052 +"zero","after",80.0000000000001,79.4999999999995,82.0000000000007 +"positive","before",79.9999999999983,79.9999999999983,80.0000000000018 +"underflow","before",69.9999999999976,67.9999999999992,70.0000000000003 +"zero","before",77.9999999999959,76.0000000000005,78.000000000001 diff --git a/dev/benchmarks/stress-fixes/support-performance.R b/dev/benchmarks/stress-fixes/support-performance.R new file mode 100644 index 00000000..97acae48 --- /dev/null +++ b/dev/benchmarks/stress-fixes/support-performance.R @@ -0,0 +1,43 @@ +# Compare the optimized RC preflight with its original correct implementation. +args <- commandArgs(trailingOnly = TRUE) +output <- if (length(args)) args[[1L]] else file.path(tempdir(), "did-rc-support-performance.csv") +script <- grep("^--file=", commandArgs(), value = TRUE)[[1L]] +root <- normalizePath(file.path(dirname(sub("^--file=", "", script)), "..", "..", "..")) +setwd(root) +pkgload::load_all(root, quiet = TRUE) +old_source <- tempfile(fileext = ".R") +status <- system2("git", c("show", "1a16c46:R/utility_functions.R"), stdout = old_source) +stopifnot(status == 0L) +old <- new.env(parent = asNamespace("did")); sys.source(old_source, old) +functions <- list(before = old$rc_outcome_has_too_few_obs, + after = did:::rc_outcome_has_too_few_obs) +set.seed(20261028) +n <- 5000L +D <- rep(c(0, 0, 1, 1), length.out = n) +post <- rep(c(0, 1, 0, 1), length.out = n) +weights <- list(positive = runif(n, .5, 2), + zero = sample(c(0, 1), n, TRUE), + underflow = rep(c(1e300, 1e-300, 1, 1), length.out = n)) +for (w in weights) for (f in functions) for (i in seq_len(100L)) invisible(f(D, post, w, 3L)) +r <- list(); j <- 0L +for (iteration in seq_len(20L)) for (case in names(weights)) { + labels <- if (iteration %% 2L) names(functions) else rev(names(functions)) + for (label in labels) { + gc(FALSE) + f <- functions[[label]]; w <- weights[[case]] + tm <- system.time(for (i in seq_len(500L)) invisible(f(D, post, w, 3L))) + j <- j + 1L + r[[j]] <- data.frame(iteration, case, source = label, + elapsed = unname(tm[["elapsed"]])) + } +} +r <- do.call(rbind, r) +summary <- do.call(rbind, lapply(split(r, list(r$case, r$source)), function(z) { + data.frame(case = z$case[[1L]], source = z$source[[1L]], + median_us_per_call = median(z$elapsed) * 1e6 / 500, + q25_us_per_call = unname(quantile(z$elapsed, .25)) * 1e6 / 500, + q75_us_per_call = unname(quantile(z$elapsed, .75)) * 1e6 / 500) +})) +write.csv(summary, output, row.names = FALSE) +write.csv(r, sub("\\.csv$", "-raw.csv", output), row.names = FALSE) +print(summary, row.names = FALSE) diff --git a/dev/benchmarks/stress-fixes/support-raw-timings.csv b/dev/benchmarks/stress-fixes/support-raw-timings.csv new file mode 100644 index 00000000..2cf1d2e4 --- /dev/null +++ b/dev/benchmarks/stress-fixes/support-raw-timings.csv @@ -0,0 +1,121 @@ +"iteration","case","source","elapsed" +1,"positive","before",0.0409999999999999 +1,"positive","after",0.0249999999999999 +1,"zero","before",0.0389999999999997 +1,"zero","after",0.0390000000000001 +1,"underflow","before",0.036 +1,"underflow","after",0.0369999999999999 +2,"positive","after",0.0249999999999999 +2,"positive","before",0.0419999999999998 +2,"zero","after",0.04 +2,"zero","before",0.0390000000000001 +2,"underflow","after",0.0369999999999999 +2,"underflow","before",0.0350000000000006 +3,"positive","before",0.121 +3,"positive","after",0.0229999999999997 +3,"zero","before",0.0389999999999997 +3,"zero","after",0.0409999999999995 +3,"underflow","before",0.0350000000000001 +3,"underflow","after",0.0369999999999999 +4,"positive","after",0.0229999999999997 +4,"positive","before",0.04 +4,"zero","after",0.04 +4,"zero","before",0.0389999999999997 +4,"underflow","after",0.0369999999999999 +4,"underflow","before",0.0349999999999993 +5,"positive","before",0.04 +5,"positive","after",0.0220000000000002 +5,"zero","before",0.0350000000000001 +5,"zero","after",0.0399999999999991 +5,"underflow","before",0.0339999999999998 +5,"underflow","after",0.0369999999999999 +6,"positive","after",0.0219999999999994 +6,"positive","before",0.0399999999999991 +6,"zero","after",0.0380000000000003 +6,"zero","before",0.036999999999999 +6,"underflow","after",0.036999999999999 +6,"underflow","before",0.0350000000000001 +7,"positive","before",0.0400000000000009 +7,"positive","after",0.0220000000000002 +7,"zero","before",0.0380000000000003 +7,"zero","after",0.0399999999999991 +7,"underflow","before",0.0349999999999984 +7,"underflow","after",0.036999999999999 +8,"positive","after",0.0229999999999997 +8,"positive","before",0.0399999999999991 +8,"zero","after",0.0399999999999991 +8,"zero","before",0.0390000000000015 +8,"underflow","after",0.0370000000000008 +8,"underflow","before",0.0350000000000001 +9,"positive","before",0.0400000000000009 +9,"positive","after",0.0220000000000002 +9,"zero","before",0.0380000000000003 +9,"zero","after",0.0400000000000009 +9,"underflow","before",0.0350000000000001 +9,"underflow","after",0.0370000000000008 +10,"positive","after",0.0230000000000015 +10,"positive","before",0.0399999999999991 +10,"zero","after",0.0410000000000004 +10,"zero","before",0.0399999999999991 +10,"underflow","after",0.0380000000000003 +10,"underflow","before",0.0340000000000007 +11,"positive","before",0.0399999999999991 +11,"positive","after",0.0229999999999997 +11,"zero","before",0.0380000000000003 +11,"zero","after",0.0560000000000009 +11,"underflow","before",0.0350000000000001 +11,"underflow","after",0.0370000000000008 +12,"positive","after",0.0229999999999997 +12,"positive","before",0.0399999999999991 +12,"zero","after",0.0429999999999993 +12,"zero","before",0.0400000000000009 +12,"underflow","after",0.0370000000000026 +12,"underflow","before",0.0349999999999966 +13,"positive","before",0.0399999999999991 +13,"positive","after",0.0229999999999997 +13,"zero","before",0.0380000000000003 +13,"zero","after",0.119 +13,"underflow","before",0.0350000000000001 +13,"underflow","after",0.0370000000000026 +14,"positive","after",0.0229999999999997 +14,"positive","before",0.0399999999999991 +14,"zero","after",0.0400000000000027 +14,"zero","before",0.0390000000000015 +14,"underflow","after",0.036999999999999 +14,"underflow","before",0.0339999999999989 +15,"positive","before",0.0410000000000004 +15,"positive","after",0.0229999999999997 +15,"zero","before",0.0399999999999991 +15,"zero","after",0.0410000000000004 +15,"underflow","before",0.0339999999999989 +15,"underflow","after",0.0360000000000014 +16,"positive","after",0.0219999999999985 +16,"positive","before",0.0390000000000015 +16,"zero","after",0.0390000000000015 +16,"zero","before",0.0389999999999979 +16,"underflow","after",0.0360000000000014 +16,"underflow","before",0.0339999999999989 +17,"positive","before",0.0399999999999991 +17,"positive","after",0.0229999999999997 +17,"zero","before",0.0389999999999979 +17,"zero","after",0.0389999999999979 +17,"underflow","before",0.0339999999999989 +17,"underflow","after",0.0360000000000014 +18,"positive","after",0.0229999999999997 +18,"positive","before",0.0399999999999991 +18,"zero","after",0.0410000000000004 +18,"zero","before",0.0389999999999979 +18,"underflow","after",0.0390000000000015 +18,"underflow","before",0.0359999999999978 +19,"positive","before",0.0399999999999991 +19,"positive","after",0.022000000000002 +19,"zero","before",0.036999999999999 +19,"zero","after",0.0399999999999991 +19,"underflow","before",0.0339999999999989 +19,"underflow","after",0.0370000000000026 +20,"positive","after",0.0229999999999997 +20,"positive","before",0.0399999999999991 +20,"zero","after",0.0380000000000003 +20,"zero","before",0.0380000000000003 +20,"underflow","after",0.0370000000000026 +20,"underflow","before",0.0340000000000025 diff --git a/man/DIDparams.Rd b/man/DIDparams.Rd index 2ea0dc02..54c8bce4 100644 --- a/man/DIDparams.Rd +++ b/man/DIDparams.Rd @@ -87,7 +87,10 @@ date (net of \code{anticipation}) onward are dropped, with a warning.} \item{anticipation}{The number of time periods before participating in the treatment where units can anticipate participating in the -treatment and therefore it can affect their untreated potential outcomes} +treatment and therefore it can affect their untreated potential outcomes. +Periods \code{g - anticipation}, ..., \code{g - 1} use the base period +\code{g - anticipation - 1} (under either \code{base_period}) and are not used +in the pre-test of parallel trends.} \item{weightsname}{The name of the column containing the sampling weights. If not set, all observations have same weight. When weights are @@ -100,7 +103,7 @@ the default behavior differs by panel type: \describe{ \item{Balanced panel}{Each 2x2 DiD comparison uses the weight from the earlier of the two time periods involved. For post-treatment cells, -this is the base period (g-1). For pre-treatment cells with +this is the base period (g-anticipation-1). For pre-treatment cells with \code{base_period="varying"}, this is the pre-treatment period itself. The panel DRDID estimators are used.} \item{Repeated cross sections and unbalanced panels}{Both periods' @@ -119,7 +122,7 @@ Options: \describe{ \item{\code{NULL} (default)}{For balanced panel: uses the weight from the earlier of the two time periods in each 2x2 comparison. For -post-treatment cells, this is the base period (g-1). For +post-treatment cells, this is the base period (g-anticipation-1). For pre-treatment cells, this depends on the \code{base_period} setting. For RC/unbalanced panel: uses per-observation weights from both periods.} @@ -226,7 +229,7 @@ determines the estimates.} \item{base_period}{Whether to use a "varying" base period or a "universal" base period. Either choice results in the same post-treatment estimates of ATT(g,t)'s. In pre-treatment -periods, using a varying base period amounts to computing a +periods (t < g - anticipation), using a varying base period amounts to computing a pseudo-ATT in each treatment period by comparing the change in outcomes for a particular group relative to its comparison group in the pre-treatment periods (i.e., in pre-treatment diff --git a/man/MP.Rd b/man/MP.Rd index a7009276..dfb7d4d9 100644 --- a/man/MP.Rd +++ b/man/MP.Rd @@ -46,7 +46,8 @@ align rows by rowname, never by position.} \item{n}{the number of unique cross-sectional units (unique values of idname)} -\item{W}{the Wald statistic for pre-testing the common trends assumption} +\item{W}{the Wald statistic for pre-testing the common trends assumption +(from the pre-treatment cells, t < g - anticipation)} \item{Wpval}{the p-value of the Wald statistic for pre-testing the common trends assumption} diff --git a/man/aggte.Rd b/man/aggte.Rd index 47a27e10..f8ca2adb 100644 --- a/man/aggte.Rd +++ b/man/aggte.Rd @@ -39,8 +39,12 @@ time period).} the sample with respect to event time. For example, if \code{balance_e=2}, \code{aggte} will drop groups that are not exposed to treatment for at least three periods. (the initial period when \code{e=0} as well as the -next two periods when \code{e=1} and the \code{e=2}). This ensures that -the composition of groups does not change when event time changes.} +next two periods when \code{e=1} and the \code{e=2}). Each retained group must +have a non-missing ATT estimate at every observed post-treatment event +time through \code{balance_e}, so the composition of groups does not change +across those event times. On irregular time grids, only event times +observed among the eligible groups are required. Balancing is done before +applying \code{min_e} and \code{max_e}; an error is returned if no group remains.} \item{min_e}{The smallest event time (length of exposure, \code{e = t - g}, measured in the same units as \code{tname}) to include. By default, diff --git a/man/att_gt.Rd b/man/att_gt.Rd index 06fc3c7c..344d1c17 100644 --- a/man/att_gt.Rd +++ b/man/att_gt.Rd @@ -101,7 +101,10 @@ date (net of \code{anticipation}) onward are dropped, with a warning.} \item{anticipation}{The number of time periods before participating in the treatment where units can anticipate participating in the -treatment and therefore it can affect their untreated potential outcomes} +treatment and therefore it can affect their untreated potential outcomes. +Periods \code{g - anticipation}, ..., \code{g - 1} use the base period +\code{g - anticipation - 1} (under either \code{base_period}) and are not used +in the pre-test of parallel trends.} \item{weightsname}{The name of the column containing the sampling weights. If not set, all observations have same weight. When weights are @@ -114,7 +117,7 @@ the default behavior differs by panel type: \describe{ \item{Balanced panel}{Each 2x2 DiD comparison uses the weight from the earlier of the two time periods involved. For post-treatment cells, -this is the base period (g-1). For pre-treatment cells with +this is the base period (g-anticipation-1). For pre-treatment cells with \code{base_period="varying"}, this is the pre-treatment period itself. The panel DRDID estimators are used.} \item{Repeated cross sections and unbalanced panels}{Both periods' @@ -133,7 +136,7 @@ Options: \describe{ \item{\code{NULL} (default)}{For balanced panel: uses the weight from the earlier of the two time periods in each 2x2 comparison. For -post-treatment cells, this is the base period (g-1). For +post-treatment cells, this is the base period (g-anticipation-1). For pre-treatment cells, this depends on the \code{base_period} setting. For RC/unbalanced panel: uses per-observation weights from both periods.} @@ -222,7 +225,7 @@ determines the estimates.} \item{base_period}{Whether to use a "varying" base period or a "universal" base period. Either choice results in the same post-treatment estimates of ATT(g,t)'s. In pre-treatment -periods, using a varying base period amounts to computing a +periods (t < g - anticipation), using a varying base period amounts to computing a pseudo-ATT in each treatment period by comparing the change in outcomes for a particular group relative to its comparison group in the pre-treatment periods (i.e., in pre-treatment diff --git a/man/conditional_did_pretest.Rd b/man/conditional_did_pretest.Rd index eda9face..91359a59 100644 --- a/man/conditional_did_pretest.Rd +++ b/man/conditional_did_pretest.Rd @@ -1,165 +1,170 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/conditional_did_pretest.R -\name{conditional_did_pretest} -\alias{conditional_did_pretest} -\title{Pre-Test of Conditional Parallel Trends Assumption} -\usage{ -conditional_did_pretest( - yname, - tname, - idname = NULL, - gname, - xformla = NULL, - data, - panel = TRUE, - allow_unbalanced_panel = FALSE, - control_group = c("nevertreated", "notyettreated"), - weightsname = NULL, - alp = 0.05, - bstrap = TRUE, - cband = TRUE, - biters = 1000, - clustervars = NULL, - est_method = "ipw", - print_details = FALSE, - pl = FALSE, - cores = 1 -) -} -\arguments{ -\item{yname}{The name of the outcome variable} - -\item{tname}{The name of the column containing the time periods} - -\item{idname}{The individual (cross-sectional unit) id name} - -\item{gname}{The name of the variable in \code{data} that -contains the first period when a particular observation is treated. -This should be a positive number for all observations in treated groups. -It defines which "group" a unit belongs to. It should be 0 for units -in the untreated group.} - -\item{xformla}{A formula for the covariates to include in the -model. It should be of the form \code{~ X1 + X2}. Default -is NULL which is equivalent to \code{xformla=~1}. This is -used to create a matrix of covariates which is then passed -to the 2x2 DID estimator chosen in \code{est_method}. - -For time-varying covariates: (1) With balanced panel data, -in each 2x2 comparison, the covariates -are taken to be the value of the covariates in the earlier time -period, and all of the underlying computations involve changes in Y -as a function of those values of covariates. (2) With repeated cross -sections data and unbalanced panel data, the covariates are taken -from each time period and computations involve Y_post conditional -on X_post minus Y_pre conditional on X_pre. A byproduct of this -is that, with balanced panel data and in the presence of -time-varying covariates, it is possible to get different numerical -results according to whether or not \code{allow_unbalanced_panel=TRUE} or -\code{FALSE}.} - -\item{data}{The name of the data.frame that contains the data} - -\item{panel}{Whether or not the data is a panel dataset. -The panel dataset should be provided in long format -- that -is, where each row corresponds to a unit observed at a -particular point in time. The default is TRUE. When -using a panel dataset, the variable \code{idname} must -be set. When \code{panel=FALSE}, the data is treated -as repeated cross sections.} - -\item{allow_unbalanced_panel}{Whether or not function should -"balance" the panel with respect to time and id. The default -value is \code{FALSE} which means that \code{\link[=att_gt]{att_gt()}} will drop -all units where data is not observed in all periods. -The advantage of this is that the computations are faster -(sometimes substantially). A cohort that loses all of its units -this way is dropped with a warning. This argument is ignored when -\code{panel=FALSE}: repeated cross sections have no panel structure -to balance.} - -\item{control_group}{Which units to use as the control group. -The default is "nevertreated" which sets the control group -to be the group of units that never participate in the -treatment. This group does not change across groups or -time periods. The other option is to set -\code{group="notyettreated"}. In this case, the control group -is set to the group of units that have not yet participated -in the treatment in that time period. This includes all -never treated units, but it includes additional units that -eventually participate in the treatment, but have not -participated yet. If there are no never-treated units (in the -data, or after the panel is balanced), the last treated cohort -serves as the comparison group and periods from its treatment -date (net of \code{anticipation}) onward are dropped, with a warning.} - -\item{weightsname}{The name of the column containing the sampling weights. -If not set, all observations have same weight.} - -\item{alp}{the significance level, default is 0.05} - -\item{bstrap}{Not used by the pre-test. Critical values and p-values -for the Cramer von Mises and Kolmogorov-Smirnov test statistics are -always computed with the multiplier bootstrap (using \code{biters} -iterations), regardless of this argument, and no standard errors are -reported.} - -\item{cband}{Boolean for whether or not to compute a uniform confidence -band that covers all of the group-time average treatment effects -with fixed probability \code{1-alp}. In order to compute uniform confidence -bands, \code{bstrap} must also be set to \code{TRUE}. The default is -\code{TRUE}. When the data contain only two time periods, uniform confidence -bands coincide with pointwise confidence intervals, so \code{cband} is set to -\code{FALSE} (with a message) and the pointwise critical value is reported.} - -\item{biters}{The number of multiplier bootstrap iterations used to -simulate the limiting distribution of the test statistics. The -default is 1000.} - -\item{clustervars}{A vector of variable names to cluster on (the -multiplier bootstrap then draws cluster-level multipliers). Clustering -at the individual (\code{idname}) level is automatic, so \code{idname} may be -included here but is redundant and is ignored; beyond \code{idname}, at most -one (coarser) clustering variable is supported (otherwise an error is -thrown).} - -\item{est_method}{the method to compute group-time average treatment -effects. The default for \code{conditional_did_pretest} is "ipw" for -inverse probability weighting. Other built-in methods include "dr" -which uses the doubly robust approach in the \code{DRDID} package and "reg" -for first step regression estimators.} - -\item{print_details}{Not used by the pre-test. Progress details are -suppressed during the pre-test computations regardless of this argument.} - -\item{pl}{Not used by the pre-test. The multiplier bootstrap used here -(\code{test.mboot}) is vectorized and runs in a single process; only \code{cores} -below has an effect.} - -\item{cores}{The number of cores to use for parallel processing. This -parallelizes Step 1 (computing the test statistic); Step 2's multiplier -bootstrap is vectorized and runs in a single process.} -} -\value{ -an \code{\link{MP.TEST}} object -} -\description{ -An integrated moments test for the conditional parallel trends -assumption holding in all pre-treatment time periods for all groups -} -\examples{ -\dontrun{ -data(mpdta) -pre.test <- conditional_did_pretest(yname="lemp", - tname="year", - idname="countyreal", - gname="first.treat", - xformla=~lpop, - data=mpdta) -summary(pre.test) -} - -} -\references{ -Callaway, Brantly and Sant'Anna, Pedro H. C. "Difference-in-Differences with Multiple Time Periods and an Application on the Minimum Wage and Employment." Working Paper \url{https://arxiv.org/abs/1803.09015v2} (2018). -} +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/conditional_did_pretest.R +\name{conditional_did_pretest} +\alias{conditional_did_pretest} +\title{Pre-Test of Conditional Parallel Trends Assumption} +\usage{ +conditional_did_pretest( + yname, + tname, + idname = NULL, + gname, + xformla = NULL, + data, + panel = TRUE, + allow_unbalanced_panel = FALSE, + control_group = c("nevertreated", "notyettreated"), + weightsname = NULL, + alp = 0.05, + bstrap = TRUE, + cband = TRUE, + biters = 1000, + clustervars = NULL, + est_method = "ipw", + print_details = FALSE, + pl = FALSE, + cores = 1 +) +} +\arguments{ +\item{yname}{The name of the outcome variable} + +\item{tname}{The name of the column containing the time periods} + +\item{idname}{The individual (cross-sectional unit) id name} + +\item{gname}{The name of the variable in \code{data} that +contains the first period when a particular observation is treated. +This should be a positive number for all observations in treated groups. +It defines which "group" a unit belongs to. It should be 0 for units +in the untreated group.} + +\item{xformla}{A formula such as \code{~ X1 + X2} specifying covariates for the +propensity score model and the indicator weighting functions. The default +\code{NULL} is equivalent to \code{~1}. For panel data, use conditioning variables +that are fixed within each unit across periods. If raw covariates vary +over time, create explicit unit-level baseline copies and include those +copies in \code{xformla}; the test then conditions on those fixed baseline +variables. For repeated cross sections (\code{panel = FALSE}), each observation's +observed covariates are used.} + +\item{data}{The name of the data.frame that contains the data} + +\item{panel}{Whether or not the data is a panel dataset. +The panel dataset should be provided in long format -- that +is, where each row corresponds to a unit observed at a +particular point in time. The default is TRUE. When +using a panel dataset, the variable \code{idname} must +be set. When \code{panel=FALSE}, the data is treated +as repeated cross sections.} + +\item{allow_unbalanced_panel}{Passed to panel preprocessing. The default +\code{FALSE} balances the panel by dropping units observed in only some periods. +With \code{TRUE}, an already balanced panel is accepted, but a genuinely +unbalanced panel is not supported by this conditional pre-test. Ignored +when \code{panel = FALSE}.} + +\item{control_group}{Which units to use as the control group. +The default is "nevertreated" which sets the control group +to be the group of units that never participate in the +treatment. This group does not change across groups or +time periods. The other option is to set +\code{group="notyettreated"}. In this case, the control group +is set to the group of units that have not yet participated +in the treatment in that time period. This includes all +never treated units, but it includes additional units that +eventually participate in the treatment, but have not +participated yet. If there are no never-treated units (in the +data, or after the panel is balanced), the last treated cohort +serves as the comparison group and periods from its treatment +date (net of \code{anticipation}) onward are dropped, with a warning.} + +\item{weightsname}{The name of the column containing the sampling weights. +If not set, all observations have same weight.} + +\item{alp}{the significance level, default is 0.05} + +\item{bstrap}{Not used by the pre-test. Critical values and p-values +for the Cramer von Mises and Kolmogorov-Smirnov test statistics are +always computed with the multiplier bootstrap (using \code{biters} +iterations), regardless of this argument, and no standard errors are +reported.} + +\item{cband}{Boolean for whether or not to compute a uniform confidence +band that covers all of the group-time average treatment effects +with fixed probability \code{1-alp}. In order to compute uniform confidence +bands, \code{bstrap} must also be set to \code{TRUE}. The default is +\code{TRUE}. When the data contain only two time periods, uniform confidence +bands coincide with pointwise confidence intervals, so \code{cband} is set to +\code{FALSE} (with a message) and the pointwise critical value is reported.} + +\item{biters}{The number of multiplier bootstrap iterations used to +simulate the limiting distribution of the test statistics. The +default is 1000.} + +\item{clustervars}{A vector of variable names to cluster on (the +multiplier bootstrap then draws cluster-level multipliers). Clustering +at the individual (\code{idname}) level is automatic, so \code{idname} may be +included here but is redundant and is ignored; beyond \code{idname}, at most +one (coarser) clustering variable is supported (otherwise an error is +thrown).} + +\item{est_method}{The estimation method for the conditional moments. +Only "ipw" (the default), for inverse probability weighting, is supported. +This test requires a correctly specified propensity score model. +Regression ("reg"), doubly robust ("dr"), and custom estimation methods +are not supported by this legacy test. For pre-treatment tests using +those estimation methods, see the tests reported by \code{\link[=att_gt]{att_gt()}}.} + +\item{print_details}{Not used by the pre-test. Progress details are +suppressed during the pre-test computations regardless of this argument.} + +\item{pl}{Not used by the pre-test. The multiplier bootstrap used here +(\code{test.mboot}) is vectorized and runs in a single process; only \code{cores} +below has an effect.} + +\item{cores}{The number of cores to use for parallel processing. This +parallelizes Step 1 (computing the test statistic); Step 2's multiplier +bootstrap is vectorized and runs in a single process.} +} +\value{ +an \code{\link{MP.TEST}} object +} +\description{ +An integrated moments test for the conditional parallel trends +assumption holding in all pre-treatment time periods for all groups +} +\details{ +The conditional moments use outcomes multiplied by covariate indicator +functions. A correctly specified outcome regression for the original +outcome need not remain correctly specified for every indicator-weighted +outcome. Consequently, refitting the original regression model to those +outcomes does not provide a valid regression or doubly robust conditional +test under correctness of the original outcome model alone. This function +implements the inverse probability weighted test only. + +For panel data, the indicator weighting functions use the covariate vector +from the first observed period. The conditional null is formulated using +a fixed vector of conditioning variables throughout the pre-treatment +comparisons. The legacy test is not justified as a test of separate +conditional nulls whose conditioning variables change with the baseline +period of each group-time comparison. Supplying time-varying raw panel +covariates does not establish that different interpretation; choose and +supply fixed baseline variables explicitly when that is the intended null. +} +\examples{ +\dontrun{ +data(mpdta) +pre.test <- conditional_did_pretest(yname="lemp", + tname="year", + idname="countyreal", + gname="first.treat", + xformla=~lpop, + data=mpdta) +summary(pre.test) +} + +} +\references{ +Callaway, Brantly and Sant'Anna, Pedro H. C. "Difference-in-Differences with Multiple Time Periods and an Application on the Minimum Wage and Employment." Working Paper \url{https://arxiv.org/abs/1803.09015v2} (2018). +} diff --git a/man/pre_process_did.Rd b/man/pre_process_did.Rd index 5c3bc64b..0b516fa1 100644 --- a/man/pre_process_did.Rd +++ b/man/pre_process_did.Rd @@ -100,7 +100,10 @@ date (net of \code{anticipation}) onward are dropped, with a warning.} \item{anticipation}{The number of time periods before participating in the treatment where units can anticipate participating in the -treatment and therefore it can affect their untreated potential outcomes} +treatment and therefore it can affect their untreated potential outcomes. +Periods \code{g - anticipation}, ..., \code{g - 1} use the base period +\code{g - anticipation - 1} (under either \code{base_period}) and are not used +in the pre-test of parallel trends.} \item{weightsname}{The name of the column containing the sampling weights. If not set, all observations have same weight. When weights are @@ -113,7 +116,7 @@ the default behavior differs by panel type: \describe{ \item{Balanced panel}{Each 2x2 DiD comparison uses the weight from the earlier of the two time periods involved. For post-treatment cells, -this is the base period (g-1). For pre-treatment cells with +this is the base period (g-anticipation-1). For pre-treatment cells with \code{base_period="varying"}, this is the pre-treatment period itself. The panel DRDID estimators are used.} \item{Repeated cross sections and unbalanced panels}{Both periods' @@ -132,7 +135,7 @@ Options: \describe{ \item{\code{NULL} (default)}{For balanced panel: uses the weight from the earlier of the two time periods in each 2x2 comparison. For -post-treatment cells, this is the base period (g-1). For +post-treatment cells, this is the base period (g-anticipation-1). For pre-treatment cells, this depends on the \code{base_period} setting. For RC/unbalanced panel: uses per-observation weights from both periods.} @@ -221,7 +224,7 @@ determines the estimates.} \item{base_period}{Whether to use a "varying" base period or a "universal" base period. Either choice results in the same post-treatment estimates of ATT(g,t)'s. In pre-treatment -periods, using a varying base period amounts to computing a +periods (t < g - anticipation), using a varying base period amounts to computing a pseudo-ATT in each treatment period by comparing the change in outcomes for a particular group relative to its comparison group in the pre-treatment periods (i.e., in pre-treatment diff --git a/man/pre_process_did2.Rd b/man/pre_process_did2.Rd index e59ea6b8..212028b4 100644 --- a/man/pre_process_did2.Rd +++ b/man/pre_process_did2.Rd @@ -100,7 +100,10 @@ date (net of \code{anticipation}) onward are dropped, with a warning.} \item{anticipation}{The number of time periods before participating in the treatment where units can anticipate participating in the -treatment and therefore it can affect their untreated potential outcomes} +treatment and therefore it can affect their untreated potential outcomes. +Periods \code{g - anticipation}, ..., \code{g - 1} use the base period +\code{g - anticipation - 1} (under either \code{base_period}) and are not used +in the pre-test of parallel trends.} \item{weightsname}{The name of the column containing the sampling weights. If not set, all observations have same weight. When weights are @@ -113,7 +116,7 @@ the default behavior differs by panel type: \describe{ \item{Balanced panel}{Each 2x2 DiD comparison uses the weight from the earlier of the two time periods involved. For post-treatment cells, -this is the base period (g-1). For pre-treatment cells with +this is the base period (g-anticipation-1). For pre-treatment cells with \code{base_period="varying"}, this is the pre-treatment period itself. The panel DRDID estimators are used.} \item{Repeated cross sections and unbalanced panels}{Both periods' @@ -132,7 +135,7 @@ Options: \describe{ \item{\code{NULL} (default)}{For balanced panel: uses the weight from the earlier of the two time periods in each 2x2 comparison. For -post-treatment cells, this is the base period (g-1). For +post-treatment cells, this is the base period (g-anticipation-1). For pre-treatment cells, this depends on the \code{base_period} setting. For RC/unbalanced panel: uses per-observation weights from both periods.} @@ -221,7 +224,7 @@ determines the estimates.} \item{base_period}{Whether to use a "varying" base period or a "universal" base period. Either choice results in the same post-treatment estimates of ATT(g,t)'s. In pre-treatment -periods, using a varying base period amounts to computing a +periods (t < g - anticipation), using a varying base period amounts to computing a pseudo-ATT in each treatment period by comparing the change in outcomes for a particular group relative to its comparison group in the pre-treatment periods (i.e., in pre-treatment diff --git a/tests/testthat/test-anticipation-window.R b/tests/testthat/test-anticipation-window.R new file mode 100644 index 00000000..d327f864 --- /dev/null +++ b/tests/testthat/test-anticipation-window.R @@ -0,0 +1,116 @@ +# Anticipation periods (g - anticipation <= t < g) use the base period +# g - anticipation - 1 under either base_period and are not used in the pre-test. +# Under base_period = "varying" they used t - 1 (so with anticipation >= 2 they were +# changes between two anticipation periods), and they entered the pre-test, which +# then rejected parallel trends because of the anticipation the user had declared. + +# true parallel trends; cohort g has anticipation effect a_eff in the delta periods +# before g and treatment effect tau from g on +make_antic <- function(seed, n_per, periods = 1:6, g = 4, delta = 2, a_eff = 1, tau = 2) { + set.seed(seed) + d <- expand.grid(id = seq_len(2 * n_per), time = periods) + d$g <- ifelse(d$id <= n_per, 0, g) + d$y <- rnorm(nrow(d)) + 0.5 * d$time + (d$id %% 7) / 10 + + ifelse(d$g == g & d$time >= g - delta & d$time < g, a_eff, 0) + ifelse(d$g == g & d$time >= g, tau, 0) + d +} +fit_a <- function(d, ...) suppressWarnings(suppressMessages( + att_gt(yname = "y", tname = "time", idname = "id", gname = "g", data = d, bstrap = FALSE, cband = FALSE, ...))) + +test_that("anticipation periods use the base period g - anticipation - 1 under both base periods", { + d <- make_antic(seed = 1, n_per = 2000) + for (delta in c(1, 2)) for (fm in c(TRUE, FALSE)) { + v <- fit_a(d, anticipation = delta, base_period = "varying", faster_mode = fm) + u <- fit_a(d, anticipation = delta, base_period = "universal", faster_mode = fm) + wv <- which(v$t >= v$group - delta); wu <- which(u$t >= u$group - delta) + expect_gte(length(wv), 3L) + expect_identical(v$t[wv], u$t[wu]) + expect_equal(v$att[wv], u$att[wu], tolerance = 1e-10) + expect_equal(v$se[wv], u$se[wu], tolerance = 1e-10) + } + # anticipation effect 1 in periods 2-3, treatment effect 2 from period 4 + v <- fit_a(d, anticipation = 2) + expect_lt(abs(v$att[v$t == 2] - 1), 0.2) + expect_lt(abs(v$att[v$t == 3] - 1), 0.2) # was ~0: the change between two anticipation periods + for (tt in 4:6) expect_lt(abs(v$att[v$t == tt] - 2), 0.2) +}) + +test_that("the pre-test uses only cells with t < g - anticipation", { + # periods 1:7, cohort 5: anticipation 2 leaves the single pre-treatment cell (5,2) + d <- make_antic(seed = 2, n_per = 500, periods = 1:7, g = 5, delta = 2) + for (fm in c(TRUE, FALSE)) { + f2 <- fit_a(d, anticipation = 2, faster_mode = fm) + i <- which(f2$group == 5 & f2$t == 2) + expect_length(as.numeric(f2$W), 1L) + expect_equal(as.numeric(f2$W), (f2$att[i] / f2$se[i])^2, tolerance = 1e-8) + expect_identical(as.numeric(f2$Wpval), round(1 - pchisq(as.numeric(f2$W), df = 1), 5)) + f0 <- fit_a(d, anticipation = 0, faster_mode = fm) + expect_length(as.numeric(f0$W), 1L) + expect_identical(as.numeric(f0$Wpval), round(1 - pchisq(as.numeric(f0$W), df = 3), 5)) + } + # universal base: the normalized base cell (att 0, se NA) is not a pre-treatment estimate + w <- capture_warnings(suppressMessages( + fu <- att_gt(yname = "y", tname = "time", idname = "id", gname = "g", data = make_antic(3, 200, periods = 1:6, g = 4), + anticipation = 2, base_period = "universal", bstrap = FALSE, cband = FALSE))) + expect_null(fu$W) + expect_true(any(grepl("No pre-treatment periods available", w))) + expect_false(any(grepl("missing or zero variance", w))) +}) + +test_that("declared anticipation no longer makes the pre-test reject true parallel trends", { + # periods 1:8, cohort 6, anticipation in periods 4-5: the pre-test uses (6,2), (6,3) + p_declared <- vapply(1:20, function(s) fit_a(make_antic(s, 300, periods = 1:8, g = 6), anticipation = 2)$Wpval, numeric(1)) + expect_lte(mean(p_declared < 0.05), 0.3) # was 20/20 + # the same effects, not declared, are pre-trend violations and are still detected + p_ignored <- vapply(1:20, function(s) fit_a(make_antic(s, 300, periods = 1:8, g = 6), anticipation = 0)$Wpval, numeric(1)) + expect_gte(mean(p_ignored < 0.05), 0.7) +}) + +test_that("the pretest-only cell filter follows the same rule", { + fit <- fit_a(make_antic(4, 100, periods = 1:7, g = 5), anticipation = 2, faster_mode = FALSE) + dp <- fit$DIDparams + dp$pretreatment_cells_only <- TRUE + cells <- suppressWarnings(did:::compute.att_gt(dp))$attgt.list + expect_gte(length(cells), 1L) + expect_true(all(vapply(cells, function(x) x$year < x$group - 2, logical(1)))) +}) + +test_that("event-study aggregation reports the anticipation effects; the overall ATT averages t >= g only", { + d <- make_antic(seed = 4, n_per = 2000) + for (fm in c(TRUE, FALSE)) { + fit <- fit_a(d, anticipation = 2, faster_mode = fm) + dyn <- suppressWarnings(aggte(fit, type = "dynamic", cband = FALSE)) + expect_true(all(c(-2, -1, 0, 1, 2) %in% dyn$egt)) + expect_lt(abs(dyn$att.egt[dyn$egt == -1] - 1), 0.2) + expect_lt(abs(dyn$overall.att - 2), 0.2) + expect_lt(abs(suppressWarnings(aggte(fit, type = "simple", cband = FALSE))$overall.att - 2), 0.2) + } +}) + +test_that("anticipation = 0 keeps the plain definitions (pseudo-ATTs with base t - 1; pre-test over t < g)", { + d <- make_antic(seed = 5, n_per = 400, delta = 0, a_eff = 0) + for (fm in c(TRUE, FALSE)) { + v <- fit_a(d, anticipation = 0, base_period = "varying", faster_mode = fm) + u <- fit_a(d, anticipation = 0, base_period = "universal", faster_mode = fm) + expect_false(isTRUE(all.equal(v$att[v$t == 2], u$att[u$t == 2]))) + expect_equal(v$att[v$t >= 4], u$att[u$t >= 4], tolerance = 1e-10) + expect_length(as.numeric(v$W), 1L) + expect_identical(as.numeric(v$Wpval), round(1 - pchisq(as.numeric(v$W), df = sum(v$t < v$group)), 5)) + } +}) + +test_that("universal-base diagnostics use the actual base on irregular grids", { + d <- expand.grid(id = seq_len(40), time = c(1, 3, 4)) + d$g <- ifelse(d$id <= 20, 0, 3) + d$y <- d$time + sin(d$id * d$time) + for (fm in c(TRUE, FALSE)) { + warnings <- capture_warnings(suppressMessages( + out <- att_gt("y", "time", "id", "g", data = d, + base_period = "universal", faster_mode = fm, + est_method = "reg", bstrap = FALSE, cband = FALSE) + )) + expect_null(out$W) + expect_true(any(grepl("No pre-treatment periods available", warnings))) + expect_false(any(grepl("missing or zero variance", warnings))) + } +}) diff --git a/tests/testthat/test-conditional-ipw-only.R b/tests/testthat/test-conditional-ipw-only.R new file mode 100644 index 00000000..14641aab --- /dev/null +++ b/tests/testthat/test-conditional-ipw-only.R @@ -0,0 +1,103 @@ +test_that("conditional pre-test rejects unsupported methods before processing data", { + methods <- list("reg", "dr", function(...) stop("must not be called"), + "unknown", c("ipw", "dr"), NA_character_, NULL) + for (method in methods) { + expect_error(conditional_did_pretest( + yname = "Y", tname = "period", idname = "id", gname = "G", + data = stop("must not process data for an unsupported method"), + est_method = method + ), 'only supports est_method = "ipw"', fixed = TRUE) + } +}) + +test_that("correct original outcome regression cannot validate the legacy regression test", { + # Conditional trends are exactly X in both groups. A correct linear model + # for that change becomes misspecified after multiplication by 1{X <= 0}. + x <- c(-1, 0, 1) + controls <- c(20, 80, 20) + treated <- c(80, 20, 80) + X <- cbind(1, x) + beta <- stats::lm.wfit(X, x, controls)$coefficients + h <- as.numeric(x <= 0) + original_residual_moment <- stats::weighted.mean(h * (x - X %*% beta), treated) + transformed_beta <- stats::lm.wfit(X, h * x, controls)$coefficients + transformed_moment <- stats::weighted.mean(h * x - X %*% transformed_beta, treated) + expect_equal(as.numeric(beta), c(0, 1), tolerance = 1e-12) + expect_equal(original_residual_moment, 0, tolerance = 1e-12) + expect_equal(transformed_moment, -5 / 18, tolerance = 1e-12) + + units <- data.frame(X = rep(x, each = 100), + G = c(rep(4, 80), rep(0, 20), rep(4, 20), rep(0, 80), + rep(4, 80), rep(0, 20)), id = seq_len(300)) + d <- merge(units, data.frame(period = 1:4)) + d$Y <- d$period * d$X + (d$G == 4 & d$period >= 4) + for (method in c("reg", "dr")) { + # Those methods remain supported by att_gt(), where the ORIGINAL linear + # outcome model correctly yields zero pre-effects and a unit post-effect. + fit <- suppressWarnings(att_gt(yname = "Y", tname = "period", + idname = "id", gname = "G", data = d, xformla = ~X, + est_method = method, bstrap = FALSE, cband = FALSE)) + expect_equal(fit$att, c(0, 0, 1), tolerance = 1e-12) + expect_error(conditional_did_pretest(yname = "Y", tname = "period", + idname = "id", gname = "G", data = d, xformla = ~X, + est_method = method), 'only supports est_method = "ipw"', fixed = TRUE) + } +}) + +test_that("default and explicit IPW conditional pre-tests remain identical", { + set.seed(490) + d <- expand.grid(id = seq_len(45), period = seq_len(4)) + d$G <- rep(c(0, 3, 4), each = 15)[d$id] + d$X <- rnorm(45)[d$id] + d$Y <- d$period + d$X + rnorm(nrow(d)) + + (d$G > 0 & d$period >= d$G) + run <- function(panel, method = NULL) { + set.seed(491) + args <- list(yname = "Y", tname = "period", gname = "G", data = d, + panel = panel, idname = if (panel) "id" else NULL, + xformla = ~X, biters = 31) + if (!is.null(method)) args$est_method <- method + invisible(capture.output(answer <- suppressMessages( + do.call(conditional_did_pretest, args)))) + answer + } + for (panel in c(TRUE, FALSE)) { + default <- run(panel) + explicit <- run(panel, "ipw") + expect_identical(default$CvM, explicit$CvM) + expect_identical(default$CvMb, explicit$CvMb) + expect_identical(default$CvMcval, explicit$CvMcval) + expect_identical(default$CvMpval, explicit$CvMpval) + } +}) + +test_that("conditional pre-test uses the resolved panel structure", { + set.seed(490) + d <- expand.grid(id = seq_len(45), period = seq_len(4)) + d$G <- rep(c(0, 3, 4), each = 15)[d$id] + d$X <- rnorm(45)[d$id] + d$Y <- d$period + d$X + rnorm(nrow(d)) + + (d$G > 0 & d$period >= d$G) + run <- function(panel, allow) { + set.seed(491) + invisible(capture.output(answer <- suppressMessages( + conditional_did_pretest(yname = "Y", tname = "period", gname = "G", + data = d, panel = panel, idname = if (panel) "id" else NULL, + allow_unbalanced_panel = allow, xformla = ~X, biters = 31)))) + answer + } + for (panel in c(TRUE, FALSE)) { + # A balanced panel resolves TRUE to FALSE; genuine cross sections ignore + # it. Neither case should be mistaken for an unsupported incomplete panel. + expected <- run(panel, FALSE) + actual <- run(panel, TRUE) + for (field in c("CvM", "CvMb", "CvMcval", "CvMpval")) { + expect_identical(actual[[field]], expected[[field]]) + } + } + d <- d[-1L, ] + expect_error(suppressMessages(conditional_did_pretest( + yname = "Y", tname = "period", idname = "id", gname = "G", + data = d, allow_unbalanced_panel = TRUE, xformla = ~X, biters = 31)), + "conditional pre-test is not supported for unbalanced panel data") +}) diff --git a/tests/testthat/test-inference-outcome-units.R b/tests/testthat/test-inference-outcome-units.R new file mode 100644 index 00000000..8e07f05d --- /dev/null +++ b/tests/testthat/test-inference-outcome-units.R @@ -0,0 +1,176 @@ +.inference_units_data <- function() { + set.seed(730) + n <- 120L + d <- expand.grid(id = seq_len(n), period = seq_len(4L)) + d$G <- rep(rep(c(0, 3, 4), each = n / 3), 4L) + d$cluster <- (d$id - 1L) %% 20L + 1L + d$X <- stats::rnorm(n)[d$id] + d$weight <- exp(0.2 * d$X) + shock <- matrix(stats::rnorm(20L * 4L), 20L, 4L) + d$Y <- stats::rnorm(n)[d$id] + 0.3 * d$period + 0.2 * d$X * d$period + + shock[cbind(d$cluster, d$period)] + stats::rnorm(nrow(d)) + + (d$G > 0 & d$period >= d$G) + d +} + +.inference_units_fit <- function(d, fm, bootstrap, clustered = FALSE, ...) { + set.seed(731) + suppressWarnings(suppressMessages(att_gt( + "Y", "period", "id", "G", data = d, xformla = ~X, + weightsname = "weight", faster_mode = fm, bstrap = bootstrap, + cband = bootstrap, biters = 199L, + clustervars = if (clustered) "cluster" else NULL, ... + ))) +} + +.inference_units_aggregate <- function(mp, type) { + set.seed(732) + suppressWarnings(aggte(mp, type = type)) +} + +test_that("outcome units preserve SEs, Wald tests, and simultaneous bands", { + d <- .inference_units_data() + for (fm in c(TRUE, FALSE)) for (bootstrap in c(FALSE, TRUE)) { + for (clustered in c(FALSE, TRUE)) { + ref <- .inference_units_fit(d, fm, bootstrap, clustered) + expect_true(all(is.finite(ref$se) & ref$se > 0)) + if (!bootstrap || !clustered) expect_false(is.null(ref$W)) + if (bootstrap) { + set.seed(731) + direct <- mboot(ref$inffunc, ref$DIDparams, return_V = FALSE) + expect_equal(ref$c, direct$crit.val, tolerance = 1e-12) + } + types <- c("simple", "dynamic", "group", "calendar") + ref_agg <- lapply(types, function(type) .inference_units_aggregate(ref, type)) + + for (scale in c(1e-12, 1e-6, 1e6)) { + scaled_data <- d + scaled_data$Y <- d$Y * scale + out <- .inference_units_fit(scaled_data, fm, bootstrap, clustered) + expect_equal(out$att / scale, ref$att, tolerance = 1e-9) + expect_equal(out$se / scale, ref$se, tolerance = 1e-9) + expect_equal(out$c, ref$c, tolerance = 1e-9) + expect_equal(out$W, ref$W, tolerance = 1e-9) + expect_equal(out$Wpval, ref$Wpval) + expect_equal(out$DIDparams$cband, ref$DIDparams$cband) + + for (j in seq_along(types)) { + a <- .inference_units_aggregate(out, types[j]) + expect_true(is.finite(a$overall.se) && a$overall.se > 0) + expect_equal(a$overall.att / scale, ref_agg[[j]]$overall.att, + tolerance = 1e-9) + expect_equal(a$overall.se / scale, ref_agg[[j]]$overall.se, + tolerance = 1e-9) + if (types[j] != "simple") { + expect_equal(a$att.egt / scale, ref_agg[[j]]$att.egt, + tolerance = 1e-9) + expect_equal(a$se.egt / scale, ref_agg[[j]]$se.egt, + tolerance = 1e-9) + expect_equal(a$crit.val.egt, ref_agg[[j]]$crit.val.egt, + tolerance = 1e-9) + } + } + } + } + } +}) + +test_that("one low-variance cohort is not filtered relative to other cohorts", { + d <- .inference_units_data() + set.seed(733) + noise_scale <- ifelse(d$G == 4, 1, 1e-11) + d$Y <- stats::rnorm(120L)[d$id] + noise_scale * stats::rnorm(nrow(d)) + + (d$G > 0 & d$period >= d$G) + for (fm in c(TRUE, FALSE)) { + out <- .inference_units_fit(d, fm, FALSE) + small <- out$se[out$group == 3] + large <- out$se[out$group == 4] + expect_true(all(is.finite(small) & small > 0)) + expect_lt(max(small), 1e-10) + expect_gt(min(large), 1e-2) + } +}) + +test_that("exactly zero influence functions remain unavailable in panels and RC", { + d <- .inference_units_data() + d$Y <- 0 + zero_estimator <- function(D, ...) { + list(ATT = 0, att = 0, att.inf.func = rep(0, length(D))) + } + for (fm in c(TRUE, FALSE)) for (panel in c(TRUE, FALSE)) { + dd <- d + if (!panel) dd$id <- seq_len(nrow(dd)) + for (method in list("dr", "reg", "ipw", zero_estimator)) { + out <- .inference_units_fit(dd, fm, TRUE, panel = panel, est_method = method) + expect_true(all(out$att == 0)) + expect_true(all(as.matrix(out$inffunc) == 0)) + expect_true(all(is.na(out$se))) + expect_null(out$W) + expect_false(out$DIDparams$cband) + expect_equal(out$c, stats::qnorm(0.975)) + for (type in c("simple", "dynamic", "group", "calendar")) { + a <- suppressWarnings(aggte(out, type = type, cband = TRUE)) + expect_equal(a$overall.att, 0) + expect_true(is.na(a$overall.se)) + if (type != "simple") { + expect_true(all(is.na(a$se.egt))) + expect_false(a$DIDparams$cband) + expect_equal(a$crit.val.egt, stats::qnorm(0.975)) + } + } + } + } +}) + +test_that("non-finite custom influence functions do not produce finite inference", { + d <- .inference_units_data() + invalid_if <- function(D, ...) { + list(ATT = 0, att = 0, att.inf.func = rep(Inf, length(D))) + } + for (fm in c(TRUE, FALSE)) for (bootstrap in c(FALSE, TRUE)) { + out <- .inference_units_fit(d, fm, bootstrap, est_method = invalid_if) + expect_true(all(is.na(out$se))) + expect_null(out$W) + expect_true(is.finite(out$c)) + if (bootstrap) expect_false(out$DIDparams$cband) + a <- suppressWarnings(aggte(out, type = "simple")) + expect_true(is.na(a$overall.se)) + } +}) + +test_that("zero-IQR bootstrap draws are not reported as zero standard errors", { + dp <- list(idname = "id", clustervars = NULL, biters = 1L, + tname = "period", alp = 0.05, panel = TRUE, + allow_unbalanced_panel = FALSE, faster_mode = FALSE) + set.seed(734) + out <- mboot(matrix(stats::rnorm(20L), ncol = 1L), dp) + expect_true(is.na(out$se)) + expect_true(is.na(out$crit.val)) +}) + +test_that("a single bootstrap draw retains columns and cannot estimate their scales", { + dp <- list(idname = "id", clustervars = NULL, biters = 1L, + tname = "period", alp = 0.05, panel = TRUE, + allow_unbalanced_panel = FALSE, faster_mode = FALSE) + x <- cbind(seq_len(20), 2 * seq_len(20), 3 * seq_len(20), 0, Inf, NA_real_) + set.seed(834) + out <- mboot(x, dp) + expect_identical(dim(out$bres), c(1L, 3L)) + expect_length(out$se, ncol(x)) + expect_true(all(is.na(out$se))) + expect_true(is.na(out$crit.val)) + + d <- .inference_units_data() + for (fm in c(TRUE, FALSE)) { + set.seed(835) + out <- suppressWarnings(att_gt( + "Y", "period", "id", "G", data = d, xformla = ~X, + faster_mode = fm, bstrap = TRUE, cband = TRUE, biters = 1L + )) + expect_true(all(is.na(out$se))) + expect_false(out$DIDparams$cband) + expect_equal(out$c, stats::qnorm(0.975)) + # The analytical covariance still supports the separate Wald pre-test. + expect_true(is.finite(as.numeric(out$W))) + } +}) diff --git a/tests/testthat/test-preprocess-name-collisions.R b/tests/testthat/test-preprocess-name-collisions.R new file mode 100644 index 00000000..4128c140 --- /dev/null +++ b/tests/testthat/test-preprocess-name-collisions.R @@ -0,0 +1,109 @@ +# Used data columns may share names with local preprocessing variables. +.name_collision_fit <- function(d, names, faster = TRUE, ...) { + suppressWarnings(suppressMessages(att_gt( + yname = names[["lemp"]], tname = names[["year"]], + idname = names[["countyreal"]], gname = names[["first.treat"]], + xformla = reformulate(names[["lpop"]]), data = d, + bstrap = FALSE, cband = FALSE, faster_mode = faster, ... + ))) +} + +test_that("fast preprocessing respects columns named args and tlist", { + data(mpdta, package = "did") + column_names <- setNames(names(mpdta), names(mpdta)) + ref <- .name_collision_fit(mpdta, column_names) + for (local_name in c("args", "tlist")) { + for (column in c("lemp", "year", "countyreal", "first.treat", "lpop")) { + d <- mpdta + names(d)[names(d) == column] <- local_name + renamed <- column_names + renamed[[column]] <- local_name + actual <- .name_collision_fit(d, renamed) + expect_equal(actual$att, ref$att, tolerance = 1e-10) + expect_equal(actual$se, ref$se, tolerance = 1e-10) + } + } +}) + +test_that("argument name collisions work across preprocessing branches", { + data(mpdta, package = "did") + column_names <- setNames(names(mpdta), names(mpdta)) + for (structure in c("unbalanced", "rc", "no_never", "balance_drop", "missing")) { + d <- mpdta + extra <- list() + if (structure == "unbalanced") { + d <- d[-1L, ] + extra$allow_unbalanced_panel <- TRUE + } else if (structure == "rc") { + d$countyreal <- seq_len(nrow(d)) + extra$panel <- FALSE + } else if (structure == "no_never") { + d <- d[d$first.treat > 0, ] + } else if (structure == "balance_drop") { + d <- d[!(d$first.treat == 0 & d$year == min(d$year)), ] + } else { + d$lemp[1L] <- Inf + } + ref <- do.call(.name_collision_fit, c(list(d, column_names), extra)) + for (column in c("year", "countyreal", "first.treat", "lpop")) { + renamed_data <- d + names(renamed_data)[names(renamed_data) == column] <- "args" + renamed <- column_names + renamed[[column]] <- "args" + actual <- do.call(.name_collision_fit, c(list(renamed_data, renamed), extra)) + expect_equal(actual$att, ref$att, tolerance = 1e-10) + expect_equal(actual$se, ref$se, tolerance = 1e-10) + } + } +}) + +test_that("group size checks use counts even if the cohort is named V1", { + data(mpdta, package = "did") + ids <- unique(mpdta$countyreal[mpdta$first.treat == 0]) + d <- mpdta[mpdta$first.treat > 0 | mpdta$countyreal %in% ids[1:3], ] + names(d)[names(d) == "first.treat"] <- "V1" + expect_error(suppressWarnings(att_gt( + "lemp", "year", "countyreal", "V1", xformla = ~lpop, + data = d, bstrap = FALSE, cband = FALSE + )), "never-treated group is too small") +}) + + +test_that("args can name sampling weights or a cluster column", { + data(mpdta, package = "did") + column_names <- setNames(names(mpdta), names(mpdta)) + for (role in c("weightsname", "clustervars")) { + for (structure in c("balanced", "unbalanced", "rc")) { + d <- mpdta + d$sample_var <- if (role == "weightsname") + 1 + d$countyreal %% 5 else d$countyreal %% 31 + extra <- setNames(list("sample_var"), role) + if (structure == "unbalanced") { + d <- d[-1L, ] + extra$allow_unbalanced_panel <- TRUE + } else if (structure == "rc") { + d$countyreal <- seq_len(nrow(d)) + extra$panel <- FALSE + } + ref <- do.call(.name_collision_fit, c(list(d, column_names), extra)) + names(d)[names(d) == "sample_var"] <- "args" + extra[[role]] <- "args" + actual <- do.call(.name_collision_fit, c(list(d, column_names), extra)) + expect_equal(actual$att, ref$att, tolerance = 1e-10) + expect_equal(actual$se, ref$se, tolerance = 1e-10) + } + } +}) + + +test_that("last-cohort filtering uses the cutoff argument rather than a covariate", { + data(mpdta, package = "did") + d <- mpdta[mpdta$first.treat > 0, ] + column_names <- setNames(names(d), names(d)) + ref <- .name_collision_fit(d, column_names) + names(d)[names(d) == "lpop"] <- "cutoff_t" + column_names[["lpop"]] <- "cutoff_t" + actual <- .name_collision_fit(d, column_names) + expect_equal(actual$att, ref$att, tolerance = 1e-10) + expect_equal(actual$se, ref$se, tolerance = 1e-10) +}) diff --git a/tests/testthat/test-rc-normalized-support.R b/tests/testthat/test-rc-normalized-support.R new file mode 100644 index 00000000..4e8a9688 --- /dev/null +++ b/tests/testthat/test-rc-normalized-support.R @@ -0,0 +1,138 @@ +test_that("RC support counts use the weights after DRDID normalization", { + D <- c(0, 0, 1, 1) + post <- c(0, 1, 0, 1) + for (rescale in c(1, 2^-900)) { + w <- c(2^1000, 2^1000, 2^-100, 2^-100) * rescale + expect_true(all(is.finite(w) & w > 0)) + expect_identical(w / mean(w), c(2, 2, 0, 0)) + expect_true(rc_outcome_has_too_few_obs(D, post, w, 1L)) + # Regression-only needs the two controls, which still have positive mass. + expect_false(rc_outcome_has_too_few_obs(D, post, w, 1L, FALSE)) + expect_true(rc_outcome_has_too_few_obs(D, post, rev(w), 1L, FALSE)) + } + # Do not impose a positive-weight tolerance: the smallest subnormal still + # counts, while an exactly rounded zero does not. No native fit is invoked. + retained <- c(2^1000, 2^1000, 2^-75, 2^-75) + lost <- c(2^1000, 2^1000, 2^-76, 2^-76) + expect_identical((retained / mean(retained))[3], 2^-1074) + expect_false(rc_outcome_has_too_few_obs(D, post, retained, 1L)) + expect_identical((lost / mean(lost))[3], 0) + expect_true(rc_outcome_has_too_few_obs(D, post, lost, 1L)) + for (level in c(2^-1000, 1, 2^1000)) { + expect_false(rc_outcome_has_too_few_obs(D, post, rep(level, 4), 1L)) + } + for (w in list(numeric(), rep(0, 4), rep(Inf, 4), rep(NA_real_, 4))) { + expect_true(rc_outcome_has_too_few_obs(D[seq_along(w)], post[seq_along(w)], + w, 1L)) + } +}) + +test_that("normalization loss is rejected before DR/reg backend entry", { + # Enough physical rows for the existing global cohort checks and every OR. + d <- expand.grid(j = 1:16, G = c(0, 2, 3), period = 1:3) + d$id <- seq_len(nrow(d)) + d$Y <- 0.2 * d$period + (d$G > 0 & d$period >= d$G) + 0.1 * sin(d$id) + originals <- list(dr = DRDID::drdid_rc, reg = DRDID::reg_did_rc) + entries <- list() + make_backend <- function(method) { + force(method) + function(y, post, D, covariates, i.weights, ...) { + normalized <- i.weights / mean(i.weights) + required <- if (method == "dr") rep(TRUE, length(D)) else D == 0 + lost <- any(!is.finite(normalized[required]) | normalized[required] <= 0) + entries[[length(entries) + 1L]] <<- lost + # Safely fail on a missing preflight. Never run a vulnerable native fit. + if (lost) stop("normalized support loss reached the backend") + originals[[method]](y = y, post = post, D = D, + covariates = covariates, i.weights = i.weights, ...) + } + } + dr_backend <- make_backend("dr") + reg_backend <- make_backend("reg") + local_mocked_bindings(drdid_rc = dr_backend, reg_did_rc = reg_backend, + .package = "DRDID") + local_mocked_bindings(drdid_rc = dr_backend, reg_did_rc = reg_backend, + .package = "did") + + for (small_group in c(2, 0)) { + for (method in if (small_group == 2) "dr" else c("dr", "reg")) { + for (rescale in c(1, 2^-900)) { + d$w <- ifelse(d$G == small_group, 2^-100, 2^1000) * rescale + expect_true(all(is.finite(d$w) & d$w > 0)) + for (mode in c(FALSE, TRUE)) { + for (do_inf in c(FALSE, TRUE)) { + warnings <- capture_warnings(r <- suppressMessages(att_gt( + yname = "Y", tname = "period", idname = "id", gname = "G", + xformla = ~1, data = d, panel = FALSE, weightsname = "w", + est_method = method, faster_mode = mode, compute_inffunc = do_inf, + bstrap = FALSE, cband = FALSE))) + failed <- if (small_group == 2) r$group == 2 else rep(TRUE, length(r$att)) + expect_equal(sum(grepl("fewer positive-weight observations", warnings)), + sum(failed)) + expect_false(any(grepl("reached the backend", warnings))) + expect_identical(is.na(r$att), failed) + expect_true(all(is.finite(r$att[!failed]))) + if (do_inf) { + expect_identical(is.na(r$se), failed) + expect_true(all(is.finite(r$se[!failed]))) + expect_true(all(is.na(as.matrix(r$inffunc)[, failed]))) + } else expect_null(r$inffunc) + } + } + } + } + } + expect_length(entries, 16L) + expect_false(any(unlist(entries))) +}) + +test_that("a second local normalization is checked on every RC route", { + d <- expand.grid(j = 1:16, period = 1:4, D = 0:1) + d$G <- 2 * d$D + d$Y <- d$period + d$w <- ifelse(d$D == 0 & d$period %in% 1:2, 6, 2^-1074) + # Global mean1.5 leaves the smallest subnormal positive. In the period1/2 + # cell the normalized weights have mean2, so local division rounds it to0. + expect_identical(mean(d$w), 1.5) + expect_true(all(d$w / mean(d$w) > 0)) + seen_times <- numeric() + backend <- function(y, post, ...) { + seen_times <<- c(seen_times, unique(y[post == 1])) + # All cells stay behind a safe probe; this fixture estimates no ATT. + stop("safe non-target backend probe") + } + local_mocked_bindings(drdid_rc = backend, .package = "DRDID") + local_mocked_bindings(drdid_rc = backend, .package = "did") + for (route in c("rc", "unbalanced", "varying")) { + dat <- d + dat$id <- if (route == "rc") seq_len(nrow(dat)) else 1000 * (dat$D + 1) + dat$j + if (route == "unbalanced") { + extra <- dat[dat$D == 1 & dat$period == 4 & dat$j == 16, ] + extra$id <- 3000 + dat <- rbind(dat, extra) + } + for (mode in c(FALSE, TRUE)) { + for (do_inf in c(FALSE, TRUE)) { + seen_times <- numeric() + warnings <- capture_warnings(r <- suppressMessages(att_gt( + yname = "Y", tname = "period", idname = "id", gname = "G", + xformla = ~1, data = dat, panel = route != "rc", weightsname = "w", + est_method = "dr", faster_mode = mode, compute_inffunc = do_inf, + allow_unbalanced_panel = route == "unbalanced", + fix_weights = if (route == "varying") "varying" else NULL, + bstrap = FALSE, cband = FALSE))) + expect_equal(seen_times, c(3, 4)) + expect_true(any(grepl(paste0("fewer positive-weight observations.*", + "group 2 in time period 2"), warnings))) + col <- which(r$t == 2) + expect_true(is.na(r$att[col])) + if (do_inf) { + expect_true(is.na(r$se[col])) + expect_true(all(is.na(r$inffunc[, col]))) + expect_equal(nrow(r$inffunc), if (route == "rc") nrow(dat) else + length(unique(dat$id))) + } else expect_null(r$inffunc) + } + } + } +}) diff --git a/tests/testthat/test-stress-balance-support.R b/tests/testthat/test-stress-balance-support.R new file mode 100644 index 00000000..31d45599 --- /dev/null +++ b/tests/testthat/test-stress-balance-support.R @@ -0,0 +1,100 @@ +.balance_support_data <- function(periods = 1:5, groups = c(0, 2, 3)) { + set.seed(711) + n <- 180L + d <- expand.grid(id = seq_len(n), t = periods) + d$g <- rep(rep(groups, each = n / length(groups)), length(periods)) + d$y <- 0.1 * d$t + ifelse(d$g > 0 & d$t >= d$g, d$g, 0) + + stats::rnorm(nrow(d), sd = 0.1) + d +} + +.balance_support_fit <- function(d, faster_mode) { + suppressWarnings(suppressMessages(att_gt( + "y", "t", "id", "g", data = d, faster_mode = faster_mode, + est_method = "reg", allow_unbalanced_panel = TRUE, + bstrap = FALSE, cband = FALSE + ))) +} + +test_that("balance_e retains the same cohorts after missing cells are removed", { + d <- .balance_support_data() + d <- d[!(d$g == 2 & d$t == 3), ] + for (fm in c(TRUE, FALSE)) { + mp <- .balance_support_fit(d, fm) + expect_true(is.na(mp$att[mp$group == 2 & mp$t == 3])) + a <- aggte(mp, "dynamic", balance_e = 1, min_e = 0, na.rm = TRUE) + k <- vapply(0:1, function(e) which(mp$group == 3 & mp$t == 3 + e), integer(1)) + expect_equal(a$egt, 0:1) + expect_equal(unname(a$att.egt), mp$att[k], tolerance = 1e-10) + expect_equal(unname(a$se.egt), mp$se[k], tolerance = 1e-10) + expect_equal(unname(a$inf.function$dynamic.inf.func.e), + unname(as.matrix(mp$inffunc[, k])), tolerance = 1e-10) + + # Restricting the displayed window does not redefine which cohorts have + # complete support through balance_e: g=2 is still excluded at e=0. + a0 <- aggte(mp, "dynamic", balance_e = 1, min_e = 0, max_e = 0, + na.rm = TRUE) + expect_equal(a0$att.egt, mp$att[k[1]], tolerance = 1e-10) + } +}) + +test_that("balance_e requires common event-time support on irregular grids", { + d <- .balance_support_data(c(1, 2, 4, 5), c(0, 2, 4)) + for (fm in c(TRUE, FALSE)) { + mp <- .balance_support_fit(d, fm) + a <- aggte(mp, "dynamic", balance_e = 1, min_e = 0) + # g=2 reaches the horizon but has no e=1 cell. Only g=4 contributes. + k <- vapply(0:1, function(e) which(mp$group == 4 & mp$t == 4 + e), integer(1)) + expect_equal(a$egt, 0:1) + expect_equal(unname(a$att.egt), mp$att[k], tolerance = 1e-10) + } +}) + +test_that("balance_e uses observed support without inventing absent event times", { + d <- .balance_support_data(c(1, 3, 5), c(0, 3, 5)) + for (fm in c(TRUE, FALSE)) { + mp <- .balance_support_fit(d, fm) + a <- aggte(mp, "dynamic", balance_e = 2, min_e = 0) + k <- vapply(c(0, 2), function(e) which(mp$group == 3 & mp$t == 3 + e), integer(1)) + expect_equal(a$egt, c(0, 2)) + expect_equal(unname(a$att.egt), mp$att[k], tolerance = 1e-10) + } +}) + +test_that("balance_e errors when no cohort has complete observed support", { + d <- .balance_support_data() + d <- d[!(d$g == 2 & d$t == 3) & !(d$g == 3 & d$t == 3), ] + for (fm in c(TRUE, FALSE)) { + mp <- .balance_support_fit(d, fm) + expect_error(aggte(mp, "dynamic", balance_e = 1, min_e = 0, na.rm = TRUE), + "No groups have non-missing ATT") + expect_error(aggte(mp, "dynamic", balance_e = 10, na.rm = TRUE), + "No groups have non-missing ATT") + } +}) + +test_that("an event time with all missing estimates still has to be balanced", { + d <- .balance_support_data() + d <- d[!(d$g == 2 & d$t == 3) & !(d$g == 3 & d$t == 4), ] + for (fm in c(TRUE, FALSE)) { + mp <- .balance_support_fit(d, fm) + k <- which(mp$t - mp$group == 1) + expect_equal(length(k), 2) + expect_true(all(is.na(mp$att[k]))) + expect_error(aggte(mp, "dynamic", balance_e = 1, min_e = 0, na.rm = TRUE), + "No groups have non-missing ATT") + } +}) + +test_that("balance_e preserves the complete regularly spaced aggregation", { + d <- .balance_support_data() + for (fm in c(TRUE, FALSE)) { + mp <- .balance_support_fit(d, fm) + a <- aggte(mp, "dynamic", balance_e = 1, min_e = 0) + for (e in 0:1) { + k <- which(mp$t - mp$group == e) + # The two treated cohorts have equal population shares. + expect_equal(a$att.egt[a$egt == e], mean(mp$att[k]), tolerance = 1e-10) + } + } +}) diff --git a/tests/testthat/test-stress-bootstrap-scale.R b/tests/testthat/test-stress-bootstrap-scale.R new file mode 100644 index 00000000..90e26349 --- /dev/null +++ b/tests/testthat/test-stress-bootstrap-scale.R @@ -0,0 +1,54 @@ +test_that("bootstrap degeneracy does not depend on the units of the outcome", { + set.seed(501) + inf <- cbind(matrix(rnorm(120 * 3), nrow = 120), 0, NA_real_) + dp <- list(idname = "id", clustervars = NULL, biters = 199, + tname = "period", alp = 0.05, panel = TRUE, + allow_unbalanced_panel = FALSE, faster_mode = FALSE) + for (clustered in c(FALSE, TRUE)) { + dp$clustervars <- if (clustered) "cluster" else NULL + dp$cluster_vector <- if (clustered) rep(seq_len(30), each = 4) else NULL + set.seed(502) + reference <- mboot(inf, dp) + expect_true(all(is.finite(reference$se[1:3]))) + expect_true(all(is.na(reference$se[4:5]))) + for (scale in c(1e-6, 1e-100, 1e6)) { + set.seed(502) + scaled <- mboot(inf * scale, dp) + expect_identical(dim(scaled$bres), dim(reference$bres)) + expect_equal(scaled$se / scale, reference$se, tolerance = 1e-12) + expect_equal(scaled$bres / scale, reference$bres, tolerance = 1e-12) + expect_equal(scaled$crit.val, reference$crit.val, tolerance = 1e-12) + } + } +}) + +test_that("rescaling outcomes preserves att_gt and aggte bootstrap inference", { + set.seed(813) + d <- expand.grid(id = seq_len(120), t = seq_len(4)) + d$g <- rep(c(0, 2, 3, 4), each = 30)[d$id] + d$y <- 0.1 * d$t + (d$g > 0 & d$t >= d$g) * d$g + rnorm(nrow(d)) + scale <- 3e-6 + for (faster in c(TRUE, FALSE)) { + run <- function(multiplier) { + dd <- d + dd$y <- dd$y * multiplier + set.seed(123) + att_gt("y", "t", "id", "g", data = dd, bstrap = TRUE, + cband = TRUE, biters = 1000, est_method = "reg", + faster_mode = faster) + } + reference <- run(1) + scaled <- run(scale) + # These SEs exceed the separate near-zero-SE cutoff in att_gt/aggte. + expect_true(all(reference$se * scale > sqrt(.Machine$double.eps) * 10)) + expect_equal(scaled$att / scale, reference$att, tolerance = 1e-10) + expect_equal(scaled$se / scale, reference$se, tolerance = 1e-10) + expect_equal(scaled$c, reference$c, tolerance = 1e-10) + set.seed(503) + a <- aggte(reference, "simple") + set.seed(503) + b <- aggte(scaled, "simple") + expect_gt(a$overall.se * scale, sqrt(.Machine$double.eps) * 10) + expect_equal(b$overall.se / scale, a$overall.se, tolerance = 1e-10) + } +}) diff --git a/tests/testthat/test-stress-pretest-defaults.R b/tests/testthat/test-stress-pretest-defaults.R new file mode 100644 index 00000000..d0088b20 --- /dev/null +++ b/tests/testthat/test-stress-pretest-defaults.R @@ -0,0 +1,44 @@ +test_that("conditional pre-test resolves omitted covariates and observation ids", { + set.seed(401) + d <- expand.grid(id = seq_len(45), period = seq_len(4)) + d$G <- rep(c(0, 3, 4), each = 15)[d$id] + d$X <- rnorm(45)[d$id] + d$Y <- d$period + d$X + rnorm(nrow(d)) + + (d$G > 0 & d$period >= d$G) + + run <- function(panel, idname, xformla) { + set.seed(402) + suppressMessages(capture.output( + ans <- conditional_did_pretest( + yname = "Y", tname = "period", gname = "G", data = d, + panel = panel, idname = idname, xformla = xformla, biters = 31 + ) + )) + ans + } + + explicit <- run(TRUE, "id", ~1) + default <- run(TRUE, "id", NULL) + expect_equal(default$CvM, explicit$CvM) + expect_equal(default$CvMb, explicit$CvMb) + expect_equal(default$CvMpval, explicit$CvMpval) + + # With an intercept-only weighting function every indicator is one, so the + # conditional statistic reduces to n times the squared pre-treatment ATTs. + fit <- att_gt(yname = "Y", tname = "period", idname = "id", gname = "G", + data = d, est_method = "ipw", bstrap = FALSE, cband = FALSE) + expect_equal(default$CvM, + fit$n * sum(fit$att[fit$t < fit$group]^2)) + + # Treat these observations as independent cross sections; a per-row id and + # the internally generated id must give the same statistic and bootstrap. + d$row_id <- seq_len(nrow(d)) + for (formula in list(~1, ~X)) { + explicit <- run(FALSE, "row_id", formula) + default <- run(FALSE, NULL, formula) + expect_gt(explicit$CvM, 0) + expect_equal(default$CvM, explicit$CvM) + expect_equal(default$CvMb, explicit$CvMb) + expect_equal(default$CvMpval, explicit$CvMpval) + } +}) diff --git a/tests/testthat/test-tiny-rc-outcome-designs.R b/tests/testthat/test-tiny-rc-outcome-designs.R new file mode 100644 index 00000000..24259cf6 --- /dev/null +++ b/tests/testthat/test-tiny-rc-outcome-designs.R @@ -0,0 +1,183 @@ +# Issue #64: the default RC DR estimator needs treated outcome regressions. +# Its old regression test lost the default-DR call after a native fastglm crash +# (a811d46). Use deterministic data and verify rejection BEFORE that backend. +tiny_rc_data <- function(n_treated = 3L) { + cube <- as.matrix(expand.grid(x1 = c(-1, 1), x2 = c(-1, 1), + x3 = c(-1, 1))) + regular <- cube[rep(1:8, 2), ] + tiny <- if (n_treated == 3L) { + rbind(c(-1, -1, -1), c(1, 1, -1), c(0, 0, 1)) + } else if (n_treated == 4L) { + rbind(c(0, 0, 0), c(1, 0, 0), c(0, 1, 0), c(0, 0, 1)) + } else { + regular + } + cells <- lapply(1:3, function(t) { + do.call(rbind, lapply(c(0, 2, 3), function(g) { + x <- if (g == 2) tiny else regular + data.frame(period = t, G = g, X = x[, 1], X2 = x[, 2], X3 = x[, 3]) + })) + }) + d <- do.call(rbind, cells) + d$id <- seq_len(nrow(d)) + d$w <- 1 + d$Y <- 0.2 * d$period + 0.3 * d$X - 0.1 * d$X2 + 0.2 * d$X3 + + (d$G > 0 & d$period >= d$G) + 0.1 * sin(d$id) + d +} + +fit_tiny_rc <- function(d, mode, ...) { + suppressMessages(att_gt(yname = "Y", tname = "period", idname = "id", + gname = "G", xformla = ~X + X2 + X3, data = d, panel = FALSE, + faster_mode = mode, bstrap = FALSE, cband = FALSE, ...)) +} + +test_that("small RC treated groups are rejected before the DR backend", { + d <- tiny_rc_data() + original <- DRDID::drdid_rc + calls <- integer() + backend <- function(y, post, D, ...) { + calls <<- c(calls, sum(D)) + # A failed preflight must never reach an unsafe native fit, even on an old + # dependency. Fail safely here so this regression test cannot segfault R. + if (sum(D) == 6L) stop("undersized treated regression reached the backend") + original(y = y, post = post, D = D, ...) + } + local_mocked_bindings(drdid_rc = backend, .package = "DRDID") + local_mocked_bindings(drdid_rc = backend, .package = "did") + + results <- lapply(c(FALSE, TRUE), function(mode) { + w <- capture_warnings(r <- fit_tiny_rc(d, mode, est_method = "dr")) + failures <- grep("fewer positive-weight observations than covariates", w, + value = TRUE) + expect_length(failures, 2L) + expect_true(any(grepl("group 2 in time period 2", failures))) + expect_true(any(grepl("group 2 in time period 3", failures))) + expect_false(any(grepl("reached the backend", w))) + expect_identical(is.na(r$att), r$group == 2) + expect_identical(is.na(r$se), r$group == 2) + expect_true(all(is.na(as.matrix(r$inffunc)[, r$group == 2]))) + expect_true(all(is.finite(r$att[r$group == 3]))) + expect_true(all(is.finite(r$se[r$group == 3]))) + expect_true(all(is.finite(as.matrix(r$inffunc)[, r$group == 3]))) + r + }) + expect_equal(calls, rep(32L, 4)) + expect_equal(results[[1]]$att, results[[2]]$att, tolerance = 1e-10) + expect_equal(results[[1]]$se, results[[2]]$se, tolerance = 1e-10) +}) + +test_that("RC methods without treated outcome regressions retain small groups", { + d <- tiny_rc_data() + for (mode in c(FALSE, TRUE)) { + for (method in list("reg", "ipw", DRDID::drdid_rc1)) { + w <- capture_warnings(r <- fit_tiny_rc(d, mode, est_method = method)) + expect_false(any(grepl("fewer positive-weight observations", w))) + expect_true(all(is.finite(r$att))) + expect_true(all(is.finite(r$se))) + expect_true(all(is.finite(r$inffunc))) + } + } +}) + +test_that("zero-weight rows cannot identify RC treated outcome regressions", { + d <- tiny_rc_data(16L) + for (t in 1:3) { + rows <- which(d$G == 2 & d$period == t) + d$w[rows[-(1:3)]] <- 0 + } + for (mode in c(FALSE, TRUE)) { + w <- capture_warnings(r <- fit_tiny_rc(d, mode, est_method = "dr", + weightsname = "w")) + expect_equal(sum(grepl("fewer positive-weight observations", w)), 2L) + expect_identical(is.na(r$att), r$group == 2) + expect_identical(is.na(r$se), r$group == 2) + expect_true(all(is.finite(r$att[r$group == 3]))) + } +}) + +test_that("square full-rank RC treated regressions are allowed", { + d <- tiny_rc_data(4L) + for (mode in c(FALSE, TRUE)) { + w <- capture_warnings(r <- fit_tiny_rc(d, mode, est_method = "dr")) + expect_false(any(grepl("fewer positive-weight observations", w))) + expect_true(all(is.finite(r$att))) + expect_true(all(is.finite(r$se))) + expect_true(all(is.finite(r$inffunc))) + + # A direct 2x2 reference fixes both the estimator and IF normalization for + # an estimable cell, rather than only comparing fast and slow code paths. + cell <- d$G %in% c(0, 2) & d$period %in% 1:2 + ref <- DRDID::drdid_rc(d$Y[cell], as.numeric(d$period[cell] == 2), + as.numeric(d$G[cell] == 2), model.matrix(~X + X2 + X3, d[cell, ]), + inffunc = TRUE) + col <- which(r$group == 2 & r$t == 2) + expect_equal(r$att[col], ref$ATT, tolerance = 1e-10) + expected_if <- numeric(nrow(d)) + expected_if[cell] <- nrow(d) / sum(cell) * ref$att.inf.func + # The modes use different row orders; compare by sampling-unit id. + aligned_if <- expected_if[match(rownames(r$inffunc), as.character(d$id))] + expect_equal(as.numeric(r$inffunc[, col]), aligned_if, tolerance = 1e-10) + expect_equal(r$se[col], sqrt(mean(expected_if^2) / nrow(d)), + tolerance = 1e-10) + } +}) + +test_that("undersized control-period regressions fail only affected RC cells", { + d <- tiny_rc_data(16L) + rows <- which(d$G == 0 & d$period == 1) + d$w[rows[-(1:3)]] <- 0 + for (mode in c(FALSE, TRUE)) { + for (method in c("dr", "reg")) { + w <- capture_warnings(r <- fit_tiny_rc(d, mode, est_method = method, + weightsname = "w")) + # Group 2 uses period 1 as its base in both post-treatment cells; + # group 3 uses it only in its period-2 placebo cell. + failed <- r$group == 2 | r$t == 2 + expect_equal(sum(grepl("fewer positive-weight observations", w)), 3L) + expect_identical(is.na(r$att), failed) + expect_identical(is.na(r$se), failed) + expect_true(all(is.finite(r$att[!failed]))) + expect_true(all(is.finite(r$se[!failed]))) + } + } +}) + +test_that("unbalanced and varying-weight panel RC routes check treated regressions", { + d <- tiny_rc_data() + d$id <- (d$G + 1) * 1000 + ave(d$id, d$G, d$period, FUN = seq_along) + d$w <- 1 + d$period / 10 + for (mode in c(FALSE, TRUE)) { + for (route in c("unbalanced", "varying")) { + dat <- if (route == "unbalanced") { + d[-which(d$G == 3 & d$period == 1)[1], ] + } else d + for (do_inf in c(TRUE, FALSE)) { + w <- capture_warnings(r <- suppressMessages(att_gt(yname = "Y", + tname = "period", idname = "id", gname = "G", xformla = ~X + X2 + X3, + data = dat, panel = TRUE, faster_mode = mode, bstrap = FALSE, + cband = FALSE, weightsname = "w", est_method = "dr", + allow_unbalanced_panel = route == "unbalanced", + fix_weights = if (route == "varying") "varying" else NULL, + compute_inffunc = do_inf))) + expect_equal(sum(grepl("fewer positive-weight observations", w)), 2L) + expect_identical(is.na(r$att), r$group == 2) + expect_true(all(is.finite(r$att[r$group == 3]))) + if (do_inf) { + expect_identical(is.na(r$se), r$group == 2) + expect_true(all(is.finite(r$se[r$group == 3]))) + expect_true(all(is.na(as.matrix(r$inffunc)[, r$group == 2]))) + } else expect_null(r$inffunc) + } + } + + # Ordinary balanced-panel DR only fits a control outcome regression. + w <- capture_warnings(r <- suppressMessages(att_gt(yname = "Y", + tname = "period", idname = "id", gname = "G", xformla = ~X + X2 + X3, + data = d, panel = TRUE, faster_mode = mode, bstrap = FALSE, + cband = FALSE, weightsname = "w", est_method = "dr"))) + expect_false(any(grepl("fewer positive-weight observations", w))) + expect_true(all(is.finite(r$att))) + expect_true(all(is.finite(r$se))) + } +}) diff --git a/tests/testthat/test-user_bug_fixes.R b/tests/testthat/test-user_bug_fixes.R index 0d4df872..36f7f46f 100644 --- a/tests/testthat/test-user_bug_fixes.R +++ b/tests/testthat/test-user_bug_fixes.R @@ -50,27 +50,6 @@ test_that("missing covariates", { }) -test_that("repeated cross sections small groups with covariates", { - # from https://github.com/bcallaway11/did/issues/64 - sp <- did::reset.sim(time.periods=3) - data <- build_sim_dataset(sp, panel=FALSE) - data$X2 <- rnorm(nrow(data)) - data$X3 <- rnorm(nrow(data)) - dropids <- unique(subset(data, G==2)$id) - # interestingly, this seems to return all NA's (which "works") - # if you only keep one or two observations - dropids <- dropids[-c(1,2,3)] # keep three observations from group 2 - data <- subset(data, !(id %in% dropids)) - - expect_warning(res_dr <- att_gt(yname="Y", xformla=~X+X2+X3, data=data, tname="period", idname="id", - gname="G", est_method=DRDID::drdid_rc1, panel=FALSE), "very few observations") - - expect_true(is.numeric(res_dr$se[1])) - - skip("known bug, code crashes in this case, fix is probably in DRDID package") -}) - - test_that("fewer time periods than groups", { # from https://github.com/bcallaway11/did/issues/56 # not sure if this is actually a bug, diff --git a/tests/testthat/test-zero-weight-guards.R b/tests/testthat/test-zero-weight-guards.R new file mode 100644 index 00000000..41098a03 --- /dev/null +++ b/tests/testthat/test-zero-weight-guards.R @@ -0,0 +1,123 @@ +# A zero-weight observation does not constrain a weighted 2x2 estimator's +# overlap or regression design, even when its covariates are extreme. +zero_weight_guard_data <- function() { + set.seed(50) + units <- data.frame(id = 1:100, G = rep(c(0, 3), each = 50), + X = rnorm(100), w = 1, a = rnorm(100)) + d <- merge(units, data.frame(period = 1:4)) + d$Y <- d$a + d$period * (0.2 + 0.3 * d$X) + rnorm(nrow(d)) + + 2 * (d$G == 3 & d$period >= 3) + d +} + +test_that("zero-weight covariates cannot cause false overlap or rank failures", { + d <- zero_weight_guard_data() + for (method in c("reg", "dr", "ipw")) { + # A zero-weight control spoils the unweighted control Gram matrix; a + # zero-weight treated observation spoils the preliminary overlap logit. + extra <- data.frame(id = 101, G = if (method == "reg") 0 else 3, + X = if (method == "reg") 1e12 else 1000, + w = 0, a = 0, period = 1:4, Y = 0) + for (structure in c("panel", "varying", "rc", "unbalanced")) { + original <- d + if (structure == "unbalanced") original <- original[-seq(17, nrow(d), 17), ] + augmented <- rbind(original, extra) + if (structure == "rc") { + original$id <- seq_len(nrow(original)) + augmented$id <- seq_len(nrow(augmented)) + } + for (faster in c(TRUE, FALSE)) { + run <- function(dat) suppressMessages(att_gt( + yname = "Y", tname = "period", idname = "id", gname = "G", + xformla = ~X, data = dat, weightsname = "w", est_method = method, + panel = structure != "rc", allow_unbalanced_panel = structure == "unbalanced", + fix_weights = if (structure == "varying") "varying" else NULL, + faster_mode = faster, bstrap = FALSE, cband = FALSE)) + expected <- run(original) + expect_warning(actual <- run(augmented), NA) + expect_true(all(is.finite(actual$att))) + expect_equal(actual$att, expected$att, tolerance = 1e-10, + info = paste(method, structure, faster)) + } + } + } +}) + +test_that("zero-weight guard fix also applies without positional precompute or guard cache", { + d <- zero_weight_guard_data() + extra <- data.frame(id = 101, G = 3, X = 1000, w = 0, + a = 0, period = 1:4, Y = 0) + withr::local_options(list(did.disable_precompute = TRUE, did.disable_check_cache = TRUE)) + for (faster in c(TRUE, FALSE)) { + run <- function(dat) att_gt(yname = "Y", tname = "period", idname = "id", + gname = "G", xformla = ~X, data = dat, weightsname = "w", + faster_mode = faster, bstrap = FALSE, cband = FALSE) + expected <- run(d) + expect_warning(actual <- run(rbind(d, extra)), NA) + expect_equal(actual$att, expected$att, tolerance = 1e-10) + # Independently assemble the post-treatment (g=3,t=3) 2x2 estimator. + z <- rbind(d, extra) + pre <- z[z$period == 2, ] + post <- z[z$period == 3, ] + pre <- pre[order(pre$id), ] + post <- post[order(post$id), ] + reference <- DRDID::drdid_panel(y1 = post$Y, y0 = pre$Y, + D = as.numeric(pre$G == 3), covariates = model.matrix(~X, pre), + i.weights = pre$w, inffunc = TRUE) + expect_equal(actual$att[actual$t == 3], reference$ATT, tolerance = 1e-10) + } +}) + +test_that("guard support follows normalization and still rejects unidentified designs", { + X <- cbind(1, c(-1, 0, 1, 1000)) + D <- c(0, 1, 0, 1) + w <- c(1e200, 1e200, 1e200, 1e-200) + # The last positive raw weight becomes zero in the estimator's normalization. + expect_identical(did:::overlap_check_fail(X, D, i.weights = w), + did:::overlap_check_fail(X[1:3, ], D[1:3])) + expect_true(did:::overlap_check_fail(matrix(1, 3, 1), c(0, 1, 1), + intercept_only = TRUE, i.weights = c(0, 1, 1))) + expect_true(did:::rcond_check_fail(matrix(1, 3, 1), c(0, 1, 1), + intercept_only = TRUE, i.weights = c(0, 1, 1))) + expect_true(did:::overlap_check_fail(matrix(1, 3, 1), c(0, 1, 1), + intercept_only = TRUE, i.weights = rep(0, 3))) + expect_true(did:::rcond_check_fail(matrix(1, 3, 1), c(0, 1, 1), + intercept_only = TRUE, i.weights = rep(0, 3))) +}) + +test_that("regression feasibility uses the actual weighted control design", { + d <- zero_weight_guard_data() + extra <- data.frame(id = 101, G = 0, X = 1e12, w = 1e-24, + a = 0, period = 1:4, Y = 0) + z <- rbind(d, extra) + pre <- z[z$period == 2, ] + post <- z[z$period == 3, ] + pre <- pre[order(pre$id), ] + post <- post[order(post$id), ] + reference <- DRDID::reg_did_panel(y1 = post$Y, y0 = pre$Y, + D = as.numeric(pre$G == 3), covariates = model.matrix(~X, pre), + i.weights = pre$w, inffunc = TRUE) + for (faster in c(TRUE, FALSE)) { + expect_warning(actual <- att_gt(yname = "Y", tname = "period", + idname = "id", gname = "G", xformla = ~X, data = z, + weightsname = "w", est_method = "reg", faster_mode = faster, + bstrap = FALSE, cband = FALSE), NA) + expect_equal(actual$att[actual$t == 3], reference$ATT, tolerance = 1e-10) + } + + # Conversely, the unweighted control design is full-rank but its second + # covariate direction has negligible weight in the actual regression. + X <- cbind(1, c(-1, 1, -1, 1, 0, 0)) + D <- c(0, 0, 0, 0, 1, 1) + w <- c(1, 1e-20, 1, 1e-20, 1, 1) + expect_false(did:::rcond_check_fail(X, D)) + expect_true(did:::rcond_check_fail(X, D, i.weights = w)) + for (scale in c(1e-100, 1, 1e100)) { + expect_identical(did:::rcond_check_fail(X, D, i.weights = w * scale), TRUE) + expect_identical(did:::rcond_check_fail(X, D, i.weights = rep(scale, 6)), FALSE) + } + # The weighted regression itself has the same numerical feasibility limit. + expect_error(DRDID::reg_did_panel(y1 = 1:6, y0 = rep(0, 6), D = D, + covariates = X, i.weights = w, inffunc = TRUE), + "singular|coefficients have NA") +})