diff --git a/.github/RELEASING.md b/.github/RELEASING.md index 35a3419..d61c401 100644 --- a/.github/RELEASING.md +++ b/.github/RELEASING.md @@ -31,6 +31,26 @@ first: an interrupted upload is completed only if its existing artifact matches. A reused version with changed source, unexpected build configuration or yanked files fails visibly and needs a new version. +After an upload, a separate clean checkout of the tested commit verifies the +public wheel and source archive again, including source, metadata and PyPI +hashes. Their filenames, hashes and download URLs must also appear in PyPI's +public Simple index. This catches a stale preflight 404 followed by an upload +that skipped an already existing artifact with different contents. It also +waits for the release to become visible to ordinary package installers. +The existing-release skip path performs the same bounded public confirmation in +the read-only gate, so a later scheduled retry can recover from an earlier +visibility timeout without a build, upload or publishing token. + +Missing or partially visible publication data can retry six times, with waits +of 5, 10, 20, 40 and 60 seconds (135 seconds total backoff). Visible source, +metadata, hash or index identity mismatches fail immediately. The verification +step has an eight-minute hard timeout including network calls. A timeout or +mismatch fails the publisher even when PyPI accepted the upload; it never +deletes or replaces immutable files. Scheduled retries can finish a matching +partial release after visibility recovers. Client-side checks cannot make the +preflight, upload and public index update one atomic transaction, or prevent a +later owner action from changing publication availability. + ## One-time PyPI configuration The PyPI `csdid` project must register this GitHub Trusted Publisher: diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 81b8528..276ac5b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -53,7 +53,7 @@ directory, and `FASTER_MODE.md` for scope and reproduction instructions. ### Key Conventions - Version is defined in `csdid/_version.py` as `__version`; update it there for releases. -- The README is auto-generated: edit `README.qmd` (Quarto), not `readme.md` directly. CI renders and commits the output on push to `main`. +- The README is auto-generated: edit `README.qmd` (Quarto), not `README.md` directly. CI renders it on pushes to `main` and proposes a PR for generated changes. Keep badges, examples and citation instructions in `README.qmd`; do not create a second case-variant README file. - PyPI publishing is automated via CI on push to `main` using `python setup.py sdist` + Twine. ## Subprojects diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index cb4f6e2..ca506fb 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -41,6 +41,11 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 + - name: Check README source and generated content + run: | + python scripts/check_readme.py + python -m unittest discover -s scripts/tests -p 'test_*.py' + - name: Read reviewed Python DRDID source id: peer run: | diff --git a/.github/workflows/gen_readme.yml b/.github/workflows/gen_readme.yml index 7aff770..b64cf59 100644 --- a/.github/workflows/gen_readme.yml +++ b/.github/workflows/gen_readme.yml @@ -6,51 +6,106 @@ on: - main workflow_dispatch: +concurrency: + group: readme-generation-${{ github.ref }} + cancel-in-progress: true + jobs: CreateQuarto: - runs-on: ubuntu-latest + # Manual runs on other branches validate a preview; only main publishes PRs. + runs-on: ubuntu-24.04 + timeout-minutes: 15 permissions: - # `contents: write` lets the action create the branch; `pull-requests: write` - # lets it open the PR. We no longer push to `main` directly because `main` - # is a protected branch (direct pushes are rejected with GH006). contents: write pull-requests: write + defaults: + run: + working-directory: csdid + env: + MPLBACKEND: Agg + PYTHONHASHSEED: '0' + OPENBLAS_NUM_THREADS: '1' + OMP_NUM_THREADS: '1' + MKL_NUM_THREADS: '1' steps: - - name: Chequear el código - uses: actions/checkout@v4 + - name: Check out the source being documented + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + path: csdid + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: csdid/docs/requirements-readme.txt + + - name: Check README source content + run: python scripts/check_readme.py --source-only + + - name: Read the reviewed Python DRDID source + id: peer + run: >- + python scripts/check_release_ci.py lock + --lock csdid/test_csdid/joint_sync/python_drdid.json + + - name: Check out the reviewed Python DRDID + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ steps.peer.outputs.repository }} + ref: ${{ steps.peer.outputs.commit }} + path: python-drdid + persist-credentials: false - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@v2 - - - name: Configurar Python - uses: actions/setup-python@v5 with: - python-version: "3.10" - cache: "pip" + version: '1.8.25' - - name: Install csdid and dependencies + - name: Install the checked-out package and rendering dependencies run: | - pip uninstall -y csdid || true - pip install git+https://github.com/d2cml-ai/csdid/ - pip install nbclient nbformat PyYAML - pip install git+https://github.com/d2cml-ai/DRDID - pip install -r requirements.txt + python -m venv "$RUNNER_TEMP/readme-venv" + echo "$RUNNER_TEMP/readme-venv/bin" >> "$GITHUB_PATH" + echo "QUARTO_PYTHON=$RUNNER_TEMP/readme-venv/bin/python" >> "$GITHUB_ENV" + "$RUNNER_TEMP/readme-venv/bin/python" -m pip install \ + -r docs/requirements-readme.txt "$GITHUB_WORKSPACE/python-drdid" . + "$RUNNER_TEMP/readme-venv/bin/python" -m pip check - - name: Generar quarto doc - run: quarto render README.qmd + - name: Render twice and reject unstable output + run: | + quarto render README.qmd + cp README.md "$RUNNER_TEMP/README-first.md" + find README_files/figure-commonmark -type f -name '*.png' -print0 \ + | sort -z | xargs -0 sha256sum > "$RUNNER_TEMP/readme-first.sha256" + quarto render README.qmd + cmp "$RUNNER_TEMP/README-first.md" README.md + find README_files/figure-commonmark -type f -name '*.png' -print0 \ + | sort -z | xargs -0 sha256sum > "$RUNNER_TEMP/readme-second.sha256" + cmp "$RUNNER_TEMP/readme-first.sha256" "$RUNNER_TEMP/readme-second.sha256" + + - name: Check generated content and regression guards + run: | + python scripts/check_readme.py + python -m unittest discover -s scripts/tests -p 'test_*.py' - name: Open a PR with the regenerated README + if: github.ref == 'refs/heads/main' uses: peter-evans/create-pull-request@v7 with: + path: csdid token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "docs: update generated README" - title: "docs: update generated README" + add-paths: | + README.md + README_files/figure-commonmark/*.png + commit-message: 'docs: update generated README' + title: 'docs: update generated README' body: | - Automated regeneration of `readme.md` from `README.qmd` by the - **Actualizar Readme** workflow. + Automated regeneration of `README.md` from `README.qmd` using + the checked-out package, pinned rendering dependencies, and a fixed + example seed. Content guards and two identical renders passed. - Merge this PR to publish the updated README. If there are no changes - to the rendered README, no PR is created. + Edit `README.qmd` to change authored documentation. This PR contains + only the rendered README and its generated figures. branch: auto/update-readme delete-branch: true author: "github-actions[bot] " diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 43e18d4..007a70c 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -54,6 +54,7 @@ jobs: - name: Skip an identical release or wait for its backend if: steps.verify.outputs.ready == 'true' id: release + timeout-minutes: 8 working-directory: source run: python3 scripts/check_release_ci.py release @@ -88,6 +89,7 @@ jobs: - name: Recheck release state after any environment delay if: steps.verify.outputs.ready == 'true' id: release + timeout-minutes: 8 working-directory: source run: python3 scripts/check_release_ci.py release @@ -146,9 +148,25 @@ jobs: - name: Publish to PyPI if: steps.final-ci.outputs.ready == 'true' + id: publish uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 with: packages-dir: source/dist/ # Safely finish a partial upload. The gate verifies existing artifacts # instead of silently accepting changed code under an existing version. skip-existing: true + + - name: Check out clean reviewed source for public verification + if: steps.publish.outcome == 'success' + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ needs.gate.outputs.sha }} + path: publication-source + persist-credentials: false + + - name: Verify public artifacts and installer index after upload + if: steps.publish.outcome == 'success' + # Includes network time, in addition to the bounded 135s of backoff. + timeout-minutes: 8 + working-directory: publication-source + run: python3 scripts/check_release_ci.py published diff --git a/MANIFEST.in b/MANIFEST.in index 7b5044a..f9e2cb2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include readme.md +include README.md include LICENSE include CHANGELOG_v0.3.0.md include CHANGELOG_v0.4.3.md diff --git a/README.md b/README.md index 1289a7d..bd375ec 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,16 @@ # Difference in Difference in Python -[![PyPI version](https://img.shields.io/pypi/v/csdid.svg?color=blue)](https://pypi.org/project/csdid/) -[![Downloads](https://static.pepy.tech/personalized-badge/csdid?period=total&units=international_system&left_color=blue&right_color=grey&left_text=Downloads)](https://pepy.tech/project/csdid) -[![Last commit](https://img.shields.io/github/last-commit/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/commits/main) -[![GitHub stars](https://img.shields.io/github/stars/d2cml-ai/csdid.svg?style=social)](https://github.com/d2cml-ai/csdid/stargazers) -[![GitHub issues](https://img.shields.io/github/issues/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/issues) -[![License](https://img.shields.io/github/license/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/blob/main/LICENSE) +[![PyPI +version](https://img.shields.io/pypi/v/csdid.svg?color=blue)](https://pypi.org/project/csdid/) +Downloads +[![Last +commit](https://img.shields.io/github/last-commit/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/commits/main) +[![GitHub +stars](https://img.shields.io/github/stars/d2cml-ai/csdid.svg?style=social)](https://github.com/d2cml-ai/csdid/stargazers) +[![GitHub +issues](https://img.shields.io/github/issues/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/issues) +[![License](https://img.shields.io/github/license/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/blob/main/LICENSE) The **csdid** package contains tools for computing average treatment effect parameters in a Difference-in-Differences setup allowing for @@ -43,12 +47,12 @@ There has been some recent work on DiD with multiple time periods. The “Difference-in-Differences with Multiple Time Periods.” Journal of Econometrics, Vol. 225, No. 2, pp. 200-230, 2021.](https://doi.org/10.1016/j.jeconom.2020.12.001) or - \[arXiv\](https://arxiv.org/abs/1803.09015 + [arXiv](https://arxiv.org/abs/1803.09015). This project is based on the original [did R package](https://github.com/bcallaway11/did). -## Instalation +## Installation You can install **csdid** from `pypi` with: @@ -60,10 +64,12 @@ or via github: ### Dependencies -Additionally, I have created an additional library called `drdid`, which -can be installed via GitHub. - - pip install git+https://github.com/d2cml-ai/DRDID +The `drdid` dependency is installed automatically with `csdid`. CSDID +0.4.5 requires the reviewed DRDID 1.1.7 release. The Python backend is +maintained in [DRDIDpy](https://github.com/d2cml-ai/DRDIDpy); +development checkouts use the exact peer revision recorded in +[`python_drdid.json`](csdid/test_csdid/joint_sync/python_drdid.json). +Update both packages together using their matching reviewed versions. ## Basic Example @@ -75,18 +81,21 @@ from [Callaway and Sant’Anna - [More detailed examples are also available](https://bcallaway11.github.io/did/articles/did-basics.html) -A subset of the data is available in the package and can be loaded by +A subset of the data is available at [data/mpdta.csv](data/mpdta.csv). +Run the following examples from the repository root: ``` python from csdid.att_gt import ATTgt +import numpy as np import pandas as pd -data = pd.read_csv("https://raw.githubusercontent.com/d2cml-ai/csdid/function-aggte/data/mpdta.csv") +data = pd.read_csv("data/mpdta.csv") +np.random.seed(12345) ``` -The dataset contains 500 observations of county-level teen employment -rates from 2003-2007. Some states are first treated in 2004, some in -2006, and some in 2007 (see the paper for more details). The important -variables in the dataset are +The dataset contains 2,500 observations for 500 counties, with teen +employment rates from 2003-2007. Some states are first treated in 2004, +some in 2006, and some in 2007 (see the paper for more details). The +important variables in the dataset are - **lemp** This is the log of county-level teen employment. It is the outcome variable @@ -134,18 +143,18 @@ out.summ_attgt().summary2 | | Group | Time | ATT(g, t) | Post | Std. Error | \[95% Pointwise | Conf. Band\] | | |-----|-------|------|-----------|------|------------|-----------------|--------------|-----| -| 0 | 2004 | 2004 | -0.0105 | 0 | 0.0241 | -0.0781 | 0.0571 | | -| 1 | 2004 | 2005 | -0.0704 | 0 | 0.0324 | -0.1612 | 0.0204 | | -| 2 | 2004 | 2006 | -0.1373 | 0 | 0.0393 | -0.2476 | -0.0269 | \* | -| 3 | 2004 | 2007 | -0.1008 | 0 | 0.0360 | -0.2017 | 0.0001 | | -| 4 | 2006 | 2004 | 0.0065 | 0 | 0.0238 | -0.0601 | 0.0732 | | -| 5 | 2006 | 2005 | -0.0028 | 0 | 0.0188 | -0.0554 | 0.0499 | | -| 6 | 2006 | 2006 | -0.0046 | 0 | 0.0172 | -0.0528 | 0.0437 | | -| 7 | 2006 | 2007 | -0.0412 | 0 | 0.0201 | -0.0976 | 0.0152 | | -| 8 | 2007 | 2004 | 0.0305 | 0 | 0.0147 | -0.0108 | 0.0719 | | -| 9 | 2007 | 2005 | -0.0027 | 0 | 0.0160 | -0.0476 | 0.0421 | | -| 10 | 2007 | 2006 | -0.0311 | 0 | 0.0173 | -0.0796 | 0.0174 | | -| 11 | 2007 | 2007 | -0.0261 | 0 | 0.0171 | -0.0740 | 0.0219 | | +| 0 | 2004 | 2004 | -0.0105 | 1 | 0.0243 | -0.0581 | 0.0371 | | +| 1 | 2004 | 2005 | -0.0704 | 1 | 0.0348 | -0.1387 | -0.0022 | \* | +| 2 | 2004 | 2006 | -0.1373 | 1 | 0.0412 | -0.2179 | -0.0566 | \* | +| 3 | 2004 | 2007 | -0.1008 | 1 | 0.0356 | -0.1706 | -0.0311 | \* | +| 4 | 2006 | 2004 | 0.0065 | 0 | 0.0249 | -0.0424 | 0.0554 | | +| 5 | 2006 | 2005 | -0.0028 | 0 | 0.0197 | -0.0414 | 0.0359 | | +| 6 | 2006 | 2006 | -0.0046 | 1 | 0.0173 | -0.0386 | 0.0294 | | +| 7 | 2006 | 2007 | -0.0412 | 1 | 0.0197 | -0.0799 | -0.0026 | \* | +| 8 | 2007 | 2004 | 0.0305 | 0 | 0.0146 | 0.0019 | 0.0591 | \* | +| 9 | 2007 | 2005 | -0.0027 | 0 | 0.0170 | -0.0361 | 0.0306 | | +| 10 | 2007 | 2006 | -0.0311 | 0 | 0.0177 | -0.0657 | 0.0035 | | +| 11 | 2007 | 2007 | -0.0261 | 1 | 0.0181 | -0.0615 | 0.0094 | | @@ -156,14 +165,14 @@ class and the graph information. out.plot_attgt(); ``` - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) +
+ + + +Figure 1 -![](README_files/figure-commonmark/cell-5-output-2.png) +
``` python out.aggte(typec='calendar'); @@ -173,15 +182,15 @@ out.aggte(typec='calendar'); Overall summary of ATT's based on calendar time aggregation: ATT Std. Error [95.0% Conf. Int.] - -0.0417 0.0169 -0.0748 -0.0086 * + -0.0417 0.0164 -0.0738 -0.0096 * Time Effects (calendar): Time Estimate Std. Error [95.0% Simult. Conf. Band - 0 2004 -0.0105 0.0244 -0.0584 0.0374 - 1 2005 -0.0704 0.0307 -0.1305 -0.0103 * - 2 2006 -0.0488 0.0210 -0.0900 -0.0076 * - 3 2007 -0.0371 0.0136 -0.0637 -0.0105 * + 0 2004 -0.0105 0.0242 -0.0580 0.0370 + 1 2005 -0.0704 0.0325 -0.1342 -0.0067 * + 2 2006 -0.0488 0.0195 -0.0871 -0.0105 * + 3 2007 -0.0371 0.0146 -0.0657 -0.0084 * --- Signif. codes: `*' confidence band does not cover 0 Control Group: Never Treated , @@ -192,13 +201,16 @@ out.aggte(typec='calendar'); out.plot_aggte(); ``` - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) +
+ + -![](README_files/figure-commonmark/cell-7-output-2.png) +Figure 2 +
-**Event Studies** +## Event studies Although in the current example it is pretty easy to directly interpret the group-time average treatment effects, there are many cases where it @@ -207,7 +219,7 @@ a small number of parameters. A main type of aggregation is into an *event study* plot. To make an event study plot in the **csdid** package, one can use the -**aggte** function with **dynamic** option +**aggte** function with the **dynamic** option: ``` python out.aggte(typec='dynamic'); @@ -216,19 +228,19 @@ out.aggte(typec='dynamic'); Overall summary of ATT's based on event-study/dynamic aggregation: - ATT Std. Error [95.0% Conf. Int.] - -0.0772 0.0207 -0.1179 -0.0366 * + ATT Std. Error [95.0% Conf. Int.] + -0.0772 0.0212 -0.1187 -0.0357 * Dynamic Effects: - Event time Estimate Std. Error [95.0% Simult. Conf. Band - 0 -3 0.0305 0.0146 0.0019 0.0591 * - 1 -2 -0.0006 0.0129 -0.0259 0.0248 - 2 -1 -0.0245 0.0141 -0.0521 0.0032 - 3 0 -0.0199 0.0117 -0.0428 0.0030 - 4 1 -0.0510 0.0154 -0.0811 -0.0208 * - 5 2 -0.1373 0.0366 -0.2091 -0.0655 * - 6 3 -0.1008 0.0337 -0.1669 -0.0347 * + Event time Estimate Std. Error [95.0% Simult. Conf. Band + 0 -3 0.0305 0.0156 -0.0001 0.0612 + 1 -2 -0.0006 0.0128 -0.0257 0.0246 + 2 -1 -0.0245 0.0141 -0.0522 0.0033 + 3 0 -0.0199 0.0118 -0.0430 0.0032 + 4 1 -0.0510 0.0184 -0.0869 -0.0150 * + 5 2 -0.1373 0.0408 -0.2172 -0.0574 * + 6 3 -0.1008 0.0367 -0.1727 -0.0289 * --- Signif. codes: `*' confidence band does not cover 0 Control Group: Never Treated , @@ -236,41 +248,45 @@ out.aggte(typec='dynamic'); Estimation Method: Doubly Robust The column `event time` is for each group relative to when they first -participate in the treatment. To give some examples, `event time=0` -corresponds to the *on impact* effect, and `event time=-1` is the -*effect* in the period before a unit becomes treated (checking that this -is equal to 0 is potentially useful as a pre-test). +participate in the treatment. For example, `event time=0` corresponds to +the *on impact* effect, and `event time=-1` is the *effect* in the +period before a unit becomes treated. Checking that this is equal to +zero is potentially useful as a pre-test when no anticipation is +specified. + +To plot the event study, use **plot_aggte**: -To plot the event study, use **plot_aggte** method ``` python out.plot_aggte(); ``` - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) +
-![](README_files/figure-commonmark/cell-8-output-2.png) + + +Figure 3 + +
The figure here is very similar to the group-time average treatment effects. Red dots are pre-treatment periods, blue dots are post-treatment periods. The difference is that the x-axis is in event time. -**Overall Effect of Participating in the Treatment** +## Overall effect of participating in treatment The event study above reported an overall effect of participating in the treatment. This was computed by averaging the average effects computed at each length of exposure. In many cases, a more general purpose overall treatment effect parameter -is give by computing the average treatment effect for each group, and -then averaging across groups. This sort of procedure provides an average -treatment effect parameter with a very similar interpretation to the -Average Treatment Effect on the Treated (ATT) in the two period and two -group case. - -To compute this overall average treatment effect parameter, use +is given by computing the average treatment effect for each group, and +then averaging across groups. This procedure provides an average +treatment effect parameter with a similar interpretation to the Average +Treatment Effect on the Treated (ATT) in the two-period, two-group case. +To compute this overall average treatment effect parameter, use: ``` python out.aggte(typec='group'); @@ -279,15 +295,15 @@ out.aggte(typec='group'); Overall summary of ATT's based on group/cohort aggregation: - ATT Std. Error [95.0% Conf. Int.] - -0.031 0.0124 -0.0553 -0.0067 * + ATT Std. Error [95.0% Conf. Int.] + -0.031 0.013 -0.0565 -0.0055 * Group Effects: - Group Estimate Std. Error [95.0% Simult. Conf. Band - 0 2004 -0.0797 0.0301 -0.1387 -0.0208 * - 1 2006 -0.0229 0.0172 -0.0567 0.0109 - 2 2007 -0.0261 0.0174 -0.0601 0.0080 + Group Estimate Std. Error [95.0% Simult. Conf. Band + 0 2004 -0.0797 0.0293 -0.1371 -0.0224 * + 1 2006 -0.0229 0.0166 -0.0554 0.0096 + 2 2007 -0.0261 0.0163 -0.0581 0.0059 --- Signif. codes: `*' confidence band does not cover 0 Control Group: Never Treated , @@ -296,16 +312,51 @@ out.aggte(typec='group'); Of particular interest is the `Overall ATT` in the results. Here, we estimate that increasing the minimum wage decreased teen employment by -3.1% and the effect is marginally statistically significant. +approximately 3.1% in this example. + +## Additional options + +`ATTgt(...)` and `.fit(...)` support several options for inference, +performance, and weighting: + +- **Control group**: `control_group="nevertreated"` (default) or + `"notyettreated"`. +- **Estimation method**: `.fit(est_method="dr")` (doubly robust, + default), `"reg"` (outcome regression), `"ipw"` (inverse-probability + weighting), or a Python callable. +- **Base period**: `.fit(base_period="varying")` (default) or + `"universal"`. +- **Clustered inference**: pass `clustervar="..."` for cluster-robust + standard errors (analytical when `bstrap=False`, or via the multiplier + bootstrap). +- **Point estimates only**: `compute_inffunc=False` skips + influence-function and standard-error computation for faster, lighter + exploratory runs. +- **Time-varying weights** (`weights_name`): `fix_weights` controls + which period’s weights each 2×2 comparison uses — `None` (earlier + period, default), `"base_period"`, `"first_period"`, or `"varying"`. +- **Faster computation**: `faster_mode=True` is the default starting in + 0.4.3. It precomputes panel outcomes, weights, and covariates by + period. Pass `faster_mode=False` to use the standard computation path. + Both modes use the same estimators and inference; speed gains depend + on the data and options. + +``` python +# Cluster-robust SEs using the default faster computation path: +out = ATTgt(yname="lemp", gname="first.treat", idname="countyreal", tname="year", + xformla="lemp~1", data=data, + clustervar="countyreal").fit(est_method="dr") +``` + +## How to cite -# How to cite If you want to cite CSDID, you can use the following BibTeX entry: -``` python +``` bibtex @software{csdid, author = {Callaway, Brantly and Sant'Anna, Pedro HC and Quispe, Alexander and Guevara, Carlos}, title = {{csdid: Difference-in-Differences with Multiple Time Periods in Python}}, year = {2024}, url = {https://github.com/d2cml-ai/csdid} } -``` \ No newline at end of file +``` diff --git a/README.qmd b/README.qmd index 63c54cf..e5ccad4 100644 --- a/README.qmd +++ b/README.qmd @@ -1,8 +1,19 @@ --- title: Difference in Difference in Python format: gfm +output-file: README.md +execute: + cache: false + error: false --- +[![PyPI version](https://img.shields.io/pypi/v/csdid.svg?color=blue)](https://pypi.org/project/csdid/) +Downloads +[![Last commit](https://img.shields.io/github/last-commit/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/commits/main) +[![GitHub stars](https://img.shields.io/github/stars/d2cml-ai/csdid.svg?style=social)](https://github.com/d2cml-ai/csdid/stargazers) +[![GitHub issues](https://img.shields.io/github/issues/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/issues) +[![License](https://img.shields.io/github/license/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/blob/main/LICENSE) + The **csdid** package contains tools for computing average treatment effect parameters in a Difference-in-Differences setup allowing for @@ -32,17 +43,17 @@ treatment effects or event-study-type estimands. ## Getting Started There has been some recent work on DiD with multiple time periods. The -**did** package implements the framework put forward in +**csdid** package implements the framework put forward in - [Callaway, Brantly and Pedro H.C. Sant’Anna. "Difference-in-Differences with Multiple Time Periods." Journal of Econometrics, Vol. 225, No. 2, pp. 200-230, 2021.](https://doi.org/10.1016/j.jeconom.2020.12.001) - or [arXiv](https://arxiv.org/abs/1803.09015 + or [arXiv](https://arxiv.org/abs/1803.09015). This project is based on the original [did R package](https://github.com/bcallaway11/did). -## Instalation +## Installation You can install **csdid** from `pypi` with: @@ -58,11 +69,12 @@ pip install git+https://github.com/d2cml-ai/csdid/ ### Dependencies -Additionally, I have created an additional library called `drdid`, which can be installed via GitHub. - -``` -pip install git+https://github.com/d2cml-ai/DRDID -``` +The `drdid` dependency is installed automatically with `csdid`. CSDID 0.4.5 +requires the reviewed DRDID 1.1.7 release. The Python backend is maintained in +[DRDIDpy](https://github.com/d2cml-ai/DRDIDpy); development checkouts use the +exact peer revision recorded in +[`python_drdid.json`](csdid/test_csdid/joint_sync/python_drdid.json). +Update both packages together using their matching reviewed versions. ## Basic Example @@ -75,15 +87,18 @@ from [Callaway and Sant’Anna - [More detailed examples are also available](https://bcallaway11.github.io/did/articles/did-basics.html) -A subset of the data is available in the package and can be loaded by +A subset of the data is available at [data/mpdta.csv](data/mpdta.csv). Run the +following examples from the repository root: ```{python} from csdid.att_gt import ATTgt +import numpy as np import pandas as pd -data = pd.read_csv("https://raw.githubusercontent.com/d2cml-ai/csdid/function-aggte/data/mpdta.csv") +data = pd.read_csv("data/mpdta.csv") +np.random.seed(12345) ``` -The dataset contains 500 observations of county-level teen employment +The dataset contains 2,500 observations for 500 counties, with teen employment rates from 2003-2007. Some states are first treated in 2004, some in 2006, and some in 2007 (see the paper for more details). The important variables in the dataset are @@ -116,13 +131,14 @@ out = ATTgt(yname = "lemp", Summary table -```{python, eval = False} +```{python} out.summ_attgt().summary2 ``` In the graphs, a semicolon `;` should be added to prevent printing the class and the graph information. ```{python} +#| label: fig-attgt out.plot_attgt(); ``` @@ -133,9 +149,66 @@ out.aggte(typec='calendar'); ```{python} +#| label: fig-calendar out.plot_aggte(); ``` +## Event studies + +Although in the current example it is pretty easy to directly interpret +the group-time average treatment effects, there are many cases where it +is convenient to aggregate the group-time average treatment effects into +a small number of parameters. A main type of aggregation is into an +*event study* plot. + +To make an event study plot in the **csdid** package, one can use the +**aggte** function with the **dynamic** option: + +```{python} +out.aggte(typec='dynamic'); +``` + +The column `event time` is for each group relative to when they first +participate in the treatment. For example, `event time=0` corresponds to +the *on impact* effect, and `event time=-1` is the *effect* in the period +before a unit becomes treated. Checking that this is equal to zero is +potentially useful as a pre-test when no anticipation is specified. + +To plot the event study, use **plot_aggte**: + +```{python} +#| label: fig-dynamic +out.plot_aggte(); +``` + +The figure here is very similar to the group-time average treatment +effects. Red dots are pre-treatment periods, blue dots are +post-treatment periods. The difference is that the x-axis is in event +time. + +## Overall effect of participating in treatment + +The event study above reported an overall effect of participating in the +treatment. This was computed by averaging the average effects computed +at each length of exposure. + +In many cases, a more general purpose overall treatment effect parameter +is given by computing the average treatment effect for each group, and +then averaging across groups. This procedure provides an average +treatment effect parameter with a similar interpretation to the +Average Treatment Effect on the Treated (ATT) in the two-period, +two-group case. + +To compute this overall average treatment effect parameter, use: + +```{python} +out.aggte(typec='group'); +``` + +Of particular interest is the `Overall ATT` in the results. Here, we +estimate that increasing the minimum wage decreased teen employment by +approximately 3.1% in this example. + ## Additional options `ATTgt(...)` and `.fit(...)` support several options for inference, performance, @@ -158,9 +231,23 @@ and weighting: `faster_mode=False` to use the standard computation path. Both modes use the same estimators and inference; speed gains depend on the data and options. -```{python, eval = False} +```{python} +#| eval: false # Cluster-robust SEs using the default faster computation path: out = ATTgt(yname="lemp", gname="first.treat", idname="countyreal", tname="year", xformla="lemp~1", data=data, clustervar="countyreal").fit(est_method="dr") ``` + +## How to cite + +If you want to cite CSDID, you can use the following BibTeX entry: + +```bibtex +@software{csdid, + author = {Callaway, Brantly and Sant'Anna, Pedro HC and Quispe, Alexander and Guevara, Carlos}, + title = {{csdid: Difference-in-Differences with Multiple Time Periods in Python}}, + year = {2024}, + url = {https://github.com/d2cml-ai/csdid} +} +``` diff --git a/README_files/figure-commonmark/cell-5-output-1.png b/README_files/figure-commonmark/cell-5-output-1.png deleted file mode 100644 index 6516da2..0000000 Binary files a/README_files/figure-commonmark/cell-5-output-1.png and /dev/null differ diff --git a/README_files/figure-commonmark/cell-5-output-2.png b/README_files/figure-commonmark/cell-5-output-2.png deleted file mode 100644 index d27701f..0000000 Binary files a/README_files/figure-commonmark/cell-5-output-2.png and /dev/null differ diff --git a/README_files/figure-commonmark/cell-7-output-1.png b/README_files/figure-commonmark/cell-7-output-1.png deleted file mode 100644 index e56fb17..0000000 Binary files a/README_files/figure-commonmark/cell-7-output-1.png and /dev/null differ diff --git a/README_files/figure-commonmark/cell-7-output-2.png b/README_files/figure-commonmark/cell-7-output-2.png deleted file mode 100644 index 444a9f1..0000000 Binary files a/README_files/figure-commonmark/cell-7-output-2.png and /dev/null differ diff --git a/README_files/figure-commonmark/cell-8-output-2.png b/README_files/figure-commonmark/cell-8-output-2.png deleted file mode 100644 index 5c14980..0000000 Binary files a/README_files/figure-commonmark/cell-8-output-2.png and /dev/null differ diff --git a/README_files/figure-commonmark/fig-attgt-output-1.png b/README_files/figure-commonmark/fig-attgt-output-1.png new file mode 100644 index 0000000..618fb63 Binary files /dev/null and b/README_files/figure-commonmark/fig-attgt-output-1.png differ diff --git a/README_files/figure-commonmark/fig-calendar-output-1.png b/README_files/figure-commonmark/fig-calendar-output-1.png new file mode 100644 index 0000000..823bb1a Binary files /dev/null and b/README_files/figure-commonmark/fig-calendar-output-1.png differ diff --git a/README_files/figure-commonmark/fig-dynamic-output-1.png b/README_files/figure-commonmark/fig-dynamic-output-1.png new file mode 100644 index 0000000..60e42de Binary files /dev/null and b/README_files/figure-commonmark/fig-dynamic-output-1.png differ diff --git a/csdid/_version.py b/csdid/_version.py index 104a82d..949ce82 100644 --- a/csdid/_version.py +++ b/csdid/_version.py @@ -1 +1 @@ -__version = '0.4.4' +__version = '0.4.5' diff --git a/csdid/test_csdid/joint_sync/run.py b/csdid/test_csdid/joint_sync/run.py index e9c41b4..b5f796e 100644 --- a/csdid/test_csdid/joint_sync/run.py +++ b/csdid/test_csdid/joint_sync/run.py @@ -334,6 +334,13 @@ def command(arguments, name, env=None, checked=True): shutil.copytree(sources / "csdid/scripts", csdid_plain / "scripts", ignore=ignore) if (csdid_plain / "scripts/tests").is_dir(): extra_test_roots.append(csdid_plain / "scripts/tests") + # Repository-level documentation guards read the real authored and + # rendered README fixtures from their plain snapshot's root. + for name in ("README.qmd", "README.md"): + if (sources / "csdid" / name).is_file(): + shutil.copyfile(sources / "csdid" / name, csdid_plain / name) + if (sources / "csdid/README_files").is_dir(): + shutil.copytree(sources / "csdid/README_files", csdid_plain / "README_files", ignore=ignore) (peer_plain / "scripts").mkdir() bridge = peer_plain / "scripts/check_csdid_integration.py" shutil.copyfile(sources / "drdid/scripts/check_csdid_integration.py", bridge) diff --git a/docs/readme-generation-validation-0.4.4.json b/docs/readme-generation-validation-0.4.4.json new file mode 100644 index 0000000..73adab0 --- /dev/null +++ b/docs/readme-generation-validation-0.4.4.json @@ -0,0 +1,71 @@ +{ + "audit_date": "2026-10-27", + "base_commit": "86bf96225eb23f6aeffa0a6bb928b378485de383", + "quarto": "1.8.25", + "python": "3.12.12", + "fixed_seed": 12345, + "direct_dependency_resolution": "passed using reviewed peer source plus all pinned rendering requirements in one install", + "consecutive_renders_bit_identical": true, + "rendered_sha256": { + "README.md": "aecd536061c0a6765aed0827523e5621cf7917e5bc7d626e447a943873bb531a", + "README_files/figure-commonmark/fig-attgt-output-1.png": "af5e8b557dea1a0281dd27de6ee8d30a784dc8f63bac18709e34843f9a2b4a17", + "README_files/figure-commonmark/fig-calendar-output-1.png": "c3cc20bb076900c677fcebd841e01e272cd926775b0396269158e873cff3463f", + "README_files/figure-commonmark/fig-dynamic-output-1.png": "b6d1c34b9b043d0d3e0f0a6336283f1ce9c13f0993e0499fd8fc1220cbcb7d63" + }, + "guard_tests_passed": 20, + "observed_bot_regression_commit": "53e8fc985afaf3057caba0aa83ca0a9a336dc5b1", + "observed_bot_content_rejections": [ + "bot README: the Python package must be identified as csdid", + "bot README: the Python package was renamed to did", + "bot README: missing section 'Event studies'", + "bot README: missing section 'Overall effect of participating in treatment'", + "bot README: missing section 'How to cite'", + "bot README: missing dynamic aggregation example", + "bot README: missing group aggregation example", + "bot README: missing PyPI version badge for csdid", + "bot README: missing Downloads badge for csdid", + "bot README: missing Last commit badge for csdid", + "bot README: missing GitHub stars badge for csdid", + "bot README: missing GitHub issues badge for csdid", + "bot README: missing License badge for csdid", + "bot README: missing csdid software citation" + ], + "package_metadata_checks": { + "name": "csdid", + "version": "0.4.4", + "requires_drdid": "==1.1.7", + "wheel_long_description_matches_canonical_readme": true, + "sdist_has_only_canonical_readme": true + }, + "actionlint": "passed", + "estimator_runtime_changed": false, + "scope": "Authored source and generated documentation are preserved; numerical outputs intentionally regenerate from the checkout. Font/OS differences between platforms are outside the same-environment byte-replay claim.", + "historical_validation": "readme-generation-validation-20261026.json", + "versions": { + "csdid": "0.4.4", + "drdid": "1.1.7", + "numpy": "2.5.4", + "pandas": "3.1.0", + "scipy": "1.18.1", + "statsmodels": "0.15.0", + "matplotlib": "3.11.2", + "plotnine": "0.15.8" + }, + "peer_lock": { + "schema_version": 1, + "repository": "d2cml-ai/DRDIDpy", + "commit": "53a6b764724c9e3c7221909502de95761eda9c6d", + "version": "1.1.7" + }, + "pip_check": "passed", + "readme_guard_tests": 10, + "release_guard_tests": 10, + "joint_snapshot_documentation_fixtures": [ + "README.qmd", + "README.md", + "README_files" + ], + "estimator_runtime_changed_vs_current_main": false, + "figure_outputs_identical_to_previous_validation": true, + "render_requirements_sha256": "95c970e2b0e418bc70cefbc8fef317f94a01ace5a1743dadb5b1225f6edaa647" +} diff --git a/docs/readme-generation-validation-20261026.json b/docs/readme-generation-validation-20261026.json new file mode 100644 index 0000000..24f7c4c --- /dev/null +++ b/docs/readme-generation-validation-20261026.json @@ -0,0 +1,41 @@ +{ + "audit_date": "2026-10-26", + "base_commit": "b2e3b0baae0f4226e90e016b73cb0baf9456c3e2", + "quarto": "1.8.25", + "python": "3.12.12 (main, Oct 9 2025, 11:07:00) [Clang 17.0.0 (clang-1700.4.4.1)]", + "fixed_seed": 12345, + "direct_dependency_resolution": "passed", + "consecutive_renders_bit_identical": true, + "rendered_sha256": { + "README.md": "13d979e420c4d24c622a0c2ea8b76e015aee68190c850acf43df55fde9480ea7", + "README_files/figure-commonmark/fig-attgt-output-1.png": "af5e8b557dea1a0281dd27de6ee8d30a784dc8f63bac18709e34843f9a2b4a17", + "README_files/figure-commonmark/fig-calendar-output-1.png": "c3cc20bb076900c677fcebd841e01e272cd926775b0396269158e873cff3463f", + "README_files/figure-commonmark/fig-dynamic-output-1.png": "b6d1c34b9b043d0d3e0f0a6336283f1ce9c13f0993e0499fd8fc1220cbcb7d63" + }, + "guard_tests_passed": 10, + "observed_bot_regression_commit": "53e8fc985afaf3057caba0aa83ca0a9a336dc5b1", + "observed_bot_content_rejections": [ + "bot README: the Python package must be identified as csdid", + "bot README: the Python package was renamed to did", + "bot README: missing section 'Event studies'", + "bot README: missing section 'Overall effect of participating in treatment'", + "bot README: missing section 'How to cite'", + "bot README: missing dynamic aggregation example", + "bot README: missing group aggregation example", + "bot README: missing PyPI version badge for csdid", + "bot README: missing Downloads badge for csdid", + "bot README: missing Last commit badge for csdid", + "bot README: missing GitHub stars badge for csdid", + "bot README: missing GitHub issues badge for csdid", + "bot README: missing License badge for csdid", + "bot README: missing csdid software citation" + ], + "package_metadata_checks": { + "name": "csdid", + "wheel_long_description_matches_canonical_readme": true, + "sdist_has_only_canonical_readme": true + }, + "actionlint": "passed", + "estimator_runtime_changed": false, + "scope": "Authored source and generated documentation are preserved; numerical outputs intentionally regenerate from the checkout. Font/OS differences between platforms are outside the same-environment byte-replay claim." +} diff --git a/docs/readme-generation-validation.json b/docs/readme-generation-validation.json new file mode 100644 index 0000000..e77f0f1 --- /dev/null +++ b/docs/readme-generation-validation.json @@ -0,0 +1,112 @@ +{ + "audit_date": "2026-10-27", + "base_commit": "d5123fbf315f593221e9589b33cad25aa9741dd4", + "quarto": "1.8.25", + "python": "3.12.12", + "fixed_seed": 12345, + "direct_dependency_resolution": "passed using reviewed peer source plus all pinned rendering requirements in one install", + "consecutive_renders_bit_identical": true, + "rendered_sha256": { + "README.md": "cc4fe7a9d9596725053305dc376fb6aa65a05ede350fca73317e312f5e45cfdd", + "README_files/figure-commonmark/fig-attgt-output-1.png": "af5e8b557dea1a0281dd27de6ee8d30a784dc8f63bac18709e34843f9a2b4a17", + "README_files/figure-commonmark/fig-calendar-output-1.png": "c3cc20bb076900c677fcebd841e01e272cd926775b0396269158e873cff3463f", + "README_files/figure-commonmark/fig-dynamic-output-1.png": "b6d1c34b9b043d0d3e0f0a6336283f1ce9c13f0993e0499fd8fc1220cbcb7d63" + }, + "guard_tests_passed": 41, + "observed_bot_regression_commit": "53e8fc985afaf3057caba0aa83ca0a9a336dc5b1", + "observed_bot_content_rejections": [ + "bot README: the Python package must be identified as csdid", + "bot README: the Python package was renamed to did", + "bot README: missing section 'Event studies'", + "bot README: missing section 'Overall effect of participating in treatment'", + "bot README: missing section 'How to cite'", + "bot README: missing dynamic aggregation example", + "bot README: missing group aggregation example", + "bot README: missing PyPI version badge for csdid", + "bot README: missing Downloads badge for csdid", + "bot README: missing Last commit badge for csdid", + "bot README: missing GitHub stars badge for csdid", + "bot README: missing GitHub issues badge for csdid", + "bot README: missing License badge for csdid", + "bot README: missing csdid software citation" + ], + "package_metadata_checks": { + "name": "csdid", + "version": "0.4.5", + "requires_drdid": "==1.1.7", + "wheel_long_description_matches_canonical_readme": true, + "sdist_long_description_matches_canonical_readme": true, + "sdist_has_only_canonical_readme": true, + "wheel_source_and_sdist_source_match_current_checkout": true + }, + "actionlint": "passed using actionlint with shellcheck disabled", + "estimator_runtime_changed": false, + "scope": "Canonical README packaging and rendering correction with CSDID 0.4.5, plus strict bounded verification of public PyPI distributions and installer index after upload or an existing-release retry. Estimator logic, backend dependency and reviewed peer lock remain unchanged.", + "historical_validation": "readme-generation-validation-20261026.json", + "versions": { + "csdid": "0.4.5", + "drdid": "1.1.7", + "numpy": "2.5.4", + "pandas": "3.1.0", + "scipy": "1.18.1", + "statsmodels": "0.15.0", + "matplotlib": "3.11.2", + "plotnine": "0.15.8", + "fonttools": "4.68.0", + "build": "1.6.1", + "twine": "7.0.0" + }, + "peer_lock": { + "schema_version": 1, + "repository": "d2cml-ai/DRDIDpy", + "commit": "53a6b764724c9e3c7221909502de95761eda9c6d", + "version": "1.1.7" + }, + "pip_check": "passed", + "readme_guard_tests": 10, + "release_guard_tests": 31, + "joint_snapshot_documentation_fixtures": [ + "README.qmd", + "README.md", + "README_files" + ], + "estimator_runtime_changed_vs_current_main": false, + "figure_outputs_identical_to_previous_validation": true, + "render_requirements_sha256": "95c970e2b0e418bc70cefbc8fef317f94a01ace5a1743dadb5b1225f6edaa647", + "distribution_sha256": { + "csdid-0.4.5-py3-none-any.whl": "38830453cead0b8119cb988ce07c88a51dec85bf9aeccd2fbc551c31ef2a34b9", + "csdid-0.4.5.tar.gz": "4029860d99bcf51ef0b76d6e4ae7e03f3b4e7e42c26bedf3178ab01c78447611" + }, + "artifact_source_verification": "passed using the actual wheel/sdist and production release verification functions", + "validated_at_utc": "2026-10-27T21:37:00.662417+00:00", + "validation_parent_commit": "80274ff650c92a0421c159126d2f3844bbc71a9d", + "previous_current_validation": "docs/readme-generation-validation-0.4.4.json", + "version_file_changed": true, + "release_version_reason": "The canonical README path changes setup.py and MANIFEST.in. Publish this packaging/documentation correction as 0.4.5 instead of reusing 0.4.4.", + "backend_pypi_verification": "Published drdid 1.1.7 wheel matches every reviewed Python source byte and PyPI SHA256", + "distribution_metadata_validation": "python -m twine check passed for the actual wheel and sdist", + "release_preflight_on_clean_checkout": { + "ready": true, + "message": "CSDID 0.4.5 needs publication; the locked backend is available for byte verification.", + "mode": "read-only release state preflight; required CI is checked separately by the publication workflow" + }, + "integrated_validation": { + "validated_at_utc": "2026-10-27T21:48:27.054847+00:00", + "source_commit": "6d432108d06f9843ecf3c9ccafa0fcda3e87e86f", + "root_guards_passed": 41, + "readme_guard_tests": 10, + "release_guard_tests": 31, + "canonical_readme_guard": "passed", + "source_readme_guard": "passed", + "actionlint": "passed with shellcheck disabled", + "release_preflight_on_clean_checkout": { + "version": "0.4.5", + "ready": true, + "message": "CSDID 0.4.5 needs publication; the locked backend is available for byte verification." + }, + "backend_pypi_verification": "Published drdid 1.1.7 wheel matches every reviewed Python source byte and PyPI SHA256", + "packaged_inputs_unchanged_since": "560d0c9dc5568db2d5cc8b693ffc36f1b405c260", + "render_and_distribution_hashes_rechecked": "passed against the actual saved files; no rerender or rebuild was needed for changes outside packaged inputs", + "render_and_build_timestamp": "Retain top-level validated_at_utc for the original real render/build verification; this integrated timestamp covers the later release hardening checks." + } +} diff --git a/docs/readme-generation.md b/docs/readme-generation.md new file mode 100644 index 0000000..45ad184 --- /dev/null +++ b/docs/readme-generation.md @@ -0,0 +1,52 @@ +# Maintaining the generated README + +Edit `README.qmd`. It is the source for authored prose, badges, examples and +citation instructions. `README.md` is generated; a second `readme.md` must not +be tracked because the two paths alias on case-insensitive filesystems. + +The renderer uses Quarto 1.8.25, Python 3.12, the packages pinned in +`docs/requirements-readme.txt`, the checked-out csdid source, and the exact +Python DRDID peer selected by `csdid/test_csdid/joint_sync/python_drdid.json`. +The workflow checks out both repositories into sibling directories and installs +the pair together with the pinned rendering dependencies. It therefore works +before the reviewed backend is published, while retaining the exact package +requirement. A future peer upgrade must update the rendering pin too. Examples use +the checked-in `data/mpdta.csv` and set NumPy's seed before bootstrap estimation. +Numerical outputs can change after a reviewed estimator correction; random +render-to-render variation should not create documentation updates. + +From the repository root, with Quarto installed: + +```sh +python3.12 -m venv /tmp/csdid-readme-venv +# Keep the Python DRDID checkout outside this repository. Fetch the exact SHA +# recorded in csdid/test_csdid/joint_sync/python_drdid.json and check it out. +/tmp/csdid-readme-venv/bin/python -m pip install \ + -r docs/requirements-readme.txt /path/to/reviewed/python-drdid . +/tmp/csdid-readme-venv/bin/python -m pip check +QUARTO_PYTHON=/tmp/csdid-readme-venv/bin/python \ + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \ + MPLBACKEND=Agg PYTHONHASHSEED=0 quarto render README.qmd +python scripts/check_readme.py +python -m unittest discover -s scripts/tests -p 'test_*.py' +``` + +The README workflow renders twice and compares Markdown and PNG hashes before +opening a PR. It may commit only `README.md` and generated figure PNGs. The +content guard also runs in both mandatory Python CI jobs; it rejects the +observed package-name regression, missing badges/examples/citation, duplicate +case-variant README paths, broken figure links and the corrupted Downloads +badge query. Maintainers should still review proposed wording and results. +The joint installed-wheel gate copies these root documentation fixtures into +its immutable plain test snapshot, so the README tests run there too while the +runtime packages continue to come only from the installed wheels. + +The existing manual workflow trigger can validate a feature branch before +merging: `gh workflow run gen_readme.yml --ref BRANCH`. Such a run executes the +renders and guards but skips publishing a bot PR. Automatic main-branch runs +retain PR publication. + +The bot uses `GITHUB_TOKEN`. GitHub does not start push-triggered workflows +from its automated pushes, so a generated PR still needs the required checks +to run before it can merge. This change does not add credentials or change the +repository's push-only upstream parity trigger. diff --git a/docs/requirements-readme.txt b/docs/requirements-readme.txt new file mode 100644 index 0000000..f3a260f --- /dev/null +++ b/docs/requirements-readme.txt @@ -0,0 +1,16 @@ +# Rendering environment tested with Python 3.12 and Quarto 1.8.25. +# Update these pins and regenerate the README together in a reviewed PR. +numpy==2.5.4 +pandas==3.1.0 +scipy==1.18.1 +patsy==1.0.3 +plotnine==0.15.8 +matplotlib==3.11.2 +mizani==0.14.6 +statsmodels==0.15.0 +joblib==1.6.0 +drdid==1.1.7 +nbclient==0.11.0 +nbformat==5.11.1 +PyYAML==6.0.3 +ipykernel==7.4.0 diff --git a/readme.md b/readme.md deleted file mode 100644 index 1289a7d..0000000 --- a/readme.md +++ /dev/null @@ -1,311 +0,0 @@ -# Difference in Difference in Python - -[![PyPI version](https://img.shields.io/pypi/v/csdid.svg?color=blue)](https://pypi.org/project/csdid/) -[![Downloads](https://static.pepy.tech/personalized-badge/csdid?period=total&units=international_system&left_color=blue&right_color=grey&left_text=Downloads)](https://pepy.tech/project/csdid) -[![Last commit](https://img.shields.io/github/last-commit/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/commits/main) -[![GitHub stars](https://img.shields.io/github/stars/d2cml-ai/csdid.svg?style=social)](https://github.com/d2cml-ai/csdid/stargazers) -[![GitHub issues](https://img.shields.io/github/issues/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/issues) -[![License](https://img.shields.io/github/license/d2cml-ai/csdid.svg)](https://github.com/d2cml-ai/csdid/blob/main/LICENSE) - - -The **csdid** package contains tools for computing average treatment -effect parameters in a Difference-in-Differences setup allowing for - -- More than two time periods - -- Variation in treatment timing (i.e., units can become treated at - different points in time) - -- Treatment effect heterogeneity (i.e, the effect of participating in - the treatment can vary across units and exhibit potentially complex - dynamics, selection into treatment, or time effects) - -- The parallel trends assumption holds only after conditioning on - covariates - -The main parameters are **group-time average treatment effects**. These -are the average treatment effect for a particular group (group is -defined by treatment timing) in a particular time period. These -parameters are a natural generalization of the average treatment effect -on the treated (ATT) which is identified in the textbook case with two -periods and two groups to the case with multiple periods. - -Group-time average treatment effects are also natural building blocks -for more aggregated treatment effect parameters such as overall -treatment effects or event-study-type estimands. - -## Getting Started - -There has been some recent work on DiD with multiple time periods. The -**csdid** package implements the framework put forward in - -- [Callaway, Brantly and Pedro H.C. Sant’Anna. - “Difference-in-Differences with Multiple Time Periods.” Journal of - Econometrics, Vol. 225, No. 2, pp. 200-230, - 2021.](https://doi.org/10.1016/j.jeconom.2020.12.001) or - \[arXiv\](https://arxiv.org/abs/1803.09015 - -This project is based on the original [did R -package](https://github.com/bcallaway11/did). - -## Instalation - -You can install **csdid** from `pypi` with: - - pip install csdid - -or via github: - - pip install git+https://github.com/d2cml-ai/csdid/ - -### Dependencies - -Additionally, I have created an additional library called `drdid`, which -can be installed via GitHub. - - pip install git+https://github.com/d2cml-ai/DRDID - -## Basic Example - -The following is a simplified example of the effect of states increasing -their minimum wages on county-level teen employment rates which comes -from [Callaway and Sant’Anna -(2021)](https://authors.elsevier.com/a/1cFzc15Dji4pnC). - -- [More detailed examples are also - available](https://bcallaway11.github.io/did/articles/did-basics.html) - -A subset of the data is available in the package and can be loaded by - -``` python -from csdid.att_gt import ATTgt -import pandas as pd -data = pd.read_csv("https://raw.githubusercontent.com/d2cml-ai/csdid/function-aggte/data/mpdta.csv") -``` - -The dataset contains 500 observations of county-level teen employment -rates from 2003-2007. Some states are first treated in 2004, some in -2006, and some in 2007 (see the paper for more details). The important -variables in the dataset are - -- **lemp** This is the log of county-level teen employment. It is the - outcome variable - -- **first.treat** This is the period when a state first increases its - minimum wage. It can be 2004, 2006, or 2007. It is the variable that - defines *group* in this application - -- **year** This is the year and is the *time* variable - -- **countyreal** This is an id number for each county and provides the - individual identifier in this panel data context - -To estimate group-time average treatment effects, use the -**ATTgt().fit()** method - -``` python -out = ATTgt(yname = "lemp", - gname = "first.treat", - idname = "countyreal", - tname = "year", - xformla = f"lemp~1", - data = data, - ).fit(est_method = 'dr') -``` - -Summary table - -``` python -out.summ_attgt().summary2 -``` - -
- - -| | Group | Time | ATT(g, t) | Post | Std. Error | \[95% Pointwise | Conf. Band\] | | -|-----|-------|------|-----------|------|------------|-----------------|--------------|-----| -| 0 | 2004 | 2004 | -0.0105 | 0 | 0.0241 | -0.0781 | 0.0571 | | -| 1 | 2004 | 2005 | -0.0704 | 0 | 0.0324 | -0.1612 | 0.0204 | | -| 2 | 2004 | 2006 | -0.1373 | 0 | 0.0393 | -0.2476 | -0.0269 | \* | -| 3 | 2004 | 2007 | -0.1008 | 0 | 0.0360 | -0.2017 | 0.0001 | | -| 4 | 2006 | 2004 | 0.0065 | 0 | 0.0238 | -0.0601 | 0.0732 | | -| 5 | 2006 | 2005 | -0.0028 | 0 | 0.0188 | -0.0554 | 0.0499 | | -| 6 | 2006 | 2006 | -0.0046 | 0 | 0.0172 | -0.0528 | 0.0437 | | -| 7 | 2006 | 2007 | -0.0412 | 0 | 0.0201 | -0.0976 | 0.0152 | | -| 8 | 2007 | 2004 | 0.0305 | 0 | 0.0147 | -0.0108 | 0.0719 | | -| 9 | 2007 | 2005 | -0.0027 | 0 | 0.0160 | -0.0476 | 0.0421 | | -| 10 | 2007 | 2006 | -0.0311 | 0 | 0.0173 | -0.0796 | 0.0174 | | -| 11 | 2007 | 2007 | -0.0261 | 0 | 0.0171 | -0.0740 | 0.0219 | | - -
- -In the graphs, a semicolon `;` should be added to prevent printing the -class and the graph information. - -``` python -out.plot_attgt(); -``` - - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - -![](README_files/figure-commonmark/cell-5-output-2.png) - -``` python -out.aggte(typec='calendar'); -``` - - - - Overall summary of ATT's based on calendar time aggregation: - ATT Std. Error [95.0% Conf. Int.] - -0.0417 0.0169 -0.0748 -0.0086 * - - - Time Effects (calendar): - Time Estimate Std. Error [95.0% Simult. Conf. Band - 0 2004 -0.0105 0.0244 -0.0584 0.0374 - 1 2005 -0.0704 0.0307 -0.1305 -0.0103 * - 2 2006 -0.0488 0.0210 -0.0900 -0.0076 * - 3 2007 -0.0371 0.0136 -0.0637 -0.0105 * - --- - Signif. codes: `*' confidence band does not cover 0 - Control Group: Never Treated , - Anticipation Periods: 0 - Estimation Method: Doubly Robust - -``` python -out.plot_aggte(); -``` - - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - -![](README_files/figure-commonmark/cell-7-output-2.png) - - -**Event Studies** - -Although in the current example it is pretty easy to directly interpret -the group-time average treatment effects, there are many cases where it -is convenient to aggregate the group-time average treatment effects into -a small number of parameters. A main type of aggregation is into an -*event study* plot. - -To make an event study plot in the **csdid** package, one can use the -**aggte** function with **dynamic** option - -``` python -out.aggte(typec='dynamic'); -``` - - - - Overall summary of ATT's based on event-study/dynamic aggregation: - ATT Std. Error [95.0% Conf. Int.] - -0.0772 0.0207 -0.1179 -0.0366 * - - - Dynamic Effects: - Event time Estimate Std. Error [95.0% Simult. Conf. Band - 0 -3 0.0305 0.0146 0.0019 0.0591 * - 1 -2 -0.0006 0.0129 -0.0259 0.0248 - 2 -1 -0.0245 0.0141 -0.0521 0.0032 - 3 0 -0.0199 0.0117 -0.0428 0.0030 - 4 1 -0.0510 0.0154 -0.0811 -0.0208 * - 5 2 -0.1373 0.0366 -0.2091 -0.0655 * - 6 3 -0.1008 0.0337 -0.1669 -0.0347 * - --- - Signif. codes: `*' confidence band does not cover 0 - Control Group: Never Treated , - Anticipation Periods: 0 - Estimation Method: Doubly Robust - -The column `event time` is for each group relative to when they first -participate in the treatment. To give some examples, `event time=0` -corresponds to the *on impact* effect, and `event time=-1` is the -*effect* in the period before a unit becomes treated (checking that this -is equal to 0 is potentially useful as a pre-test). - -To plot the event study, use **plot_aggte** method -``` python -out.plot_aggte(); -``` - - /home/runner/work/csdid/csdid/csdid/plots/gplot.py:19: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['2004' '2005' '2006' '2007']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first. - ssresults.loc[:, 'year'] = ssresults['year'].astype(int).astype(str) - -![](README_files/figure-commonmark/cell-8-output-2.png) - -The figure here is very similar to the group-time average treatment -effects. Red dots are pre-treatment periods, blue dots are -post-treatment periods. The difference is that the x-axis is in event -time. - -**Overall Effect of Participating in the Treatment** - -The event study above reported an overall effect of participating in the -treatment. This was computed by averaging the average effects computed -at each length of exposure. - -In many cases, a more general purpose overall treatment effect parameter -is give by computing the average treatment effect for each group, and -then averaging across groups. This sort of procedure provides an average -treatment effect parameter with a very similar interpretation to the -Average Treatment Effect on the Treated (ATT) in the two period and two -group case. - -To compute this overall average treatment effect parameter, use - - -``` python -out.aggte(typec='group'); -``` - - - - Overall summary of ATT's based on group/cohort aggregation: - ATT Std. Error [95.0% Conf. Int.] - -0.031 0.0124 -0.0553 -0.0067 * - - - Group Effects: - Group Estimate Std. Error [95.0% Simult. Conf. Band - 0 2004 -0.0797 0.0301 -0.1387 -0.0208 * - 1 2006 -0.0229 0.0172 -0.0567 0.0109 - 2 2007 -0.0261 0.0174 -0.0601 0.0080 - --- - Signif. codes: `*' confidence band does not cover 0 - Control Group: Never Treated , - Anticipation Periods: 0 - Estimation Method: Doubly Robust - -Of particular interest is the `Overall ATT` in the results. Here, we -estimate that increasing the minimum wage decreased teen employment by -3.1% and the effect is marginally statistically significant. - -# How to cite -If you want to cite CSDID, you can use the following BibTeX entry: - -``` python -@software{csdid, - author = {Callaway, Brantly and Sant'Anna, Pedro HC and Quispe, Alexander and Guevara, Carlos}, - title = {{csdid: Difference-in-Differences with Multiple Time Periods in Python}}, - year = {2024}, - url = {https://github.com/d2cml-ai/csdid} -} -``` \ No newline at end of file diff --git a/scripts/check_readme.py b/scripts/check_readme.py new file mode 100644 index 0000000..561b967 --- /dev/null +++ b/scripts/check_readme.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Reject known README generation regressions before the bot opens a PR.""" + +import argparse +from html import unescape +from html.parser import HTMLParser +from pathlib import Path +import re +import subprocess +import sys +from urllib.parse import parse_qs, urlsplit + + +REQUIRED_SECTIONS = ( + "Getting Started", + "Basic Example", + "Event studies", + "Overall effect of participating in treatment", + "Additional options", + "How to cite", +) +REQUIRED_BADGES = { + "PyPI version": "https://img.shields.io/pypi/v/csdid.svg", + "Downloads": "https://static.pepy.tech/personalized-badge/csdid?", + "Last commit": "https://img.shields.io/github/last-commit/d2cml-ai/csdid.svg", + "GitHub stars": "https://img.shields.io/github/stars/d2cml-ai/csdid.svg", + "GitHub issues": "https://img.shields.io/github/issues/d2cml-ai/csdid.svg", + "License": "https://img.shields.io/github/license/d2cml-ai/csdid.svg", +} + + +class _ImageParser(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.images = [] + + def handle_starttag(self, tag, attrs): + if tag.casefold() == "img": + attributes = dict(attrs) + self.images.append((attributes.get("alt", ""), attributes.get("src", ""))) + + +def image_references(content): + """Read Markdown images and HTML images, including escaped query strings.""" + parser = _ImageParser() + parser.feed(content) + references = re.findall(r"!\[([^\]]*)\]\(([^\s)]+)\)", content) + references.extend(parser.images) + return [(" ".join((alt or "").split()), unescape(source or "")) for alt, source in references] + + +def check_document(content, name): + """Check authored content that the old bot source silently discarded.""" + errors = [] + normalized = " ".join(content.split()) + if not re.search(r"\*\*csdid\*\* package implements", normalized): + errors.append(f"{name}: the Python package must be identified as csdid") + if re.search(r"\*\*did\*\* package", normalized): + errors.append(f"{name}: the Python package was renamed to did") + + headings = { + " ".join(match.split()).casefold() + for match in re.findall(r"^#{1,6}\s+(.+?)\s*#*\s*$", content, re.MULTILINE) + } + for heading in REQUIRED_SECTIONS: + if heading.casefold() not in headings: + errors.append(f"{name}: missing section '{heading}'") + for aggregation in ("dynamic", "group"): + if not re.search(r"\.aggte\(\s*typec\s*=\s*['\"]" + aggregation + r"['\"]", content): + errors.append(f"{name}: missing {aggregation} aggregation example") + images = image_references(content) + for badge, image_url in REQUIRED_BADGES.items(): + sources = [source for alt, source in images if alt == badge and source.startswith(image_url)] + if not sources: + errors.append(f"{name}: missing {badge} badge for csdid") + if badge == "Downloads": + for source in sources: + if parse_qs(urlsplit(source).query).get("left_text") != ["Downloads"]: + errors.append(f"{name}: Downloads badge left_text must be exactly Downloads") + if not re.search(r"@software\s*\{csdid\s*,", content): + errors.append(f"{name}: missing csdid software citation") + return errors + + +def check_readme_paths(paths): + """Git can track case aliases that a macOS checkout cannot distinguish.""" + aliases = sorted(path for path in paths if path.casefold() == "readme.md") + if aliases != ["README.md"]: + return [f"keep one canonical README.md; tracked README paths: {aliases}"] + return [] + + +def check_images(content, root): + errors = [] + references = [source for _, source in image_references(content) if source.startswith("README_files/")] + if not references: + errors.append("README.md: no generated example figures") + for reference in references: + path = (root / reference).resolve() + try: + path.relative_to(root.resolve()) + except ValueError: + errors.append(f"README.md: figure path escapes the repository: {reference}") + continue + if not path.is_file(): + errors.append(f"README.md: missing generated figure {reference}") + return errors + + +def check_repository(root, source_only=False): + errors = [] + tracked = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z"], + check=True, capture_output=True, text=True, + ).stdout.split("\0") + errors.extend(check_readme_paths(tracked)) + names = ["README.qmd"] if source_only else ["README.qmd", "README.md"] + for name in names: + path = root / name + if not path.is_file(): + errors.append(f"missing {name}") + continue + content = path.read_text(encoding="utf-8") + errors.extend(check_document(content, name)) + if name == "README.md": + errors.extend(check_images(content, root)) + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--source-only", action="store_true") + args = parser.parse_args() + errors = check_repository(args.root, args.source_only) + if errors: + for error in errors: + print(error, file=sys.stderr) + return 1 + print("README content, canonical path, and generated figures are valid.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_release_ci.py b/scripts/check_release_ci.py index 65aaffd..daaee35 100644 --- a/scripts/check_release_ci.py +++ b/scripts/check_release_ci.py @@ -12,6 +12,7 @@ import subprocess import sys import tarfile +import time from urllib.error import HTTPError from urllib.parse import urlparse from urllib.request import Request, urlopen @@ -66,8 +67,8 @@ def candidate(event_name, event, repository, repository_id, ref, sha): return sha -def fetch_json(url, token=None): - headers = {"Accept": "application/json", "User-Agent": "csdid-release-check"} +def fetch_json(url, token=None, accept="application/json"): + headers = {"Accept": accept, "User-Agent": "csdid-release-check"} if token: headers["Accept"] = "application/vnd.github+json" headers["Authorization"] = "Bearer " + token @@ -227,7 +228,19 @@ def verify_sdist(value, version, expected_files, build_files, requirements): raise GateError("Published csdid source archive metadata differs; use a new version") -def release_ready(source_path): +def csdid_version(source_bytes): + assignments = [node for node in ast.parse(source_bytes).body + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "__version" for target in node.targets)] + if len(assignments) != 1: + raise GateError("Expected one CSDID version assignment") + version = ast.literal_eval(assignments[0].value) + if not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise GateError("Expected a complete CSDID release version") + return version + + +def release_ready(source_path, release=None, confirm_publication=False): """Skip a complete identical release, or wait for its locked backend. This is a read-only preflight. Publication still rechecks CI and verifies @@ -246,14 +259,7 @@ def git(*args): for parent in Path(path).parents if str(parent) != ".")} if "csdid/_version.py" not in expected: raise GateError("Reviewed CSDID Python source is incomplete") - assignments = [node for node in ast.parse(expected["csdid/_version.py"]).body - if isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == "__version" for target in node.targets)] - if len(assignments) != 1: - raise GateError("Expected one CSDID version assignment") - version = ast.literal_eval(assignments[0].value) - if not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): - raise GateError("Expected a complete CSDID release version") + version = csdid_version(expected["csdid/_version.py"]) setup = ast.parse(git("show", "HEAD:setup.py")) build_paths = git("ls-files", "-z", "--", "setup.py", "setup.cfg", "pyproject.toml", "MANIFEST.in").decode().split("\0") build_files = {path: git("show", "HEAD:" + path) for path in build_paths if path} @@ -271,7 +277,8 @@ def git(*args): if backend_requirements != ["drdid==" + lock["version"]]: raise GateError("setup.py must require exactly the reviewed backend release") try: - release = fetch_json("https://pypi.org/pypi/csdid/" + version + "/json") + if release is None: + release = fetch_json("https://pypi.org/pypi/csdid/" + version + "/json") except HTTPError as error: if error.code != 404: raise @@ -291,6 +298,10 @@ def git(*args): if sdists: verify_sdist(download_artifact(sdists[0]), version, expected, build_files, requirements) if wheels and sdists: + if confirm_publication: + # An earlier postcheck may have timed out after JSON became + # visible but before installers could see the Simple entries. + wait_for_publication(source) return False, "Skipped: csdid " + version + " is already published with matching Python source, build inputs and dependencies." try: backend = fetch_json("https://pypi.org/pypi/drdid/" + lock["version"] + "/json") @@ -303,6 +314,59 @@ def git(*args): return True, "CSDID " + version + " needs publication; the locked backend is available for byte verification." +def publication_ready(source_path): + """Require verified artifacts and their public installer index entries.""" + version = csdid_version((Path(source_path) / "csdid/_version.py").read_bytes()) + try: + release = fetch_json("https://pypi.org/pypi/csdid/" + version + "/json") + if release.get("info", {}).get("name", "").lower() != "csdid" or release["info"].get("version") != version: + raise GateError("Unexpected public CSDID publication identity") + if not release["urls"]: + return False, "Waiting: the public CSDID release does not list its artifacts yet." + if any(item.get("packagetype") not in ("bdist_wheel", "sdist") for item in release["urls"]): + raise GateError("Unexpected CSDID publication artifact") + # Reuse the full clean-source, wheel/sdist, dependency and digest checks. + # Supplying this public snapshot avoids a second inconsistent JSON read. + ready, message = release_ready(source_path, release=release) + if ready or {item["packagetype"] for item in release["urls"]} != {"bdist_wheel", "sdist"}: + return False, "Waiting: both verified CSDID publication artifacts are not visible yet. " + message + simple = fetch_json("https://pypi.org/simple/csdid/", accept="application/vnd.pypi.simple.v1+json") + if simple.get("name", "").lower() != "csdid": + raise GateError("Unexpected CSDID Simple index project name") + missing = [] + for artifact in release["urls"]: + entries = [item for item in simple["files"] if item["filename"] == artifact["filename"]] + if not entries: + missing.append(artifact["filename"]) + continue + if len(entries) != 1: + raise GateError("Duplicate CSDID Simple index entries") + entry = entries[0] + if (entry.get("yanked", False) is not False + or entry.get("hashes", {}).get("sha256") != artifact["digests"]["sha256"] + or entry.get("url") != artifact["url"]): + raise GateError("Published CSDID Simple index identity differs from verified artifacts") + if missing: + return False, "Waiting: the public Simple index does not list these artifacts yet: " + ", ".join(missing) + except HTTPError as error: + if error.code == 404: + return False, "Waiting: public CSDID publication data is not visible yet (HTTP 404)." + raise + return True, "Verified public csdid " + version + " wheel, source archive and Simple index against reviewed source and PyPI hashes." + + +def wait_for_publication(source_path, delays=(5, 10, 20, 40, 60)): + """Six checks; only absent/partial public visibility can consume a retry.""" + for attempt in range(len(delays) + 1): + ready, message = publication_ready(source_path) + if ready: + return message + print(message, flush=True) + if attempt == len(delays): + raise GateError("Timed out waiting for verified public CSDID publication") + time.sleep(delays[attempt]) + + def read_lock(lock_path): lock = json.loads(Path(lock_path).read_text()) # The release lock is authored and reviewed alongside the package version. @@ -359,13 +423,16 @@ def output(message, ready=None, sha=None): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("mode", choices=("ci", "backend", "lock", "release")) + parser.add_argument("mode", choices=("ci", "backend", "lock", "release", "published")) parser.add_argument("--lock") parser.add_argument("--peer") args = parser.parse_args() try: + if args.mode == "published": + output(wait_for_publication(Path.cwd())) + return 0 if args.mode == "release": - ready, message = release_ready(Path.cwd()) + ready, message = release_ready(Path.cwd(), confirm_publication=True) output(message, ready) return 0 if args.mode == "lock": diff --git a/scripts/tests/test_check_readme.py b/scripts/tests/test_check_readme.py new file mode 100644 index 0000000..0e830ba --- /dev/null +++ b/scripts/tests/test_check_readme.py @@ -0,0 +1,101 @@ +"""Exercise the content losses observed in the generated README bot PR.""" + +import importlib.util +from pathlib import Path +import re +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("check_readme", ROOT / "scripts/check_readme.py") +guard = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(guard) + + +class ReadmeGuardTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.source = (ROOT / "README.qmd").read_text(encoding="utf-8") + cls.rendered = (ROOT / "README.md").read_text(encoding="utf-8") + + def test_repaired_source_and_rendered_readme_are_accepted(self): + self.assertEqual(guard.check_document(self.source, "README.qmd"), []) + self.assertEqual(guard.check_document(self.rendered, "README.md"), []) + self.assertEqual(guard.check_images(self.rendered, ROOT), []) + + def test_old_bot_package_rename_is_rejected(self): + damaged = self.rendered.replace("**csdid** package implements", "**did** package implements") + self.assertTrue(damaged != self.rendered) + self.assertTrue(any("renamed to did" in error for error in guard.check_document(damaged, "bot"))) + + def test_each_lost_badge_is_rejected(self): + for badge in guard.REQUIRED_BADGES: + with self.subTest(badge=badge): + alt = r"\s+".join(re.escape(word) for word in badge.split()) + damaged = re.sub(r"\[!\[" + alt + r"\]\([^)]+\)\]\([^)]+\)\s*", "", self.rendered) + damaged = re.sub(r"]*\balt=['\"]" + alt + r"['\"])[^>]*>", "", damaged, + flags=re.IGNORECASE) + self.assertTrue(damaged != self.rendered) + self.assertTrue(any(badge + " badge" in error for error in guard.check_document(damaged, "bot"))) + + def test_html_download_badge_accepts_escaped_query(self): + for content in (self.source, self.rendered): + with self.subTest(document="source" if content == self.source else "rendered"): + images = dict(guard.image_references(content)) + original = images["Downloads"] + escaped = original.replace("&", "&") + html_badge = 'Downloads' + candidate = re.sub(r"]*\balt=['\"]Downloads['\"])[^>]*>", html_badge, content, + flags=re.IGNORECASE) + candidate = re.sub(r"!\[Downloads\]\([^)]+\)", html_badge, candidate) + self.assertEqual(guard.check_document(candidate, "HTML badge"), []) + + def test_real_quarto_download_label_corruption_is_rejected(self): + # Quarto appended .png to this exact extensionless SVG URL during regeneration. + corrupt_url = ( + "https://static.pepy.tech/personalized-badge/csdid?period=total" + "&units=international_system&left_color=blue&right_color=grey" + "&left_text=Downloads.png" + ) + for image in ('![Downloads](' + corrupt_url + ')', + 'Downloads'): + with self.subTest(markup=image[:15]): + errors = guard.check_document(self.rendered + "\n" + image, "corrupted badge") + self.assertTrue(any("left_text must be exactly Downloads" in error for error in errors)) + + def test_lost_examples_and_citation_are_rejected(self): + for title in ("Event studies", "Overall effect of participating in treatment", "How to cite"): + with self.subTest(section=title): + damaged = re.sub(r"^## " + re.escape(title) + r"\n.*?(?=^## |\Z)", "", self.rendered, + flags=re.MULTILINE | re.DOTALL | re.IGNORECASE) + self.assertTrue(damaged != self.rendered) + self.assertTrue(any("missing section '" + title + "'" in error + for error in guard.check_document(damaged, "bot"))) + + def test_citation_text_is_required_even_when_heading_remains(self): + damaged = self.rendered.replace("@software{csdid,", "@software{other,") + self.assertTrue(damaged != self.rendered) + self.assertTrue(any("software citation" in error for error in guard.check_document(damaged, "bot"))) + + def test_example_code_is_required_even_when_headings_remain(self): + for aggregation in ("dynamic", "group"): + with self.subTest(aggregation=aggregation): + damaged = re.sub(r"\.aggte\(\s*typec\s*=\s*['\"]" + aggregation + r"['\"]", ".removed()", self.rendered) + self.assertTrue(damaged != self.rendered) + self.assertTrue(any(aggregation + " aggregation example" in error + for error in guard.check_document(damaged, "bot"))) + + def test_case_variant_readmes_are_rejected(self): + self.assertEqual(guard.check_readme_paths(["README.md", "README.qmd"]), []) + self.assertTrue(guard.check_readme_paths(["README.md", "readme.md"])) + self.assertTrue(guard.check_readme_paths(["readme.md"])) + + def test_missing_generated_image_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + errors = guard.check_images(self.rendered, Path(directory)) + self.assertTrue(any("missing generated figure" in error for error in errors)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_check_release_ci.py b/scripts/tests/test_check_release_ci.py index 7f72c20..3cd27f3 100644 --- a/scripts/tests/test_check_release_ci.py +++ b/scripts/tests/test_check_release_ci.py @@ -273,7 +273,7 @@ def wheel(self, dependencies=("numpy", "drdid==1.1.7"), files=None, sdist=True): "url": "https://files.pythonhosted.org/packages/reviewed.whl", "digests": {"sha256": hashlib.sha256(value).hexdigest()}} self.downloads[artifact["url"]] = value - return value, {"info": {"version": "0.4.4"}, "urls": [artifact] + ([self.sdist(dependencies=dependencies)] if sdist else [])} + return value, {"info": {"name": "csdid", "version": "0.4.4"}, "urls": [artifact] + ([self.sdist(dependencies=dependencies)] if sdist else [])} def sdist(self, dependencies=("numpy", "drdid==1.1.7"), changes=None): files = dict(self.files, **{"setup.py": (self.root / "setup.py").read_bytes(), @@ -372,6 +372,106 @@ def test_dependency_lock_mismatch_and_dirty_checkout_are_blocked(self): with self.assertRaisesRegex(gate.GateError, "must be clean"): gate.release_ready(self.root) + def simple(self, release): + return {"name": "csdid", "files": [dict(filename=item["filename"], url=item["url"], + hashes={"sha256": item["digests"]["sha256"]}, yanked=False) + for item in release["urls"]]} + + def test_post_upload_cached_404_waits_then_verifies_public_artifacts_and_simple(self): + _, release = self.wheel() + with patch.object(gate, "fetch_json", side_effect=[self.missing(), release, self.simple(release)]) as fetch, patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + self.assertIn("Verified public csdid 0.4.4", gate.wait_for_publication(self.root)) + sleep.assert_called_once_with(5) + self.assertEqual(fetch.call_args.kwargs["accept"], "application/vnd.pypi.simple.v1+json") + + def test_post_upload_empty_release_file_list_is_bounded_visibility_pending(self): + _, release = self.wheel() + empty = dict(release, urls=[]) + with patch.object(gate, "fetch_json", side_effect=[empty, release, self.simple(release)]), patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + self.assertIn("Verified public", gate.wait_for_publication(self.root)) + sleep.assert_called_once_with(5) + + def test_post_upload_waits_for_either_partial_artifact_order_and_simple_visibility(self): + _, release = self.wheel() + for index in (0, 1): + partial = dict(release, urls=[release["urls"][index]]) + partial_simple = self.simple(partial) + responses = [partial, {"urls": [{"packagetype": "bdist_wheel"}]}, + release, partial_simple, release, self.simple(release)] + with self.subTest(first_artifact=index), patch.object(gate, "fetch_json", side_effect=responses), patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + self.assertIn("Verified public", gate.wait_for_publication(self.root)) + self.assertEqual([call.args[0] for call in sleep.call_args_list], [5, 10]) + + def test_post_upload_mismatched_existing_source_fails_without_retry(self): + _, release = self.wheel(files=dict(self.files, **{"csdid/att_gt.py": b"old = True\n"})) + with patch.object(gate, "fetch_json", return_value=release), patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + with self.assertRaisesRegex(gate.GateError, "use a new version"): + gate.wait_for_publication(self.root) + sleep.assert_not_called() + + def test_post_upload_simple_hash_url_yank_and_duplicate_mismatches_fail_without_retry(self): + _, release = self.wheel() + for mutation in ("hash", "url", "yanked", "duplicate", "name"): + simple = self.simple(release) + if mutation == "hash": + simple["files"][0]["hashes"]["sha256"] = "0" * 64 + elif mutation == "url": + simple["files"][0]["url"] = "https://attacker.invalid/unreviewed.whl" + elif mutation == "yanked": + simple["files"][0]["yanked"] = True + elif mutation == "name": + simple["name"] = "another-project" + else: + simple["files"].append(dict(simple["files"][0])) + with self.subTest(mutation=mutation), patch.object(gate, "fetch_json", side_effect=[release, simple]), patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + with self.assertRaises(gate.GateError): + gate.wait_for_publication(self.root) + sleep.assert_not_called() + + def test_post_upload_missing_simple_entry_cannot_hide_visible_mismatch(self): + _, release = self.wheel() + simple = self.simple(release) + simple["files"] = simple["files"][1:] + simple["files"][0]["hashes"]["sha256"] = "0" * 64 + with patch.object(gate, "fetch_json", side_effect=[release, simple]), patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + with self.assertRaisesRegex(gate.GateError, "identity differs"): + gate.wait_for_publication(self.root) + sleep.assert_not_called() + + def test_post_upload_persistent_absence_times_out_after_six_attempts(self): + with patch.object(gate, "fetch_json", side_effect=self.missing()) as fetch, patch.object(gate.time, "sleep") as sleep: + with self.assertRaisesRegex(gate.GateError, "Timed out"): + gate.wait_for_publication(self.root) + self.assertEqual(fetch.call_count, 6) + self.assertEqual([call.args[0] for call in sleep.call_args_list], [5, 10, 20, 40, 60]) + + def test_post_upload_non_visibility_errors_are_not_retried(self): + for error in (HTTPError("url", 503, "Unavailable", {}, None), ValueError("malformed response")): + with self.subTest(error=error), patch.object(gate, "fetch_json", side_effect=error), patch.object(gate.time, "sleep") as sleep: + with self.assertRaises(type(error)): + gate.wait_for_publication(self.root) + sleep.assert_not_called() + + def test_existing_complete_noop_rechecks_installer_visibility_before_skipping_build(self): + _, release = self.wheel() + missing_simple = dict(self.simple(release), files=[]) + responses = [release, release, missing_simple, release, self.simple(release)] + with patch.object(gate, "fetch_json", side_effect=responses) as fetch, patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + ready, message = gate.release_ready(self.root, confirm_publication=True) + self.assertFalse(ready) + self.assertIn("already published", message) + sleep.assert_called_once_with(5) + self.assertFalse(any('/drdid/' in call.args[0] for call in fetch.call_args_list)) + + def test_existing_complete_noop_cannot_hide_persistent_unconfirmed_simple_index(self): + _, release = self.wheel() + def fetch(url, **kwargs): + return dict(self.simple(release), files=[]) if '/simple/' in url else release + with patch.object(gate, "fetch_json", side_effect=fetch), patch.object(gate, "urlopen", side_effect=self.download), patch.object(gate.time, "sleep") as sleep: + with self.assertRaisesRegex(gate.GateError, "Timed out"): + gate.release_ready(self.root, confirm_publication=True) + self.assertEqual([call.args[0] for call in sleep.call_args_list], [5, 10, 20, 40, 60]) + if __name__ == "__main__": unittest.main() diff --git a/setup.py b/setup.py index b9a9576..7672984 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ # Long description for PyPI (rendered from the project README). _here = os.path.abspath(os.path.dirname(__file__)) try: - with open(os.path.join(_here, 'readme.md'), encoding='utf-8') as _f: + with open(os.path.join(_here, 'README.md'), encoding='utf-8') as _f: long_description = _f.read() except Exception: long_description = 'Difference in Difference in Python'