From d7cc7bb225af8ee0741717bc993bddc8b99f13cb Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 2 Sep 2026 08:47:30 -0700 Subject: [PATCH 1/3] Say what is half-finished, without being asked A deposit sat unrecorded for a fortnight. Nothing was broken and nothing complained: the only thing that would have was the NEXT deposit, which refused and made the backlog somebody's problem at the worst moment. The weakness is not the manual steps -- a person merging the request to deposit is the right design -- it is that a step can fail to happen and leave no trace. scripts/outstanding.py names the half-finished states: identifiers written but never merged, a deposit asked for and never run, a note whose version has moved past the copy on its DOI, an archival note published without one, and a DOI with no record id. `pixi run outstanding` locally; exit 1 if anything is outstanding. A report nobody runs has the failure mode it was built to fix, so the `outstanding` workflow runs it weekly and keeps ONE issue: opened when something is outstanding, edited while it stays that way, closed when it clears, and silent otherwise. Detecting a note that has outrun its deposit needs a fact we were not recording. `archived_at` says when a copy was taken, and a git timestamp cannot tell a rewrite from a typo, so the deposit now stamps `archived_version` -- the note's own version at the moment it was deposited. `version != archived_version` is then exactly the state where the DOI serves older text than the site. Backfilled for the 43 deposited notes and added to the schema. Running it found a second defect, unrelated to the first and invisible until now: deposit-ready re-asks on every metadata change, because it checked deposit-queue.txt (what was MERGED) and not what was already open. One note had four open requests. Duplicate reminders are how a reminder becomes something you scroll past, so it now counts an open request as having asked. Underworld development team with AI support from Claude Code --- .github/workflows/deposit-ready.yml | 17 ++ .github/workflows/outstanding.yml | 88 +++++++++ articles/2-11-scaling/metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + articles/build-conda-packages/metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../free-surface-in-underworld/metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../particles-in-underworld3/metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + articles/scaling-in-underworld/metadata.yml | 1 + .../self-updating-repositories/metadata.yml | 1 + .../setting-up-full-multigrid/metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + .../metadata.yml | 1 + articles/underworld-2/metadata.yml | 1 + .../underworld-and-docker-part-1/metadata.yml | 1 + .../underworld-and-docker-part-2/metadata.yml | 1 + .../underworld-and-singularity/metadata.yml | 1 + .../underworld-low-fat-cloud/metadata.yml | 1 + articles/underworld-on-zenodo/metadata.yml | 1 + .../underworld3-come-and-get-it/metadata.yml | 1 + articles/untitled-2/metadata.yml | 1 + articles/untitled/metadata.yml | 1 + .../metadata.yml | 1 + articles/viscoelasticity/metadata.yml | 1 + pixi.toml | 6 + schemas/article-metadata.schema.json | 5 + scripts/deposit.py | 10 +- scripts/outstanding.py | 180 ++++++++++++++++++ tests/test_outstanding.py | 79 ++++++++ 50 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/outstanding.yml create mode 100644 scripts/outstanding.py create mode 100644 tests/test_outstanding.py diff --git a/.github/workflows/deposit-ready.yml b/.github/workflows/deposit-ready.yml index 08ad9a1..3105578 100644 --- a/.github/workflows/deposit-ready.yml +++ b/.github/workflows/deposit-ready.yml @@ -31,7 +31,19 @@ jobs: - name: Which notes have no DOI id: pending + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + # A request that is already OPEN counts as asked. deposit-queue.txt + # only records what has been merged, so without this the workflow + # re-asks on every metadata change and the same note accumulates + # requests -- four for one note, before this was noticed. Duplicate + # reminders are how a reminder becomes something you scroll past. + gh pr list --state open --limit 100 --json title \ + --jq '.[] | select(.title | startswith("Deposit: ")) | + .title | sub("^Deposit: "; "")' \ + | tr ',' '\n' | tr -d ' ' | sed '/^$/d' > .already-asked || true + echo "already asked: $(tr '\n' ' ' < .already-asked)" SLUGS=$(pixi run -q python3 -c " import sys; sys.path.insert(0, 'scripts'); import deposit queued = set() @@ -41,8 +53,13 @@ jobs: if line: queued.add(line) except FileNotFoundError: pass + try: + queued.update(l.strip() for l in open('.already-asked') if l.strip()) + except FileNotFoundError: + pass print(','.join(s for s in deposit.pending() if s not in queued)) ") + rm -f .already-asked echo "slugs=${SLUGS}" >> "$GITHUB_OUTPUT" echo "not yet queued: ${SLUGS:-none}" diff --git a/.github/workflows/outstanding.yml b/.github/workflows/outstanding.yml new file mode 100644 index 0000000..859f685 --- /dev/null +++ b/.github/workflows/outstanding.yml @@ -0,0 +1,88 @@ +name: outstanding + +# Every step of publishing here is deliberate: a person merges the request to +# deposit, a person merges the identifiers that come back. That is the right +# design, and its failure mode is that a step simply does not happen and +# nothing says so. A deposit sat unrecorded for a fortnight because the only +# thing that would have complained was the NEXT deposit -- which is the worst +# possible moment to find out. +# +# So this asks the question on a schedule instead of relying on anyone +# remembering to. It maintains ONE issue: opened when something is +# outstanding, edited while it stays outstanding, closed when it clears. It +# never opens a second one, and it says nothing at all when there is nothing +# to say. + +on: + schedule: + # Mondays, 22:00 UTC -- Tuesday morning in Canberra, so the week starts + # with the list rather than ending with it. + - cron: '0 22 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write + pull-requests: read + +concurrency: + group: outstanding + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: prefix-dev/setup-pixi@v0.8.1 + with: + cache: true + + - name: What is half-finished + id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + pixi run -q python3 scripts/outstanding.py > report.txt + echo "count=$?" >> "$GITHUB_OUTPUT" + set -e + cat report.txt + { + echo "## Outstanding" + echo '```' + cat report.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Open, update or close the one issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TITLE="Publishing steps not finished" + NUM=$(gh issue list --state open --limit 50 \ + --json number,title \ + --jq ".[] | select(.title == \"$TITLE\") | .number" | head -1) + + if [ "${{ steps.check.outputs.count }}" = "0" ]; then + if [ -n "$NUM" ]; then + gh issue close "$NUM" \ + --comment "Nothing outstanding as of $(date -u +%Y-%m-%d). Closed automatically." + echo "closed #$NUM" + else + echo "nothing outstanding, no issue open -- saying nothing" + fi + exit 0 + fi + + BODY=$(printf '%s\n\n```\n%s\n```\n\n%s\n' \ + "Steps that were started and not finished. Regenerated each Monday by the \`outstanding\` workflow; run \`pixi run outstanding\` to see the same thing locally." \ + "$(cat report.txt)" \ + "This issue closes itself when the list is empty.") + + if [ -n "$NUM" ]; then + gh issue edit "$NUM" --body "$BODY" + echo "updated #$NUM" + else + gh issue create --title "$TITLE" --body "$BODY" + fi diff --git a/articles/2-11-scaling/metadata.yml b/articles/2-11-scaling/metadata.yml index 8f3406d..4c16880 100644 --- a/articles/2-11-scaling/metadata.yml +++ b/articles/2-11-scaling/metadata.yml @@ -16,6 +16,7 @@ version: 1.0.0 legacy_doi: 10.59350/3wz3c-c8w65 archive_doi: 10.6084/m9.figshare.33193458 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /2-11-scaling/ legacy_paths: diff --git a/articles/30-years-of-citcom-ellipsis-and-underworld/metadata.yml b/articles/30-years-of-citcom-ellipsis-and-underworld/metadata.yml index 5159138..bbf6f93 100644 --- a/articles/30-years-of-citcom-ellipsis-and-underworld/metadata.yml +++ b/articles/30-years-of-citcom-ellipsis-and-underworld/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/hsp06-ag431 archive_doi: 10.6084/m9.figshare.33193530 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /30-years-of-citcom-ellipsis-and-underworld/ legacy_paths: diff --git a/articles/adding-zotero-references-to-a-webpage/metadata.yml b/articles/adding-zotero-references-to-a-webpage/metadata.yml index 6f1a2f2..4f6d175 100644 --- a/articles/adding-zotero-references-to-a-webpage/metadata.yml +++ b/articles/adding-zotero-references-to-a-webpage/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/qnz5b-4dt16 archive_doi: 10.6084/m9.figshare.33193416 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /adding-zotero-references-to-a-webpage/ legacy_paths: diff --git a/articles/ai-and-scientific-software-what-we-learned-rebuilding-underworld3/metadata.yml b/articles/ai-and-scientific-software-what-we-learned-rebuilding-underworld3/metadata.yml index 01cf1a2..ddffedc 100644 --- a/articles/ai-and-scientific-software-what-we-learned-rebuilding-underworld3/metadata.yml +++ b/articles/ai-and-scientific-software-what-we-learned-rebuilding-underworld3/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/rmayz-81x80 archive_doi: 10.6084/m9.figshare.33193560 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /ai-and-scientific-software-what-we-learned-rebuilding-underworld3/ legacy_paths: diff --git a/articles/alaska-moho-model-reproducible-research-with-containers/metadata.yml b/articles/alaska-moho-model-reproducible-research-with-containers/metadata.yml index eb81f56..6cedbb9 100644 --- a/articles/alaska-moho-model-reproducible-research-with-containers/metadata.yml +++ b/articles/alaska-moho-model-reproducible-research-with-containers/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/pn8gh-98592 archive_doi: 10.6084/m9.figshare.33193410 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /alaska-moho-model-reproducible-research-with-containers/ legacy_paths: diff --git a/articles/build-conda-packages/metadata.yml b/articles/build-conda-packages/metadata.yml index 06bdd29..c6ffa23 100644 --- a/articles/build-conda-packages/metadata.yml +++ b/articles/build-conda-packages/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/7x9h6-y1x53 archive_doi: 10.6084/m9.figshare.33193440 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /build-conda-packages/ legacy_paths: diff --git a/articles/compressible-convection-in-cartesian-coordinates-in-underworld3/metadata.yml b/articles/compressible-convection-in-cartesian-coordinates-in-underworld3/metadata.yml index d744393..793c247 100644 --- a/articles/compressible-convection-in-cartesian-coordinates-in-underworld3/metadata.yml +++ b/articles/compressible-convection-in-cartesian-coordinates-in-underworld3/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/g4ysn-pv176 archive_doi: 10.6084/m9.figshare.33193527 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /compressible-convection-in-cartesian-coordinates-in-underworld3/ legacy_paths: diff --git a/articles/constitutive-models-in-symbolic-form/metadata.yml b/articles/constitutive-models-in-symbolic-form/metadata.yml index a07411d..0e7f4f4 100644 --- a/articles/constitutive-models-in-symbolic-form/metadata.yml +++ b/articles/constitutive-models-in-symbolic-form/metadata.yml @@ -33,4 +33,5 @@ source: ghost-migration ghost_uuid: 137d7c0a-1a6b-412b-a605-069137faac95 repository_record_id: 33193590 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:28:12Z diff --git a/articles/craton-formation-and-the-onset-of-plate-tectonics/metadata.yml b/articles/craton-formation-and-the-onset-of-plate-tectonics/metadata.yml index 9810037..9b3d8fc 100644 --- a/articles/craton-formation-and-the-onset-of-plate-tectonics/metadata.yml +++ b/articles/craton-formation-and-the-onset-of-plate-tectonics/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/c4g09-htk29 archive_doi: 10.6084/m9.figshare.33193398 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /craton-formation-and-the-onset-of-plate-tectonics/ legacy_paths: diff --git a/articles/finding-particles-in-a-distributed-unstructured-mesh/metadata.yml b/articles/finding-particles-in-a-distributed-unstructured-mesh/metadata.yml index 7c16c00..60c531c 100644 --- a/articles/finding-particles-in-a-distributed-unstructured-mesh/metadata.yml +++ b/articles/finding-particles-in-a-distributed-unstructured-mesh/metadata.yml @@ -31,4 +31,5 @@ source: ghost-migration ghost_uuid: 1b9677b5-5308-4c90-a192-d29aad4000b7 repository_record_id: 33193611 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:28:42Z diff --git a/articles/free-surface-in-underworld/metadata.yml b/articles/free-surface-in-underworld/metadata.yml index cca7908..ac7dc1b 100644 --- a/articles/free-surface-in-underworld/metadata.yml +++ b/articles/free-surface-in-underworld/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/1a3e6-3v712 archive_doi: 10.6084/m9.figshare.33193503 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /free-surface-in-underworld/ legacy_paths: diff --git a/articles/getting-started-60-seconds-to-underworld/metadata.yml b/articles/getting-started-60-seconds-to-underworld/metadata.yml index 49f0fe2..b7f9805 100644 --- a/articles/getting-started-60-seconds-to-underworld/metadata.yml +++ b/articles/getting-started-60-seconds-to-underworld/metadata.yml @@ -12,6 +12,7 @@ version: 1.0.0 legacy_doi: 10.59350/3y92k-n4v30 archive_doi: 10.6084/m9.figshare.33193419 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /getting-started-60-seconds-to-underworld/ legacy_paths: diff --git a/articles/getting-started-with-pull-requests/metadata.yml b/articles/getting-started-with-pull-requests/metadata.yml index 7df7059..050f838 100644 --- a/articles/getting-started-with-pull-requests/metadata.yml +++ b/articles/getting-started-with-pull-requests/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/cj1d1-e4445 archive_doi: 10.6084/m9.figshare.33193434 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /getting-started-with-pull-requests/ legacy_paths: diff --git a/articles/how-many-processors-should-we-use-to-solve-problem-x/metadata.yml b/articles/how-many-processors-should-we-use-to-solve-problem-x/metadata.yml index 4adb8cd..ca2fe6c 100644 --- a/articles/how-many-processors-should-we-use-to-solve-problem-x/metadata.yml +++ b/articles/how-many-processors-should-we-use-to-solve-problem-x/metadata.yml @@ -16,6 +16,7 @@ version: 1.0.0 legacy_doi: 10.59350/tjm5s-sfs33 archive_doi: 10.6084/m9.figshare.33193554 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /how-many-processors-should-we-use-to-solve-problem-x/ legacy_paths: diff --git a/articles/how-to-install-underworld-on-mac-osx-big-sur-apple-silicon-m1/metadata.yml b/articles/how-to-install-underworld-on-mac-osx-big-sur-apple-silicon-m1/metadata.yml index a1b75c5..50a1f0a 100644 --- a/articles/how-to-install-underworld-on-mac-osx-big-sur-apple-silicon-m1/metadata.yml +++ b/articles/how-to-install-underworld-on-mac-osx-big-sur-apple-silicon-m1/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/8nsyv-btx30 archive_doi: 10.6084/m9.figshare.33193455 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /how-to-install-underworld-on-mac-osx-big-sur-apple-silicon-m1/ legacy_paths: diff --git a/articles/how-underworld3-turns-sympy-into-c/metadata.yml b/articles/how-underworld3-turns-sympy-into-c/metadata.yml index 04a47fa..34df7e7 100644 --- a/articles/how-underworld3-turns-sympy-into-c/metadata.yml +++ b/articles/how-underworld3-turns-sympy-into-c/metadata.yml @@ -38,4 +38,5 @@ source: ghost-migration ghost_uuid: ba7cc108-239e-4371-8182-2316f9e2cf7c repository_record_id: 33193572 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:27:41Z diff --git a/articles/ismip-hom-benchmark-experiments-using-underworld/metadata.yml b/articles/ismip-hom-benchmark-experiments-using-underworld/metadata.yml index 4d0e41e..4f23ef0 100644 --- a/articles/ismip-hom-benchmark-experiments-using-underworld/metadata.yml +++ b/articles/ismip-hom-benchmark-experiments-using-underworld/metadata.yml @@ -12,6 +12,7 @@ version: 1.0.0 legacy_doi: 10.59350/8196m-xmj49 archive_doi: 10.6084/m9.figshare.33193518 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /ismip-hom-benchmark-experiments-using-underworld/ legacy_paths: diff --git a/articles/mesh-variables-and-petsc-vectors-keeping-arrays-in-sync/metadata.yml b/articles/mesh-variables-and-petsc-vectors-keeping-arrays-in-sync/metadata.yml index 586f0db..70f18de 100644 --- a/articles/mesh-variables-and-petsc-vectors-keeping-arrays-in-sync/metadata.yml +++ b/articles/mesh-variables-and-petsc-vectors-keeping-arrays-in-sync/metadata.yml @@ -30,4 +30,5 @@ source: ghost-migration ghost_uuid: 28b160de-c9a0-4942-9a7e-74c84d21c7a4 repository_record_id: 33193578 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:27:52Z diff --git a/articles/moving-the-mesh-without-remaking-it/metadata.yml b/articles/moving-the-mesh-without-remaking-it/metadata.yml index a765a65..ae7a9df 100644 --- a/articles/moving-the-mesh-without-remaking-it/metadata.yml +++ b/articles/moving-the-mesh-without-remaking-it/metadata.yml @@ -37,4 +37,5 @@ source: native repository_record_id: 33241170 archive_doi: 10.6084/m9.figshare.33241170 archived_at: 2026-08-13T11:10:11Z +archived_version: 1.0.0 archive_published_at: 2026-08-13T11:11:48Z diff --git a/articles/new-features-of-the-surface-coupling-framework-in-underworld-2/metadata.yml b/articles/new-features-of-the-surface-coupling-framework-in-underworld-2/metadata.yml index 601991d..5002d7b 100644 --- a/articles/new-features-of-the-surface-coupling-framework-in-underworld-2/metadata.yml +++ b/articles/new-features-of-the-surface-coupling-framework-in-underworld-2/metadata.yml @@ -31,4 +31,5 @@ source: ghost-migration ghost_uuid: ba04ae00-5f7f-4655-89ef-c58d6060c9c7 repository_record_id: 33193557 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:27:06Z diff --git a/articles/our-journey-from-underworld2-to-underworld3/metadata.yml b/articles/our-journey-from-underworld2-to-underworld3/metadata.yml index e3a2620..84acebd 100644 --- a/articles/our-journey-from-underworld2-to-underworld3/metadata.yml +++ b/articles/our-journey-from-underworld2-to-underworld3/metadata.yml @@ -41,4 +41,5 @@ source: ghost-migration ghost_uuid: 6bfa4eab-e83e-4a57-8246-5c6ae2ecc353 repository_record_id: 33193566 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:27:30Z diff --git a/articles/particles-in-underworld3/metadata.yml b/articles/particles-in-underworld3/metadata.yml index cc16c94..de7180c 100644 --- a/articles/particles-in-underworld3/metadata.yml +++ b/articles/particles-in-underworld3/metadata.yml @@ -31,4 +31,5 @@ source: ghost-migration ghost_uuid: d7e1f38a-c62a-4b5f-bc55-0133fff3b2c3 repository_record_id: 33193599 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:28:32Z diff --git a/articles/physical-units-in-computational-geodynamics/metadata.yml b/articles/physical-units-in-computational-geodynamics/metadata.yml index e83c228..097f69e 100644 --- a/articles/physical-units-in-computational-geodynamics/metadata.yml +++ b/articles/physical-units-in-computational-geodynamics/metadata.yml @@ -33,4 +33,5 @@ source: ghost-migration ghost_uuid: f8e03b51-3f07-4bfd-92bc-cf4f468b78a9 repository_record_id: 33193587 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:28:02Z diff --git a/articles/running-underworld-in-a-browser/metadata.yml b/articles/running-underworld-in-a-browser/metadata.yml index 5387318..870b668 100644 --- a/articles/running-underworld-in-a-browser/metadata.yml +++ b/articles/running-underworld-in-a-browser/metadata.yml @@ -20,6 +20,7 @@ archive_doi: 10.6084/m9.figshare.33216996 repository_provider: figshare repository_record_id: 33216996 archived_at: 2026-08-31T17:36:06Z +archived_version: 1.1.0 archive_published_at: 2026-08-31T17:38:10Z subjects: methods: diff --git a/articles/scaling-in-underworld/metadata.yml b/articles/scaling-in-underworld/metadata.yml index 80abebd..49a7f7a 100644 --- a/articles/scaling-in-underworld/metadata.yml +++ b/articles/scaling-in-underworld/metadata.yml @@ -19,6 +19,7 @@ version: 1.0.0 legacy_doi: 10.59350/qvgm0-yz754 archive_doi: 10.6084/m9.figshare.33193452 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /scaling-in-underworld/ legacy_paths: diff --git a/articles/self-updating-repositories/metadata.yml b/articles/self-updating-repositories/metadata.yml index 323a108..f0d07f6 100644 --- a/articles/self-updating-repositories/metadata.yml +++ b/articles/self-updating-repositories/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/wftyb-vgs67 archive_doi: 10.6084/m9.figshare.33193428 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /self-updating-repositories/ legacy_paths: diff --git a/articles/setting-up-full-multigrid/metadata.yml b/articles/setting-up-full-multigrid/metadata.yml index 9eeeb44..4044b81 100644 --- a/articles/setting-up-full-multigrid/metadata.yml +++ b/articles/setting-up-full-multigrid/metadata.yml @@ -39,4 +39,5 @@ source: native repository_record_id: 33273984 archive_doi: 10.6084/m9.figshare.33273984 archived_at: 2026-08-17T21:37:48Z +archived_version: 1.0.0 archive_published_at: 2026-08-17T21:39:35Z diff --git a/articles/setting-up-underworld-dependencies/metadata.yml b/articles/setting-up-underworld-dependencies/metadata.yml index 7ccf419..c6fcf13 100644 --- a/articles/setting-up-underworld-dependencies/metadata.yml +++ b/articles/setting-up-underworld-dependencies/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/t7ghx-8f823 archive_doi: 10.6084/m9.figshare.33193509 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /setting-up-underworld-dependencies/ legacy_paths: diff --git a/articles/shear-bands-with-dilatancy-modelled-with-underworld/metadata.yml b/articles/shear-bands-with-dilatancy-modelled-with-underworld/metadata.yml index 7c94fe1..61256c1 100644 --- a/articles/shear-bands-with-dilatancy-modelled-with-underworld/metadata.yml +++ b/articles/shear-bands-with-dilatancy-modelled-with-underworld/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/awc90-63186 archive_doi: 10.6084/m9.figshare.33193365 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /shear-bands-with-dilatancy-modelled-with-underworld/ legacy_paths: diff --git a/articles/stress-recovery-in-underworld/metadata.yml b/articles/stress-recovery-in-underworld/metadata.yml index 09321e2..60b4a49 100644 --- a/articles/stress-recovery-in-underworld/metadata.yml +++ b/articles/stress-recovery-in-underworld/metadata.yml @@ -12,6 +12,7 @@ version: 1.0.0 legacy_doi: 10.59350/97evt-ays09 archive_doi: 10.6084/m9.figshare.33193446 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /stress-recovery-in-underworld/ legacy_paths: diff --git a/articles/symbolic-time-derivatives-in-underworld3/metadata.yml b/articles/symbolic-time-derivatives-in-underworld3/metadata.yml index 72c0b7d..fdc2c5e 100644 --- a/articles/symbolic-time-derivatives-in-underworld3/metadata.yml +++ b/articles/symbolic-time-derivatives-in-underworld3/metadata.yml @@ -32,4 +32,5 @@ source: ghost-migration ghost_uuid: 1eb496b6-8422-4b09-9443-2894629c35cf repository_record_id: 33193596 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 archive_published_at: 2026-08-10T04:28:22Z diff --git a/articles/the-dynamics-of-continental-accretion/metadata.yml b/articles/the-dynamics-of-continental-accretion/metadata.yml index de6361e..ff78023 100644 --- a/articles/the-dynamics-of-continental-accretion/metadata.yml +++ b/articles/the-dynamics-of-continental-accretion/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/73ded-8k350 archive_doi: 10.6084/m9.figshare.33193179 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /the-dynamics-of-continental-accretion/ legacy_paths: diff --git a/articles/underworld-2/metadata.yml b/articles/underworld-2/metadata.yml index 381bb42..94ef812 100644 --- a/articles/underworld-2/metadata.yml +++ b/articles/underworld-2/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/fsmnk-jat73 archive_doi: 10.6084/m9.figshare.33193233 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld-2/ legacy_paths: diff --git a/articles/underworld-and-docker-part-1/metadata.yml b/articles/underworld-and-docker-part-1/metadata.yml index 45aef4f..4289da4 100644 --- a/articles/underworld-and-docker-part-1/metadata.yml +++ b/articles/underworld-and-docker-part-1/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/y8762-pe280 archive_doi: 10.6084/m9.figshare.33193329 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld-and-docker-part-1/ legacy_paths: diff --git a/articles/underworld-and-docker-part-2/metadata.yml b/articles/underworld-and-docker-part-2/metadata.yml index efb6a3d..cae2911 100644 --- a/articles/underworld-and-docker-part-2/metadata.yml +++ b/articles/underworld-and-docker-part-2/metadata.yml @@ -16,6 +16,7 @@ version: 1.0.0 legacy_doi: 10.59350/4cqwc-rth67 archive_doi: 10.6084/m9.figshare.33193341 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld-and-docker-part-2/ legacy_paths: diff --git a/articles/underworld-and-singularity/metadata.yml b/articles/underworld-and-singularity/metadata.yml index 632e1fa..fd222da 100644 --- a/articles/underworld-and-singularity/metadata.yml +++ b/articles/underworld-and-singularity/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/hfgvn-dkk05 archive_doi: 10.6084/m9.figshare.33193545 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld-and-singularity/ legacy_paths: diff --git a/articles/underworld-low-fat-cloud/metadata.yml b/articles/underworld-low-fat-cloud/metadata.yml index 804b67b..752bd1e 100644 --- a/articles/underworld-low-fat-cloud/metadata.yml +++ b/articles/underworld-low-fat-cloud/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/mw43c-vn265 archive_doi: 10.6084/m9.figshare.33193422 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld-low-fat-cloud/ legacy_paths: diff --git a/articles/underworld-on-zenodo/metadata.yml b/articles/underworld-on-zenodo/metadata.yml index 82b7d43..9c009b3 100644 --- a/articles/underworld-on-zenodo/metadata.yml +++ b/articles/underworld-on-zenodo/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/at0ev-7re42 archive_doi: 10.6084/m9.figshare.33193404 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld-on-zenodo/ legacy_paths: diff --git a/articles/underworld3-come-and-get-it/metadata.yml b/articles/underworld3-come-and-get-it/metadata.yml index 2d8d426..41e7f6d 100644 --- a/articles/underworld3-come-and-get-it/metadata.yml +++ b/articles/underworld3-come-and-get-it/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/z0c1r-rpg80 archive_doi: 10.6084/m9.figshare.33193533 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /underworld3-come-and-get-it/ legacy_paths: diff --git a/articles/untitled-2/metadata.yml b/articles/untitled-2/metadata.yml index 38a13ac..8ed06a2 100644 --- a/articles/untitled-2/metadata.yml +++ b/articles/untitled-2/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/ts8za-rv858 archive_doi: 10.6084/m9.figshare.33193380 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /untitled-2/ legacy_paths: diff --git a/articles/untitled/metadata.yml b/articles/untitled/metadata.yml index db55daa..b53e9f1 100644 --- a/articles/untitled/metadata.yml +++ b/articles/untitled/metadata.yml @@ -12,6 +12,7 @@ version: 1.0.0 legacy_doi: 10.59350/x638s-dpr14 archive_doi: 10.6084/m9.figshare.33193386 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /untitled/ legacy_paths: diff --git a/articles/using-python-virtual-environment-for-underworld-development/metadata.yml b/articles/using-python-virtual-environment-for-underworld-development/metadata.yml index 89c6572..c09efbf 100644 --- a/articles/using-python-virtual-environment-for-underworld-development/metadata.yml +++ b/articles/using-python-virtual-environment-for-underworld-development/metadata.yml @@ -13,6 +13,7 @@ version: 1.0.0 legacy_doi: 10.59350/681vw-w9h32 archive_doi: 10.6084/m9.figshare.33193515 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /using-python-virtual-environment-for-underworld-development/ legacy_paths: diff --git a/articles/viscoelasticity/metadata.yml b/articles/viscoelasticity/metadata.yml index d252b4e..b803722 100644 --- a/articles/viscoelasticity/metadata.yml +++ b/articles/viscoelasticity/metadata.yml @@ -12,6 +12,7 @@ version: 1.0.0 legacy_doi: 10.59350/3atx2-v4j54 archive_doi: 10.6084/m9.figshare.33191160 archived_at: 2026-08-10T04:19:43Z +archived_version: 1.0.0 license: CC-BY-4.0 canonical_path: /viscoelasticity/ legacy_paths: diff --git a/pixi.toml b/pixi.toml index f8acce0..cce2b00 100644 --- a/pixi.toml +++ b/pixi.toml @@ -87,6 +87,12 @@ test-unit = "pytest -q tests/" # densities whose right value depends on what the document is, and that is # the author's call rather than the build's. check-style = "python3 scripts/check_style.py" + +# What is half-finished: deposits started and not recorded, requests never +# merged, notes that have outrun their archival copy. Exits 1 if anything is +# outstanding, which is what lets the scheduled `outstanding` workflow raise +# an issue. Not part of `test`: an unfinished deposit is not a broken build. +outstanding = "python3 scripts/outstanding.py" test = { depends-on = ["test-unit", "validate", "test-dois", "test-assets"] } # Stage 0 migration tooling. Read-only against the live Ghost site. diff --git a/schemas/article-metadata.schema.json b/schemas/article-metadata.schema.json index f32faf1..055e603 100644 --- a/schemas/article-metadata.schema.json +++ b/schemas/article-metadata.schema.json @@ -253,6 +253,11 @@ ], "description": "When the archival PDF was made, ISO-8601 UTC. Stamped by the deposit at the moment the copy is taken, and again for each new version, so it always names the snapshot the record actually holds." }, + "archived_version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$", + "description": "The note's `version` at the moment it was last deposited. `version` ahead of this means the note has outrun its archival copy and wants a new deposit." + }, "archive_published_at": { "type": [ "string", diff --git a/scripts/deposit.py b/scripts/deposit.py index 4dcd085..37a768b 100644 --- a/scripts/deposit.py +++ b/scripts/deposit.py @@ -616,8 +616,16 @@ def run(slug, provider, live, publish, new_version, delete_draft, if (new_version and rebuild) or not meta.get("archived_at"): stamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) set_field(slug, "archived_at", stamp.isoformat().replace("+00:00", "Z")) + # The note's own version, recorded as deposited. This is what makes a + # STALE deposit visible: `version` moves when an author decides the + # content has materially changed, so `version != archived_version` is + # the note having outrun its archival copy. Nothing else in the + # metadata says that -- archived_at only says when the copy was taken, + # and a git timestamp cannot tell a rewrite from a typo. + set_field(slug, "archived_version", meta.get("version") or "0.0.0") meta = load(slug) - steps.append("stamped archived_at %s" % stamp.isoformat()) + steps.append("stamped archived_at %s (version %s)" + % (stamp.isoformat(), meta.get("archived_version"))) # The reserved DOI has to be on the title page of the document it # identifies, so the PDF is rebuilt between reserving and uploading. In the diff --git a/scripts/outstanding.py b/scripts/outstanding.py new file mode 100644 index 0000000..37e33ce --- /dev/null +++ b/scripts/outstanding.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""What is half-finished right now. + +Every step of publishing a note is deliberate -- a person merges the +request to deposit, a person merges the identifiers that come back -- and +that is the right design. Its failure mode is that a step can simply not +happen, and nothing says so. A deposit sat unrecorded for a fortnight +because the only thing that would have complained was the NEXT deposit, +which is exactly when it is least welcome. + +So this names the half-finished states. It reads metadata and, if `gh` is +available, open pull requests; it changes nothing. + + deposited, not recorded an identifiers pull request is still open, so + the repository does not know a record exists + and the duplicate-mint guard is blind + asked for, not deposited a deposit request is open on the queue + note ahead of its copy `version` has moved past `archived_version`: + the note has outrun what is on the DOI + never deposited archival, published, and has no DOI + guard blind an `archive_doi` with no `repository_record_id` + +Exit 1 if anything is outstanding, so a scheduled run can raise it. + +Usage: + python3 scripts/outstanding.py # report + python3 scripts/outstanding.py --json # for a workflow + python3 scripts/outstanding.py --no-net # metadata only +""" + +import argparse +import json +import pathlib +import re +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +ARTICLES = ROOT / "articles" +sys.path.insert(0, str(ROOT / "scripts")) + + +def field(text, key): + m = re.search(r"^%s:\s*(.+?)\s*$" % re.escape(key), text, re.M) + if not m: + return None + v = m.group(1).strip().strip('"').strip("'") + return None if v in ("null", "~", "") else v + + +def open_branches(prefix): + """Open pull requests whose branch starts with `prefix`. + + Returns None -- not [] -- when `gh` cannot answer, so the report can + say "not checked" rather than "nothing outstanding". The difference + matters: this exists because silence was mistaken for good news. + """ + try: + out = subprocess.run( + ["gh", "pr", "list", "--state", "open", "--limit", "100", + "--json", "number,title,headRefName,createdAt"], + capture_output=True, text=True, timeout=60, cwd=str(ROOT)) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + try: + prs = json.loads(out.stdout or "[]") + except ValueError: + return None + return [p for p in prs if p["headRefName"].startswith(prefix)] + + +def survey(check_net=True): + import build_index + build_index.TYPES.update(build_index.article_types()) + + findings = {"unrecorded": None, "queued": None, + "stale": [], "undeposited": [], "blind": []} + + for md in sorted(ARTICLES.glob("*/metadata.yml")): + slug = md.parent.name + text = md.read_text(encoding="utf-8") + meta = build_index.read_yaml(md) + doi = field(text, "archive_doi") + rid = field(text, "repository_record_id") + version = field(text, "version") + archived = field(text, "archived_version") + + if doi and not rid: + findings["blind"].append(slug) + if not build_index.is_archival(meta): + continue + if not doi: + if field(text, "status") == "published": + findings["undeposited"].append(slug) + continue + # archived_version is absent on nothing after the backfill, but a + # note deposited by an older workflow would have none; say so + # rather than guessing it matches. + if version and archived and version != archived: + findings["stale"].append((slug, archived, version)) + elif version and not archived: + findings["stale"].append((slug, "unrecorded", version)) + + if check_net: + findings["unrecorded"] = open_branches("deposit/identifiers-") + findings["queued"] = open_branches("deposit/queue-") + return findings + + +def report(f): + lines, outstanding = [], 0 + + def head(title, n): + lines.append("") + lines.append("%s (%s)" % (title, n)) + + if f["unrecorded"] is None: + head("deposited, not recorded", "not checked -- gh unavailable") + elif f["unrecorded"]: + head("deposited, not recorded", len(f["unrecorded"])) + for p in f["unrecorded"]: + lines.append(" #%-5d %s (opened %s)" + % (p["number"], p["title"][:60], p["createdAt"][:10])) + lines.append(" -> merge these: until they land the repository does " + "not know the records exist") + outstanding += len(f["unrecorded"]) + + if f["queued"]: + head("asked for, not deposited", len(f["queued"])) + for p in f["queued"]: + lines.append(" #%-5d %s (opened %s)" + % (p["number"], p["title"][:60], p["createdAt"][:10])) + outstanding += len(f["queued"]) + + if f["stale"]: + head("note ahead of its archival copy", len(f["stale"])) + for slug, was, now in f["stale"]: + lines.append(" %-52s deposited %s, now %s" % (slug[:52], was, now)) + lines.append(" -> deposit a new version, or the DOI serves the " + "older text") + outstanding += len(f["stale"]) + + if f["undeposited"]: + head("published, never deposited", len(f["undeposited"])) + for slug in f["undeposited"]: + lines.append(" %s" % slug) + outstanding += len(f["undeposited"]) + + if f["blind"]: + head("archive_doi with no record id", len(f["blind"])) + for slug in f["blind"]: + lines.append(" %s" % slug) + outstanding += len(f["blind"]) + + if not outstanding: + lines.append("Nothing outstanding.") + return "\n".join(lines).lstrip("\n"), outstanding + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--json", action="store_true") + ap.add_argument("--no-net", action="store_true", + help="metadata only; do not ask gh about pull requests") + args = ap.parse_args() + + f = survey(check_net=not args.no_net) + text, n = report(f) + if args.json: + print(json.dumps({"outstanding": n, "report": text, "findings": f}, + indent=2, default=list)) + else: + print(text) + return 1 if n else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_outstanding.py b/tests/test_outstanding.py new file mode 100644 index 0000000..be156d6 --- /dev/null +++ b/tests/test_outstanding.py @@ -0,0 +1,79 @@ +"""The outstanding report names the half-finished states, and stays quiet +otherwise. + +The negative controls matter more than usual here. This exists because +silence was mistaken for good news, so a check that cannot answer must +say so rather than report nothing, and a clean repository must produce +no noise at all. +""" +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent + / "scripts")) +import outstanding # noqa: E402 + +CLEAN = {"unrecorded": [], "queued": [], "stale": [], "undeposited": [], + "blind": []} + + +def test_a_clean_repository_says_nothing(): + text, n = outstanding.report(dict(CLEAN)) + assert n == 0 + assert text == "Nothing outstanding." + + +def test_an_unanswerable_check_says_so_rather_than_nothing(): + """None is not []. If gh cannot answer, the report must not read as + 'nothing outstanding' -- that is the mistake this tool exists for.""" + f = dict(CLEAN, unrecorded=None) + text, n = outstanding.report(f) + assert "not checked" in text + assert n == 0 # unknown is not counted as outstanding + + +def test_unrecorded_identifiers_are_reported_with_the_reason(): + f = dict(CLEAN, unrecorded=[ + {"number": 25, "title": "Deposit identifiers from run 1", + "createdAt": "2026-08-17T21:39:38Z"}]) + text, n = outstanding.report(f) + assert n == 1 + assert "#25" in text and "2026-08-17" in text + assert "duplicate-mint" in text or "does not know" in text + + +def test_a_note_ahead_of_its_deposit_is_reported(): + f = dict(CLEAN, stale=[("running-underworld-in-a-browser", "1.0.0", "1.1.0")]) + text, n = outstanding.report(f) + assert n == 1 + assert "deposited 1.0.0, now 1.1.0" in text + + +def test_a_deposit_with_no_recorded_version_is_not_assumed_current(): + """A note deposited before archived_version existed must be reported, + not silently taken to match.""" + f = dict(CLEAN, stale=[("old-note", "unrecorded", "1.0.0")]) + text, n = outstanding.report(f) + assert n == 1 and "unrecorded" in text + + +def test_the_counts_add_up(): + f = {"unrecorded": [{"number": 1, "title": "t", "createdAt": "2026-01-01T00:00:00Z"}], + "queued": [{"number": 2, "title": "t", "createdAt": "2026-01-01T00:00:00Z"}], + "stale": [("a", "1.0.0", "1.1.0")], + "undeposited": ["b"], + "blind": ["c"]} + _text, n = outstanding.report(f) + assert n == 5 + + +def test_the_real_repository_surveys_without_network(): + """The metadata half must work with no gh and no network at all.""" + f = outstanding.survey(check_net=False) + assert f["unrecorded"] is None and f["queued"] is None + assert isinstance(f["stale"], list) + # every deposited note carries archived_version after the backfill, so + # nothing should be reported as having an unrecorded deposit version + assert not [s for s in f["stale"] if s[1] == "unrecorded"], f["stale"] + # and validate already forbids a doi with no record id + assert f["blind"] == [] From 7c25078f6d7ba5bf5690dccfdfbf29e626677a1b Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 2 Sep 2026 12:05:25 -0700 Subject: [PATCH 2/3] A short slug was never truncated, so it has nothing to restore The preview build died on this branch with "AMBIGUOUS: /underworld-2/ could belong to joss-publication-underworld-2 or underworld-2-10", an ambiguity that does not exist: those are two different notes with two different pages. fix_slugs restores URLs MyST truncated at 50 characters, by matching a slug against the pages that were built. A preview builds only the notes a branch changes, so in a preview most slugs have no built page -- and this branch's metadata backfill touched 43 articles, which made the build large enough for three related slugs to be in play at once. Matched loosely, `underworld-2-10` starts with `underworld-2` and `joss-publication-underworld-2` ends with it, so both claimed the same page and the second one to arrive raised the ambiguity. Neither could have been truncated: both are shorter than the cap. So the fix is to say that -- a slug at or under the cap is skipped, because there is nothing to restore, and only a longer one is matched at all. The cap is now a named constant rather than a literal in two places. Underworld development team with AI support from Claude Code --- scripts/fix_slugs.py | 17 ++++++++++++++++- tests/test_migration.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/scripts/fix_slugs.py b/scripts/fix_slugs.py index fd7b865..e00b4db 100644 --- a/scripts/fix_slugs.py +++ b/scripts/fix_slugs.py @@ -31,6 +31,9 @@ import shutil import sys +# MyST truncates a page's URL at this many characters. +SLUG_CAP = 50 + ROOT = pathlib.Path(__file__).resolve().parent.parent # Files whose contents may reference a page URL. @@ -59,7 +62,7 @@ def main(): # MyST would disambiguate with a numeric suffix. Catch that before renaming. prefixes = {} for slug in slugs: - key = slug[:50] + key = slug[:SLUG_CAP] prefixes.setdefault(key, []).append(slug) collisions = {k: v for k, v in prefixes.items() if len(v) > 1} if collisions: @@ -72,6 +75,18 @@ def main(): for slug in slugs: if slug in built: continue # already correct, nothing to do + # Only a slug LONGER than the cap can have been truncated. A shorter + # one that is missing from `built` simply was not built -- which is + # normal in a preview, where only the notes a branch changes are. + # Matching it by prefix or suffix anyway lets it claim somebody else's + # page: `underworld-2-10` starts with `underworld-2` and + # `joss-publication-underworld-2` ends with it, so on a partial build + # both claimed /underworld-2/ and the run died reporting an ambiguity + # that does not exist. + if len(slug) <= SLUG_CAP: + if slug not in built: + print(" (not in this build: %s)" % slug, file=sys.stderr) + continue heads = [b for b in built if slug.startswith(b) and b != slug] tails = [b for b in built if slug.endswith(b) and b != slug] candidates = heads + tails diff --git a/tests/test_migration.py b/tests/test_migration.py index 561edb4..d829403 100644 --- a/tests/test_migration.py +++ b/tests/test_migration.py @@ -2277,3 +2277,37 @@ def test_display_math_that_needs_a_blank_line_has_one(): "blank line before the `$$`." % (path.name, opened + 1, risky[0].strip()[:40])) opened = None + + +def test_a_partial_build_does_not_let_one_slug_claim_anothers_page(): + """`fix_slugs` restores URLs MyST truncated at 50 characters. + + A preview builds only the notes a branch changes, so most slugs have no + built page at all -- and a slug SHORTER than the cap was never truncated, + so it has nothing to restore. Matching it by prefix or suffix anyway let + `underworld-2-10` and `joss-publication-underworld-2` both claim the built + page `underworld-2`, and the run died reporting an ambiguity that did not + exist. Seen on #42, whose metadata backfill touched 43 articles and so + made the preview a large partial build. + """ + import importlib.util + spec = importlib.util.spec_from_file_location( + "fix_slugs", ROOT / "scripts" / "fix_slugs.py") + fix_slugs = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fix_slugs) + + assert fix_slugs.SLUG_CAP == 50 + built = {"underworld-2", "underworld-2-9"} + for slug in ("underworld-2-10", "joss-publication-underworld-2"): + assert len(slug) <= fix_slugs.SLUG_CAP, \ + "the case only holds for slugs shorter than the cap" + assert slug not in built + # the old matching; what the fix must now refuse to act on + loose = ([b for b in built if slug.startswith(b) and b != slug] + + [b for b in built if slug.endswith(b) and b != slug]) + assert loose == ["underworld-2"], \ + "this is the collision the fix exists to prevent" + + src = (ROOT / "scripts" / "fix_slugs.py").read_text(encoding="utf-8") + assert "if len(slug) <= SLUG_CAP:" in src, \ + "short slugs must be skipped before any prefix/suffix matching" From f4056738a3d9911c1d324e2331b0de747fd29df3 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Wed, 2 Sep 2026 12:18:30 -0700 Subject: [PATCH 3/3] Match the two manglings MyST actually does, and nothing looser The preview died on this branch reporting an ambiguity that does not exist: "/underworld-2/ could belong to joss-publication-underworld-2 or underworld-2-10". Those are three separate notes. fix_slugs restores URLs MyST mangled, by matching a slug against the pages that were built. A preview builds only the notes a branch changes, so most slugs have no built page at all -- and this branch's metadata backfill touched 43 articles, which put three related slugs in play at once. Matched loosely, `underworld-2-10` claimed `underworld-2` merely by starting with it and `joss-publication-underworld-2` by ending with it. My first attempt at this was wrong, and the build caught it: I assumed the only mangling was truncation at 50 characters and skipped every shorter slug, which broke the fifteen that are mangled a different way. MyST also drops a LEADING NUMBER, so `2-11-scaling` is served as /scaling/ and `30-years-of-citcom-...` as /years-of-citcom-.../. So both manglings are now matched exactly -- truncation only when the slug is longer than the cap, and a leading strip only when what was removed is a number -- and anything looser is refused. Checked against a real full build: 43 slugs already correct, 15 renamed, none unmatched, no ambiguity, and test-dois reports 50 resolved and 0 broken. Underworld development team with AI support from Claude Code --- scripts/fix_slugs.py | 31 ++++++++++++---------- tests/test_migration.py | 57 ++++++++++++++++++++++++----------------- 2 files changed, 51 insertions(+), 37 deletions(-) diff --git a/scripts/fix_slugs.py b/scripts/fix_slugs.py index e00b4db..8dcc4c8 100644 --- a/scripts/fix_slugs.py +++ b/scripts/fix_slugs.py @@ -75,20 +75,23 @@ def main(): for slug in slugs: if slug in built: continue # already correct, nothing to do - # Only a slug LONGER than the cap can have been truncated. A shorter - # one that is missing from `built` simply was not built -- which is - # normal in a preview, where only the notes a branch changes are. - # Matching it by prefix or suffix anyway lets it claim somebody else's - # page: `underworld-2-10` starts with `underworld-2` and - # `joss-publication-underworld-2` ends with it, so on a partial build - # both claimed /underworld-2/ and the run died reporting an ambiguity - # that does not exist. - if len(slug) <= SLUG_CAP: - if slug not in built: - print(" (not in this build: %s)" % slug, file=sys.stderr) - continue - heads = [b for b in built if slug.startswith(b) and b != slug] - tails = [b for b in built if slug.endswith(b) and b != slug] + # MyST mangles a URL in exactly two ways, and matching anything looser + # lets one note claim another's page. On a preview -- which builds only + # the notes a branch changes, so most slugs have no page at all -- + # `underworld-2-10` was matching `underworld-2` merely by starting with + # it, and `joss-publication-underworld-2` by ending with it, so both + # claimed /underworld-2/ and the run died on an ambiguity that does not + # exist. Neither is a mangling of the other; they are three notes. + # + # truncation the URL is cut at SLUG_CAP characters + # leading strip a leading NUMBER is dropped, so `2-11-scaling` + # is served as /scaling/ and + # `30-years-of-citcom-...` as /years-of-citcom-.../ + heads = [b for b in built + if len(slug) > SLUG_CAP and b == slug[:SLUG_CAP]] + tails = [b for b in built + if b != slug and slug.endswith(b) + and re.fullmatch(r"[0-9]+(-[0-9]+)*-", slug[:len(slug) - len(b)])] candidates = heads + tails if not candidates: print(" WARNING: no built page found for %s" % slug, file=sys.stderr) diff --git a/tests/test_migration.py b/tests/test_migration.py index d829403..4391569 100644 --- a/tests/test_migration.py +++ b/tests/test_migration.py @@ -2280,34 +2280,45 @@ def test_display_math_that_needs_a_blank_line_has_one(): def test_a_partial_build_does_not_let_one_slug_claim_anothers_page(): - """`fix_slugs` restores URLs MyST truncated at 50 characters. - - A preview builds only the notes a branch changes, so most slugs have no - built page at all -- and a slug SHORTER than the cap was never truncated, - so it has nothing to restore. Matching it by prefix or suffix anyway let - `underworld-2-10` and `joss-publication-underworld-2` both claim the built - page `underworld-2`, and the run died reporting an ambiguity that did not - exist. Seen on #42, whose metadata backfill touched 43 articles and so - made the preview a large partial build. + """`fix_slugs` restores URLs MyST mangled, and matches only the two ways + it actually mangles them: truncation at 50 characters, and dropping a + LEADING NUMBER (`2-11-scaling` is served as /scaling/). + + Matched any looser, one note claims another's page. A preview builds only + the notes a branch changes, so most slugs have no built page at all, and + there `underworld-2-10` matched `underworld-2` merely by starting with it + while `joss-publication-underworld-2` matched by ending with it. Both + claimed /underworld-2/ and the run died on an ambiguity that does not + exist -- they are three separate notes. Seen on #42, whose metadata + backfill touched 43 articles and so made the preview a large partial build. """ import importlib.util + import re as _re spec = importlib.util.spec_from_file_location( "fix_slugs", ROOT / "scripts" / "fix_slugs.py") fix_slugs = importlib.util.module_from_spec(spec) spec.loader.exec_module(fix_slugs) - - assert fix_slugs.SLUG_CAP == 50 + cap = fix_slugs.SLUG_CAP + assert cap == 50 + + def candidates(slug, built): + heads = [b for b in built if len(slug) > cap and b == slug[:cap]] + tails = [b for b in built + if b != slug and slug.endswith(b) + and _re.fullmatch(r"[0-9]+(-[0-9]+)*-", + slug[:len(slug) - len(b)])] + return heads + tails + + # the collision, on the partial build that exposed it built = {"underworld-2", "underworld-2-9"} for slug in ("underworld-2-10", "joss-publication-underworld-2"): - assert len(slug) <= fix_slugs.SLUG_CAP, \ - "the case only holds for slugs shorter than the cap" - assert slug not in built - # the old matching; what the fix must now refuse to act on - loose = ([b for b in built if slug.startswith(b) and b != slug] - + [b for b in built if slug.endswith(b) and b != slug]) - assert loose == ["underworld-2"], \ - "this is the collision the fix exists to prevent" - - src = (ROOT / "scripts" / "fix_slugs.py").read_text(encoding="utf-8") - assert "if len(slug) <= SLUG_CAP:" in src, \ - "short slugs must be skipped before any prefix/suffix matching" + assert candidates(slug, built) == [], \ + "%s must not claim another note's page" % slug + + # and the two manglings that ARE real still resolve + assert candidates("2-11-scaling", {"scaling"}) == ["scaling"] + assert candidates("30-years-of-citcom-ellipsis-and-underworld", + {"years-of-citcom-ellipsis-and-underworld"}) == \ + ["years-of-citcom-ellipsis-and-underworld"] + long_slug = "a" * 60 + assert candidates(long_slug, {"a" * cap}) == ["a" * cap]