From 71607d2a52c23a51ebc3d72345a50ac475aa58d6 Mon Sep 17 00:00:00 2001 From: Paullo9 Date: Tue, 27 Oct 2026 23:51:40 +0100 Subject: [PATCH] Public repo: neutral wording for private references, CI scan, Dependabot Rephrase references to the private suite folder (archive paths, the suite's process file, the memory-cap wrapper, the reference-setup script) in neutral public wording; behaviour unchanged. The memory-capped harness scripts now take the cap wrapper from MEMORY_CAP (a command prefix, or none) and stop if it is unset. Add a CI job "No private references" that fails on those patterns in tracked files, and a weekly Dependabot check for GitHub Actions. Done: private references scrubbed, scan job and Dependabot added. Next: publish the branch and watch the new CI job pass. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 6 ++++++ .github/workflows/ci.yml | 11 +++++++++++ .github/workflows/docs.yml | 2 +- Sources/FormatoEMPAD/FormatoEMPAD.docc/Scope.md | 2 +- Tests/Fixtures/README.md | 2 +- tools/acq-metadata/generate.py | 2 +- tools/benchmarks/bench.py | 2 +- tools/bragg-writer/check.py | 4 ++-- tools/bragg-writer/generate.py | 2 +- tools/dm4-gather/generate.py | 4 ++-- tools/emd-bundle/check.py | 4 ++-- tools/emd-bundle/oracle/run.sh | 6 +++++- tools/empad/generate.py | 2 +- tools/hdf5-4d/generate.py | 2 +- tools/hdf5-append/generate.py | 2 +- tools/hdf5-write/generate.py | 2 +- tools/mib/generate.py | 2 +- tools/png/check.py | 2 +- tools/step-1/owner-check.sh | 8 ++++++-- tools/step-1/pins.py | 2 +- tools/step-4/owner-check.sh | 8 ++++++-- 21 files changed, 53 insertions(+), 24 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bd65a2..df9f77b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,3 +46,14 @@ jobs: top=$(grep -m1 -E '^## [0-9]+\.[0-9]+\.[0-9]+' CHANGELOG.md | sed -E 's/^## ([0-9]+\.[0-9]+\.[0-9]+).*/\1/') tag=${GITHUB_REF_NAME#v} [ "$top" = "$tag" ] || { echo "CHANGELOG top is $top, tag is $tag"; exit 1; } + + private-refs: + name: No private references + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Scan tracked files + run: | + if git grep -n -I -E '_archive|CLAUDE\.md|memcap|macEM-suite|setup-python-refs' -- ':!.gitignore' ':!.github/workflows/ci.yml'; then + echo "Private references above; this repo is public."; exit 1 + fi diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index de7c3b0..b76a214 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,7 +2,7 @@ # published to GitHub Pages at https://mac4dstem.github.io/FormatoKit/documentation/ on every tag. # On a pull request the site is built and uploaded as an artifact only, so a broken catalog fails before the tag. # -# No swift-docc-plugin: the kit takes no package dependencies (CLAUDE.md). `xcodebuild docbuild` builds the archives +# No swift-docc-plugin: the kit takes no package dependencies. `xcodebuild docbuild` builds the archives # from the package's own schemes; `docc merge` (Xcode 16+) joins them. Needs Pages enabled with source "GitHub Actions" # (repository Settings > Pages). Copied from CuboKit's pattern; the module list is read from the .docc catalogs. name: Docs diff --git a/Sources/FormatoEMPAD/FormatoEMPAD.docc/Scope.md b/Sources/FormatoEMPAD/FormatoEMPAD.docc/Scope.md index 6be5389..b9c24b6 100644 --- a/Sources/FormatoEMPAD/FormatoEMPAD.docc/Scope.md +++ b/Sources/FormatoEMPAD/FormatoEMPAD.docc/Scope.md @@ -17,6 +17,6 @@ shapes, invalid numeric calibrations and incorrect raw lengths throw `EMPADError reader, trailing raw bytes are refused. Arbitrary Python literal expressions in XML are refused. Repeated XML scalar paths retain their last value. The committed synthetic pair and complete byte/axis/metadata pins are produced by `tools/empad/generate.py` -under the pinned FormatoKit venv with `../tools/memcap.sh`. Tests inflate the small gzip fixtures in a temporary directory; +under the pinned FormatoKit venv and a memory cap. Tests inflate the small gzip fixtures in a temporary directory; no Python dependency is needed to run them. NaN, infinity and signed-zero bits are preserved; `remove_nans` is not ported. 4D HDF5 reading remains future work. diff --git a/Tests/Fixtures/README.md b/Tests/Fixtures/README.md index 67c0420..aa03aee 100644 --- a/Tests/Fixtures/README.md +++ b/Tests/Fixtures/README.md @@ -3,7 +3,7 @@ One row per committed fixture file under `Tests/*/Fixtures/`: size, the script under `tools/` that names it (the generator or its pin script; a few scripts write a whole directory; a script found only by file-name prefix, tool-folder name or test module is marked so), and a note where the origin is not the kit's own pinned reference. 593 files. -Licence: every fixture is generated from this kit's own code and the pinned Python reference (see `CLAUDE.md`, Reference) and ships under the kit's GPL-3.0-or-later licence, except the rows marked *third party*. Nothing here is the owner's data: the committed text fixtures carry no owner path or name (checked by search), and real datasets are read from `$MACEM_DATA` and never copied into the repo. +Licence: every fixture is generated from this kit's own code and the pinned Python reference (named in the README) and ships under the kit's GPL-3.0-or-later licence, except the rows marked *third party*. Nothing here is the owner's data: the committed text fixtures carry no owner path or name (checked by search), and real datasets are read from `$MACEM_DATA` and never copied into the repo. ## Third party diff --git a/tools/acq-metadata/generate.py b/tools/acq-metadata/generate.py index 23fbf25..342f130 100644 --- a/tools/acq-metadata/generate.py +++ b/tools/acq-metadata/generate.py @@ -1,6 +1,6 @@ """Acquisition-metadata fixtures and pins (FormatoCore AcquisitionMetadata; FormatoDM/MRC/TIFF/EMD readers). - MACEM_SUITE= /tools/memcap.sh -- /References/FormatoKit/rsciio-venv/bin/python -I -B tools/acq-metadata/generate.py + MACEM_SUITE= /References/FormatoKit/rsciio-venv/bin/python -I -B tools/acq-metadata/generate.py Writes small synthetic DM4, MRC, TIFF and Velox EMD files (struct/h5py only, byte by byte) under Tests/*/Fixtures/acq-metadata/ and, for each, two pins: diff --git a/tools/benchmarks/bench.py b/tools/benchmarks/bench.py index c9c79bf..0a87e38 100644 --- a/tools/benchmarks/bench.py +++ b/tools/benchmarks/bench.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """FormatoKit vs RosettaSciIO benchmark driver (docs/benchmarks.md). - tools/memcap.sh -- References/rsciio-venv/bin/python -I -B tools/benchmarks/bench.py run --work DIR [--large] [--only PREFIX] + References/rsciio-venv/bin/python -I -B tools/benchmarks/bench.py run --work DIR [--large] [--only PREFIX] References/rsciio-venv/bin/python -I -B tools/benchmarks/bench.py report --work DIR --out docs/benchmarks.md `gen` writes the inputs under DIR (a scratch folder, never the repo) and DIR/cases.json; `run` generates what is missing, then for diff --git a/tools/bragg-writer/check.py b/tools/bragg-writer/check.py index 0771d2f..5caa3ea 100644 --- a/tools/bragg-writer/check.py +++ b/tools/bragg-writer/check.py @@ -1,8 +1,8 @@ """Checks the Swift-written BraggVectors fixtures against py4DSTEM and h5py. Run (after generate.py, and after the Swift -test wrote swift-*.h5 into the fixtures folder) with References/python/py4dstem/bin/python -I -B, under memcap.sh: +test wrote swift-*.h5 into the fixtures folder) with References/python/py4dstem/bin/python -I -B, under a memory cap: FORMATO_BRAGG_FIXTURE_DIR=$PWD/Tests/FormatoHDF5Tests/Fixtures/bragg swift test -c release --jobs 4 --filter BraggVectorsTests - MACEM_SUITE=... MPLCONFIGDIR=$PWD/.build/matplotlib tools/memcap.sh -- .../python -I -B tools/bragg-writer/check.py + MACEM_SUITE=... MPLCONFIGDIR=$PWD/.build/matplotlib .../python -I -B tools/bragg-writer/check.py For each swift-*.h5: py4DSTEM.read returns a BraggVectors whose peaks equal the py4DSTEM-written pins exactly, its calibration equals the pins, and the h5py group/dataset/attribute structure equals the py4DSTEM-written file's (extra attributes diff --git a/tools/bragg-writer/generate.py b/tools/bragg-writer/generate.py index 61e83c5..65a5254 100644 --- a/tools/bragg-writer/generate.py +++ b/tools/bragg-writer/generate.py @@ -1,5 +1,5 @@ """Pinned py4DSTEM BraggVectors (EMD 1.0) fixtures. Run using References/python/py4dstem/bin/python -I -B, -under the suite tools/memcap.sh. MACEM_SUITE may locate References from a worktree. +under a memory cap. MACEM_SUITE may locate References from a worktree. Writes Tests/FormatoHDF5Tests/Fixtures/bragg/: py4DSTEM-written files, the peaks as little-endian float64 (qx, qy, intensity) triples in scan row-major order, and pins.json with the h5py structure and the values. diff --git a/tools/dm4-gather/generate.py b/tools/dm4-gather/generate.py index 0e43297..19ebdbc 100644 --- a/tools/dm4-gather/generate.py +++ b/tools/dm4-gather/generate.py @@ -1,5 +1,5 @@ -"""Synthetic 4D DM4 and byte/axis pins. MACEM_SUITE= /tools/memcap.sh -- -/References/FormatoKit/rsciio-venv/bin/python -I -B tools/dm4-gather/generate.py. +"""Synthetic 4D DM4 and byte/axis pins. Run under a memory cap: +MACEM_SUITE= /References/FormatoKit/rsciio-venv/bin/python -I -B tools/dm4-gather/generate.py. App output comes from its exact pinned gather and decode methods, compiled with identity-view scaffolding. """ import os, pathlib, subprocess, tempfile, struct, json, hashlib diff --git a/tools/emd-bundle/check.py b/tools/emd-bundle/check.py index 22f5c40..9310212 100644 --- a/tools/emd-bundle/check.py +++ b/tools/emd-bundle/check.py @@ -1,9 +1,9 @@ """Checks the Swift-written EMD bundle fixtures against the first consumer's own files, h5py and py4DSTEM. -Run (after the Swift test wrote swift-*.h5 into the fixtures folder) with the py4DSTEM venv, under memcap.sh: +Run (after the Swift test wrote swift-*.h5 into the fixtures folder) with the py4DSTEM venv, under a memory cap: FORMATO_EMD_FIXTURE_DIR=$PWD/Tests/FormatoHDF5Tests/Fixtures/emd-bundle swift test -c release --jobs 4 --filter EMDBundleTests - MPLCONFIGDIR=$PWD/.build/matplotlib /tools/memcap.sh -- /References/python/py4dstem/bin/python -I -B tools/emd-bundle/check.py + MPLCONFIGDIR=$PWD/.build/matplotlib /References/python/py4dstem/bin/python -I -B tools/emd-bundle/check.py app-*.h5 are mac4DSTEM's own writers' output on the same inputs (tools/emd-bundle/oracle). For each swift file: structure_equal h5py shows the same groups, datasets, dtypes (string encodings, bool storage), shapes, chunking-free attributes diff --git a/tools/emd-bundle/oracle/run.sh b/tools/emd-bundle/oracle/run.sh index cc12a81..d5aec91 100755 --- a/tools/emd-bundle/oracle/run.sh +++ b/tools/emd-bundle/oracle/run.sh @@ -9,6 +9,10 @@ # (the app's Package.swift pins). SwiftPM warns that the kits' own dependency URLs conflict with the local copies: expected. # Takes about two minutes (a release build of the app's Core and the kits). The output folder defaults to the kit's fixtures. set -euo pipefail +# MEMORY_CAP: the command prefix that caps the run's memory (a wrapper that stops the process tree past its limit, with its +# argument separator), as an absolute path; MEMORY_CAP=none runs without a cap. +: ${MEMORY_CAP:?set MEMORY_CAP to a memory-cap command prefix, or none} +cap=(); [[ $MEMORY_CAP == none ]] || cap=(${=MEMORY_CAP}) here=${0:A:h}; suite=${here:h:h:h:h} scratch=${1:?scratch dir}; out=${2:-${here:h:h:h}/Tests/FormatoHDF5Tests/Fixtures/emd-bundle} mkdir -p $scratch/kits $scratch/Sources/oracle @@ -20,4 +24,4 @@ mkdir -p $scratch/export && git -C $suite/mac4DSTEM archive c47a2b7a mac4DSTEM/C rm -rf $scratch/Sources/DSTEMCore && mv $scratch/export/mac4DSTEM/Core $scratch/Sources/DSTEMCore cp $here/Package.swift $scratch/Package.swift; cp $here/main.swift $scratch/Sources/oracle/main.swift cd $scratch && swift build -c release --jobs 4 -$suite/tools/memcap.sh -- .build/release/oracle $out +$cap .build/release/oracle $out diff --git a/tools/empad/generate.py b/tools/empad/generate.py index 8bb8f3f..4fd4aee 100644 --- a/tools/empad/generate.py +++ b/tools/empad/generate.py @@ -1,4 +1,4 @@ -"""Run from FormatoKit with ../tools/memcap.sh -- ../References/FormatoKit/rsciio-venv/bin/python -I -B tools/empad/generate.py.""" +"""Run from FormatoKit with ../References/FormatoKit/rsciio-venv/bin/python -I -B tools/empad/generate.py.""" import gzip, json, pathlib, subprocess, tempfile import numpy as np import rsciio diff --git a/tools/hdf5-4d/generate.py b/tools/hdf5-4d/generate.py index 3641258..900c89f 100644 --- a/tools/hdf5-4d/generate.py +++ b/tools/hdf5-4d/generate.py @@ -1,5 +1,5 @@ """Pinned py4DSTEM/EMD 1.0 fixtures. Run using References/python/py4dstem/bin/python -I -B, -under the suite tools/memcap.sh. MACEM_SUITE may locate References from a worktree. +under a memory cap. MACEM_SUITE may locate References from a worktree. """ import os, pathlib, subprocess, json import numpy as np diff --git a/tools/hdf5-append/generate.py b/tools/hdf5-append/generate.py index 89437b6..f0ffae9 100644 --- a/tools/hdf5-append/generate.py +++ b/tools/hdf5-append/generate.py @@ -1,6 +1,6 @@ """Fixtures and h5py pins for FormatoHDF5's low-level reader (HDF5Reader) and append writer (HDF5AppendWriter). - /tools/memcap.sh -- /References/FormatoKit/rsciio-venv/bin/python -I -B tools/hdf5-append/generate.py + /References/FormatoKit/rsciio-venv/bin/python -I -B tools/hdf5-append/generate.py (a) READ. Writes with h5py (the reference HDF5 binding) Tests/FormatoHDF5Tests/Fixtures/low-level/{assorted,latest}.h5: many dtypes (both byte orders, chunked and compressed, scalar, empty), fixed and variable-length strings, attributes of every shape the diff --git a/tools/hdf5-write/generate.py b/tools/hdf5-write/generate.py index edd8bed..889f3a8 100644 --- a/tools/hdf5-write/generate.py +++ b/tools/hdf5-write/generate.py @@ -1,6 +1,6 @@ """HDF5 root-attribute fixtures and h5py pins for FormatoHDF5.HDF5RootAttributes. - /tools/memcap.sh -- /References/FormatoKit/rsciio-venv/bin/python -I -B tools/hdf5-write/generate.py + /References/FormatoKit/rsciio-venv/bin/python -I -B tools/hdf5-write/generate.py Writes with h5py (the reference HDF5 binding) the files the reader must read or refuse, and records how h5py reads the golden file the kit's own writer produced (Tests/FormatoHDF5Tests/Fixtures/root-attributes/kit-written.h5, regenerated by diff --git a/tools/mib/generate.py b/tools/mib/generate.py index 74c636d..97a8896 100644 --- a/tools/mib/generate.py +++ b/tools/mib/generate.py @@ -1,4 +1,4 @@ -"""Run with MACEM_SUITE= /tools/memcap.sh -- /References/FormatoKit/rsciio-venv/bin/python -I -B tools/mib/generate.py. +"""Run with MACEM_SUITE= /References/FormatoKit/rsciio-venv/bin/python -I -B tools/mib/generate.py. Processed 1/6/12/24-bit synthetic frames and raw refusal pins. App pins compile the exact pinned initializer/readPattern with identity-view scaffolding (no crop/bin); no app files are changed or app targets built. """ diff --git a/tools/png/check.py b/tools/png/check.py index dc0c8e3..50e6ce3 100644 --- a/tools/png/check.py +++ b/tools/png/check.py @@ -1,7 +1,7 @@ """Checks the Swift-written PNG fixtures (FormatoPNG) with Pillow and zlib, and the oracle's. FORMATO_PNG_FIXTURE_DIR=$PWD/Tests/FormatoCoreTests/Fixtures/png swift test -c release --jobs 4 --filter PNGTests - /tools/memcap.sh -- /References/FormatoKit/rsciio-venv/bin/python -I -B tools/png/check.py + /References/FormatoKit/rsciio-venv/bin/python -I -B tools/png/check.py For each swift-*.png and for mactem-inserted.png (the first consumer's own insertion, tools/png/oracle.sh): crc_valid every chunk's CRC-32 equals zlib.crc32 of its type and body; the signature and IEND are there. diff --git a/tools/step-1/owner-check.sh b/tools/step-1/owner-check.sh index 57536fc..fbe9194 100755 --- a/tools/step-1/owner-check.sh +++ b/tools/step-1/owner-check.sh @@ -5,9 +5,13 @@ # its numbers are cited with this log in docs/status.md, and every file read has a line in ../data/README.md. # owner-check.sh prints the files read, the result line, exit 0 when every signal agrees. set -euo pipefail -# The suite's memory rule (../CLAUDE.md, 2026-10-15): the oracle runs under ../tools/memcap.sh (8 GB over its process tree), +# The memory rule (2026-10-15): the oracle runs under a memory cap (8 GB over its process tree, MEMORY_CAP below), # with tools/step-1/pins.py's own watchdog as a second line (macOS refuses setrlimit); the owner's inputs are under 2 GB, and the Swift # side's arrays are sized from them (the SI's dense array is 1.6 GB). +# MEMORY_CAP: the command prefix that caps the run's memory (a wrapper that stops the process tree past its limit, with its +# argument separator), as an absolute path; MEMORY_CAP=none runs without a cap. +: ${MEMORY_CAP:?set MEMORY_CAP to a memory-cap command prefix, or none} +cap=(); [[ $MEMORY_CAP == none ]] || cap=(${=MEMORY_CAP}) cd "${0:A:h}/../.." data=${MACEM_DATA:-../data} out=$PWD/.build/owner @@ -16,7 +20,7 @@ find $data/eels $data/tem $data/stem-edx \( -iname '*.dm3' -o -iname '*.dm4' -o while IFS= read -r f; do print -r -- "${f:A}"; done < $out/files.txt > $out/files-abs.txt print "owner-check: $(wc -l < $out/files-abs.txt | tr -d ' ') files" if [[ ! -s $out/owner-pins.json || $out/owner-pins.json -ot $out/files-abs.txt && ${OWNER_REUSE_PINS:-0} == 0 ]]; then - tr '\n' '\0' < $out/files-abs.txt | xargs -0 ../tools/memcap.sh -- References/rsciio-venv/bin/python -I -B tools/step-1/pins.py --work $out --out $out/owner-pins.json --owner + tr '\n' '\0' < $out/files-abs.txt | xargs -0 $cap References/rsciio-venv/bin/python -I -B tools/step-1/pins.py --work $out --out $out/owner-pins.json --owner fi export FORMATO_OWNER_PINS=$out/owner-pins.json FORMATO_OWNER_FILES=$out/files-abs.txt [[ -d dist/xcframeworks/FormatoLibHDF5.xcframework ]] || tools/xcframework/build.sh --local > /dev/null 2>&1 diff --git a/tools/step-1/pins.py b/tools/step-1/pins.py index d8f3f28..41d2ba7 100644 --- a/tools/step-1/pins.py +++ b/tools/step-1/pins.py @@ -30,7 +30,7 @@ ROOT = Path(__file__).resolve().parents[2] def cap_memory(): - """The suite's rule (../CLAUDE.md, owner 2026-10-15 12:30): every oracle or helper process caps its memory first, 8 GB by + """The memory rule (2026-10-15): every oracle or helper process caps its memory first, 8 GB by default (FORMATO_MEM_CAP_GB to change it), so an oversized case fails loudly instead of swapping. macOS refuses setrlimit(RLIMIT_AS / RLIMIT_DATA) below infinity ("current limit exceeds maximum limit", checked 2026-10-15 on this Mac), so the cap is tried and, in any case, enforced by a watchdog thread on the process's own peak resident size (ru_maxrss, bytes diff --git a/tools/step-4/owner-check.sh b/tools/step-4/owner-check.sh index a188439..b189835 100755 --- a/tools/step-4/owner-check.sh +++ b/tools/step-4/owner-check.sh @@ -5,9 +5,13 @@ # compared as the public pins are. Never part of the gate; its numbers are cited with this log in docs/status.md, and the # file read has a line in ../data/README.md. set -euo pipefail -# The suite's memory rule (../CLAUDE.md, 2026-10-15): the oracle runs under ../tools/memcap.sh (8 GB over its process tree), +# The memory rule (2026-10-15): the oracle runs under a memory cap (8 GB over its process tree, MEMORY_CAP below), # with tools/step-1/pins.py's own watchdog as a second line (macOS refuses setrlimit); the owner's inputs are under 2 GB, and the Swift # side's arrays are sized from them (the SI's dense array is 1.6 GB). +# MEMORY_CAP: the command prefix that caps the run's memory (a wrapper that stops the process tree past its limit, with its +# argument separator), as an absolute path; MEMORY_CAP=none runs without a cap. +: ${MEMORY_CAP:?set MEMORY_CAP to a memory-cap command prefix, or none} +cap=(); [[ $MEMORY_CAP == none ]] || cap=(${=MEMORY_CAP}) cd "${0:A:h}/../.." data=${MACEM_DATA:-../data} out=$PWD/.build/owner-step4 @@ -17,7 +21,7 @@ f=$(find $data/stem-edx -iname '*.emd' -type f | sort | head -1) print -r -- "${f:A}" > $out/file.txt print "owner-check step 4: ${f:t}" if [[ ! -s $out/owner-pins.json || ${OWNER_REUSE_PINS:-0} == 0 ]]; then - /usr/bin/time -p ../tools/memcap.sh -- References/rsciio-venv/bin/python -I -B tools/step-4/pins.py --owner "${f:A}" --out $out/owner-pins.json --options O1,O4,O11 2>&1 | grep -v WARNING + /usr/bin/time -p $cap References/rsciio-venv/bin/python -I -B tools/step-4/pins.py --owner "${f:A}" --out $out/owner-pins.json --options O1,O4,O11 2>&1 | grep -v WARNING fi export FORMATO_OWNER_SI_PINS=$out/owner-pins.json FORMATO_OWNER_SI_FILE=${f:A} [[ -d dist/xcframeworks/FormatoLibHDF5.xcframework ]] || tools/xcframework/build.sh --local > /dev/null 2>&1