From b5e8d9f0080c025cc75adcc719fb6bee18885c62 Mon Sep 17 00:00:00 2001 From: Paullo9 Date: Tue, 27 Oct 2026 23:51:39 +0100 Subject: [PATCH] Public repo: neutral wording for private references, CI scan, Dependabot Rephrase references to the private suite folder (archive paths, the suite's process file, the memory-cap wrapper, the reference-setup script) in neutral public wording; behaviour unchanged. Add a CI job "No private references" that fails on those patterns in tracked files, and a weekly Dependabot check for GitHub Actions. Done: private references scrubbed, scan job and Dependabot added. Next: publish the branch and watch the new CI job pass. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 6 ++++++ .github/workflows/ci.yml | 11 +++++++++++ .github/workflows/docs.yml | 2 +- Sources/CuboStrain/ConsensusNumerics.swift | 2 +- Tests/Fixtures/README.md | 2 +- docs/bragg.md | 2 +- docs/embedding.md | 4 ++-- docs/hot-pixels.md | 2 +- docs/idpc.md | 2 +- docs/parallax-alignment.md | 2 +- docs/ptycho.md | 2 +- docs/strain.md | 6 +++--- tools/benchmarks/bench.py | 6 +++--- tools/bragg/pins.py | 4 ++-- tools/braggmask/pins.py | 2 +- tools/calibration/pins.py | 4 ++-- tools/classification/pins.py | 2 +- tools/cubeview/pins.py | 2 +- tools/dpcviews/pins.py | 2 +- tools/embedding/pins.py | 4 ++-- tools/hot-pixels/pins.py | 2 +- tools/idpc/pins.py | 2 +- tools/origin/trimmed_pins.py | 2 +- tools/parallax/pins.py | 2 +- tools/parallax/preprocess_pins.py | 2 +- tools/parallax/reconstruct_pins.py | 2 +- tools/parallax/subpixel_pins.py | 2 +- tools/polar-ellipse/pins.py | 4 ++-- tools/ptycho/engines_pins.py | 2 +- tools/ptycho/odd_pins.py | 2 +- tools/ptycho/options_pins.py | 2 +- tools/ptycho/pins.py | 2 +- tools/ptycho/position_pins.py | 2 +- tools/ptycho/preprocess_pins.py | 2 +- tools/rereference/pins.py | 2 +- tools/slice-4/pins.py | 8 ++++---- tools/strain/mapping_pins.py | 2 +- tools/strain/pins.py | 2 +- 38 files changed, 65 insertions(+), 48 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cda92d7..5c97c3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,3 +43,14 @@ jobs: top=$(grep -m1 -E '^## [0-9]+\.[0-9]+\.[0-9]+' CHANGELOG.md | sed -E 's/^## ([0-9]+\.[0-9]+\.[0-9]+).*/\1/') tag=${GITHUB_REF_NAME#v} [ "$top" = "$tag" ] || { echo "CHANGELOG top is $top, tag is $tag"; exit 1; } + + private-refs: + name: No private references + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Scan tracked files + run: | + if git grep -n -I -E '_archive|CLAUDE\.md|memcap|macEM-suite|setup-python-refs' -- ':!.gitignore' ':!.github/workflows/ci.yml'; then + echo "Private references above; this repo is public."; exit 1 + fi diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3dec733..8ec0ecd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,7 +2,7 @@ # published to GitHub Pages at https://mac4dstem.github.io/CuboKit/documentation/ on every tag. # On a pull request the site is built and uploaded as an artifact only, so a broken catalog fails before the tag. # -# No swift-docc-plugin: the kit takes no package dependencies (CLAUDE.md). `xcodebuild docbuild` builds the archives +# No swift-docc-plugin: the kit takes no package dependencies. `xcodebuild docbuild` builds the archives # from the package's own schemes; `docc merge` (Xcode 16+) joins them. Needs Pages enabled with source "GitHub Actions" # (repository Settings > Pages). Pattern first proposed for CuboKit; the other kits copy it with their module list. name: Docs diff --git a/Sources/CuboStrain/ConsensusNumerics.swift b/Sources/CuboStrain/ConsensusNumerics.swift index c8ba2f5..074e779 100644 --- a/Sources/CuboStrain/ConsensusNumerics.swift +++ b/Sources/CuboStrain/ConsensusNumerics.swift @@ -269,7 +269,7 @@ enum ConsensusNumerics { /// peaks. Basis row for a peak is [1, h, k]; solves the 3×3 normal /// equations for each of the x and y coordinates. /// - /// DEVIATION (measured 2026-08-04, ../_archive/mac4DSTEM/tools/training-dataset-campaign parity): + /// DEVIATION (measured 2026-08-04, training-dataset campaign parity): /// py4DSTEM's fit_lattice_vectors multiplies both the design matrix and /// the data by the intensity weight before lstsq, which minimizes /// Σ w²·r². These normal equations weight by w once, minimizing Σ w·r² — diff --git a/Tests/Fixtures/README.md b/Tests/Fixtures/README.md index 39c46d2..ecb9aa2 100644 --- a/Tests/Fixtures/README.md +++ b/Tests/Fixtures/README.md @@ -3,7 +3,7 @@ One row per committed fixture file under `Tests/*/Fixtures/`: size, the script under `tools/` that names it (the generator or its pin script; a few scripts write a whole directory; a script found only by file-name prefix, tool-folder name or test module is marked so), and a note where the origin is not the kit's own pinned reference. 152 files. -Licence: every fixture is generated from this kit's own code and the pinned Python reference (see `CLAUDE.md`, Reference) and ships under the kit's GPL-3.0-or-later licence, except the rows marked *third party*. Nothing here is the owner's data: the committed text fixtures carry no owner path or name (checked by search), and real datasets are read from `$MACEM_DATA` and never copied into the repo. +Licence: every fixture is generated from this kit's own code and the pinned Python reference (named in the README) and ships under the kit's GPL-3.0-or-later licence, except the rows marked *third party*. Nothing here is the owner's data: the committed text fixtures carry no owner path or name (checked by search), and real datasets are read from `$MACEM_DATA` and never copied into the repo. ## Needs an origin line diff --git a/docs/bragg.md b/docs/bragg.md index 9c328b0..74d93f0 100644 --- a/docs/bragg.md +++ b/docs/bragg.md @@ -149,5 +149,5 @@ measurement), pinned to scipy in `GaussianEdgeTests`. Regenerate from a suite worktree: ```sh -../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/bragg/pins.py +../../References/python/py4dstem/bin/python -I -B tools/bragg/pins.py ``` diff --git a/docs/embedding.md b/docs/embedding.md index 802c4a8..88c427f 100644 --- a/docs/embedding.md +++ b/docs/embedding.md @@ -94,14 +94,14 @@ numerics (sliced from the git object at the commit above, access qualifiers remo cancellation token and error type) with `swiftc`, then runs them. It never imports or executes CuboEmbedding. ```sh -../../tools/memcap.sh -- ../../References/CuboKit/py4dstem-venv/bin/python -I -B tools/embedding/pins.py +../../References/CuboKit/py4dstem-venv/bin/python -I -B tools/embedding/pins.py ``` It checks Python 3.12.4, NumPy 2.5.0 and scikit-learn 1.4.2 and records versions, SHA256 of the scikit-learn sources and of the app file in `Fixtures/embedding.json`. Data: planted-spectrum Gaussian matrices (60 × 8, and 6 × 10 with fewer rows than columns), three Gaussian blobs (90 × 3), a pattern set with a negative count, an all-zero pattern, and a detector smaller than the binned size. Cases cover an empty cluster in iteration 1, the iteration cap, and the centre-shift stop. Binary -little-endian f64, 41 KB in all. The oracle peaked at 371 MB under the 8 GB memcap. +little-endian f64, 41 KB in all. The oracle peaked at 371 MB under an 8 GB memory cap. Measured gaps against tolerances (tests print them): diff --git a/docs/hot-pixels.md b/docs/hot-pixels.md index 253486b..d544369 100644 --- a/docs/hot-pixels.md +++ b/docs/hot-pixels.md @@ -25,7 +25,7 @@ input/intermediate mean, nonfinite/unrepresentable threshold or rank outside Regenerate from this worktree: ```sh -MPLCONFIGDIR="$PWD/.build/matplotlib" ../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/hot-pixels/pins.py +MPLCONFIGDIR="$PWD/.build/matplotlib" ../../References/python/py4dstem/bin/python -I -B tools/hot-pixels/pins.py ``` The generator verifies the pinned commit, installed oracle source bytes, Python diff --git a/docs/idpc.md b/docs/idpc.md index eb5d3ed..1714e89 100644 --- a/docs/idpc.md +++ b/docs/idpc.md @@ -23,7 +23,7 @@ The result is projected phase in radians up to its fixed zero-mean gauge. Regeneration from this worktree: ```sh -MPLCONFIGDIR="$PWD/.build/matplotlib" ../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/idpc/pins.py +MPLCONFIGDIR="$PWD/.build/matplotlib" ../../References/python/py4dstem/bin/python -I -B tools/idpc/pins.py ``` Fixtures cover 7×10 periodic with anisotropic sampling, 8×8 at 2× padding and diff --git a/docs/parallax-alignment.md b/docs/parallax-alignment.md index 381de9e..a315663 100644 --- a/docs/parallax-alignment.md +++ b/docs/parallax-alignment.md @@ -39,7 +39,7 @@ files to it. Versions are enforced and recorded in `Tests/CuboParallaxTests/Fixt Python 3.12.4, NumPy 2.5.0, SciPy 1.18.0, py4DSTEM 0.14.19. Regenerate from the package root: ```sh -../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/parallax/pins.py +../References/python/py4dstem/bin/python -I -B tools/parallax/pins.py ``` Four small three-frame stacks cover integer/wrapped shifts (8 × 10), fractional shifts on odd axes diff --git a/docs/ptycho.md b/docs/ptycho.md index a46e4c2..494db0c 100644 --- a/docs/ptycho.md +++ b/docs/ptycho.md @@ -14,7 +14,7 @@ See the module's DocC Scope for the precise supported subset. Regenerate from this worktree: ```sh -MPLCONFIGDIR="$PWD/.build/matplotlib" ../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/ptycho/pins.py +MPLCONFIGDIR="$PWD/.build/matplotlib" ../../References/python/py4dstem/bin/python -I -B tools/ptycho/pins.py ``` The script verifies py4DSTEM f050d207 / 0.14.19, NumPy 2.5.0, and the installed diff --git a/docs/strain.md b/docs/strain.md index 2c9d004..781d8b8 100644 --- a/docs/strain.md +++ b/docs/strain.md @@ -66,7 +66,7 @@ minimal value shims. It never imports or executes CuboStrain to generate pins. Regenerate from a suite worktree: ```sh -../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/strain/pins.py +../../References/python/py4dstem/bin/python -I -B tools/strain/pins.py ``` The generator checks Python 3.12.4, NumPy 2.5.0, py4DSTEM 0.14.19, the reference @@ -75,7 +75,7 @@ unequal-intensity noise that distinguishes weighting, a reference population that distinguishes MAD rejection, repeated vector clusters, and peaks measured on public `small_datacube.dm4` crop [0,0,200:312,200:312]. Its input SHA256 and source SHA256 are in `Fixtures/strain.json`. Numeric arrays are little-endian -binary f64, under 5 KB total. Oracle execution under the 8 GB memcap peaked at +binary f64, under 5 KB total. Oracle execution under an 8 GB memory cap peaked at 689 MB. The Swift source oracle is also a descendant of that capped process. Absolute tolerance 1e-10 for primary fits/strains (no bit-exact claim across SVD @@ -130,7 +130,7 @@ would index out of range or return NaN, the kit refuses with the reason. `tools/strain/mapping_pins.py` (a sibling of `pins.py`, which is unchanged, as are its fixtures) runs py4DSTEM's real `BraggVectors.histogram`, `StrainMap.choose_basis_vectors`, `fit_basis_vectors` and `get_strain` (Agg backend, the plots discarded) and `get_maxima_2D`, and writes `Tests/CuboStrainTests/Fixtures/map_*.f64` and `strain_mapping.json` -(61 little-endian f64 arrays, 277 KB; the oracle peaked at 199 MB under the 8 GB memcap). Datasets: +(61 little-endian f64 arrays, 277 KB; the oracle peaked at 199 MB under an 8 GB memory cap). Datasets: - d1: noise-free planted lattice (7 x 8 scan, 48 x 48 detector, fractional origin, non-orthogonal reference lattice, planted exx, eyy, exy, theta ramps, columns 0 to 2 exactly unstrained). diff --git a/tools/benchmarks/bench.py b/tools/benchmarks/bench.py index 767e655..5d507bc 100644 --- a/tools/benchmarks/bench.py +++ b/tools/benchmarks/bench.py @@ -1,7 +1,7 @@ """py4DSTEM side of the CuboKit benchmark, and the merge into docs/benchmarks.md. -Run from the suite folder (one benchmark at a time machine-wide, under memcap): - tools/memcap.sh -- References/python/py4dstem/bin/python -I -B CuboKit/tools/benchmarks/bench.py make # writes /{small,mid,typical}.npy - tools/memcap.sh -- References/python/py4dstem/bin/python -I -B CuboKit/tools/benchmarks/bench.py python # writes /python.json +Run from the suite folder (one benchmark at a time machine-wide, under a memory cap): + References/python/py4dstem/bin/python -I -B CuboKit/tools/benchmarks/bench.py make # writes /{small,mid,typical}.npy + References/python/py4dstem/bin/python -I -B CuboKit/tools/benchmarks/bench.py python # writes /python.json CUBOKIT_BENCH_DIR= swift test -c release --filter BenchmarkHarness # writes /swift.json References/python/py4dstem/bin/python -I -B CuboKit/tools/benchmarks/bench.py merge # prints the markdown table Speed round (docs/benchmarks.md): `bench.py speed [calls]` prints one `SPEED name median_ms= best_ms=` line per case for the Python side (default 15 calls); diff --git a/tools/bragg/pins.py b/tools/bragg/pins.py index 7ff245c..3fed909 100644 --- a/tools/bragg/pins.py +++ b/tools/bragg/pins.py @@ -1,5 +1,5 @@ -"""Generate CuboBragg pins. Run with memcap and the pinned venv, from the package root: -../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/bragg/pins.py +"""Generate CuboBragg pins. Run under a memory cap with the pinned venv, from the package root: +../../References/python/py4dstem/bin/python -I -B tools/bragg/pins.py Two independent oracles, neither is Swift written for the kit: 1. py4DSTEM 0.14.19 (f050d207) `find_Bragg_disks`, called as a public function, is the primary reference. diff --git a/tools/braggmask/pins.py b/tools/braggmask/pins.py index b6d07d0..f618638 100644 --- a/tools/braggmask/pins.py +++ b/tools/braggmask/pins.py @@ -1,5 +1,5 @@ """Pinned py4DSTEM DataCube.get_braggmask and get_local_ave_dp (braggmask and plain). Run from the suite folder: - tools/memcap.sh -- References/python/py4dstem/bin/python -I -B CuboKit/tools/braggmask/pins.py + References/python/py4dstem/bin/python -I -B CuboKit/tools/braggmask/pins.py Output: CuboKit/Tests/CuboBraggTests/Fixtures/braggmask.json """ from pathlib import Path diff --git a/tools/calibration/pins.py b/tools/calibration/pins.py index 006a04c..cbfca12 100644 --- a/tools/calibration/pins.py +++ b/tools/calibration/pins.py @@ -1,8 +1,8 @@ """Pinned py4DSTEM calibration model and cube reshaping: DataCube.calibrate (dim vectors, meshgrids, qxxs), Calibration setters, and preprocess.set_scan_shape, swap_RQ, swap_Rxy, swap_Qxy, bin_data_real, thin_data_real, pad_data_diffraction, resample_data_diffraction (bilinear), median_filter_masked_pixels on a Float32 DataCube. -Run from the suite folder (References/python/py4dstem from tools/setup-python-refs.sh): - tools/memcap.sh -- References/python/py4dstem/bin/python -I -B CuboKit/tools/calibration/pins.py +Run from the suite folder (References/python/py4dstem, a pinned venv): + References/python/py4dstem/bin/python -I -B CuboKit/tools/calibration/pins.py Output: CuboKit/Tests/CuboCoreTests/Fixtures/calibration.json """ from pathlib import Path diff --git a/tools/classification/pins.py b/tools/classification/pins.py index 41ad700..892c2b4 100644 --- a/tools/classification/pins.py +++ b/tools/classification/pins.py @@ -3,7 +3,7 @@ produced by py4DSTEM itself on small planted inputs; no CuboKit Swift is used. Run from the CuboKit folder with the suite's reference environment under the memory cap: - ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/classification/pins.py [featurization|classification|virtualimage|all] + ../References/python/py4dstem/bin/python -I -B tools/classification/pins.py [featurization|classification|virtualimage|all] Outputs (floats as repr, so they round-trip exactly): Tests/CuboEmbeddingTests/Fixtures/featurization.json diff --git a/tools/cubeview/pins.py b/tools/cubeview/pins.py index d23d015..098d125 100644 --- a/tools/cubeview/pins.py +++ b/tools/cubeview/pins.py @@ -1,6 +1,6 @@ """Pinned py4DSTEM crop and bin: preprocess.crop_data_real, crop_data_diffraction, bin_data_diffraction on a Float32 DataCube. Run from the kit root: - ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/cubeview/pins.py + ../References/python/py4dstem/bin/python -I -B tools/cubeview/pins.py Output: Tests/CuboCoreTests/Fixtures/cubeview.json """ from pathlib import Path diff --git a/tools/dpcviews/pins.py b/tools/dpcviews/pins.py index d5c7284..7c77182 100644 --- a/tools/dpcviews/pins.py +++ b/tools/dpcviews/pins.py @@ -1,6 +1,6 @@ """NumPy reference for the DPC views: magnitude np.hypot and the CoM angle np.angle(x + 1j y), as py4DSTEM's show_complex_CoM builds the complex CoM (`com_x + 1j*com_y`, x = first axis). Wrapped to [0, 2 pi) with +0 canonical. -Run from the kit root: ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/dpcviews/pins.py +Run from the kit root: ../References/python/py4dstem/bin/python -I -B tools/dpcviews/pins.py Output: Tests/CuboPhaseTests/Fixtures/dpcviews.json """ from pathlib import Path diff --git a/tools/embedding/pins.py b/tools/embedding/pins.py index 392b10f..4924f7d 100644 --- a/tools/embedding/pins.py +++ b/tools/embedding/pins.py @@ -1,7 +1,7 @@ -"""Generate CuboEmbedding pins. Run with memcap and the OLD pinned venv (scikit-learn 1.4.2; References/python/py4dstem has 1.9.1, +"""Generate CuboEmbedding pins. Run under a memory cap with the OLD pinned venv (scikit-learn 1.4.2; References/python/py4dstem has 1.9.1, whose PCA fixtures differ, so a regeneration there is a fixture move of its own): - ../../tools/memcap.sh -- ../../References/CuboKit/py4dstem-venv/bin/python -I -B tools/embedding/pins.py + ../../References/CuboKit/py4dstem-venv/bin/python -I -B tools/embedding/pins.py Primary pins come from scikit-learn (PCA svd_solver="full", KMeans init=, n_init=1, algorithm="lloyd"). App pins come from the ORIGINAL mac4DSTEM DiffractionEmbedding.swift at APP: its numeric helpers are sliced out of the diff --git a/tools/hot-pixels/pins.py b/tools/hot-pixels/pins.py index a0ee4d7..312e261 100644 --- a/tools/hot-pixels/pins.py +++ b/tools/hot-pixels/pins.py @@ -1,5 +1,5 @@ """Actual pinned py4DSTEM DataCube.filter_hot_pixels; Float32 cube semantics. -Run under ../../tools/memcap.sh using the pinned isolated Python venv. +Run under a memory cap using the pinned isolated Python venv. """ from pathlib import Path import json, subprocess, platform diff --git a/tools/idpc/pins.py b/tools/idpc/pins.py index 3802c23..fb9a15b 100644 --- a/tools/idpc/pins.py +++ b/tools/idpc/pins.py @@ -1,7 +1,7 @@ """NumPy float64 transcription of the app's Fourier-integration iDPC. This is NOT py4DSTEM iterative DPC. Periodic/centered zero-padded boundary, mean subtraction before padding and after cropping, regularized inverse gradient. -Run with the pinned venv under ../../tools/memcap.sh, from this worktree. +Run with the pinned venv under a memory cap, from this worktree. """ from pathlib import Path import json, platform, hashlib, subprocess diff --git a/tools/origin/trimmed_pins.py b/tools/origin/trimmed_pins.py index c36fefa..9b06fb3 100644 --- a/tools/origin/trimmed_pins.py +++ b/tools/origin/trimmed_pins.py @@ -1,6 +1,6 @@ """Pins for OriginFit.fitTrimmed: the MAD-trimmed origin fit of mac4DSTEM's calibration (OriginCalibration.fitOriginTrimmed). Run from the kit root: - MPLCONFIGDIR="$PWD/.build/matplotlib" ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/origin/trimmed_pins.py + MPLCONFIGDIR="$PWD/.build/matplotlib" ../References/python/py4dstem/bin/python -I -B tools/origin/trimmed_pins.py Two references, both recorded in the fixture: * `app`: a line-by-line NumPy transcription of the app's Swift source (mac4DSTEM origin/main, Core/Analysis/OriginCalibration.swift, `fitOriginTrimmed` and `fit2D`), in Float32 where the app is Float (residuals, median, MAD, cutoff, RMS), the least-squares surface in diff --git a/tools/parallax/pins.py b/tools/parallax/pins.py index 489471c..489145f 100644 --- a/tools/parallax/pins.py +++ b/tools/parallax/pins.py @@ -1,5 +1,5 @@ """Regenerate with the pinned venv, from the package root: -../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/parallax/pins.py +../References/python/py4dstem/bin/python -I -B tools/parallax/pins.py Only public py4DSTEM test data is cropped; no owner's dataset is copied. """ diff --git a/tools/parallax/preprocess_pins.py b/tools/parallax/preprocess_pins.py index a75f4b4..daea160 100644 --- a/tools/parallax/preprocess_pins.py +++ b/tools/parallax/preprocess_pins.py @@ -1,4 +1,4 @@ -"""Pinned Parallax.preprocess oracle; run with suite memcap and pinned Python -I -B.""" +"""Pinned Parallax.preprocess oracle; run under a memory cap and pinned Python -I -B.""" import json, platform, subprocess, hashlib from pathlib import Path import numpy as np diff --git a/tools/parallax/reconstruct_pins.py b/tools/parallax/reconstruct_pins.py index de8a692..5c3e1a6 100644 --- a/tools/parallax/reconstruct_pins.py +++ b/tools/parallax/reconstruct_pins.py @@ -1,7 +1,7 @@ """Pinned Parallax.reconstruct oracle (alignment schedule, regularised shifts, running average) and one end-to-end subpixel case. Real py4DSTEM `Parallax(...).preprocess().reconstruct()` on small synthetic bright-field cubes. From the clone root: - ../../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/parallax/reconstruct_pins.py + ../References/python/py4dstem/bin/python -I -B tools/parallax/reconstruct_pins.py Bright-field disks are small (at most 16 groups per bin): NumPy sorts up to 16 elements with a stable insertion sort. Planted data: one smooth random object seen through every bright-field pixel with a linear shift of the pixel offset (a parallax of defocus type), so correlation maxima are unique (no float32 ties). For every bin the script asserts diff --git a/tools/parallax/subpixel_pins.py b/tools/parallax/subpixel_pins.py index d9108fc..5bfc36b 100644 --- a/tools/parallax/subpixel_pins.py +++ b/tools/parallax/subpixel_pins.py @@ -1,5 +1,5 @@ """Pinned subpixel_alignment KDE oracle (bilinear and Lanczos deposition, Gaussian blur, sinc low-pass, default factor -heuristic). Position correction is not pinned here. Run with the pinned venv, python -I -B, under tools/memcap.sh.""" +heuristic). Position correction is not pinned here. Run with the pinned venv, python -I -B, under a memory cap.""" import json, warnings from pathlib import Path src=Path(__file__).with_name('aberration_pins.py').read_text().split("cases=[]")[0] diff --git a/tools/polar-ellipse/pins.py b/tools/polar-ellipse/pins.py index cd742be..478ef6e 100644 --- a/tools/polar-ellipse/pins.py +++ b/tools/polar-ellipse/pins.py @@ -3,8 +3,8 @@ Calls py4DSTEM itself: PolarDatacube, DataCube.get_radial_bkgrnd / get_radial_bksb_dp / get_local_ave_dp(radial_bksb=True), fit_ellipse_1D, fit_ellipse_amorphous_ring, constrain_degenerate_ellipse, ellipse_err, double_sided_gaussian(_fiterr), convert_ellipse_params(_r), fourier_resample and resample_data_diffraction(method="fourier"). -Run from the suite folder (References/python/py4dstem from tools/setup-python-refs.sh): - tools/memcap.sh -- References/python/py4dstem/bin/python -I -B CuboKit/tools/polar-ellipse/pins.py +Run from the suite folder (References/python/py4dstem, a pinned venv): + References/python/py4dstem/bin/python -I -B CuboKit/tools/polar-ellipse/pins.py Output: CuboKit/Tests/CuboVirtualTests/Fixtures/polar-radial-pins.json and CuboKit/Tests/CuboCoreTests/Fixtures/ellipse-resample-pins.json Conventions: py4DSTEM's x is the kit's ROW and y its COLUMN, so the (x0, y0, a, b, theta) here are the kit's (row, column, a, b, theta) with theta of the major axis from the +row axis towards the +column axis. planted.py measures theta of the major axis from +column diff --git a/tools/ptycho/engines_pins.py b/tools/ptycho/engines_pins.py index 568d996..a4ce899 100644 --- a/tools/ptycho/engines_pins.py +++ b/tools/ptycho/engines_pins.py @@ -1,6 +1,6 @@ """Pinned py4DSTEM oracle for the mixed-state and multislice CPU engines (CuboPtycho). Run from the kit root: - MPLCONFIGDIR="$PWD/.build/matplotlib" ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/ptycho/engines_pins.py + MPLCONFIGDIR="$PWD/.build/matplotlib" ../References/python/py4dstem/bin/python -I -B tools/ptycho/engines_pins.py The real MixedstatePtychography / MultislicePtychography classes run `reconstruct` on numerically injected preprocessing state (the same method as tools/ptycho/pins.py); orthogonalisation, probe initialisation (seeded NumPy RNG) and the propagator arrays are also called directly. Output: Tests/CuboPtychoTests/Fixtures/engines.json diff --git a/tools/ptycho/odd_pins.py b/tools/ptycho/odd_pins.py index 7e1be8d..9006662 100644 --- a/tools/ptycho/odd_pins.py +++ b/tools/ptycho/odd_pins.py @@ -1,6 +1,6 @@ """Pinned py4DSTEM oracle for ODD, non-square detector / probe / object sizes (CuboPtycho preprocessing and the three engines). Run from the kit root: - MPLCONFIGDIR="$PWD/.build/matplotlib" ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/ptycho/odd_pins.py + MPLCONFIGDIR="$PWD/.build/matplotlib" ../References/python/py4dstem/bin/python -I -B tools/ptycho/odd_pins.py Same method as preprocess_pins.py and engines_pins.py (real py4DSTEM methods on numerically injected state; the engine data go through py4DSTEM's _normalize_diffraction_intensities), with a 15 x 13 probe on a 31 x 29 object (single, mixed, multislice) and a 9 x 7 detector in the preprocessing pin. Odd lengths change fftfreq (no Nyquist bin), the corner-centred patch offsets, the diff --git a/tools/ptycho/options_pins.py b/tools/ptycho/options_pins.py index c58dee7..3331014 100644 --- a/tools/ptycho/options_pins.py +++ b/tools/ptycho/options_pins.py @@ -1,6 +1,6 @@ """Pinned oracle for the single-slice options the mac4DSTEM app uses (CuboPtycho 0.6.0 candidates). Run from the worktree root: - MPLCONFIGDIR="$PWD/.build/matplotlib" ../../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/ptycho/options_pins.py + MPLCONFIGDIR="$PWD/.build/matplotlib" ../References/python/py4dstem/bin/python -I -B tools/ptycho/options_pins.py Needs Tests/CuboPtychoTests/Fixtures/ptycho.json (pins.py) as its input data; writes Fixtures/options.json. 1. constrainObjectAmplitude = false (the app's toggle). py4DSTEM has no switch: its _object_threshold_constraint is unconditional. diff --git a/tools/ptycho/pins.py b/tools/ptycho/pins.py index 8434fd7..9ff31ef 100644 --- a/tools/ptycho/pins.py +++ b/tools/ptycho/pins.py @@ -1,5 +1,5 @@ """Pinned py4DSTEM CPU single-slice oracle, no app transcription. -Run from the worktree: ../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/ptycho/pins.py +Run from the worktree: ../../References/python/py4dstem/bin/python -I -B tools/ptycho/pins.py Only preprocessed numerical arrays cross the API; no calibration/state/UI is ported. """ from pathlib import Path diff --git a/tools/ptycho/position_pins.py b/tools/ptycho/position_pins.py index 05cbeeb..5e9adc4 100644 --- a/tools/ptycho/position_pins.py +++ b/tools/ptycho/position_pins.py @@ -1,7 +1,7 @@ """Pinned py4DSTEM oracle for position correction in the single-slice engine (fix_positions=False, positions_step_size, max_position_update_distance, max_position_total_distance, fix_positions_com). Run from the kit root: - MPLCONFIGDIR="$PWD/.build/matplotlib" ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/ptycho/position_pins.py + MPLCONFIGDIR="$PWD/.build/matplotlib" ../References/python/py4dstem/bin/python -I -B tools/ptycho/position_pins.py The real SingleslicePtychography.reconstruct runs on numerically injected preprocessing state (as pins.py). The data are generated at positions that differ from the starting positions by up to 0.3 px (uniform), so the correction has work to do; they go through py4DSTEM's _normalize_diffraction_intensities from a raw cube with a mild offset and noise. Output: Fixtures/positions.json diff --git a/tools/ptycho/preprocess_pins.py b/tools/ptycho/preprocess_pins.py index 80d6555..e8a31a9 100644 --- a/tools/ptycho/preprocess_pins.py +++ b/tools/ptycho/preprocess_pins.py @@ -1,6 +1,6 @@ """Pinned py4DSTEM preprocessing oracle for CuboPtycho (amplitudes, scan positions, object shape, probe normalisation). Run from the kit root: - MPLCONFIGDIR="$PWD/.build/matplotlib" ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/ptycho/preprocess_pins.py + MPLCONFIGDIR="$PWD/.build/matplotlib" ../References/python/py4dstem/bin/python -I -B tools/ptycho/preprocess_pins.py Actual py4DSTEM methods (_normalize_diffraction_intensities, _calculate_scan_positions_in_pixels, _initialize_object, _initialize_probe) are called on a SingleslicePtychography whose preprocessing attributes are set directly; centre of mass, rotation search and calibration extraction are outside this pin (the origins and the rotation are inputs). diff --git a/tools/rereference/pins.py b/tools/rereference/pins.py index 4811e69..980c404 100644 --- a/tools/rereference/pins.py +++ b/tools/rereference/pins.py @@ -1,7 +1,7 @@ """Pins for CuboCore FrameTransform. py4DSTEM (f050d207) moves no origin on crop/bin, so the primary pin for those is the app source CalibrationReReference.swift at mac4DSTEM c809679c, transcribed here in Float32 (never from the Swift port). py4DSTEM contributes the crop slice and the Q-pixel-size x bin rule, asserted from its source text. -Run: ../../tools/memcap.sh -- ../../References/python/py4dstem/bin/python -I -B tools/rereference/pins.py""" +Run: ../../References/python/py4dstem/bin/python -I -B tools/rereference/pins.py""" import json, pathlib, numpy as np f = np.float32 ref = pathlib.Path('../References/CuboKit/py4DSTEM-f050d207/py4DSTEM/preprocess/preprocess.py') diff --git a/tools/slice-4/pins.py b/tools/slice-4/pins.py index 54dab69..7df7f3b 100644 --- a/tools/slice-4/pins.py +++ b/tools/slice-4/pins.py @@ -1,10 +1,10 @@ """CuboKit slice 4: the pin generator (docs/registration-slice-4.md). - ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/slice-4/pins.py synthetic - ../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/slice-4/pins.py export-bullseye - ../tools/memcap.sh [--gb 16] -- ../References/python/py4dstem/bin/python -I -B tools/slice-4/pins.py real [name ...] + ../References/python/py4dstem/bin/python -I -B tools/slice-4/pins.py synthetic + ../References/python/py4dstem/bin/python -I -B tools/slice-4/pins.py export-bullseye + ../References/python/py4dstem/bin/python -I -B tools/slice-4/pins.py real [name ...] -The suite's memcap.sh (a watchdog on the process tree, 8 GB by default) is the enforcer; the watchdog thread below is a +An external memory cap (a watchdog on the process tree, 8 GB by default) is the enforcer; the watchdog thread below is a second guard. One worker at a time; almgsi and auref need --gb 16 (and CUBOKIT_PINS_MEM_GB=16). synthetic -> Tests/CuboVirtualTests/Fixtures/slice-4-pins.json (+ planted cubes as .npy), before any Swift; diff --git a/tools/strain/mapping_pins.py b/tools/strain/mapping_pins.py index e851b3d..8595f1b 100644 --- a/tools/strain/mapping_pins.py +++ b/tools/strain/mapping_pins.py @@ -3,7 +3,7 @@ public simulated Au dataset. Sibling of pins.py (which is untouched); writes only `map_*.f64` and `strain_mapping.json` under Tests/CuboStrainTests/Fixtures. Run from the clone root: - ../../tools/memcap.sh -- ../References/python/py4dstem/bin/python -I -B tools/strain/mapping_pins.py + ../References/python/py4dstem/bin/python -I -B tools/strain/mapping_pins.py Peaks are stored as py4DSTEM sees them (centred, calibrated vectors from `get_vectors(center=True)`), so the Swift side starts from the same doubles. Rows are py4DSTEM x, columns y. diff --git a/tools/strain/pins.py b/tools/strain/pins.py index ff85ea9..a498d9a 100644 --- a/tools/strain/pins.py +++ b/tools/strain/pins.py @@ -1,4 +1,4 @@ -"""Generate reference and original-app pins. Run with memcap and the pinned venv. +"""Generate reference and original-app pins. Run under a memory cap with the pinned venv. Paths resolve from either the primary checkout or a worktree. No new Swift implementation is used. """ import hashlib, json, platform, subprocess, tempfile