diff --git a/.github/workflows/sync-satellites.yml b/.github/workflows/sync-satellites.yml index a609fae..d47d386 100644 --- a/.github/workflows/sync-satellites.yml +++ b/.github/workflows/sync-satellites.yml @@ -1,23 +1,99 @@ name: Sync template satellites on: + push: + branches: [main] workflow_dispatch: + inputs: + full_catalog: + description: Synchronize every ready satellite instead of only changed sources + required: true + default: true + type: boolean permissions: contents: read +concurrency: + group: template-satellite-sync-${{ github.ref }} + cancel-in-progress: false + jobs: - sync: + prepare: if: github.repository == 'Tuurio/auth_samples' runs-on: ubuntu-latest + outputs: + ids: ${{ steps.selection.outputs.ids }} + count: ${{ steps.selection.outputs.count }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24 + cache: npm + cache-dependency-path: distribution/package-lock.json + - name: Install distribution tooling + working-directory: distribution + run: npm ci + - name: Select affected templates + id: selection + working-directory: distribution + env: + BEFORE_SHA: ${{ github.event.before }} + EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.sha }} + FULL_CATALOG: ${{ inputs.full_catalog }} + run: | + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + if [ "$FULL_CATALOG" = "true" ]; then + node scripts/affected-templates.mjs --all > selection.json + elif base_sha="$(git rev-parse "${HEAD_SHA}^" 2>/dev/null)"; then + node scripts/affected-templates.mjs --base "$base_sha" --head "$HEAD_SHA" > selection.json + else + node scripts/affected-templates.mjs --all > selection.json + fi + elif [ -z "$BEFORE_SHA" ]; then + node scripts/affected-templates.mjs --all > selection.json + else + node scripts/affected-templates.mjs --base "$BEFORE_SHA" --head "$HEAD_SHA" > selection.json + fi + cat selection.json + node --input-type=module >> "$GITHUB_OUTPUT" <<'NODE' + import { readFileSync } from "node:fs"; + const selection = JSON.parse(readFileSync("selection.json", "utf8")); + console.log(`ids=${JSON.stringify(selection.ids)}`); + console.log(`count=${selection.count}`); + NODE + { + echo "### Satellite selection" + echo + echo "Selected $(node -p 'require(\"./selection.json\").count') template(s)." + echo + echo '```json' + cat selection.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + sync: + needs: prepare + if: needs.prepare.outputs.count != '0' + runs-on: ubuntu-latest timeout-minutes: 20 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + template: ${{ fromJSON(needs.prepare.outputs.ids) }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 20 + node-version: 24 cache: npm cache-dependency-path: distribution/package-lock.json - name: Require narrowly scoped satellite token @@ -31,9 +107,37 @@ jobs: - name: Install distribution tooling working-directory: distribution run: npm ci - - name: Sync pilot repositories + - name: Configure authenticated Git transport + env: + GH_TOKEN: ${{ secrets.SATELLITE_SYNC_TOKEN }} + run: gh auth setup-git + - name: Synchronize ${{ matrix.template }} env: GH_TOKEN: ${{ secrets.SATELLITE_SYNC_TOKEN }} + TEMPLATE_ID: ${{ matrix.template }} working-directory: distribution - run: npm run sync -- --id react-vite --id nextjs --id spring-boot --apply - + run: | + set +e + npm run --silent sync -- --id "$TEMPLATE_ID" --apply > sync-result.json 2> sync-error.log + status=$? + set -e + { + echo "### $TEMPLATE_ID" + echo + if [ "$status" -eq 0 ]; then + echo "Synchronization completed with a normal commit (or no changes were needed)." + echo '```json' + cat sync-result.json + echo '```' + else + echo "Synchronization failed. No force-push or destructive recovery was attempted." + echo '```text' + cat sync-error.log + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + cat sync-result.json + if [ "$status" -ne 0 ]; then + cat sync-error.log >&2 + exit "$status" + fi diff --git a/.github/workflows/verify-satellites.yml b/.github/workflows/verify-satellites.yml index 80eb28d..416b0b0 100644 --- a/.github/workflows/verify-satellites.yml +++ b/.github/workflows/verify-satellites.yml @@ -3,7 +3,7 @@ name: Verify template satellites on: workflow_dispatch: schedule: - - cron: "17 5 * * 1" + - cron: "17 5 * * *" permissions: contents: read @@ -12,19 +12,45 @@ jobs: verify: if: github.repository == 'Tuurio/auth_samples' runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 20 + node-version: 24 cache: npm cache-dependency-path: distribution/package-lock.json - name: Install distribution tooling working-directory: distribution run: npm ci - - name: Verify pilot repositories + - name: Verify all managed repositories env: GH_TOKEN: ${{ github.token }} working-directory: distribution - run: npm run verify:remotes -- --id react-vite --id nextjs --id spring-boot + run: | + set +e + npm run --silent verify:remotes > verification.json 2> verification-error.log + status=$? + set -e + { + echo "### Nightly satellite drift report" + echo + if [ "$status" -eq 0 ]; then + echo "All managed files, checksums, markers, topics, descriptions, homepages, and template flags match the catalog." + else + echo "Drift or a verification failure was detected. Review the report before running a full catalog synchronization." + fi + echo '```json' + cat verification.json + echo '```' + if [ -s verification-error.log ]; then + echo '```text' + cat verification-error.log + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + cat verification.json + if [ "$status" -ne 0 ]; then + cat verification-error.log >&2 + exit "$status" + fi diff --git a/distribution/README.md b/distribution/README.md new file mode 100644 index 0000000..db66aa1 --- /dev/null +++ b/distribution/README.md @@ -0,0 +1,35 @@ +# Template distribution operations + +`auth_samples` is the source of truth for 20 public GitHub template repositories. Satellite repositories are normal repositories with preserved issues, settings, and unmanaged files; synchronization updates only files listed in `.tuurio-template.json` and never force-pushes. + +## Automation + +- A push to `main` computes the affected catalog entries from changed source paths and synchronizes only those satellites. +- Changes to shared package inputs (`LICENSE`, the generated README, the catalog, or the packager) synchronize the full catalog. +- **Sync template satellites** can be dispatched manually with `full_catalog=true` for a reviewed full-catalog repair. +- **Verify template satellites** runs nightly and can also be dispatched manually. It compares package checksums, every managed file checksum, the management marker, repository metadata, topics, and the GitHub template flag. +- Synchronization uses ordinary commits on each satellite's `main` branch. A conflict or rejected push fails visibly; automation never force-pushes or silently rewrites history. + +The synchronization workflow requires `SATELLITE_SYNC_TOKEN`, a fine-grained token scoped only to the 20 managed satellite repositories with repository contents write access. It is available only to trusted `main` pushes and manual workflow dispatches, never pull requests. + +## Local commands + +From `distribution/`: + +```bash +npm ci +npm run validate +npm run affected -- --base --head +npm run sync -- --id react-vite +npm run verify:remotes -- --id react-vite +``` + +Dry-run is the default. Add `--apply` only after reviewing the package and target repository. A manual full-catalog synchronization omits `--id`; the GitHub workflow is preferred because it provides per-template isolation and summaries. + +## Contribution policy + +Implementation changes belong in this repository under the source path named in the satellite's `.tuurio-template.json`. Open pull requests against `Tuurio/auth_samples`, run the source sample's tests plus `npm run validate` in `distribution/`, and let the post-merge workflow propagate the reviewed result. + +Do not submit generated implementation changes directly to a satellite: they can be replaced by the next synchronization. Satellite-specific issues, discussions, stars, repository settings, and files outside the management marker remain in the satellite and are intentionally preserved. + +If a synchronization is wrong, revert the normal synchronization commit in the affected satellite and fix or revert the source change here. Do not delete the marker, bypass the allow-list, or force-push a satellite. diff --git a/distribution/package.json b/distribution/package.json index 0d294c8..221f755 100644 --- a/distribution/package.json +++ b/distribution/package.json @@ -6,6 +6,7 @@ "test": "node --test scripts/*.test.mjs", "validate": "npm test && node scripts/validate-manifest.mjs", "package": "node scripts/package-template.mjs", + "affected": "node scripts/affected-templates.mjs", "bootstrap": "node scripts/bootstrap-template-repos.mjs", "sync": "node scripts/sync-template-repos.mjs", "verify:remotes": "node scripts/verify-template-repos.mjs" diff --git a/distribution/scripts/affected-templates.mjs b/distribution/scripts/affected-templates.mjs new file mode 100644 index 0000000..cad21ee --- /dev/null +++ b/distribution/scripts/affected-templates.mjs @@ -0,0 +1,67 @@ +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { loadCatalog, repositoryRoot } from "./catalog.mjs"; + +const GLOBAL_PACKAGE_INPUTS = new Set([ + "LICENSE", + "distribution/README.template.md", + "distribution/scripts/package-template.mjs", + "distribution/templates.yml", +]); + +function normalizePath(path) { + return path.replaceAll("\\", "/").replace(/^\.\//, ""); +} + +export function affectedTemplateIds(templates, changedFiles, { fullCatalog = false } = {}) { + const ready = templates.filter((template) => template.status === "ready" && template.files); + if (fullCatalog) return ready.map((template) => template.id); + const paths = changedFiles.map(normalizePath); + if (paths.some((path) => GLOBAL_PACKAGE_INPUTS.has(path))) { + return ready.map((template) => template.id); + } + return ready + .filter((template) => paths.some((path) => path === template.source || path.startsWith(`${template.source}/`))) + .map((template) => template.id); +} + +export function changedFilesBetween(base, head, { root = repositoryRoot } = {}) { + if (!base || !head) throw new Error("Both --base and --head are required unless --all is used"); + if (/^0+$/.test(base)) return ["distribution/templates.yml"]; + const output = execFileSync("git", ["diff", "--name-only", "-z", base, head, "--"], { + cwd: root, + encoding: "utf8", + }); + return output.split("\0").filter(Boolean).map(normalizePath); +} + +function parseArgs(argv) { + let base = null; + let head = null; + let fullCatalog = false; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--base" || argument === "--head") { + const value = argv[index + 1]; + if (!value) throw new Error(`${argument} requires a value`); + if (argument === "--base") base = value; + else head = value; + index += 1; + } else if (argument === "--all") { + fullCatalog = true; + } else { + throw new Error(`Unknown argument: ${argument}`); + } + } + return { base, head, fullCatalog }; +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) { + const args = parseArgs(process.argv.slice(2)); + const catalog = loadCatalog(); + const changedFiles = args.fullCatalog ? [] : changedFilesBetween(args.base, args.head); + const ids = affectedTemplateIds(catalog.templates, changedFiles, { fullCatalog: args.fullCatalog }); + console.log(JSON.stringify({ ids, count: ids.length, changedFiles })); +} diff --git a/distribution/scripts/affected-templates.test.mjs b/distribution/scripts/affected-templates.test.mjs new file mode 100644 index 0000000..6735f81 --- /dev/null +++ b/distribution/scripts/affected-templates.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { loadCatalog } from "./catalog.mjs"; +import { affectedTemplateIds } from "./affected-templates.mjs"; + +const templates = loadCatalog().templates; +const allReady = templates.filter((template) => template.status === "ready" && template.files).map((template) => template.id).sort(); + +test("selects only templates whose reviewed source changed", () => { + assert.deepEqual(affectedTemplateIds(templates, ["auth_samples_react/src/App.tsx"]), ["react-vite"]); + assert.deepEqual( + affectedTemplateIds(templates, ["auth_samples_fastapi/app/main.py", "auth_samples_django/authapp/views.py"]), + ["django", "fastapi"], + ); +}); + +test("selects the full catalog for shared package inputs and explicit full syncs", () => { + assert.deepEqual(affectedTemplateIds(templates, ["distribution/README.template.md"]).sort(), allReady); + assert.deepEqual(affectedTemplateIds(templates, [], { fullCatalog: true }).sort(), allReady); +}); + +test("does not sync satellites for distribution tooling that cannot change packages", () => { + assert.deepEqual(affectedTemplateIds(templates, ["distribution/scripts/verify-template-repos.mjs"]), []); + assert.deepEqual(affectedTemplateIds(templates, ["docs/vibe/AGENT_DIRECTORY_SUBMISSIONS.md"]), []); +}); diff --git a/distribution/scripts/package-template.mjs b/distribution/scripts/package-template.mjs index ec19518..d90dbbc 100644 --- a/distribution/scripts/package-template.mjs +++ b/distribution/scripts/package-template.mjs @@ -130,6 +130,17 @@ function checksum(path) { return createHash("sha256").update(readFileSync(path)).digest("hex"); } +function packageChecksum(managedFiles) { + const digest = createHash("sha256"); + for (const entry of managedFiles) { + digest.update(entry.path); + digest.update("\0"); + digest.update(entry.sha256); + digest.update("\n"); + } + return digest.digest("hex"); +} + export function resolveSourceSha(root = repositoryRoot) { return execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); } @@ -189,14 +200,18 @@ export function packageTemplate(template, { root = repositoryRoot, output, sourc writeFileSync(resolve(outputPath, ".stackblitzrc"), `${JSON.stringify({ startCommand: template.start }, null, 2)}\n`); } - const managedFiles = listFiles(outputPath); + const managedFiles = listFiles(outputPath).map((path) => ({ + path, + sha256: checksum(resolve(outputPath, path)), + })); const marker = { - schemaVersion: 1, + schemaVersion: 2, sourceRepository: "Tuurio/auth_samples", sourcePath: template.source, sourceSha, + packageSha256: packageChecksum(managedFiles), templateId: template.id, - managedFiles: managedFiles.map((path) => ({ path, sha256: checksum(resolve(outputPath, path)) })), + managedFiles, }; writeFileSync(resolve(outputPath, ".tuurio-template.json"), `${JSON.stringify(marker, null, 2)}\n`); return { outputPath, marker }; diff --git a/distribution/scripts/package-template.test.mjs b/distribution/scripts/package-template.test.mjs index fb28fd4..79e6ce2 100644 --- a/distribution/scripts/package-template.test.mjs +++ b/distribution/scripts/package-template.test.mjs @@ -5,7 +5,7 @@ import { resolve } from "node:path"; import test from "node:test"; import { loadCatalog } from "./catalog.mjs"; import { packageTemplate } from "./package-template.mjs"; -import { copyPackageContents, removeManagedFiles } from "./sync-template-repos.mjs"; +import { assertNoUnmanagedCollisions, copyPackageContents, removeManagedFiles } from "./sync-template-repos.mjs"; import { compareMarkers } from "./verify-template-repos.mjs"; const manifest = loadCatalog(); @@ -30,7 +30,9 @@ test("packages the React pilot deterministically from an allow-list", () => { const firstResult = packageTemplate(react, { output: first, sourceSha }); const secondResult = packageTemplate(react, { output: second, sourceSha }); assert.deepEqual(firstResult.marker, secondResult.marker); + assert.equal(firstResult.marker.schemaVersion, 2); assert.equal(firstResult.marker.sourceSha, sourceSha); + assert.match(firstResult.marker.packageSha256, /^[a-f0-9]{64}$/); assert.ok(firstResult.marker.managedFiles.some((entry) => entry.path === "LICENSE")); assert.ok(firstResult.marker.managedFiles.some((entry) => entry.path === "package-lock.json")); assert.ok(!firstResult.marker.managedFiles.some((entry) => entry.path === ".env")); @@ -155,23 +157,44 @@ test("sync rejects unsafe paths from a tampered management marker", () => { } }); -test("remote verification detects source drift and unexpected managed files", () => { +test("sync refuses to overwrite files that are not owned by the management marker", () => { + const temporary = mkdtempSync(resolve(tmpdir(), "tuurio-sync-collision-test-")); + try { + const packaged = resolve(temporary, "packaged"); + const clone = resolve(temporary, "clone"); + mkdirSync(packaged); + mkdirSync(clone); + writeFileSync(resolve(packaged, "README.md"), "generated\n"); + writeFileSync(resolve(clone, "README.md"), "maintainer-owned\n"); + assert.throws(() => assertNoUnmanagedCollisions(packaged, clone), /overwrite an unmanaged file/); + assert.equal(readFileSync(resolve(clone, "README.md"), "utf8"), "maintainer-owned\n"); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +}); + +test("remote verification detects package drift without coupling to unrelated source commits", () => { const base = { - schemaVersion: 1, + schemaVersion: 2, sourceRepository: "Tuurio/auth_samples", sourcePath: "auth_samples_react", sourceSha: "e".repeat(40), + packageSha256: "a".repeat(64), templateId: "react-vite", managedFiles: [{ path: "README.md", sha256: "one" }], }; assert.deepEqual(compareMarkers(base, base), []); + assert.deepEqual(compareMarkers({ ...base, sourceSha: "f".repeat(40) }, base), []); + assert.deepEqual(compareMarkers({ ...base, sourceSha: "not-a-commit" }, base), [ + "marker sourceSha is not a full Git commit", + ]); const drifted = { ...base, - sourceSha: "f".repeat(40), + packageSha256: "b".repeat(64), managedFiles: [...base.managedFiles, { path: "unexpected.txt", sha256: "two" }], }; assert.deepEqual(compareMarkers(drifted, base), [ - "marker sourceSha mismatch", + "marker packageSha256 mismatch", "marker has unexpected managed file unexpected.txt", ]); }); diff --git a/distribution/scripts/sync-template-repos.mjs b/distribution/scripts/sync-template-repos.mjs index 6a124b0..2ca79d9 100644 --- a/distribution/scripts/sync-template-repos.mjs +++ b/distribution/scripts/sync-template-repos.mjs @@ -1,4 +1,4 @@ -import { cpSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { cpSync, existsSync, lstatSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { isAbsolute, relative, resolve, sep } from "node:path"; import process from "node:process"; @@ -38,6 +38,34 @@ export function copyPackageContents(packaged, clone) { } } +function packageFiles(root, current = root) { + const files = []; + for (const name of readdirSync(current).sort()) { + const absolute = resolve(current, name); + const metadata = lstatSync(absolute); + if (metadata.isDirectory()) files.push(...packageFiles(root, absolute)); + else if (metadata.isFile()) files.push(relative(root, absolute)); + else throw new Error(`Unsupported packaged entry: ${absolute}`); + } + return files; +} + +export function assertNoUnmanagedCollisions(packaged, clone) { + for (const path of packageFiles(packaged)) { + const destination = assertManagedPath(clone, path); + let parent = resolve(destination, ".."); + while (parent !== resolve(clone)) { + if (existsSync(parent) && lstatSync(parent).isSymbolicLink()) { + throw new Error(`Packaged path crosses an unmanaged symbolic link: ${path}`); + } + parent = resolve(parent, ".."); + } + if (existsSync(destination)) { + throw new Error(`Packaged path would overwrite an unmanaged file: ${path}`); + } + } +} + export function syncTemplate(template, { apply = false, initialize = false, root = repositoryRoot } = {}) { const temporary = mkdtempSync(resolve(tmpdir(), `tuurio-sync-${template.id}-`)); try { @@ -59,6 +87,7 @@ export function syncTemplate(template, { apply = false, initialize = false, root throw new Error(`${template.repository} has an invalid management marker: ${error.message}`); } } + assertNoUnmanagedCollisions(packaged, clone); copyPackageContents(packaged, clone); run("git", ["add", "--all"], { cwd: clone }); const status = run("git", ["status", "--short"], { cwd: clone }); diff --git a/distribution/scripts/verify-template-repos.mjs b/distribution/scripts/verify-template-repos.mjs index d4d0648..d8961c0 100644 --- a/distribution/scripts/verify-template-repos.mjs +++ b/distribution/scripts/verify-template-repos.mjs @@ -16,9 +16,12 @@ function managedFilesByPath(marker) { export function compareMarkers(actual, expected) { const errors = []; - for (const field of ["schemaVersion", "sourceRepository", "sourcePath", "sourceSha", "templateId"]) { + for (const field of ["schemaVersion", "sourceRepository", "sourcePath", "packageSha256", "templateId"]) { if (actual[field] !== expected[field]) errors.push(`marker ${field} mismatch`); } + if (typeof actual.sourceSha !== "string" || !/^[a-f0-9]{40}$/.test(actual.sourceSha)) { + errors.push("marker sourceSha is not a full Git commit"); + } const actualFiles = managedFilesByPath(actual); const expectedFiles = managedFilesByPath(expected); for (const [path, hash] of expectedFiles) {