diff --git a/.github/workflows/central-orchestrator.yaml b/.github/workflows/central-orchestrator.yaml index efab71f..c39497f 100644 --- a/.github/workflows/central-orchestrator.yaml +++ b/.github/workflows/central-orchestrator.yaml @@ -71,6 +71,9 @@ on: publish-on-release: required: false type: string + sync-to-public: + required: false + type: string permissions: contents: write @@ -95,6 +98,7 @@ jobs: version-check-python-version: ${{ inputs.version-check-python-version }} publish-python-version: ${{ inputs.publish-python-version }} publish-on-release: ${{ inputs.publish-on-release }} + sync-to-public: ${{ inputs.sync-to-public }} route: name: Route @@ -120,6 +124,7 @@ jobs: PUBLIC_REPO_RELEASE_BRANCH: ${{ needs.env-config.outputs.PUBLIC_REPO_RELEASE_BRANCH }} PUBLIC_REPO_INCOMING_BRANCH: ${{ needs.env-config.outputs.PUBLIC_REPO_INCOMING_BRANCH }} PUBLISH_ON_RELEASE: ${{ needs.env-config.outputs.PUBLISH_ON_RELEASE }} + SYNC_TO_PUBLIC: ${{ needs.env-config.outputs.SYNC_TO_PUBLIC }} CALLER_EVENT_NAME: ${{ inputs.event-name }} CALLER_EVENT_ACTION: ${{ inputs.event-action }} CALLER_BASE_REF: ${{ inputs.base-ref }} @@ -191,7 +196,7 @@ jobs: if [ "$EVENT_NAME" = "workflow_dispatch" ]; then if [ "$REPOSITORY_NAME" = "$PRIVATE_REPO" ] && [ "$REF_NAME" = "$PRIVATE_REPO_RELEASE_BRANCH" ]; then RUN_BACK_SYNC=true - if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ]; then + if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ] && [ "$SYNC_TO_PUBLIC" = "true" ]; then RUN_SYNC_TO_PUBLIC=true fi fi @@ -215,7 +220,7 @@ jobs: if [ "$EVENT_ACTION" = "closed" ] && [ "$PR_MERGED" = "true" ] && [ "$HEAD_REF" = "$PRIVATE_REPO_MAIN_BRANCH" ]; then RUN_BACK_SYNC=true - if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ]; then + if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ] && [ "$SYNC_TO_PUBLIC" = "true" ]; then RUN_SYNC_TO_PUBLIC=true fi fi diff --git a/.github/workflows/config.yaml b/.github/workflows/config.yaml index 5c282e5..f64f24f 100644 --- a/.github/workflows/config.yaml +++ b/.github/workflows/config.yaml @@ -50,6 +50,9 @@ on: publish-on-release: required: false type: string + sync-to-public: + required: false + type: string outputs: # Package @@ -87,6 +90,8 @@ on: value: ${{ jobs.export-env.outputs.PUBLISH_PYTHON_VERSION }} PUBLISH_ON_RELEASE: value: ${{ jobs.export-env.outputs.PUBLISH_ON_RELEASE }} + SYNC_TO_PUBLIC: + value: ${{ jobs.export-env.outputs.SYNC_TO_PUBLIC }} @@ -117,7 +122,7 @@ env: # - ensure-release-source: validates private main->release and public incoming->release routes # - pre-release-version-check: PR to PRIVATE_REPO_RELEASE_BRANCH or PUBLIC_REPO_RELEASE_BRANCH # - back-sync-release-to-main: merged PR main->release in private repo (plus manual) - # - sync-to-public: merged PR to private release (plus manual) + # - sync-to-public: merged PR to private release (plus manual, when enabled) # - publish-to-pypi: merged PR to public release (plus manual on release branch) # - sync-from-public: manual only with selected public branch input @@ -127,6 +132,7 @@ env: VERSION_CHECK_PYTHON_VERSION: ${{ inputs.version-check-python-version || '3.13' }} PUBLISH_PYTHON_VERSION: ${{ inputs.publish-python-version || '3.13' }} PUBLISH_ON_RELEASE_RAW: ${{ inputs.publish-on-release || vars.PUBLISH_ON_RELEASE || 'false' }} + SYNC_TO_PUBLIC_RAW: ${{ inputs.sync-to-public || vars.SYNC_TO_PUBLIC || 'false' }} # ================= # End Manual Config # ================= @@ -159,6 +165,7 @@ jobs: VERSION_CHECK_PYTHON_VERSION: ${{ steps.set.outputs.VERSION_CHECK_PYTHON_VERSION }} PUBLISH_PYTHON_VERSION: ${{ steps.set.outputs.PUBLISH_PYTHON_VERSION }} PUBLISH_ON_RELEASE: ${{ steps.set.outputs.PUBLISH_ON_RELEASE }} + SYNC_TO_PUBLIC: ${{ steps.set.outputs.SYNC_TO_PUBLIC }} steps: @@ -181,6 +188,12 @@ jobs: exit 1 fi + sync_to_public="${SYNC_TO_PUBLIC_RAW,,}" + if [ "$sync_to_public" != "true" ] && [ "$sync_to_public" != "false" ]; then + echo "SYNC_TO_PUBLIC must be 'true' or 'false'." + exit 1 + fi + # Package echo "PACKAGE_NAME=$package_name" >> "$GITHUB_OUTPUT" echo "PACKAGE_SLUG=$package_slug" >> "$GITHUB_OUTPUT" @@ -206,3 +219,4 @@ jobs: echo "VERSION_CHECK_PYTHON_VERSION=${{ env.VERSION_CHECK_PYTHON_VERSION }}" >> "$GITHUB_OUTPUT" echo "PUBLISH_PYTHON_VERSION=${{ env.PUBLISH_PYTHON_VERSION }}" >> "$GITHUB_OUTPUT" echo "PUBLISH_ON_RELEASE=$publish_on_release" >> "$GITHUB_OUTPUT" + echo "SYNC_TO_PUBLIC=$sync_to_public" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/self-orchestrator.yaml b/.github/workflows/self-orchestrator.yaml index 71c010d..c38e245 100644 --- a/.github/workflows/self-orchestrator.yaml +++ b/.github/workflows/self-orchestrator.yaml @@ -39,4 +39,5 @@ jobs: version-check-python-version: '3.13' publish-python-version: '3.13' publish-on-release: 'false' + sync-to-public: 'true' secrets: inherit diff --git a/CHANGELOG.md b/CHANGELOG.md index 12090d6..2ff82b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## v0.11.0 (2026-03-24) + +### Feat + +- **repo_template**: tweaked template to centralise config variables + +## v0.10.1 (2026-03-24) + +### Feat + +- **sync-to-public**: default behaviour false, with override + ## v0.10.0 (2026-03-23) ### Feat diff --git a/README.md b/README.md index ae7dff0..31dd9a6 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ This means workflow changes are tested directly from the current branch. - **Private/Public: `main/incoming_from_private` -> `release`:** Runs `ensure-release-source.yaml` and `pre-release-version-check.yaml` to ensure PR comes from `main` and version has been bumped ### PR Merged - - **Private: `main` -> `release`:** Runs `back-sync-release-to-main.yaml` and `sync-to-public.yaml` to commit the release back to main (private), and sync to the public repo. + - **Private: `main` -> `release`:** Runs `back-sync-release-to-main.yaml` and optionally `sync-to-public.yaml` (when `sync-to-public` is enabled) to commit the release back to main (private), and sync to the public repo. - **Public: `incoming_from_private` -> `release`:** Runs `publish-to-pypi.yaml` when `publish-on-release` is enabled. ### Manual Dispatch @@ -88,68 +88,9 @@ flowchart TD ``` ## Development Lifecycle - -```mermaid -flowchart TB - - subgraph LEGEND[Legend] - direction TB - L_S[Source Branch] - L_O(PR opened Workflow) - L_R{Review: n reviewers} - L_T[Target Branch] - L_P(PR Closed Workflow) - L_S --> L_O --> L_R --> L_T --> L_P - end - - subgraph PRIVATE[Private] - direction TB - P_MAIN[main] - P_DEV[any dev branch] - P_WF_BUILD(build-and-test) - P_REV_MAIN{Review: 1} - P_WF_REL(ensure-release-source pre-release-version-check) - P_REV_REL{Review: 1} - P_REL[release] - P_WF_BACK(back-sync-release-to-main) - P_WF_SYNC(sync-to-public) - end - - subgraph PUBLIC[Public] - direction LR - U_IN[incoming_from_private] - U_REV{Review: 1} - U_REL[release] - U_WF_PUB(publish-to-pypi) - end - - P_MAIN -->|Branch: any| P_DEV - P_DEV -.->|PR Opened: any to main| P_WF_BUILD - P_WF_BUILD -.-> P_REV_MAIN - P_REV_MAIN -->|PR Merged: any to main| P_MAIN - - P_MAIN -.->|PR Opened: main to release| P_WF_REL - P_WF_REL -.-> P_REV_REL - P_REV_REL -->|PR Squashed: main to release| P_REL - - P_REL -.->|Post-merge trigger| P_WF_BACK - P_REL -.->|Post-merge trigger| P_WF_SYNC - P_WF_BACK -.->|Auto back sync release to main| P_MAIN - P_WF_SYNC -.->|Sync release to incoming_from_private| U_IN - - U_IN -.->|PR Opened: incoming_from_private to release| U_REV - U_REV -->|PR Merged incoming_from_private to release| U_REL - U_REL -.->|Post-merge trigger| U_WF_PUB - - classDef branchNode fill:#ffe3e3,stroke:#c01c28,color:#7a0010,stroke-width:2px; - classDef workflowNode fill:#e6f4ff,stroke:#175cd3,color:#0b3b91,stroke-width:2px; - classDef reviewNode fill:#fff4cc,stroke:#b54708,color:#7a2e0e,stroke-width:2px; - classDef markerNode fill:#f5f5f5,stroke:#667085,color:#1f2937; - - class P_MAIN,P_REL,P_DEV,U_REL,U_IN,L_S,L_T branchNode; - class P_WF_BUILD,P_WF_REL,P_WF_BACK,P_WF_SYNC,U_WF_PUB,L_O,L_P workflowNode; - class P_REV_MAIN,P_REV_REL,U_REV,L_R reviewNode; -``` +

+ DAIR Development Lifecycle +

## Licence diff --git a/docs/images/development_lifecycle.png b/docs/images/development_lifecycle.png new file mode 100644 index 0000000..b698805 Binary files /dev/null and b/docs/images/development_lifecycle.png differ diff --git a/pyproject.toml b/pyproject.toml index d04f9f2..7b01984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "workflows" -version = "0.10.0" +version = "0.11.0" description = "Data & AI Unit centralised CI/CD workflows" authors = [{ name = "Cai Davis", email = "Cai.Davis@uhs.nhs.uk" }] license = "CC-BY-NC-4.0" diff --git a/repo_template/.github/workflows/README.md b/repo_template/.github/workflows/README.md index 310ea6d..4823797 100644 --- a/repo_template/.github/workflows/README.md +++ b/repo_template/.github/workflows/README.md @@ -11,35 +11,38 @@ Template location: 1. Copy `repo_template/.github/workflows` to `/.github/workflows` in your downstream repository. 2. Confirm these files exist: +- `/.github/workflows/config.yaml` - `/.github/workflows/orchestrator.yaml` - `/.github/workflows/sync-from-public.yaml` - `/.github/workflows/pre-install.sh` - `/.github/workflows/README.md` -## Configure `orchestrator.yaml` +## Configure `config.yaml` -Set required values: +Set configuration values once in this file: - `package-name` - `package-slug` -- `private-repo` and/or `public-repo` +- `private-repo` +- `public-repo` +- `build-smoke-python-version` +- `version-check-python-version` +- `publish-python-version` +- `publish-on-release` +- `sync-to-public` +- `test-matrix-json` (optional compact JSON; leave empty for central defaults) -Optional values: +Both `orchestrator.yaml` and `sync-from-public.yaml` read from this file. Edit placeholder values to match your setup. -- `publish-on-release: 'true'` if you want publish on public release merges (default is off) -- `test-matrix-json` if you want a custom test matrix -- branch overrides if you do not use `main` and `release` +## `orchestrator.yaml` + +No identity or runtime settings are configured in this file; it reads all values from `config.yaml`. This file calls the central stable orchestrator: - `SETT-Centre-Data-and-AI/workflows/.github/workflows/central-orchestrator.yaml@release` -## Configure `sync-from-public.yaml` - -Set: - -- `private-repo` -- `public-repo` +## `sync-from-public.yaml` Use this workflow for manual pullback from a selected public branch to private. diff --git a/repo_template/.github/workflows/config.yaml b/repo_template/.github/workflows/config.yaml new file mode 100644 index 0000000..7a28067 --- /dev/null +++ b/repo_template/.github/workflows/config.yaml @@ -0,0 +1,86 @@ +name: Repo Config + +# ============================================================ # +# Repository identity and runtime/policy configuration +# Edit values below to match your repository setup +# ============================================================ # + +on: + workflow_call: + outputs: + package-name: + value: ${{ jobs.export-identity.outputs.package-name }} + package-slug: + value: ${{ jobs.export-identity.outputs.package-slug }} + private-repo: + value: ${{ jobs.export-identity.outputs.private-repo }} + public-repo: + value: ${{ jobs.export-identity.outputs.public-repo }} + build-smoke-python-version: + value: ${{ jobs.export-identity.outputs.build-smoke-python-version }} + version-check-python-version: + value: ${{ jobs.export-identity.outputs.version-check-python-version }} + publish-python-version: + value: ${{ jobs.export-identity.outputs.publish-python-version }} + publish-on-release: + value: ${{ jobs.export-identity.outputs.publish-on-release }} + sync-to-public: + value: ${{ jobs.export-identity.outputs.sync-to-public }} + test-matrix-json: + value: ${{ jobs.export-identity.outputs.test-matrix-json }} + +jobs: + export-identity: + name: Export Config + runs-on: ubuntu-latest + outputs: + package-name: ${{ steps.set.outputs.package-name }} + package-slug: ${{ steps.set.outputs.package-slug }} + private-repo: ${{ steps.set.outputs.private-repo }} + public-repo: ${{ steps.set.outputs.public-repo }} + build-smoke-python-version: ${{ steps.set.outputs.build-smoke-python-version }} + version-check-python-version: ${{ steps.set.outputs.version-check-python-version }} + publish-python-version: ${{ steps.set.outputs.publish-python-version }} + publish-on-release: ${{ steps.set.outputs.publish-on-release }} + sync-to-public: ${{ steps.set.outputs.sync-to-public }} + test-matrix-json: ${{ steps.set.outputs.test-matrix-json }} + + # ================================================ # + # Edit values below to match your repository setup # + # ================================================ # + + env: + # Package identity + PACKAGE_NAME: your_package # e.g., your_package + PACKAGE_SLUG: your-package # e.g., your-package (URL-safe) + + # Repositories + PRIVATE_REPO: your-org/your-private-repo # Development repo e.g. SETT-Centre-Data-and-AI/your_package_development + PUBLIC_REPO: your-org/your-public-repo # Public repo e.g. SETT-Centre-Data-and-AI/your_package + + # Runtime and policy defaults + BUILD_SMOKE_PYTHON_VERSION: '3.13' + VERSION_CHECK_PYTHON_VERSION: '3.13' + PUBLISH_PYTHON_VERSION: '3.13' + PUBLISH_ON_RELEASE: 'false' # set 'true' to auto-publish on public release merges + SYNC_TO_PUBLIC: 'false' # set 'true' to auto-sync private release to public + TEST_MATRIX_JSON: '' # optional compact JSON; empty uses central defaults + + # ================================================ # + # ------------------- End Edit ------------------- # + # ================================================ # + + steps: + - id: set + shell: bash + run: | + echo "package-name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT" + echo "package-slug=$PACKAGE_SLUG" >> "$GITHUB_OUTPUT" + echo "private-repo=$PRIVATE_REPO" >> "$GITHUB_OUTPUT" + echo "public-repo=$PUBLIC_REPO" >> "$GITHUB_OUTPUT" + echo "build-smoke-python-version=$BUILD_SMOKE_PYTHON_VERSION" >> "$GITHUB_OUTPUT" + echo "version-check-python-version=$VERSION_CHECK_PYTHON_VERSION" >> "$GITHUB_OUTPUT" + echo "publish-python-version=$PUBLISH_PYTHON_VERSION" >> "$GITHUB_OUTPUT" + echo "publish-on-release=$PUBLISH_ON_RELEASE" >> "$GITHUB_OUTPUT" + echo "sync-to-public=$SYNC_TO_PUBLIC" >> "$GITHUB_OUTPUT" + echo "test-matrix-json=$TEST_MATRIX_JSON" >> "$GITHUB_OUTPUT" diff --git a/repo_template/.github/workflows/orchestrator.yaml b/repo_template/.github/workflows/orchestrator.yaml index da0095f..32fb846 100644 --- a/repo_template/.github/workflows/orchestrator.yaml +++ b/repo_template/.github/workflows/orchestrator.yaml @@ -3,6 +3,7 @@ name: Repo Orchestrator # Reminder: # grant MANAGEMENT_TOKEN to repos that use sync workflows # grant PYPI_TOKEN to public repos that publish to PyPI +# complete settings in config.yaml on: pull_request: @@ -15,8 +16,13 @@ permissions: pull-requests: write jobs: + load-identity: + name: Load Config + uses: ./.github/workflows/config.yaml + call-central-orchestrator: name: Central Orchestrator + needs: load-identity uses: SETT-Centre-Data-and-AI/workflows/.github/workflows/central-orchestrator.yaml@release with: event-name: ${{ github.event_name }} @@ -27,30 +33,15 @@ jobs: repository: ${{ github.repository }} pr-merged: ${{ github.event.pull_request.merged || false }} - # Required identity values - package-name: your_package - package-slug: your-package - private-repo: your-org/your-private-repo - public-repo: your-org/your-public-repo - - # Optional branch overrides - # private-repo-main-branch: main - # private-repo-release-branch: release - # private-repo-incoming-branch: incoming_from_public - # public-repo-release-branch: release - # public-repo-incoming-branch: incoming_from_private - - # Optional runtime overrides - # build-smoke-python-version: '3.13' - # version-check-python-version: '3.13' - # publish-python-version: '3.13' - - # Optional: disable publish workflow on public release PR merges - # publish-on-release: 'false' - - # Optional: disable publish via repo variable fallback instead - # PUBLISH_ON_RELEASE=false - - # Optional matrix override (compact JSON on one line) - # test-matrix-json: '{"include":[{"os":"ubuntu-latest","python-version":"3.13"}]}' + # Identity values from config.yaml + package-name: ${{ needs.load-identity.outputs.package-name }} + package-slug: ${{ needs.load-identity.outputs.package-slug }} + private-repo: ${{ needs.load-identity.outputs.private-repo }} + public-repo: ${{ needs.load-identity.outputs.public-repo }} + build-smoke-python-version: ${{ needs.load-identity.outputs.build-smoke-python-version }} + version-check-python-version: ${{ needs.load-identity.outputs.version-check-python-version }} + publish-python-version: ${{ needs.load-identity.outputs.publish-python-version }} + publish-on-release: ${{ needs.load-identity.outputs.publish-on-release }} + sync-to-public: ${{ needs.load-identity.outputs.sync-to-public }} + test-matrix-json: ${{ needs.load-identity.outputs.test-matrix-json }} secrets: inherit diff --git a/repo_template/.github/workflows/sync-from-public.yaml b/repo_template/.github/workflows/sync-from-public.yaml index e5dd338..9d70aec 100644 --- a/repo_template/.github/workflows/sync-from-public.yaml +++ b/repo_template/.github/workflows/sync-from-public.yaml @@ -13,12 +13,15 @@ permissions: pull-requests: write jobs: + load-identity: + name: Load Identity + uses: ./.github/workflows/config.yaml + sync-from-public: + needs: load-identity uses: SETT-Centre-Data-and-AI/workflows/.github/workflows/sync-from-public.yaml@release with: public_branch: ${{ github.event.inputs.public_branch }} - private-repo: your-org/your-private-repo - private-repo-main-branch: main - private-repo-incoming-branch: incoming_from_public - public-repo: your-org/your-public-repo + private-repo: ${{ needs.load-identity.outputs.private-repo }} + public-repo: ${{ needs.load-identity.outputs.public-repo }} secrets: inherit