Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/workflows/central-orchestrator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ on:
publish-on-release:
required: false
type: string
sync-to-public:
required: false
type: string

permissions:
contents: write
Expand All @@ -95,6 +98,7 @@ jobs:
version-check-python-version: ${{ inputs.version-check-python-version }}
publish-python-version: ${{ inputs.publish-python-version }}
publish-on-release: ${{ inputs.publish-on-release }}
sync-to-public: ${{ inputs.sync-to-public }}

route:
name: Route
Expand All @@ -120,6 +124,7 @@ jobs:
PUBLIC_REPO_RELEASE_BRANCH: ${{ needs.env-config.outputs.PUBLIC_REPO_RELEASE_BRANCH }}
PUBLIC_REPO_INCOMING_BRANCH: ${{ needs.env-config.outputs.PUBLIC_REPO_INCOMING_BRANCH }}
PUBLISH_ON_RELEASE: ${{ needs.env-config.outputs.PUBLISH_ON_RELEASE }}
SYNC_TO_PUBLIC: ${{ needs.env-config.outputs.SYNC_TO_PUBLIC }}
CALLER_EVENT_NAME: ${{ inputs.event-name }}
CALLER_EVENT_ACTION: ${{ inputs.event-action }}
CALLER_BASE_REF: ${{ inputs.base-ref }}
Expand Down Expand Up @@ -191,7 +196,7 @@ jobs:
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
if [ "$REPOSITORY_NAME" = "$PRIVATE_REPO" ] && [ "$REF_NAME" = "$PRIVATE_REPO_RELEASE_BRANCH" ]; then
RUN_BACK_SYNC=true
if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ]; then
if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ] && [ "$SYNC_TO_PUBLIC" = "true" ]; then
RUN_SYNC_TO_PUBLIC=true
fi
fi
Expand All @@ -215,7 +220,7 @@ jobs:

if [ "$EVENT_ACTION" = "closed" ] && [ "$PR_MERGED" = "true" ] && [ "$HEAD_REF" = "$PRIVATE_REPO_MAIN_BRANCH" ]; then
RUN_BACK_SYNC=true
if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ]; then
if [ -n "$PUBLIC_REPO" ] && [ "$PUBLIC_REPO" != "$PRIVATE_REPO" ] && [ "$SYNC_TO_PUBLIC" = "true" ]; then
RUN_SYNC_TO_PUBLIC=true
fi
fi
Expand Down
16 changes: 15 additions & 1 deletion .github/workflows/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ on:
publish-on-release:
required: false
type: string
sync-to-public:
required: false
type: string
outputs:

# Package
Expand Down Expand Up @@ -87,6 +90,8 @@ on:
value: ${{ jobs.export-env.outputs.PUBLISH_PYTHON_VERSION }}
PUBLISH_ON_RELEASE:
value: ${{ jobs.export-env.outputs.PUBLISH_ON_RELEASE }}
SYNC_TO_PUBLIC:
value: ${{ jobs.export-env.outputs.SYNC_TO_PUBLIC }}



Expand Down Expand Up @@ -117,7 +122,7 @@ env:
# - ensure-release-source: validates private main->release and public incoming->release routes
# - pre-release-version-check: PR to PRIVATE_REPO_RELEASE_BRANCH or PUBLIC_REPO_RELEASE_BRANCH
# - back-sync-release-to-main: merged PR main->release in private repo (plus manual)
# - sync-to-public: merged PR to private release (plus manual)
# - sync-to-public: merged PR to private release (plus manual, when enabled)
# - publish-to-pypi: merged PR to public release (plus manual on release branch)
# - sync-from-public: manual only with selected public branch input

Expand All @@ -127,6 +132,7 @@ env:
VERSION_CHECK_PYTHON_VERSION: ${{ inputs.version-check-python-version || '3.13' }}
PUBLISH_PYTHON_VERSION: ${{ inputs.publish-python-version || '3.13' }}
PUBLISH_ON_RELEASE_RAW: ${{ inputs.publish-on-release || vars.PUBLISH_ON_RELEASE || 'false' }}
SYNC_TO_PUBLIC_RAW: ${{ inputs.sync-to-public || vars.SYNC_TO_PUBLIC || 'false' }}
# =================
# End Manual Config
# =================
Expand Down Expand Up @@ -159,6 +165,7 @@ jobs:
VERSION_CHECK_PYTHON_VERSION: ${{ steps.set.outputs.VERSION_CHECK_PYTHON_VERSION }}
PUBLISH_PYTHON_VERSION: ${{ steps.set.outputs.PUBLISH_PYTHON_VERSION }}
PUBLISH_ON_RELEASE: ${{ steps.set.outputs.PUBLISH_ON_RELEASE }}
SYNC_TO_PUBLIC: ${{ steps.set.outputs.SYNC_TO_PUBLIC }}


steps:
Expand All @@ -181,6 +188,12 @@ jobs:
exit 1
fi

sync_to_public="${SYNC_TO_PUBLIC_RAW,,}"
if [ "$sync_to_public" != "true" ] && [ "$sync_to_public" != "false" ]; then
echo "SYNC_TO_PUBLIC must be 'true' or 'false'."
exit 1
fi

# Package
echo "PACKAGE_NAME=$package_name" >> "$GITHUB_OUTPUT"
echo "PACKAGE_SLUG=$package_slug" >> "$GITHUB_OUTPUT"
Expand All @@ -206,3 +219,4 @@ jobs:
echo "VERSION_CHECK_PYTHON_VERSION=${{ env.VERSION_CHECK_PYTHON_VERSION }}" >> "$GITHUB_OUTPUT"
echo "PUBLISH_PYTHON_VERSION=${{ env.PUBLISH_PYTHON_VERSION }}" >> "$GITHUB_OUTPUT"
echo "PUBLISH_ON_RELEASE=$publish_on_release" >> "$GITHUB_OUTPUT"
echo "SYNC_TO_PUBLIC=$sync_to_public" >> "$GITHUB_OUTPUT"
1 change: 1 addition & 0 deletions .github/workflows/self-orchestrator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,5 @@ jobs:
version-check-python-version: '3.13'
publish-python-version: '3.13'
publish-on-release: 'false'
sync-to-public: 'true'
secrets: inherit
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
## v0.11.0 (2026-03-24)

### Feat

- **repo_template**: tweaked template to centralise config variables

## v0.10.1 (2026-03-24)

### Feat

- **sync-to-public**: default behaviour false, with override

## v0.10.0 (2026-03-23)

### Feat
Expand Down
67 changes: 4 additions & 63 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ This means workflow changes are tested directly from the current branch.
- **Private/Public: `main/incoming_from_private` -> `release`:** Runs `ensure-release-source.yaml` and `pre-release-version-check.yaml` to ensure PR comes from `main` and version has been bumped

### PR Merged
- **Private: `main` -> `release`:** Runs `back-sync-release-to-main.yaml` and `sync-to-public.yaml` to commit the release back to main (private), and sync to the public repo.
- **Private: `main` -> `release`:** Runs `back-sync-release-to-main.yaml` and optionally `sync-to-public.yaml` (when `sync-to-public` is enabled) to commit the release back to main (private), and sync to the public repo.
- **Public: `incoming_from_private` -> `release`:** Runs `publish-to-pypi.yaml` when `publish-on-release` is enabled.

### Manual Dispatch
Expand Down Expand Up @@ -88,68 +88,9 @@ flowchart TD
```

## Development Lifecycle

```mermaid
flowchart TB

subgraph LEGEND[Legend]
direction TB
L_S[Source Branch]
L_O(PR opened Workflow)
L_R{Review: n reviewers}
L_T[Target Branch]
L_P(PR Closed Workflow)
L_S --> L_O --> L_R --> L_T --> L_P
end

subgraph PRIVATE[Private]
direction TB
P_MAIN[main]
P_DEV[any dev branch]
P_WF_BUILD(build-and-test)
P_REV_MAIN{Review: 1}
P_WF_REL(ensure-release-source pre-release-version-check)
P_REV_REL{Review: 1}
P_REL[release]
P_WF_BACK(back-sync-release-to-main)
P_WF_SYNC(sync-to-public)
end

subgraph PUBLIC[Public]
direction LR
U_IN[incoming_from_private]
U_REV{Review: 1}
U_REL[release]
U_WF_PUB(publish-to-pypi)
end

P_MAIN -->|Branch: any| P_DEV
P_DEV -.->|PR Opened: any to main| P_WF_BUILD
P_WF_BUILD -.-> P_REV_MAIN
P_REV_MAIN -->|PR Merged: any to main| P_MAIN

P_MAIN -.->|PR Opened: main to release| P_WF_REL
P_WF_REL -.-> P_REV_REL
P_REV_REL -->|PR Squashed: main to release| P_REL

P_REL -.->|Post-merge trigger| P_WF_BACK
P_REL -.->|Post-merge trigger| P_WF_SYNC
P_WF_BACK -.->|Auto back sync release to main| P_MAIN
P_WF_SYNC -.->|Sync release to incoming_from_private| U_IN

U_IN -.->|PR Opened: incoming_from_private to release| U_REV
U_REV -->|PR Merged incoming_from_private to release| U_REL
U_REL -.->|Post-merge trigger| U_WF_PUB

classDef branchNode fill:#ffe3e3,stroke:#c01c28,color:#7a0010,stroke-width:2px;
classDef workflowNode fill:#e6f4ff,stroke:#175cd3,color:#0b3b91,stroke-width:2px;
classDef reviewNode fill:#fff4cc,stroke:#b54708,color:#7a2e0e,stroke-width:2px;
classDef markerNode fill:#f5f5f5,stroke:#667085,color:#1f2937;

class P_MAIN,P_REL,P_DEV,U_REL,U_IN,L_S,L_T branchNode;
class P_WF_BUILD,P_WF_REL,P_WF_BACK,P_WF_SYNC,U_WF_PUB,L_O,L_P workflowNode;
class P_REV_MAIN,P_REV_REL,U_REV,L_R reviewNode;
```
<p align="center">
<img src="docs/images/development_lifecycle.png" alt="DAIR Development Lifecycle">
</p>

## Licence

Expand Down
Binary file added docs/images/development_lifecycle.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "workflows"
version = "0.10.0"
version = "0.11.0"
description = "Data & AI Unit centralised CI/CD workflows"
authors = [{ name = "Cai Davis", email = "Cai.Davis@uhs.nhs.uk" }]
license = "CC-BY-NC-4.0"
Expand Down
29 changes: 16 additions & 13 deletions repo_template/.github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,35 +11,38 @@ Template location:
1. Copy `repo_template/.github/workflows` to `/.github/workflows` in your downstream repository.
2. Confirm these files exist:

- `/.github/workflows/config.yaml`
- `/.github/workflows/orchestrator.yaml`
- `/.github/workflows/sync-from-public.yaml`
- `/.github/workflows/pre-install.sh`
- `/.github/workflows/README.md`

## Configure `orchestrator.yaml`
## Configure `config.yaml`

Set required values:
Set configuration values once in this file:

- `package-name`
- `package-slug`
- `private-repo` and/or `public-repo`
- `private-repo`
- `public-repo`
- `build-smoke-python-version`
- `version-check-python-version`
- `publish-python-version`
- `publish-on-release`
- `sync-to-public`
- `test-matrix-json` (optional compact JSON; leave empty for central defaults)

Optional values:
Both `orchestrator.yaml` and `sync-from-public.yaml` read from this file. Edit placeholder values to match your setup.

- `publish-on-release: 'true'` if you want publish on public release merges (default is off)
- `test-matrix-json` if you want a custom test matrix
- branch overrides if you do not use `main` and `release`
## `orchestrator.yaml`

No identity or runtime settings are configured in this file; it reads all values from `config.yaml`.

This file calls the central stable orchestrator:

- `SETT-Centre-Data-and-AI/workflows/.github/workflows/central-orchestrator.yaml@release`

## Configure `sync-from-public.yaml`

Set:

- `private-repo`
- `public-repo`
## `sync-from-public.yaml`

Use this workflow for manual pullback from a selected public branch to private.

Expand Down
86 changes: 86 additions & 0 deletions repo_template/.github/workflows/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
name: Repo Config

# ============================================================ #
# Repository identity and runtime/policy configuration
# Edit values below to match your repository setup
# ============================================================ #

on:
workflow_call:
outputs:
package-name:
value: ${{ jobs.export-identity.outputs.package-name }}
package-slug:
value: ${{ jobs.export-identity.outputs.package-slug }}
private-repo:
value: ${{ jobs.export-identity.outputs.private-repo }}
public-repo:
value: ${{ jobs.export-identity.outputs.public-repo }}
build-smoke-python-version:
value: ${{ jobs.export-identity.outputs.build-smoke-python-version }}
version-check-python-version:
value: ${{ jobs.export-identity.outputs.version-check-python-version }}
publish-python-version:
value: ${{ jobs.export-identity.outputs.publish-python-version }}
publish-on-release:
value: ${{ jobs.export-identity.outputs.publish-on-release }}
sync-to-public:
value: ${{ jobs.export-identity.outputs.sync-to-public }}
test-matrix-json:
value: ${{ jobs.export-identity.outputs.test-matrix-json }}

jobs:
export-identity:
name: Export Config
runs-on: ubuntu-latest
outputs:
package-name: ${{ steps.set.outputs.package-name }}
package-slug: ${{ steps.set.outputs.package-slug }}
private-repo: ${{ steps.set.outputs.private-repo }}
public-repo: ${{ steps.set.outputs.public-repo }}
build-smoke-python-version: ${{ steps.set.outputs.build-smoke-python-version }}
version-check-python-version: ${{ steps.set.outputs.version-check-python-version }}
publish-python-version: ${{ steps.set.outputs.publish-python-version }}
publish-on-release: ${{ steps.set.outputs.publish-on-release }}
sync-to-public: ${{ steps.set.outputs.sync-to-public }}
test-matrix-json: ${{ steps.set.outputs.test-matrix-json }}

# ================================================ #
# Edit values below to match your repository setup #
# ================================================ #

env:
# Package identity
PACKAGE_NAME: your_package # e.g., your_package
PACKAGE_SLUG: your-package # e.g., your-package (URL-safe)

# Repositories
PRIVATE_REPO: your-org/your-private-repo # Development repo e.g. SETT-Centre-Data-and-AI/your_package_development
PUBLIC_REPO: your-org/your-public-repo # Public repo e.g. SETT-Centre-Data-and-AI/your_package

# Runtime and policy defaults
BUILD_SMOKE_PYTHON_VERSION: '3.13'
VERSION_CHECK_PYTHON_VERSION: '3.13'
PUBLISH_PYTHON_VERSION: '3.13'
PUBLISH_ON_RELEASE: 'false' # set 'true' to auto-publish on public release merges
SYNC_TO_PUBLIC: 'false' # set 'true' to auto-sync private release to public
TEST_MATRIX_JSON: '' # optional compact JSON; empty uses central defaults

# ================================================ #
# ------------------- End Edit ------------------- #
# ================================================ #

steps:
- id: set
shell: bash
run: |
echo "package-name=$PACKAGE_NAME" >> "$GITHUB_OUTPUT"
echo "package-slug=$PACKAGE_SLUG" >> "$GITHUB_OUTPUT"
echo "private-repo=$PRIVATE_REPO" >> "$GITHUB_OUTPUT"
echo "public-repo=$PUBLIC_REPO" >> "$GITHUB_OUTPUT"
echo "build-smoke-python-version=$BUILD_SMOKE_PYTHON_VERSION" >> "$GITHUB_OUTPUT"
echo "version-check-python-version=$VERSION_CHECK_PYTHON_VERSION" >> "$GITHUB_OUTPUT"
echo "publish-python-version=$PUBLISH_PYTHON_VERSION" >> "$GITHUB_OUTPUT"
echo "publish-on-release=$PUBLISH_ON_RELEASE" >> "$GITHUB_OUTPUT"
echo "sync-to-public=$SYNC_TO_PUBLIC" >> "$GITHUB_OUTPUT"
echo "test-matrix-json=$TEST_MATRIX_JSON" >> "$GITHUB_OUTPUT"
Loading