From 88831ace33e20559ea574cbd9e0cf63cb2ae8580 Mon Sep 17 00:00:00 2001 From: mamahoos Date: Wed, 19 Aug 2026 13:56:28 +0330 Subject: [PATCH] feat: add composite GitHub Action, URI redaction, and OSS positioning Expose pgsync as uses: mamahoos/pgsync@v2 with thin action.yml wrapper, redact passwords in dry-run/verbose output, refresh README for CLI/Docker/Action, and update floating Action tags on semver releases (v2.1.0). --- .github/workflows/docker-publish.yml | 7 ++- README.md | 53 ++++++++++-------- action.yml | 77 ++++++++++++++++++++++++++ examples/github-action-sync.yml | 21 +++++++ pgsync.sh | 23 +++++++- tasks/SPEC-github-action.md | 82 ++++++++++++++++++++++++++++ tasks/plan.md | 40 ++++++++++++++ tasks/todo.md | 14 +++++ tests/unit/cli.bats | 2 +- tests/unit/dry_run.bats | 9 +-- tests/unit/redact.bats | 28 ++++++++++ 11 files changed, 323 insertions(+), 33 deletions(-) create mode 100644 action.yml create mode 100644 examples/github-action-sync.yml create mode 100644 tasks/SPEC-github-action.md create mode 100644 tasks/plan.md create mode 100644 tasks/todo.md create mode 100644 tests/unit/redact.bats diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d2b3762..42b1c9b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest needs: smoke permissions: - contents: read + contents: write packages: write steps: @@ -55,3 +55,8 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Update GitHub Action version tags + uses: peter-evans/action-update-version-tags@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 8496ce2..a2b0e2e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,16 @@ # pgsync -One-shot PostgreSQL logical sync: `pg_dump | psql`. rsync-shaped flags, sane defaults (`--no-owner`, `--no-privileges`, `ON_ERROR_STOP` on the target). +**One-shot PostgreSQL logical sync** — copy one database to another with `pg_dump | psql` and opinionated defaults. + +CLI • Docker • GitHub Actions + +Not incremental replication, CDC, or bidirectional sync — always a full logical dump over the wire. ## Quick start -**Script** — needs Bash 4+ and `pg_dump` / `psql` on `PATH`: +### CLI + +Needs Bash 4+ and `pg_dump` / `psql` on `PATH`: ```bash ./pgsync.sh \ @@ -12,7 +18,7 @@ One-shot PostgreSQL logical sync: `pg_dump | psql`. rsync-shaped flags, sane def -t 'postgresql://user:pass@target:5432/dbname' ``` -**Docker** — no local Postgres client needed; image includes `pg_dump` and `psql`: +### Docker ```bash docker run --rm ghcr.io/mamahoos/pgsync:latest \ @@ -20,35 +26,36 @@ docker run --rm ghcr.io/mamahoos/pgsync:latest \ -t 'postgresql://user:pass@target:5432/dbname' ``` -Published on every version tag (`v*`) to [GHCR](https://github.com/mamahoos/pgsync/pkgs/container/pgsync). Pin a release, e.g. `ghcr.io/mamahoos/pgsync:2.0.1`. - -Log output goes to **stderr**; stdout stays free for the dump pipe. +Images publish on version tags to [GHCR](https://github.com/mamahoos/pgsync/pkgs/container/pgsync). Pin a release, e.g. `ghcr.io/mamahoos/pgsync:2.1.0`. -## Install +### GitHub Actions -```bash -sudo source ./install.sh # system: /usr/local/bin/pgsync -PREFIX="${HOME}/.local" ./install.sh # user-local, no root +```yaml +- uses: mamahoos/pgsync@v2 + with: + source: ${{ secrets.PGSYNC_SOURCE_URI }} + target: ${{ secrets.PGSYNC_TARGET_URI }} ``` -Bash tab completion is installed alongside the binary. Open a new shell or `source` the completion file the installer prints. +Prefer a pinned semver (`@v2.1.0`) or commit SHA for production. Do not use `@main`. -## Docker Compose (local demo) +Reference workflow: [`examples/github-action-sync.yml`](examples/github-action-sync.yml). -Two Postgres instances plus a one-shot sync — useful for smoke tests: +Dry-run and verbose output **redact passwords** in connection URIs. Log messages go to **stderr**; stdout stays free for the dump pipe. + +## Install ```bash -docker compose run --rm pgsync +sudo source ./install.sh # /usr/local/bin/pgsync +PREFIX="${HOME}/.local" ./install.sh # user-local ``` -Custom URIs or a pre-built image: +Bash tab completion ships with the installer. -```bash -docker compose run --rm pgsync \ - -s 'postgresql://user:pass@host:5432/src' \ - -t 'postgresql://user:pass@host:5432/dst' +## Docker Compose (local demo) -PGSYNC_IMAGE=ghcr.io/mamahoos/pgsync:latest docker compose run --rm pgsync +```bash +docker compose run --rm pgsync ``` ## Options @@ -60,7 +67,7 @@ PGSYNC_IMAGE=ghcr.io/mamahoos/pgsync:latest docker compose run --rm pgsync | `-n`, `--dry-run` | Print the pipeline; do not run | | `--delete` | Drop and recreate `public` on target before restore | | `--schema-only` | Schema without data | -| `--data-only` | Data only (target table must already exist; use `--no-clean`) | +| `--data-only` | Data only (target table must exist; use `--no-clean`) | | `--no-clean` | Skip `pg_dump --clean --if-exists` | | `--single-transaction` | Wrap restore in one transaction (`psql -1`) | | `-q`, `--quiet` | Suppress the final `pgsync: ok` line | @@ -68,9 +75,7 @@ PGSYNC_IMAGE=ghcr.io/mamahoos/pgsync:latest docker compose run --rm pgsync | `-h`, `--help` | Help | | `-V`, `--version` | Version | -`--source=URI` and `--target=URI` are accepted. There is no incremental mode — always a full logical dump over the wire. - -Standard libpq env vars apply (`PGPASSWORD`, `PGSSLMODE`, …). Non-zero exit on bad args, missing tools, or `psql` errors. +`--source=URI` and `--target=URI` are accepted. Standard libpq env vars apply (`PGPASSWORD`, `PGSSLMODE`, …). ## Tests diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..3f75c21 --- /dev/null +++ b/action.yml @@ -0,0 +1,77 @@ +name: pgsync +description: One-shot PostgreSQL logical sync (pg_dump | psql) with sensible defaults. +branding: + icon: database + color: blue + +inputs: + source: + description: Source PostgreSQL connection URI + required: true + target: + description: Target PostgreSQL connection URI + required: true + dry-run: + description: Print the pipeline without running pg_dump or psql + required: false + default: "false" + delete: + description: Drop and recreate the public schema on the target before restore + required: false + default: "false" + schema-only: + description: pg_dump --schema-only + required: false + default: "false" + data-only: + description: pg_dump --data-only + required: false + default: "false" + no-clean: + description: Omit pg_dump --clean --if-exists + required: false + default: "false" + single-transaction: + description: Wrap restore in one transaction (psql -1) + required: false + default: "false" + quiet: + description: Suppress the final pgsync ok line + required: false + default: "false" + verbose: + description: Show pipeline and psql output + required: false + default: "false" + +runs: + using: composite + steps: + - name: Ensure PostgreSQL client + shell: bash + run: | + if ! command -v pg_dump >/dev/null 2>&1 || ! command -v psql >/dev/null 2>&1; then + sudo apt-get update -qq + sudo apt-get install -y -qq postgresql-client + fi + + - name: Run pgsync + shell: bash + run: | + set -euo pipefail + args=(-s "${{ inputs.source }}" -t "${{ inputs.target }}") + + is_true() { + [[ "${1,,}" == "true" ]] + } + + is_true "${{ inputs.dry-run }}" && args+=(-n) + is_true "${{ inputs.delete }}" && args+=(--delete) + is_true "${{ inputs.schema-only }}" && args+=(--schema-only) + is_true "${{ inputs.data-only }}" && args+=(--data-only) + is_true "${{ inputs.no-clean }}" && args+=(--no-clean) + is_true "${{ inputs.single-transaction }}" && args+=(--single-transaction) + is_true "${{ inputs.quiet }}" && args+=(-q) + is_true "${{ inputs.verbose }}" && args+=(-v) + + "${{ github.action_path }}/pgsync.sh" "${args[@]}" diff --git a/examples/github-action-sync.yml b/examples/github-action-sync.yml new file mode 100644 index 0000000..8b57bb0 --- /dev/null +++ b/examples/github-action-sync.yml @@ -0,0 +1,21 @@ +# Reference workflow — copy into .github/workflows/ in your repo. +# +# Required secrets: +# PGSYNC_SOURCE_URI +# PGSYNC_TARGET_URI + +name: Sync PostgreSQL + +on: + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Sync source to target + uses: mamahoos/pgsync@v2 + with: + source: ${{ secrets.PGSYNC_SOURCE_URI }} + target: ${{ secrets.PGSYNC_TARGET_URI }} + quiet: "true" diff --git a/pgsync.sh b/pgsync.sh index 159224c..b8f0954 100755 --- a/pgsync.sh +++ b/pgsync.sh @@ -9,7 +9,7 @@ # set -euo pipefail -VERSION="2.0.1" +VERSION="2.1.0" QUIET=false VERBOSE=false @@ -41,6 +41,16 @@ vmsg() { say "pgsync: $*" } +# Hide credentials in logged URIs (dry-run / verbose pipeline). +redact_uri() { + local uri="$1" + if [[ "$uri" =~ ^postgres(ql)?://[^:/@]+:[^@]+@ ]]; then + printf '%s' "$uri" | sed -E 's#^(postgres(ql)?://[^:/@]+):[^@]*@#\1:***@#' + else + printf '%s' "$uri" + fi +} + ############################################# usage() { cat <<'EOF' >&2 @@ -116,10 +126,17 @@ PSQL+=("$DST") ############################################# dry_show() { + local arg printf ' ' >&2 - printf '%q ' "${PGDUMP[@]}" >&2 + for arg in "${PGDUMP[@]}"; do + [[ "$arg" == "$SRC" ]] && arg="$(redact_uri "$SRC")" + printf '%q ' "$arg" >&2 + done printf ' | ' >&2 - printf '%q ' "${PSQL[@]}" >&2 + for arg in "${PSQL[@]}"; do + [[ "$arg" == "$DST" ]] && arg="$(redact_uri "$DST")" + printf '%q ' "$arg" >&2 + done printf '\n' >&2 } diff --git a/tasks/SPEC-github-action.md b/tasks/SPEC-github-action.md new file mode 100644 index 0000000..9a1f36d --- /dev/null +++ b/tasks/SPEC-github-action.md @@ -0,0 +1,82 @@ +# Spec: GitHub Action interface for pgsync + +## Objective + +Add a **composite GitHub Action** in the same repository as a thin wrapper around `pgsync.sh`. The Action is one distribution interface alongside CLI and Docker — not a replacement for them. + +**User stories:** +- As a CI maintainer, I can sync two PostgreSQL databases with `uses: mamahoos/pgsync@v2` and secrets for URIs. +- As an operator, dry-run/verbose output never prints database passwords. +- As a new user, README shows CLI, Docker, and GitHub Actions in under 10 seconds. + +## Tech Stack + +- Bash 4+ (`pgsync.sh` core) +- Composite Action (`action.yml` at repo root) +- Bats for unit/integration tests +- GitHub Actions + GHCR (existing) + +## Commands + +```bash +./scripts/test.sh # shellcheck + unit tests +INTEGRATION=1 ./scripts/test.sh # + integration tests +docker build -t pgsync:local . # container smoke +``` + +## Project Structure + +``` +pgsync.sh # core sync logic +action.yml # composite GitHub Action (wrapper only) +examples/ # reference workflows (not executed by CI) +tasks/ # spec, plan, todo +tests/unit/ # CLI + redaction tests +.github/workflows/ # CI (unchanged triggers except action.yml runs tests) +``` + +## Code Style + +- Single implementation: Action calls `pgsync.sh`; no duplicated pg_dump/psql logic. +- Credentials redacted in any user-visible pipeline output (`dry_show`). +- Action inputs use kebab-case matching CLI flags where possible. + +## Testing Strategy + +| Level | What | Where | +|-------|------|-------| +| Unit | URI redaction, dry-run output | `tests/unit/redact.bats`, update `dry_run.bats` | +| Unit | CLI unchanged behavior | existing bats | +| Integration | Real sync | existing `tests/integration/` | +| CI | shellcheck includes action if bash embedded | lint job | + +## Boundaries + +**Always:** +- Redact passwords in dry-run/verbose pipeline display +- Pin Action docs to version tags (`@v2.1.0`), not `@main` +- Keep Action as composite wrapper + +**Ask first:** +- Docker-based Action (heavier; defer) +- PostgreSQL version matrix + +**Never:** +- Log raw connection URIs with passwords +- Duplicate sync logic in `action.yml` +- Separate repository for the Action + +## Success Criteria + +- [ ] `action.yml` exists at repo root; composite; all CLI flags exposed as optional inputs +- [ ] `dry_show` redacts `user:password@` in URIs; bats prove password never appears in stderr +- [ ] README repositioned: one-shot sync; CLI • Docker • GitHub Actions; example `uses:` block +- [ ] `examples/github-action-sync.yml` reference workflow +- [ ] Repo description updated on GitHub +- [ ] Version bumped to 2.1.0; unit test updated +- [ ] Tag `v2.1.0` publishes Docker image and floating major tag `v2` +- [ ] `./scripts/test.sh` passes + +## Open Questions + +- None — same-repo composite Action confirmed by user analysis. diff --git a/tasks/plan.md b/tasks/plan.md new file mode 100644 index 0000000..17b2b93 --- /dev/null +++ b/tasks/plan.md @@ -0,0 +1,40 @@ +# Implementation Plan: GitHub Action interface + +## Overview + +Add composite Action + credential redaction + README/OSS positioning. One core (`pgsync.sh`), three interfaces (CLI, Docker, Action). + +## Architecture Decisions + +1. **Composite Action** on `ubuntu-latest` — installs `postgresql-client` if missing; delegates to `${{ github.action_path }}/pgsync.sh`. +2. **Credential redaction** in `dry_show` only — runtime still uses real URIs; display uses `redact_uri()`. +3. **Same repo** — Action version tracks git tags (`v2.1.0`); floating `v2` tag updated on release. +4. **examples/** — reference workflow only; not wired into CI triggers. + +## Task List + +### Phase 1: Security + core display +- [ ] Task 1: `redact_uri()` + update `dry_show` in `pgsync.sh` +- [ ] Task 2: `tests/unit/redact.bats`; fix `dry_run.bats` expectations + +### Checkpoint 1 +- [ ] `./scripts/test.sh` green + +### Phase 2: Action interface +- [ ] Task 3: `action.yml` composite with all inputs +- [ ] Task 4: `examples/github-action-sync.yml` + +### Phase 3: OSS packaging +- [ ] Task 5: README reposition + GitHub repo description +- [ ] Task 6: Bump 2.1.0; release workflow floating tag step + +### Checkpoint 2 +- [ ] Full test suite + tag `v2.1.0` + release + +## Risks + +| Risk | Mitigation | +|------|------------| +| Password in `%q` escaped output | Test asserts literal password absent | +| Runner lacks pg_dump | Action step installs postgresql-client | +| `@main` usage | README warns; document `@v2` / pinned semver | diff --git a/tasks/todo.md b/tasks/todo.md new file mode 100644 index 0000000..dd9afed --- /dev/null +++ b/tasks/todo.md @@ -0,0 +1,14 @@ +# Tasks: GitHub Action interface + +- [x] Task 1: `redact_uri()` + update `dry_show` in `pgsync.sh` +- [x] Task 2: `tests/unit/redact.bats`; fix `dry_run.bats` expectations +- [x] Task 3: `action.yml` composite with all inputs +- [x] Task 4: `examples/github-action-sync.yml` +- [x] Task 5: README reposition + GitHub repo description +- [x] Task 6: Bump 2.1.0; release workflow floating tag step + +## Checkpoint + +- [x] `./scripts/test.sh` — 18 unit tests pass +- [ ] CI on PR +- [ ] Tag `v2.1.0` + release + Docker publish diff --git a/tests/unit/cli.bats b/tests/unit/cli.bats index 41d7d9b..fb126a6 100644 --- a/tests/unit/cli.bats +++ b/tests/unit/cli.bats @@ -6,7 +6,7 @@ setup() { @test "version prints semver on stderr" { run --separate-stderr "$PGSYNC" --version [ "$status" -eq 0 ] - [ "$stderr" = "pgsync 2.0.1" ] + [ "$stderr" = "pgsync 2.1.0" ] [ -z "$output" ] } diff --git a/tests/unit/dry_run.bats b/tests/unit/dry_run.bats index b917178..4d0896b 100644 --- a/tests/unit/dry_run.bats +++ b/tests/unit/dry_run.bats @@ -11,8 +11,9 @@ setup() { [[ "$stderr" == *"would run:"* ]] [[ "$stderr" == *"pg_dump --no-owner --no-privileges --clean --if-exists"* ]] [[ "$stderr" == *"psql -v ON_ERROR_STOP=1"* ]] - [[ "$stderr" == *"$SRC"* ]] - [[ "$stderr" == *"$DST"* ]] + [[ "$stderr" == *'user:\*\*\*@source:5432/app'* ]] + [[ "$stderr" == *'user:\*\*\*@target:5432/app'* ]] + [[ "$stderr" != *":pass@"* ]] } @test "dry-run with --delete mentions public schema reset" { @@ -49,8 +50,8 @@ setup() { run --separate-stderr "$PGSYNC" -n \ --source="$SRC" --target="$DST" [ "$status" -eq 0 ] - [[ "$stderr" == *"$SRC"* ]] - [[ "$stderr" == *"$DST"* ]] + [[ "$stderr" == *'user:\*\*\*@source:5432/app'* ]] + [[ "$stderr" == *'user:\*\*\*@target:5432/app'* ]] } @test "schema-only and data-only together exit 1" { diff --git a/tests/unit/redact.bats b/tests/unit/redact.bats new file mode 100644 index 0000000..0e34fa1 --- /dev/null +++ b/tests/unit/redact.bats @@ -0,0 +1,28 @@ +setup() { + load ../helpers/common + PGSYNC="$(pgsync_bin)" + SECRET_URI='postgresql://syncuser:s3cret!@db.example.com:5432/app' + REDACTED_PATTERN='syncuser:\*\*\*@db.example.com:5432/app' +} + +@test "dry-run redacts password in source and target URIs" { + run --separate-stderr "$PGSYNC" -n -s "$SECRET_URI" -t "$SECRET_URI" + [ "$status" -eq 0 ] + [[ "$stderr" == *"$REDACTED_PATTERN"* ]] + [[ "$stderr" != *"s3cret!"* ]] + [[ "$stderr" != *"s3cret\!"* ]] +} + +@test "verbose dry-run redacts credentials in pipeline" { + run --separate-stderr "$PGSYNC" -n -v -s "$SECRET_URI" -t "$SECRET_URI" + [ "$status" -eq 0 ] + [[ "$stderr" == *"$REDACTED_PATTERN"* ]] + [[ "$stderr" != *"s3cret!"* ]] +} + +@test "URI without password is unchanged in dry-run" { + local plain='postgresql://syncuser@db.example.com:5432/app' + run --separate-stderr "$PGSYNC" -n -s "$plain" -t "$plain" + [ "$status" -eq 0 ] + [[ "$stderr" == *"$plain"* ]] +}