diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c3c64600 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,262 @@ +# CI Workflow for pglogical +# +# Runs on: +# - Push to REL2_x_STABLE and windows-build branches +# - Pull requests targeting REL2_x_STABLE +# +# Build matrix: PostgreSQL 15-18 x (Ubuntu, Windows, macOS ARM64) +# Tests: Regression tests run on all platforms + +name: CI Tests + +on: + push: + branches: + - REL2_x_STABLE + - windows-build + pull_request: + branches: + - REL2_x_STABLE + +permissions: + contents: read + checks: write + +jobs: + build: + name: PG ${{ matrix.pg-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + pg-version: [15, 16, 17, 18] + os: [ubuntu-latest, windows-2022, macos-26] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + # ============================================================ + # Linux: Install PostgreSQL from official APT repository + # ============================================================ + - name: Install PostgreSQL (Linux) + if: runner.os == 'Linux' + run: | + # Add PostgreSQL APT repository + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + + # Install PostgreSQL server, development files, and build dependencies + # Libraries required for pglogical_create_subscriber linking: + # libkrb5-dev (GSSAPI), libselinux1-dev, libxslt1-dev, libpam0g-dev, libnuma-dev + sudo apt-get install -y postgresql-${{ matrix.pg-version }} postgresql-server-dev-${{ matrix.pg-version }} \ + libkrb5-dev libselinux1-dev libxslt1-dev libpam0g-dev libnuma-dev + + # Add PostgreSQL bin to PATH + echo "/usr/lib/postgresql/${{ matrix.pg-version }}/bin" >> $GITHUB_PATH + + # ============================================================ + # macOS: Install PostgreSQL from Homebrew + # ============================================================ + - name: Install PostgreSQL (macOS) + if: runner.os == 'macOS' + run: | + # Install PostgreSQL from Homebrew + brew install postgresql@${{ matrix.pg-version }} + + # Determine Homebrew prefix (Intel vs ARM64) + if [[ "$(uname -m)" == "arm64" ]]; then + BREW_PREFIX="/opt/homebrew" + else + BREW_PREFIX="/usr/local" + fi + + # Add PostgreSQL bin to PATH + echo "${BREW_PREFIX}/opt/postgresql@${{ matrix.pg-version }}/bin" >> $GITHUB_PATH + + # ============================================================ + # Windows: Install PostgreSQL from self-hosted binaries (fast) + # Binaries hosted on GitHub releases to avoid slow EDB downloads + # ============================================================ + - name: Cache PostgreSQL (Windows) + if: runner.os == 'Windows' + uses: actions/cache@v4 + id: pg-cache + with: + path: C:\pgsql + key: postgresql-${{ matrix.pg-version }}-windows-x64-v1 + + - name: Install PostgreSQL (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + $pgMajor = "${{ matrix.pg-version }}" + $pgPath = "C:\pgsql" + + # Check cache first + if (Test-Path "$pgPath\bin\pg_config.exe") { + Write-Host "PostgreSQL $pgMajor found in cache" + } else { + # Download from self-hosted GitHub release + $zipUrl = "https://github.com/willibrandon/pglogical/releases/download/pg-binaries/postgresql-$pgMajor-windows-x64.zip" + $zipPath = "$env:TEMP\postgresql.zip" + + Write-Host "Downloading PostgreSQL $pgMajor from $zipUrl" + Invoke-WebRequest -Uri $zipUrl -OutFile $zipPath -UseBasicParsing + + Write-Host "Extracting PostgreSQL $pgMajor..." + Expand-Archive -Path $zipPath -DestinationPath "C:\" -Force + Remove-Item $zipPath -Force + } + + if (-not (Test-Path "$pgPath\bin\pg_config.exe")) { + Write-Error "PostgreSQL $pgMajor not found at $pgPath" + exit 1 + } + + # Add to PATH + echo "$pgPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + Write-Host "PostgreSQL $pgMajor ready at $pgPath" + + # ============================================================ + # Verify pg_config is available + # ============================================================ + - name: Verify pg_config + shell: bash + run: | + echo "PostgreSQL version:" + pg_config --version + echo "" + echo "PostgreSQL paths:" + echo " pkglibdir: $(pg_config --pkglibdir)" + echo " sharedir: $(pg_config --sharedir)" + echo " includedir-server: $(pg_config --includedir-server)" + + # ============================================================ + # Build: Linux and macOS use make + # ============================================================ + - name: Build extension (Linux/macOS) + if: runner.os != 'Windows' + run: | + make clean all + + # ============================================================ + # Build: Windows uses CMake with Visual Studio 2022 + # ============================================================ + - name: Build extension (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + # Get pg_config path from self-hosted binaries + $pgConfig = "C:\pgsql\bin\pg_config.exe" + + # Create build directory + New-Item -ItemType Directory -Force -Path build + Set-Location build + + # Configure with CMake + cmake -G "Visual Studio 17 2022" ` + -DPG_CONFIG="$pgConfig" ` + .. + + # Build + cmake --build . --config Release + + # ============================================================ + # Test: Run regression tests (Linux) + # ============================================================ + - name: Run regression tests (Linux) + if: runner.os == 'Linux' + run: | + # PGXS has a double-install issue: MODULE_big and MODULES both trigger + # install-lib. Work around by staging to temp dir then copying. + export DESTDIR=/tmp/pglogical-stage + make install DESTDIR=/tmp/pglogical-stage + sudo cp -rf /tmp/pglogical-stage/* / + + # Run regression tests + make check + + # ============================================================ + # Test: Run regression tests (macOS) + # ============================================================ + - name: Run regression tests (macOS) + if: runner.os == 'macOS' + run: | + # On macOS with Homebrew, the extension directories are writable + # and DESTDIR staging has issues with symlinks, so install directly + sudo make install + + # Run regression tests + make check + + # ============================================================ + # Test: Run regression tests (Windows) + # ============================================================ + - name: Run regression tests (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + # Install extension first + Set-Location build + cmake --build . --config Release --target install + + # Run regression tests using CMake target + cmake --build . --config Release --target check + + # ============================================================ + # Upload test artifacts on failure + # ============================================================ + - name: Upload regression test results on failure (Linux/macOS) + if: failure() && runner.os != 'Windows' + uses: actions/upload-artifact@v4 + with: + name: regression-results-pg${{ matrix.pg-version }}-${{ matrix.os }} + path: | + regression_output/ + tmp_check/log/ + retention-days: 7 + + - name: Upload regression test results on failure (Windows) + if: failure() && runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: regression-results-pg${{ matrix.pg-version }}-${{ matrix.os }} + path: | + regression_output/ + build/tmp_check/log/ + retention-days: 7 + + # ============================================================ + # Add build status to workflow summary + # ============================================================ + - name: Add to job summary + if: always() + shell: bash + run: | + echo "## Build Status: PG ${{ matrix.pg-version }} on ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ "${{ job.status }}" == "success" ]; then + echo "✅ Build and tests passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Build or tests failed" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "- PostgreSQL: ${{ matrix.pg-version }}" >> $GITHUB_STEP_SUMMARY + echo "- Platform: ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY + echo "- Runner OS: ${{ runner.os }}" >> $GITHUB_STEP_SUMMARY + + # Show regression diffs if test failed + if [ -f "regression_output/regression.diffs" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Regression Diffs" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + cat regression_output/regression.diffs >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..d39eb1df --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,623 @@ +# Release Workflow for pglogical +# +# Triggered by pushing a tag matching v* pattern (e.g., v2.5.0, v2.5.0-beta1) +# Creates a GitHub Release with: +# - Binary packages for all supported platforms +# - MSI installers for Windows +# - Source archives (tar.gz and zip) +# - SHA256 checksums file +# +# Prerelease: Tags containing a hyphen (e.g., v2.5.0-beta1) are marked as prerelease + +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +env: + # Version extracted from tag (without 'v' prefix) + VERSION: '' + +jobs: + # ============================================================ + # Build binaries for all platforms + # ============================================================ + build: + name: Build PG ${{ matrix.pg-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + pg-version: [15, 16, 17, 18] + os: [ubuntu-latest, windows-2022, macos-26] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Extract version from tag + id: version + shell: bash + run: | + # Strip 'v' prefix from tag + VERSION=${GITHUB_REF_NAME#v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Version: $VERSION" + + # ============================================================ + # Linux: Install PostgreSQL from official APT repository + # ============================================================ + - name: Install PostgreSQL (Linux) + if: runner.os == 'Linux' + run: | + # Add PostgreSQL APT repository + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get update + + # Install PostgreSQL server, development files, and build dependencies + # Libraries required for pglogical_create_subscriber linking: + # libkrb5-dev (GSSAPI), libselinux1-dev, libxslt1-dev, libpam0g-dev, libnuma-dev + sudo apt-get install -y postgresql-${{ matrix.pg-version }} postgresql-server-dev-${{ matrix.pg-version }} \ + libkrb5-dev libselinux1-dev libxslt1-dev libpam0g-dev libnuma-dev + + # Add PostgreSQL bin to PATH + echo "/usr/lib/postgresql/${{ matrix.pg-version }}/bin" >> $GITHUB_PATH + + # ============================================================ + # macOS: Install PostgreSQL from Homebrew + # ============================================================ + - name: Install PostgreSQL (macOS) + if: runner.os == 'macOS' + run: | + # Install PostgreSQL from Homebrew + brew install postgresql@${{ matrix.pg-version }} + + # Determine Homebrew prefix (Intel vs ARM64) + if [[ "$(uname -m)" == "arm64" ]]; then + BREW_PREFIX="/opt/homebrew" + else + BREW_PREFIX="/usr/local" + fi + + # Add PostgreSQL bin to PATH + echo "${BREW_PREFIX}/opt/postgresql@${{ matrix.pg-version }}/bin" >> $GITHUB_PATH + + # ============================================================ + # Windows: Install PostgreSQL from self-hosted binaries (fast) + # Binaries hosted on GitHub releases to avoid slow EDB downloads + # ============================================================ + - name: Cache PostgreSQL (Windows) + if: runner.os == 'Windows' + uses: actions/cache@v4 + id: pg-cache + with: + path: C:\pgsql + key: postgresql-${{ matrix.pg-version }}-windows-x64-v1 + + - name: Install PostgreSQL (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + $pgMajor = "${{ matrix.pg-version }}" + $pgPath = "C:\pgsql" + + # Check cache first + if (Test-Path "$pgPath\bin\pg_config.exe") { + Write-Host "PostgreSQL $pgMajor found in cache" + } else { + # Download from self-hosted GitHub release + $zipUrl = "https://github.com/willibrandon/pglogical/releases/download/pg-binaries/postgresql-$pgMajor-windows-x64.zip" + $zipPath = "$env:TEMP\postgresql.zip" + + Write-Host "Downloading PostgreSQL $pgMajor from $zipUrl" + Invoke-WebRequest -Uri $zipUrl -OutFile $zipPath -UseBasicParsing + + Write-Host "Extracting PostgreSQL $pgMajor..." + Expand-Archive -Path $zipPath -DestinationPath "C:\" -Force + Remove-Item $zipPath -Force + } + + if (-not (Test-Path "$pgPath\bin\pg_config.exe")) { + Write-Error "PostgreSQL $pgMajor not found at $pgPath" + exit 1 + } + + # Add to PATH + echo "$pgPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + Write-Host "PostgreSQL $pgMajor ready at $pgPath" + + # ============================================================ + # Build: Linux and macOS use make + # ============================================================ + - name: Build extension (Linux/macOS) + if: runner.os != 'Windows' + run: | + make clean all + + # ============================================================ + # Build: Windows uses CMake with Visual Studio 2022 + # ============================================================ + - name: Build extension (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + # Get pg_config path from self-hosted binaries + $pgConfig = "C:\pgsql\bin\pg_config.exe" + + # Create build directory + New-Item -ItemType Directory -Force -Path build + Set-Location build + + # Configure with CMake + cmake -G "Visual Studio 17 2022" ` + -DPG_CONFIG="$pgConfig" ` + .. + + # Build + cmake --build . --config Release + + # ============================================================ + # Test: Run regression tests (Linux) + # ============================================================ + - name: Run regression tests (Linux) + if: runner.os == 'Linux' + run: | + # PGXS has a double-install issue: MODULE_big and MODULES both trigger + # install-lib. Work around by staging to temp dir then copying. + export DESTDIR=/tmp/pglogical-stage + make install DESTDIR=/tmp/pglogical-stage + sudo cp -rf /tmp/pglogical-stage/* / + + # Run regression tests + make check + + # ============================================================ + # Test: Run regression tests (macOS) + # ============================================================ + - name: Run regression tests (macOS) + if: runner.os == 'macOS' + run: | + # On macOS with Homebrew, the extension directories are writable + # and DESTDIR staging has issues with symlinks, so install directly + sudo make install + + # Run regression tests + make check + + - name: Run regression tests (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + # Install extension first + Set-Location build + cmake --build . --config Release --target install + + # Run regression tests using CMake target + cmake --build . --config Release --target check + + # ============================================================ + # Package: Linux tar.gz + # ============================================================ + - name: Package artifact (Linux) + if: runner.os == 'Linux' + run: | + VERSION=${{ steps.version.outputs.version }} + PG_VERSION=${{ matrix.pg-version }} + PACKAGE_NAME="pglogical-${VERSION}-pg${PG_VERSION}-linux-x64" + PACKAGE_DIR="${PACKAGE_NAME}" + + # Create package directory structure + mkdir -p "${PACKAGE_DIR}/lib" + mkdir -p "${PACKAGE_DIR}/share/extension" + + # Copy shared libraries + cp pglogical.so "${PACKAGE_DIR}/lib/" + cp pglogical_output.so "${PACKAGE_DIR}/lib/" + + # Copy extension files + cp pglogical.control "${PACKAGE_DIR}/share/extension/" + cp pglogical--*.sql "${PACKAGE_DIR}/share/extension/" + cp pglogical_origin.control "${PACKAGE_DIR}/share/extension/" + cp pglogical_origin--*.sql "${PACKAGE_DIR}/share/extension/" + + # Copy install script + cp packaging/unix/install.sh "${PACKAGE_DIR}/" + chmod +x "${PACKAGE_DIR}/install.sh" + + # Create tar.gz + tar -czvf "${PACKAGE_NAME}.tar.gz" "${PACKAGE_DIR}" + + echo "Created: ${PACKAGE_NAME}.tar.gz" + + # ============================================================ + # Package: macOS tar.gz + # ============================================================ + - name: Package artifact (macOS) + if: runner.os == 'macOS' + run: | + VERSION=${{ steps.version.outputs.version }} + PG_VERSION=${{ matrix.pg-version }} + + # Determine architecture + if [[ "$(uname -m)" == "arm64" ]]; then + ARCH="arm64" + else + ARCH="x64" + fi + + PACKAGE_NAME="pglogical-${VERSION}-pg${PG_VERSION}-macos-${ARCH}" + PACKAGE_DIR="${PACKAGE_NAME}" + + # Create package directory structure + mkdir -p "${PACKAGE_DIR}/lib" + mkdir -p "${PACKAGE_DIR}/share/extension" + + # Copy shared libraries (Homebrew PostgreSQL builds use .so extension) + # Try .dylib first (some PG versions), fall back to .so + if [ -f pglogical.dylib ]; then + cp pglogical.dylib "${PACKAGE_DIR}/lib/" + cp pglogical_output.dylib "${PACKAGE_DIR}/lib/" + else + cp pglogical.so "${PACKAGE_DIR}/lib/" + cp pglogical_output.so "${PACKAGE_DIR}/lib/" + fi + + # Copy extension files + cp pglogical.control "${PACKAGE_DIR}/share/extension/" + cp pglogical--*.sql "${PACKAGE_DIR}/share/extension/" + cp pglogical_origin.control "${PACKAGE_DIR}/share/extension/" + cp pglogical_origin--*.sql "${PACKAGE_DIR}/share/extension/" + + # Copy install script + cp packaging/unix/install.sh "${PACKAGE_DIR}/" + chmod +x "${PACKAGE_DIR}/install.sh" + + # Create tar.gz + tar -czvf "${PACKAGE_NAME}.tar.gz" "${PACKAGE_DIR}" + + echo "Created: ${PACKAGE_NAME}.tar.gz" + + # ============================================================ + # Package: Windows zip + # ============================================================ + - name: Package artifact (Windows ZIP) + if: runner.os == 'Windows' + shell: powershell + run: | + $VERSION = "${{ steps.version.outputs.version }}" + $PG_VERSION = "${{ matrix.pg-version }}" + $PACKAGE_NAME = "pglogical-${VERSION}-pg${PG_VERSION}-windows-x64" + $PACKAGE_DIR = $PACKAGE_NAME + + # Create package directory structure + New-Item -ItemType Directory -Force -Path "${PACKAGE_DIR}\lib" + New-Item -ItemType Directory -Force -Path "${PACKAGE_DIR}\share\extension" + + # Copy DLLs from build output + Copy-Item "build\Release\pglogical.dll" "${PACKAGE_DIR}\lib\" + Copy-Item "build\Release\pglogical_output.dll" "${PACKAGE_DIR}\lib\" + + # Copy extension files + Copy-Item "build\pglogical.control" "${PACKAGE_DIR}\share\extension\" + Copy-Item "pglogical--*.sql" "${PACKAGE_DIR}\share\extension\" + Copy-Item "pglogical_origin.control" "${PACKAGE_DIR}\share\extension\" + Copy-Item "pglogical_origin--*.sql" "${PACKAGE_DIR}\share\extension\" + + # Copy README + Copy-Item "packaging\windows\README.md" "${PACKAGE_DIR}\" + + # Create ZIP + Compress-Archive -Path $PACKAGE_DIR -DestinationPath "${PACKAGE_NAME}.zip" + + Write-Host "Created: ${PACKAGE_NAME}.zip" + + # ============================================================ + # Build: Windows MSI installer + # ============================================================ + - name: Install WiX v5 (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + dotnet tool install --global wix --version 5.0.2 + # Add WiX UI extension for install dialogs (version must match WiX 5.x) + wix extension add WixToolset.UI.wixext/5.0.2 + # Add to PATH for subsequent steps + echo "$env:USERPROFILE\.dotnet\tools" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Build MSI installer (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + $VERSION = "${{ steps.version.outputs.version }}" + $PG_VERSION = "${{ matrix.pg-version }}" + $MSI_NAME = "pglogical-${VERSION}-pg${PG_VERSION}-windows-x64.msi" + + # Extract numeric version for MSI (strip prerelease suffix like -rc14) + $MSI_VERSION = $VERSION -replace '-.*$', '' + Write-Host "VERSION: $VERSION, MSI_VERSION: $MSI_VERSION" + + # Get absolute path to repo root for SQL file harvesting + $RepoRoot = (Get-Location).Path + + # Build MSI with WiX v5 (include UI extension for install dialogs) + wix build ` + -o $MSI_NAME ` + -ext WixToolset.UI.wixext ` + -d VERSION=$VERSION ` + -d MSI_VERSION=$MSI_VERSION ` + -d PG_VERSION=$PG_VERSION ` + -d BuildDir="$RepoRoot\build\Release" ` + -d ControlDir="$RepoRoot\build" ` + -d SqlDir="$RepoRoot" ` + packaging\windows\pglogical.wxs + + Write-Host "Created: $MSI_NAME" + + - name: Verify MSI installer (Windows) + if: runner.os == 'Windows' + shell: powershell + run: | + $VERSION = "${{ steps.version.outputs.version }}" + $PG_VERSION = "${{ matrix.pg-version }}" + $MSI_NAME = "pglogical-${VERSION}-pg${PG_VERSION}-windows-x64.msi" + + # Verify MSI exists and has reasonable size + if (-not (Test-Path $MSI_NAME)) { + Write-Error "MSI file not found: $MSI_NAME" + exit 1 + } + + $msiSize = (Get-Item $MSI_NAME).Length + Write-Host "MSI size: $($msiSize / 1KB) KB" + + if ($msiSize -lt 100KB) { + Write-Error "MSI file suspiciously small: $msiSize bytes" + exit 1 + } + + # Silent install to PostgreSQL directory (already installed from earlier step) + Write-Host "Attempting silent MSI install..." + $process = Start-Process -FilePath "msiexec" -ArgumentList "/i", $MSI_NAME, "/qn", "/norestart", "/l*v", "install.log" -Wait -PassThru + $exitCode = $process.ExitCode + + # Check install log + if (Test-Path "install.log") { + Get-Content "install.log" -Tail 50 + } + + if ($exitCode -ne 0) { + Write-Error "MSI installation failed with exit code: $exitCode" + exit 1 + } + + Write-Host "MSI installation succeeded" + + # Clean up - uninstall + Write-Host "Uninstalling MSI..." + $uninstall = Start-Process -FilePath "msiexec" -ArgumentList "/x", $MSI_NAME, "/qn", "/norestart" -Wait -PassThru + if ($uninstall.ExitCode -ne 0) { + Write-Warning "MSI uninstall returned exit code: $($uninstall.ExitCode)" + } + + Write-Host "MSI verified successfully: $MSI_NAME" + + # ============================================================ + # Upload workflow artifacts + # ============================================================ + - name: Upload artifact (Linux) + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-linux-x64 + path: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-linux-x64.tar.gz + retention-days: 1 + + - name: Upload artifact (macOS ARM64) + if: runner.os == 'macOS' && matrix.os == 'macos-26' + uses: actions/upload-artifact@v4 + with: + name: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-macos-arm64 + path: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-macos-arm64.tar.gz + retention-days: 1 + + - name: Upload artifact (Windows ZIP) + if: runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-windows-x64-zip + path: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-windows-x64.zip + retention-days: 1 + + - name: Upload artifact (Windows MSI) + if: runner.os == 'Windows' + uses: actions/upload-artifact@v4 + with: + name: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-windows-x64-msi + path: pglogical-${{ steps.version.outputs.version }}-pg${{ matrix.pg-version }}-windows-x64.msi + retention-days: 1 + + outputs: + version: ${{ steps.version.outputs.version }} + + # ============================================================ + # Create source archives + # ============================================================ + source: + name: Create Source Archives + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Extract version from tag + id: version + run: | + VERSION=${GITHUB_REF_NAME#v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Create source archives + run: | + VERSION=${{ steps.version.outputs.version }} + + # Create source directory with submodule contents + SOURCE_DIR="pglogical-${VERSION}-source" + mkdir -p "${SOURCE_DIR}" + + # Copy all source files (excluding .git directories) + rsync -av --exclude='.git' --exclude='build' --exclude='specs' . "${SOURCE_DIR}/" + + # Create tar.gz + tar -czvf "${SOURCE_DIR}.tar.gz" "${SOURCE_DIR}" + echo "Created: ${SOURCE_DIR}.tar.gz" + + # Create zip + zip -r "${SOURCE_DIR}.zip" "${SOURCE_DIR}" + echo "Created: ${SOURCE_DIR}.zip" + + - name: Upload source tar.gz + uses: actions/upload-artifact@v4 + with: + name: pglogical-${{ steps.version.outputs.version }}-source-tar + path: pglogical-${{ steps.version.outputs.version }}-source.tar.gz + retention-days: 1 + + - name: Upload source zip + uses: actions/upload-artifact@v4 + with: + name: pglogical-${{ steps.version.outputs.version }}-source-zip + path: pglogical-${{ steps.version.outputs.version }}-source.zip + retention-days: 1 + + outputs: + version: ${{ steps.version.outputs.version }} + + # ============================================================ + # Create GitHub Release + # ============================================================ + release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: [build, source] + + steps: + - name: Extract version from tag + id: version + run: | + VERSION=${GITHUB_REF_NAME#v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Prepare release assets + run: | + mkdir -p release-assets + + # Move all artifacts to release-assets with proper names + for dir in artifacts/*/; do + for file in "$dir"*; do + if [ -f "$file" ]; then + cp "$file" release-assets/ + echo "Added: $(basename "$file")" + fi + done + done + + ls -la release-assets/ + + - name: Generate checksums + run: | + cd release-assets + sha256sum * > checksums.txt + echo "" + echo "=== checksums.txt ===" + cat checksums.txt + + - name: Check if prerelease + id: prerelease + run: | + if [[ "${{ github.ref_name }}" == *"-"* ]]; then + echo "is_prerelease=true" >> $GITHUB_OUTPUT + echo "This is a prerelease" + else + echo "is_prerelease=false" >> $GITHUB_OUTPUT + echo "This is a stable release" + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: pglogical ${{ steps.version.outputs.version }} + prerelease: ${{ steps.prerelease.outputs.is_prerelease }} + generate_release_notes: true + files: release-assets/* + body: | + ## pglogical ${{ steps.version.outputs.version }} + + ### Installation + + #### Linux + ```bash + # Download and extract + tar -xzf pglogical-${{ steps.version.outputs.version }}-pg17-linux-x64.tar.gz + cd pglogical-${{ steps.version.outputs.version }}-pg17-linux-x64 + + # Install (uses pg_config to find PostgreSQL directories) + ./install.sh + ``` + + #### macOS + ```bash + # Download and extract + tar -xzf pglogical-${{ steps.version.outputs.version }}-pg17-macos-arm64.tar.gz + cd pglogical-${{ steps.version.outputs.version }}-pg17-macos-arm64 + + # Install + ./install.sh + ``` + + #### Windows (MSI Installer) + 1. Download the MSI for your PostgreSQL version + 2. Run the installer - it will auto-detect PostgreSQL location + 3. Or use the ZIP package and copy files manually + + #### Windows (ZIP Package) + 1. Extract the ZIP to a temporary location + 2. Copy files from `lib/` to your PostgreSQL `lib` directory + 3. Copy files from `share/extension/` to your PostgreSQL `share/extension` directory + + ### Enable the Extension + ```sql + CREATE EXTENSION pglogical; + ``` + + ### Verify Checksums + Download `checksums.txt` and verify your download: + ```bash + sha256sum -c checksums.txt --ignore-missing + ``` + + ### Supported Platforms + - Linux x64 (Ubuntu, Debian, RHEL, CentOS) + - Windows x64 (Windows 10, Windows 11, Windows Server 2019+) + - macOS ARM64 (Apple Silicon) + + ### PostgreSQL Versions + - PostgreSQL 15, 16, 17, 18 diff --git a/.gitignore b/.gitignore index 4400c666..eb809355 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ pglogical_create_subscriber *.o *.so *.dylib +*.bc +compile_commands.json +compile_flags.txt +.clangd +.cache/ *.gcda *.gcno *~ @@ -43,4 +48,13 @@ regression.out # Log files *.log + +# VSCode settings .vscode/c_cpp_properties.json +.vscode/settings.json + +# PostgreSQL binaries +/pg-binaries/ + +# Test results +/test-results/ diff --git a/CLAUDE.md b/CLAUDE.md index bb5718bd..cd5d2482 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,3 +117,41 @@ Key GUC variables (set in postgresql.conf or via ALTER SYSTEM): - `pglogical.conflict_resolution` - How to resolve conflicts (apply_remote, keep_local, etc.) - `pglogical.use_spi` - Use SPI instead of direct heap access for apply - `pglogical.batch_inserts` - Enable batch insert optimization + +## Release Process + +Releases are automated via GitHub Actions. Pushing a version tag triggers the release workflow. + +```bash +# Create a release (triggers build for all platforms) +git tag v2.5.0 +git push origin v2.5.0 + +# Create a prerelease (tags with hyphen are marked as prerelease) +git tag v2.5.0-rc1 +git push origin v2.5.0-rc1 +``` + +**What happens:** +1. Builds for PostgreSQL 15-18 on Linux, macOS (ARM64), and Windows +2. Creates binary packages (tar.gz for Unix, zip and MSI for Windows) +3. Creates source archives with submodules included +4. Generates SHA256 checksums +5. Publishes GitHub Release with all artifacts + +**Key files:** +- `.github/workflows/ci.yml` - PR validation (runs on all PRs and pushes to REL2_x_STABLE) +- `.github/workflows/release.yml` - Release automation (runs on v* tags) +- `packaging/unix/install.sh` - Linux/macOS installation helper +- `packaging/windows/pglogical.wxs` - Windows MSI installer definition + +## CI/CD Infrastructure + +**Build matrix:** PostgreSQL 15-18 × Linux/macOS/Windows (12 jobs) + +**Platforms:** +- Linux: ubuntu-latest, apt packages +- macOS: macos-14 (ARM64), Homebrew +- Windows: windows-2022, Chocolatey, Visual Studio 2022, WiX v5 for MSI + +**Artifacts:** Binary packages follow naming convention `pglogical-{version}-pg{pg_version}-{platform}-{arch}.{ext}` diff --git a/CMakeLists.txt b/CMakeLists.txt index ecf7657e..e9f38d26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,11 @@ if(WIN32) ) # Windows needs to link against postgres.lib - find_library(POSTGRES_LIB postgres HINTS ${PG_LIBDIR}) + # Use NO_DEFAULT_PATH to ensure we only find it in PG_LIBDIR + find_library(POSTGRES_LIB postgres + HINTS ${PG_LIBDIR} + NO_DEFAULT_PATH + ) if(NOT POSTGRES_LIB) message(FATAL_ERROR "postgres.lib not found in ${PG_LIBDIR}") endif() @@ -101,6 +105,7 @@ endif() find_library(LIBPQ_LIBRARY NAMES pq libpq HINTS ${PG_LIBDIR} + NO_DEFAULT_PATH ) if(NOT LIBPQ_LIBRARY) message(FATAL_ERROR "libpq not found in ${PG_LIBDIR}") @@ -164,6 +169,8 @@ if(WIN32) if(LIBINTL_LIBRARY) target_link_libraries(pglogical PRIVATE ${LIBINTL_LIBRARY}) endif() + # Use .def file to export PostgreSQL extension functions + target_sources(pglogical PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/pglogical.def) set_target_properties(pglogical PROPERTIES PREFIX "") else() target_link_libraries(pglogical PRIVATE ${LIBPQ_LIBRARY}) @@ -186,6 +193,7 @@ target_include_directories(pglogical_output PRIVATE ${PGLOGICAL_INCLUDES}) if(WIN32) target_link_libraries(pglogical_output PRIVATE ${POSTGRES_LIB}) + target_link_options(pglogical_output PRIVATE "/DEF:${CMAKE_SOURCE_DIR}/pglogical_output.def") set_target_properties(pglogical_output PROPERTIES PREFIX "") target_compile_options(pglogical_output PRIVATE /W3 /Zc:preprocessor /wd4267 /wd4244) else() @@ -289,7 +297,18 @@ install(FILES ${SQL_FILES} ) # Regression tests target -set(PG_REGRESS "${PG_PKGLIBDIR}/pgxs/src/test/regress/pg_regress") +# Use find_program to properly locate pg_regress (handles .exe on Windows) +find_program(PG_REGRESS pg_regress + HINTS + "${PG_PKGLIBDIR}/pgxs/src/test/regress" + "${PG_BINDIR}" + DOC "Path to pg_regress executable" +) +if(NOT PG_REGRESS) + message(WARNING "pg_regress not found - 'check' and 'installcheck' targets will not work") + set(PG_REGRESS "${PG_PKGLIBDIR}/pgxs/src/test/regress/pg_regress") +endif() +message(STATUS "pg_regress: ${PG_REGRESS}") set(REGRESS_TESTS preseed infofuncs init_fail init preseed_check basic extended conflict_secondary_unique conflict_history toasted replication_set diff --git a/Makefile b/Makefile index 880dc197..ec136a12 100644 --- a/Makefile +++ b/Makefile @@ -121,6 +121,15 @@ EXTRA_CLEAN += $(control_path) PGXS = $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +# Filter out GCC-specific flags that Clang doesn't support. +# These come from pg_config --cflags when PostgreSQL was built with GCC-compatible flags. +# Affects: macOS (Homebrew), and potentially other Clang-based builds. +CLANG_INCOMPATIBLE_FLAGS := -fexcess-precision=standard -Wno-cast-function-type-strict + +ifneq ($(findstring clang,$(shell $(CC) --version 2>&1)),) +override CFLAGS := $(filter-out $(CLANG_INCOMPATIBLE_FLAGS),$(CFLAGS)) +override PG_CFLAGS := $(filter-out $(CLANG_INCOMPATIBLE_FLAGS),$(PG_CFLAGS)) +endif ifeq ($(PGVER),94) regresscheck: ; @@ -161,7 +170,7 @@ check: install regresscheck endif pglogical_create_subscriber: pglogical_create_subscriber.o pglogical_fe.o - $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(libpq_pgport) $(filter-out -lreadline, $(LIBS)) -o $@$(X) + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) -L$(pkglibdir) $(libpq_pgport) $(filter-out -lreadline, $(LIBS)) -o $@$(X) pglogical.control: pglogical.control.in pglogical.h diff --git a/compat13/pglogical_compat.h b/compat13/pglogical_compat.h index bbb50ca7..4b7aa7aa 100644 --- a/compat13/pglogical_compat.h +++ b/compat13/pglogical_compat.h @@ -6,6 +6,7 @@ #include "access/table.h" #include "access/tableam.h" #include "replication/origin.h" +#include "replication/reorderbuffer.h" #include "utils/varlena.h" #define WaitLatchOrSocket(latch, wakeEvents, sock, timeout) \ diff --git a/compat14/pglogical_compat.h b/compat14/pglogical_compat.h index 990eb757..9f0331fa 100644 --- a/compat14/pglogical_compat.h +++ b/compat14/pglogical_compat.h @@ -5,6 +5,7 @@ #include "access/heapam.h" #include "access/table.h" #include "access/tableam.h" +#include "replication/reorderbuffer.h" #include "utils/varlena.h" #define WaitLatchOrSocket(latch, wakeEvents, sock, timeout) \ diff --git a/compat15/pglogical_compat.h b/compat15/pglogical_compat.h index 3591d941..b3665821 100644 --- a/compat15/pglogical_compat.h +++ b/compat15/pglogical_compat.h @@ -5,6 +5,7 @@ #include "access/heapam.h" #include "access/table.h" #include "access/tableam.h" +#include "replication/reorderbuffer.h" #include "utils/varlena.h" #define WaitLatchOrSocket(latch, wakeEvents, sock, timeout) \ diff --git a/compat16/pglogical_compat.h b/compat16/pglogical_compat.h index a607519e..57717d42 100644 --- a/compat16/pglogical_compat.h +++ b/compat16/pglogical_compat.h @@ -5,6 +5,7 @@ #include "access/heapam.h" #include "access/table.h" #include "access/tableam.h" +#include "replication/reorderbuffer.h" #include "utils/varlena.h" #define WaitLatchOrSocket(latch, wakeEvents, sock, timeout) \ diff --git a/compat17/pglogical_compat.h b/compat17/pglogical_compat.h index d9f4f888..12995673 100644 --- a/compat17/pglogical_compat.h +++ b/compat17/pglogical_compat.h @@ -5,6 +5,7 @@ #include "access/heapam.h" #include "access/table.h" #include "access/tableam.h" +#include "replication/reorderbuffer.h" #include "utils/varlena.h" #define WaitLatchOrSocket(latch, wakeEvents, sock, timeout) \ diff --git a/compat18/pglogical_compat.h b/compat18/pglogical_compat.h index 55d36be0..267635f6 100644 --- a/compat18/pglogical_compat.h +++ b/compat18/pglogical_compat.h @@ -5,6 +5,7 @@ #include "access/heapam.h" #include "access/table.h" #include "access/tableam.h" +#include "replication/reorderbuffer.h" #include "utils/varlena.h" #define WaitLatchOrSocket(latch, wakeEvents, sock, timeout) \ diff --git a/expected/att_list.out b/expected/att_list.out index c96ce10a..cc3bd9f7 100644 --- a/expected/att_list.out +++ b/expected/att_list.out @@ -275,6 +275,12 @@ SELECT nspname, relname, att_list, has_row_filter FROM pglogical.show_repset_tab public | basic_dml | {something} | f (1 row) +SELECT pglogical.wait_slot_confirm_lsn(NULL, NULL); + wait_slot_confirm_lsn +----------------------- + +(1 row) + \c :subscriber_dsn -- verify that columns are not automatically added for filtering unless told so. SELECT * FROM pglogical.show_subscription_table('test_subscription', 'basic_dml'); diff --git a/packaging/README-CI.md b/packaging/README-CI.md new file mode 100644 index 00000000..0ec91f7d --- /dev/null +++ b/packaging/README-CI.md @@ -0,0 +1,120 @@ +# CI/CD Documentation for pglogical + +This document describes the GitHub Actions CI/CD infrastructure for building and releasing pglogical. + +## Workflows + +### CI Workflow (`.github/workflows/ci.yml`) + +Runs on: +- Push to `REL2_x_STABLE` branch +- Push to `windows-build` branch +- Pull requests targeting `REL2_x_STABLE` + +**Build Matrix**: +| Platform | PostgreSQL Versions | Architecture | +|----------|---------------------|--------------| +| ubuntu-latest | 13, 14, 15, 16, 17, 18 | x64 | +| windows-2022 | 13, 14, 15, 16, 17, 18 | x64 | +| macos-13 | 13, 14, 15, 16, 17, 18 | x64 (Intel) | +| macos-14 | 14, 15, 16, 17, 18 | arm64 (Apple Silicon) | + +Note: macOS ARM64 (macos-14) excludes PostgreSQL 13 as Homebrew may not provide it for that architecture. + +**Build Steps**: +1. Checkout repository with submodules +2. Install PostgreSQL development files (platform-specific) +3. Build extension (make on Linux/macOS, CMake on Windows) +4. Run regression tests on all platforms +5. Upload test artifacts on failure + +### Release Workflow (`.github/workflows/release.yml`) + +Runs on: +- Push of tags matching `v*` pattern (e.g., `v2.5.0`, `v2.5.0-beta1`) + +**Outputs**: +- Binary packages for all platforms +- Source archives (tar.gz and zip) +- SHA256 checksums file +- GitHub Release with attached assets + +## Branch Protection + +To enforce CI validation before merging, configure branch protection for `REL2_x_STABLE`: + +1. Go to Repository Settings → Branches +2. Add a branch protection rule for `REL2_x_STABLE` +3. Enable: + - "Require status checks to pass before merging" + - Select all CI jobs as required checks + - "Require branches to be up to date before merging" + +## PostgreSQL Installation + +### Linux (APT) +Uses the official PostgreSQL APT repository: +```bash +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +sudo apt-get update +sudo apt-get install -y postgresql-17 postgresql-server-dev-17 +``` + +### macOS (Homebrew) +```bash +brew install postgresql@17 +# Intel path +export PATH="/usr/local/opt/postgresql@17/bin:$PATH" +# ARM64 path +export PATH="/opt/homebrew/opt/postgresql@17/bin:$PATH" +``` + +### Windows (Chocolatey) +```powershell +choco install postgresql17 -y --params '/Password:postgres' +$env:PATH = "C:\Program Files\PostgreSQL\17\bin;$env:PATH" +``` + +## Build Commands + +### Linux/macOS +```bash +make clean all +sudo make install +make check # Run regression tests +``` + +### Windows +```powershell +mkdir build && cd build +cmake -G "Visual Studio 17 2022" -DPG_CONFIG="C:\Program Files\PostgreSQL\17\bin\pg_config.exe" .. +cmake --build . --config Release +``` + +## Troubleshooting + +### PostgreSQL Version Not Available +If a PostgreSQL version is not available in the package manager: +- The matrix uses `fail-fast: false`, so other jobs continue +- Check the job logs for specific error messages +- PostgreSQL 18 may have delayed availability after release + +### Submodule Clone Failures +All submodules use HTTPS URLs (not SSH) to avoid authentication issues in CI. +If submodule checkout fails: +1. Verify the submodule URL in `.gitmodules` +2. Check if the submodule repository is public + +### Windows Build Failures +Windows builds use CMake with Visual Studio 2022: +- Ensure `pg_config.exe` is in PATH +- Check that `postgres.lib` exists in the PostgreSQL lib directory +- Review CMakeLists.txt for platform-specific settings + +### Test Failures +On failure, test artifacts are uploaded: +- `regression_output/` - Contains diff files showing expected vs actual output +- `log/` - PostgreSQL server logs + +Access these in the Actions tab under the failed job's artifacts. diff --git a/packaging/unix/README.md b/packaging/unix/README.md new file mode 100644 index 00000000..dbbbdaab --- /dev/null +++ b/packaging/unix/README.md @@ -0,0 +1,119 @@ +# Unix Installation Package + +This directory contains the installation helper script for Linux and macOS. + +## Contents + +- `install.sh` - Installation script that copies pglogical files to PostgreSQL directories + +## Usage + +### Basic Installation + +```bash +# Extract the package +tar -xzf pglogical-2.5.0-pg17-linux-x64.tar.gz +cd pglogical-2.5.0-pg17-linux-x64 + +# Run the install script (uses pg_config from PATH) +./install.sh +``` + +### Custom pg_config Location + +If pg_config is not in your PATH, specify it via environment variable: + +```bash +PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config ./install.sh +``` + +### Custom PostgreSQL Directory + +Alternatively, specify the PostgreSQL installation directory directly: + +```bash +PGDIR=/usr/lib/postgresql/17 ./install.sh +``` + +This is useful when pg_config is not available or you want to install to a specific PostgreSQL installation. + +### What the Script Does + +1. Determines PostgreSQL directories using one of: + - `PGDIR` environment variable (direct path to PostgreSQL installation) + - `PG_CONFIG` environment variable (path to pg_config executable) + - `pg_config` from PATH +2. Copies shared libraries to `lib/` directory +3. Copies extension files to `share/extension/` directory +4. Uses sudo automatically if target directories require elevated permissions + +## Package Contents + +A typical package contains: + +``` +pglogical-2.5.0-pg17-linux-x64/ +├── install.sh # This installation script +├── lib/ +│ ├── pglogical.so # Main extension library +│ └── pglogical_output.so # Output plugin library +└── share/ + └── extension/ + ├── pglogical.control # Extension control file + ├── pglogical--2.5.0.sql # Extension SQL + ├── pglogical--*.sql # Upgrade scripts + ├── pglogical_origin.control + └── pglogical_origin--1.0.0.sql +``` + +## Post-Installation + +After installation, enable the extension in your database: + +```sql +CREATE EXTENSION pglogical; +``` + +## Requirements + +- PostgreSQL development installation (pg_config must be available) +- Bash 3.2 or later (default on macOS and most Linux distributions) +- Standard coreutils (install, cp, mkdir) + +## Troubleshooting + +### pg_config not found + +Either add PostgreSQL bin directory to your PATH, or use PGDIR: + +```bash +# Option 1: Add to PATH +# Linux (APT installation) +export PATH=/usr/lib/postgresql/17/bin:$PATH + +# macOS (Homebrew on Intel) +export PATH=/usr/local/opt/postgresql@17/bin:$PATH + +# macOS (Homebrew on Apple Silicon) +export PATH=/opt/homebrew/opt/postgresql@17/bin:$PATH + +# Option 2: Use PGDIR directly +PGDIR=/usr/lib/postgresql/17 ./install.sh +``` + +### Permission denied + +The script automatically uses sudo when needed. If it still fails: + +```bash +sudo ./install.sh +``` + +### Extension not found after installation + +Verify files were copied correctly: + +```bash +ls $(pg_config --pkglibdir)/pglogical* +ls $(pg_config --sharedir)/extension/pglogical* +``` diff --git a/packaging/unix/install.sh b/packaging/unix/install.sh new file mode 100755 index 00000000..c1ad510f --- /dev/null +++ b/packaging/unix/install.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# +# pglogical Installation Script for Linux and macOS +# +# Usage: +# ./install.sh # Use pg_config from PATH +# PG_CONFIG=/path/to/pg_config ./install.sh # Use specific pg_config +# PGDIR=/path/to/postgresql ./install.sh # Use specific PostgreSQL directory +# +# Environment Variables: +# PG_CONFIG - Path to pg_config executable +# PGDIR - PostgreSQL installation directory (alternative to pg_config) +# +# This script installs pglogical extension files to the PostgreSQL directories +# determined by pg_config or PGDIR. It will use sudo if the target directories +# require elevated permissions. + +set -e + +# Colors for output (if terminal supports it) +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + NC='\033[0m' # No Color +else + RED='' + GREEN='' + YELLOW='' + NC='' +fi + +# Print colored message +info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Determine script directory (where package files are located) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Determine PostgreSQL directories +if [ -n "$PGDIR" ]; then + # User specified PostgreSQL directory directly + if [ ! -d "$PGDIR" ]; then + error "PGDIR directory does not exist: $PGDIR" + exit 1 + fi + info "Using PostgreSQL directory from PGDIR: $PGDIR" + + # Determine paths based on PGDIR + PKGLIBDIR="${PGDIR}/lib" + SHAREDIR="${PGDIR}/share" + EXTENSIONDIR="${SHAREDIR}/extension" + + # Try to get version from pg_config if available in PGDIR + if [ -x "${PGDIR}/bin/pg_config" ]; then + PG_VERSION=$("${PGDIR}/bin/pg_config" --version 2>/dev/null || echo "Unknown") + else + PG_VERSION="Unknown (PGDIR mode)" + fi + +elif [ -n "$PG_CONFIG" ]; then + # User specified pg_config path + if [ ! -x "$PG_CONFIG" ]; then + error "PG_CONFIG is set but not executable: $PG_CONFIG" + exit 1 + fi + info "Using pg_config from PG_CONFIG: $PG_CONFIG" + + PKGLIBDIR=$("$PG_CONFIG" --pkglibdir) + SHAREDIR=$("$PG_CONFIG" --sharedir) + EXTENSIONDIR="${SHAREDIR}/extension" + PG_VERSION=$("$PG_CONFIG" --version) + +elif command -v pg_config >/dev/null 2>&1; then + # Use pg_config from PATH + PG_CONFIG="pg_config" + info "Using pg_config from PATH: $(which pg_config)" + + PKGLIBDIR=$("$PG_CONFIG" --pkglibdir) + SHAREDIR=$("$PG_CONFIG" --sharedir) + EXTENSIONDIR="${SHAREDIR}/extension" + PG_VERSION=$("$PG_CONFIG" --version) + +else + error "Cannot determine PostgreSQL location" + echo "" + echo "Please use one of the following options:" + echo "" + echo " 1. Add PostgreSQL bin directory to PATH:" + echo " export PATH=/usr/lib/postgresql/17/bin:\$PATH" + echo " ./install.sh" + echo "" + echo " 2. Set PG_CONFIG environment variable:" + echo " PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config ./install.sh" + echo "" + echo " 3. Set PGDIR to PostgreSQL installation directory:" + echo " PGDIR=/usr/lib/postgresql/17 ./install.sh" + echo "" + exit 1 +fi + +info "PostgreSQL version: $PG_VERSION" +info "Library directory: $PKGLIBDIR" +info "Extension directory: $EXTENSIONDIR" + +# Check if directories exist +if [ ! -d "$PKGLIBDIR" ]; then + error "Library directory does not exist: $PKGLIBDIR" + exit 1 +fi + +if [ ! -d "$EXTENSIONDIR" ]; then + warn "Extension directory does not exist: $EXTENSIONDIR" + echo "Creating extension directory..." + if [ -w "$(dirname "$EXTENSIONDIR")" ]; then + mkdir -p "$EXTENSIONDIR" + else + sudo mkdir -p "$EXTENSIONDIR" + fi +fi + +# Determine if we need sudo +NEED_SUDO=false +if [ ! -w "$PKGLIBDIR" ] || [ ! -w "$EXTENSIONDIR" ]; then + NEED_SUDO=true + warn "Target directories require elevated permissions, using sudo" +fi + +# Copy function that uses sudo if needed +copy_file() { + local src="$1" + local dst="$2" + local mode="${3:-644}" + + if [ ! -f "$src" ]; then + error "Source file not found: $src" + return 1 + fi + + if $NEED_SUDO; then + sudo install -m "$mode" "$src" "$dst" + else + install -m "$mode" "$src" "$dst" + fi +} + +# Install shared libraries +info "Installing shared libraries..." +for lib in "$SCRIPT_DIR"/lib/*.so "$SCRIPT_DIR"/lib/*.dylib; do + if [ -f "$lib" ]; then + filename=$(basename "$lib") + copy_file "$lib" "$PKGLIBDIR/$filename" 755 + echo " Installed: $filename" + fi +done + +# Install extension files +info "Installing extension files..." +for ext in "$SCRIPT_DIR"/share/extension/*; do + if [ -f "$ext" ]; then + filename=$(basename "$ext") + copy_file "$ext" "$EXTENSIONDIR/$filename" 644 + echo " Installed: $filename" + fi +done + +echo "" +info "Installation complete!" +echo "" +echo "To enable pglogical in your database, run:" +echo " CREATE EXTENSION pglogical;" +echo "" +echo "For more information, see: https://github.com/2ndQuadrant/pglogical" diff --git a/packaging/windows/README.md b/packaging/windows/README.md new file mode 100644 index 00000000..5dd0e185 --- /dev/null +++ b/packaging/windows/README.md @@ -0,0 +1,105 @@ +# pglogical - Logical Replication Extension for PostgreSQL + +## Installation Instructions for Windows + +### Option 1: MSI Installer (Recommended) + +1. Download the MSI installer for your PostgreSQL version + - Example: `pglogical-2.5.0-pg17-windows-x64.msi` + +2. Run the installer + - It will automatically detect your PostgreSQL installation directory + - If not found, you can browse to select the correct directory + +3. The installer copies files to: + - `lib\pglogical.dll` + - `lib\pglogical_output.dll` + - `share\extension\pglogical.control` + - `share\extension\pglogical--*.sql` + +4. To uninstall, use Windows Add/Remove Programs + +### Option 2: Manual Installation (ZIP Package) + +1. Download the ZIP package for your PostgreSQL version + - Example: `pglogical-2.5.0-pg17-windows-x64.zip` + +2. Extract the ZIP to a temporary location + +3. Copy files to your PostgreSQL installation directory: + - Copy `lib\*.dll` to: `C:\Program Files\PostgreSQL\17\lib\` + - Copy `share\extension\*` to: `C:\Program Files\PostgreSQL\17\share\extension\` + + **PowerShell example:** + ```powershell + $PG_DIR = "C:\Program Files\PostgreSQL\17" + Copy-Item "lib\*.dll" "$PG_DIR\lib\" + Copy-Item "share\extension\*" "$PG_DIR\share\extension\" + ``` + +## Enabling the Extension + +After installation, enable pglogical in your database: + +```sql +CREATE EXTENSION pglogical; +``` + +To verify the installation: + +```sql +SELECT pglogical.pglogical_version(); +``` + +## Configuration + +pglogical requires these PostgreSQL settings in `postgresql.conf`: + +1. Add pglogical to shared_preload_libraries: + ``` + shared_preload_libraries = 'pglogical' + ``` + +2. Set wal_level to 'logical': + ``` + wal_level = 'logical' + ``` + +3. Increase max_worker_processes (at least 2 per database): + ``` + max_worker_processes = 10 + ``` + +4. Increase max_replication_slots: + ``` + max_replication_slots = 10 + ``` + +5. Increase max_wal_senders: + ``` + max_wal_senders = 10 + ``` + +After changing `postgresql.conf`, restart the PostgreSQL service. + +## Troubleshooting + +### "could not load library pglogical" + +- Verify `pglogical.dll` is in the PostgreSQL lib directory +- Check Windows Event Viewer for detailed error messages + +### "extension pglogical does not exist" + +- Verify `.control` and `.sql` files are in `share\extension` directory +- Check file permissions + +### PostgreSQL service won't start + +- Check `postgresql.conf` syntax +- Review PostgreSQL log files in the data directory + +## Documentation + +- Full documentation: https://github.com/2ndQuadrant/pglogical +- Report issues: https://github.com/2ndQuadrant/pglogical/issues diff --git a/packaging/windows/pglogical.wxs b/packaging/windows/pglogical.wxs new file mode 100644 index 00000000..17a1e050 --- /dev/null +++ b/packaging/windows/pglogical.wxs @@ -0,0 +1,223 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pglogical.c b/pglogical.c index 21832ad5..06fc8b80 100644 --- a/pglogical.c +++ b/pglogical.c @@ -728,9 +728,20 @@ pglogical_supervisor_main(Datum main_arg) CommitTransactionCommand(); } + /* + * Wait for latch signal or timeout. On Windows, use a shorter timeout + * to ensure faster response to subscription changes, as process startup + * and IPC can be slower than on Unix systems. + */ +#ifdef WIN32 + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 30000L); +#else rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 180000L); +#endif ResetLatch(&MyProc->procLatch); diff --git a/pglogical.def b/pglogical.def new file mode 100644 index 00000000..7f451ff3 --- /dev/null +++ b/pglogical.def @@ -0,0 +1,64 @@ +; pglogical Windows DLL exports +; Functions exported for PostgreSQL extension loading +EXPORTS + ; Module initialization (required for shared_preload_libraries) + _PG_init + _PG_output_plugin_init + + ; Module magic + Pg_magic_func + + ; Node management + pglogical_create_node + pglogical_drop_node + pglogical_alter_node_add_interface + pglogical_alter_node_drop_interface + pglogical_node_info + + ; Subscription management + pglogical_create_subscription + pglogical_drop_subscription + pglogical_alter_subscription_interface + pglogical_alter_subscription_disable + pglogical_alter_subscription_enable + pglogical_alter_subscription_add_replication_set + pglogical_alter_subscription_remove_replication_set + pglogical_alter_subscription_resynchronize_table + pglogical_alter_subscription_synchronize + pglogical_show_subscription_status + pglogical_show_subscription_table + pglogical_wait_for_subscription_sync_complete + pglogical_wait_for_table_sync_complete + + ; Replication set management + pglogical_create_replication_set + pglogical_drop_replication_set + pglogical_alter_replication_set + pglogical_replication_set_add_table + pglogical_replication_set_add_all_tables + pglogical_replication_set_remove_table + pglogical_replication_set_add_sequence + pglogical_replication_set_add_all_sequences + pglogical_replication_set_remove_sequence + pglogical_show_repset_table_info + pglogical_show_repset_table_info_by_target + + ; DDL replication + pglogical_replicate_ddl_command + pglogical_queue_truncate + + ; Utility functions + pglogical_version + pglogical_version_num + pglogical_max_proto_version + pglogical_min_proto_version + pglogical_gen_slot_name + pglogical_wait_slot_confirm_lsn + pglogical_synchronize_sequence + pglogical_table_data_filtered + pglogical_xact_commit_timestamp_origin + + ; Triggers and hooks + pglogical_truncate_trigger_add + pglogical_dependency_check_trigger + pglogical_hooks_setup diff --git a/pglogical_apply.c b/pglogical_apply.c index f7c660bd..af4399a6 100644 --- a/pglogical_apply.c +++ b/pglogical_apply.c @@ -1,2044 +1,2062 @@ -/*------------------------------------------------------------------------- - * - * pglogical_apply.c - * pglogical apply logic - * - * Copyright (c) 2015, PostgreSQL Global Development Group - * - * IDENTIFICATION - * pglogical.c - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" - -#include "miscadmin.h" -#include "libpq-fe.h" -#include "pgstat.h" - -#include "access/htup_details.h" -#include "access/xact.h" - -#include "catalog/namespace.h" - -#include "commands/async.h" -#include "commands/dbcommands.h" -#include "commands/sequence.h" -#include "commands/tablecmds.h" -#include "commands/trigger.h" - -#include "executor/executor.h" - -#include "libpq/pqformat.h" - -#include "mb/pg_wchar.h" - -#include "nodes/makefuncs.h" -#include "nodes/parsenodes.h" - -#include "optimizer/planner.h" - -#ifdef XCP -#include "pgxc/pgxcnode.h" -#endif - -#include "replication/origin.h" -#include "replication/reorderbuffer.h" - -#include "rewrite/rewriteHandler.h" - -#include "storage/ipc.h" -#include "storage/lmgr.h" -#include "storage/proc.h" - -#include "tcop/pquery.h" -#include "tcop/utility.h" - -#include "utils/builtins.h" -#include "utils/jsonb.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/snapmgr.h" - -#include "pglogical_conflict.h" -#include "pglogical_executor.h" -#include "pglogical_node.h" -#include "pglogical_queue.h" -#include "pglogical_relcache.h" -#include "pglogical_repset.h" -#include "pglogical_rpc.h" -#include "pglogical_sync.h" -#include "pglogical_worker.h" -#include "pglogical_apply.h" -#include "pglogical_apply_heap.h" -#include "pglogical_apply_spi.h" -#include "pglogical.h" - - -void PGDLLEXPORT pglogical_apply_main(Datum main_arg); - -static bool in_remote_transaction = false; -static XLogRecPtr remote_origin_lsn = InvalidXLogRecPtr; -static RepOriginId remote_origin_id = InvalidRepOriginId; -static TimeOffset apply_delay = 0; - -static Oid QueueRelid = InvalidOid; - -static List *SyncingTables = NIL; - -PGLogicalApplyWorker *MyApplyWorker = NULL; -PGLogicalSubscription *MySubscription = NULL; - -static PGconn *applyconn = NULL; - -typedef struct PGLogicalApplyFunctions -{ - pglogical_apply_begin_fn on_begin; - pglogical_apply_commit_fn on_commit; - pglogical_apply_insert_fn do_insert; - pglogical_apply_update_fn do_update; - pglogical_apply_delete_fn do_delete; - pglogical_apply_can_mi_fn can_multi_insert; - pglogical_apply_mi_add_tuple_fn multi_insert_add_tuple; - pglogical_apply_mi_finish_fn multi_insert_finish; -} PGLogicalApplyFunctions; - -static PGLogicalApplyFunctions apply_api = -{ - .on_begin = pglogical_apply_heap_begin, - .on_commit = pglogical_apply_heap_commit, - .do_insert = pglogical_apply_heap_insert, - .do_update = pglogical_apply_heap_update, - .do_delete = pglogical_apply_heap_delete, - .can_multi_insert = pglogical_apply_heap_can_mi, - .multi_insert_add_tuple = pglogical_apply_heap_mi_add_tuple, - .multi_insert_finish = pglogical_apply_heap_mi_finish -}; - -/* Number of tuples inserted after which we switch to multi-insert. */ -#define MIN_MULTI_INSERT_TUPLES 5 -static PGLogicalRelation *last_insert_rel = NULL; -static int last_insert_rel_cnt = 0; -static bool use_multi_insert = false; - -/* - * A message counter for the xact, for debugging. We don't send - * the remote change LSN with messages, so this aids identification - * of which change causes an error. - */ -static uint32 xact_action_counter; - -typedef struct PGLFlushPosition -{ - dlist_node node; - XLogRecPtr local_end; - XLogRecPtr remote_end; -} PGLFlushPosition; - -static dlist_head lsn_mapping = DLIST_STATIC_INIT(lsn_mapping); - -typedef struct ApplyExecState -{ - EState *estate; - EPQState epqstate; - ResultRelInfo *resultRelInfo; - TupleTableSlot *slot; -} ApplyExecState; - -struct ActionErrCallbackArg -{ - const char * action_name; - PGLogicalRelation *rel; - bool is_ddl_or_drop; -}; - -static struct ActionErrCallbackArg errcallback_arg; -static TransactionId remote_xid; - -static void multi_insert_finish(void); - -static void handle_queued_message(HeapTuple msgtup, bool tx_just_started); -static void handle_startup_param(const char *key, const char *value); -static bool parse_bool_param(const char *key, const char *value); -static void process_syncing_tables(XLogRecPtr end_lsn); -static void start_sync_worker(Name nspname, Name relname); - -/* - * Check if given relation is in process of being synchronized. - * - * TODO: performance - */ -static bool -should_apply_changes_for_rel(const char *nspname, const char *relname) -{ - if (list_length(SyncingTables) > 0) - { - ListCell *lc; - - foreach (lc, SyncingTables) - { - PGLogicalSyncStatus *sync = (PGLogicalSyncStatus *) lfirst(lc); - - if (namestrcmp(&sync->nspname, nspname) == 0 && - namestrcmp(&sync->relname, relname) == 0 && - (sync->status != SYNC_STATUS_READY && - !(sync->status == SYNC_STATUS_SYNCDONE && - sync->statuslsn <= replorigin_session_origin_lsn))) - return false; - } - } - - return true; -} - -/* - * Prepare apply state details for errcontext or direct logging. - * - * This callback could be invoked at all sorts of weird times - * so it should assume as little as psosible about the invoking - * context. - */ -static void -format_action_description( - StringInfo si, - const char * action_name, - PGLogicalRelation *rel, - bool is_ddl_or_drop) -{ - appendStringInfoString(si, "apply "); - appendStringInfoString(si, - action_name == NULL ? "(unknown action)" : action_name); - - if (rel != NULL && - rel->nspname != NULL - && rel->relname != NULL - && !is_ddl_or_drop) - { - appendStringInfo(si, " from remote relation %s.%s", - rel->nspname, rel->relname); - } - - appendStringInfo(si, - " in commit before %X/%X, xid %u committed at %s (action #%u)", - (uint32)(replorigin_session_origin_lsn>>32), - (uint32)replorigin_session_origin_lsn, - remote_xid, - timestamptz_to_str(replorigin_session_origin_timestamp), - xact_action_counter); - - if (replorigin_session_origin != InvalidRepOriginId) - { - appendStringInfo(si, " from node replorigin %u", - replorigin_session_origin); - } - - if (remote_origin_id != InvalidRepOriginId) - { - appendStringInfo(si, " forwarded from commit %X/%X on node %u", - (uint32)(remote_origin_lsn>>32), - (uint32)remote_origin_lsn, - remote_origin_id); - } -} - -static void -action_error_callback(void *arg) -{ - StringInfoData si; - initStringInfo(&si); - - format_action_description(&si, - errcallback_arg.action_name, - errcallback_arg.rel, - errcallback_arg.is_ddl_or_drop); - - errcontext("%s", si.data); - pfree(si.data); -} - -static bool -ensure_transaction(void) -{ - if (IsTransactionState()) - { - if (CurrentMemoryContext != MessageContext) - MemoryContextSwitchTo(MessageContext); - return false; - } - - /* - * pglogical doesn't have "statements" as such, so we'll report one - * statement per applied transaction. We must set the statement start time - * because StartTransaction() uses it to initialize the transaction cached - * timestamp used by current_timestamp. If we don't set it, every xact will - * get the same current_timestamp. See 2ndQuadrant/pglogical_internal#148 - */ - SetCurrentStatementStartTimestamp(); - - StartTransactionCommand(); - apply_api.on_begin(); - MemoryContextSwitchTo(MessageContext); - - return true; -} - -static void -handle_begin(StringInfo s) -{ - XLogRecPtr commit_lsn; - TimestampTz commit_time; - - xact_action_counter = 1; - errcallback_arg.action_name = "BEGIN"; - - pglogical_read_begin(s, &commit_lsn, &commit_time, &remote_xid); - - replorigin_session_origin_timestamp = commit_time; - replorigin_session_origin_lsn = commit_lsn; - remote_origin_id = InvalidRepOriginId; - - VALGRIND_PRINTF("PGLOGICAL_APPLY: begin %u\n", remote_xid); - - /* don't want the overhead otherwise */ - if (apply_delay > 0) - { - TimestampTz current; - current = GetCurrentIntegerTimestamp(); - - /* ensure no weirdness due to clock drift */ - if (current > replorigin_session_origin_timestamp) - { - long sec; - int usec; - - current = TimestampTzPlusMilliseconds(current, - -apply_delay); - - TimestampDifference(current, replorigin_session_origin_timestamp, - &sec, &usec); - /* FIXME: deal with overflow? */ - pg_usleep(usec + (sec * USECS_PER_SEC)); - } - } - - in_remote_transaction = true; - - pgstat_report_activity(STATE_RUNNING, NULL); -} - -/* - * Handle COMMIT message. - */ -static void -handle_commit(StringInfo s) -{ - XLogRecPtr commit_lsn; - XLogRecPtr end_lsn; - TimestampTz commit_time; - - errcallback_arg.action_name = "COMMIT"; - xact_action_counter++; - - pglogical_read_commit(s, &commit_lsn, &end_lsn, &commit_time); - - Assert(commit_time == replorigin_session_origin_timestamp); - - if (IsTransactionState()) - { - PGLFlushPosition *flushpos; - - multi_insert_finish(); - - apply_api.on_commit(); - - /* We need to write end_lsn to the commit record. */ - replorigin_session_origin_lsn = end_lsn; - - CommitTransactionCommand(); - MemoryContextSwitchTo(TopMemoryContext); - - /* Track commit lsn */ - flushpos = (PGLFlushPosition *) palloc(sizeof(PGLFlushPosition)); - flushpos->local_end = XactLastCommitEnd; - flushpos->remote_end = end_lsn; - - dlist_push_tail(&lsn_mapping, &flushpos->node); - MemoryContextSwitchTo(MessageContext); - } - - /* - * If the xact isn't from the immediate upstream, advance the slot of the - * node it originally came from so we start replay of that node's change - * data at the right place. - * - * This is only necessary when we're streaming data from one peer (A) that - * in turn receives from other peers (B, C), and we plan to later switch to - * replaying directly from B and/or C, no longer receiving forwarded xacts - * from A. When we do the switchover we need to know the right place at - * which to start replay from B and C. We don't actually do that yet, but - * we'll want to be able to do cascaded initialisation in future, so it's - * worth keeping track. - * - * A failure can occur here (see #79) if there's a cascading - * replication configuration like: - * - * X--> Y -> Z - * | ^ - * | | - * \---------/ - * - * where the direct and indirect connections from X to Z use different - * replication sets so as not to conflict, and where Y and Z are on the - * same PostgreSQL instance. In this case our attempt to advance the - * replication identifier here will ERROR because it's already in use - * for the direct connection from X to Z. So don't do that. - */ - if (remote_origin_id != InvalidRepOriginId && - remote_origin_id != replorigin_session_origin) - { -#if PG_VERSION_NUM >= 90500 - Relation replorigin_rel; -#endif - elog(DEBUG3, "advancing origin oid %u for forwarded row to %X/%X", - remote_origin_id, - (uint32)(XactLastCommitEnd>>32), (uint32)XactLastCommitEnd); - -#if PG_VERSION_NUM >= 90500 - replorigin_rel = table_open(ReplicationOriginRelationId, RowExclusiveLock); -#endif - replorigin_advance(remote_origin_id, remote_origin_lsn, - XactLastCommitEnd, false, false /* XXX ? */); -#if PG_VERSION_NUM >= 90500 - table_close(replorigin_rel, RowExclusiveLock); -#endif - } - - in_remote_transaction = false; - - /* - * Stop replay if we're doing limited replay and we've replayed up to the - * last record we're supposed to process. - */ - if (MyApplyWorker->replay_stop_lsn != InvalidXLogRecPtr - && MyApplyWorker->replay_stop_lsn <= end_lsn) - { - ereport(LOG, - (errmsg("pglogical %s finished processing; replayed to %X/%X of required %X/%X", - MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC ? "sync" : "apply", - (uint32)(end_lsn>>32), (uint32)end_lsn, - (uint32)(MyApplyWorker->replay_stop_lsn >>32), - (uint32)MyApplyWorker->replay_stop_lsn))); - - /* - * If this is sync worker, update syncing table state to done. - */ - if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC) - { - StartTransactionCommand(); - set_table_sync_status(MyApplyWorker->subid, - NameStr(MyPGLogicalWorker->worker.sync.nspname), - NameStr(MyPGLogicalWorker->worker.sync.relname), - SYNC_STATUS_SYNCDONE, end_lsn); - CommitTransactionCommand(); - } - - /* - * Flush all writes so the latest position can be reported back to the - * sender. - */ - XLogFlush(GetXLogWriteRecPtr()); - - /* - * Disconnect. - * - * This needs to happen before the pglogical_sync_worker_finish() - * call otherwise slot drop will fail. - */ - PQfinish(applyconn); - - /* - * If this is sync worker, finish it. - */ - if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC) - pglogical_sync_worker_finish(); - - /* Stop gracefully */ - proc_exit(0); - } - - VALGRIND_PRINTF("PGLOGICAL_APPLY: commit %u\n", remote_xid); - - xact_action_counter = 0; - remote_xid = InvalidTransactionId; - - process_syncing_tables(end_lsn); - -#if PG_VERSION_NUM < 150000 - /* - * Ensure any pending signals/self-notifies are sent out. - * - * (This is no longer needed in PG15 and also a no-op in later - * minor versions of PG13 and PG14. In those versions, - * notifications are sent at transaction commit.) - * - * Note that there is a possibility that this will result in an ERROR, - * which will result in the apply worker being killed and restarted. As - * the notification queues have already been flushed, the same error won't - * occur again, however if errors continue, they will dramatically slow - * down - but not stop - replication. - */ - ProcessCompletedNotifies(); -#endif - - pgstat_report_activity(STATE_IDLE, NULL); -} - -/* - * Handle ORIGIN message. - */ -static void -handle_origin(StringInfo s) -{ - char *origin; - - /* - * ORIGIN message can only come inside remote transaction and before - * any actual writes. - */ - if (!in_remote_transaction || IsTransactionState()) - elog(ERROR, "ORIGIN message sent out of order"); - - /* We have to start transaction here so that we can work with origins. */ - ensure_transaction(); - - origin = pglogical_read_origin(s, &remote_origin_lsn); - remote_origin_id = replorigin_by_name(origin, true); -} - -/* - * Handle RELATION message. - * - * Note we don't do validation against local schema here. The validation is - * posponed until first change for given relation comes. - */ -static void -handle_relation(StringInfo s) -{ - multi_insert_finish(); - - (void) pglogical_read_rel(s); -} - -static void -handle_insert(StringInfo s) -{ - PGLogicalTupleData newtup; - PGLogicalRelation *rel; - bool started_tx = ensure_transaction(); - - PushActiveSnapshot(GetTransactionSnapshot()); - - errcallback_arg.action_name = "INSERT"; - xact_action_counter++; - - rel = pglogical_read_insert(s, RowExclusiveLock, &newtup); - errcallback_arg.rel = rel; - - /* If in list of relations which are being synchronized, skip. */ - if (!should_apply_changes_for_rel(rel->nspname, rel->relname)) - { - pglogical_relation_close(rel, NoLock); - PopActiveSnapshot(); - CommandCounterIncrement(); - return; - } - - /* Handle multi_insert capabilities. */ - if (use_multi_insert) - { - if (rel != last_insert_rel) - { - multi_insert_finish(); - /* Fall through to normal insert. */ - } - else - { - apply_api.multi_insert_add_tuple(rel, &newtup); - last_insert_rel_cnt++; - return; - } - } - else if (pglogical_batch_inserts && - RelationGetRelid(rel->rel) != QueueRelid && - apply_api.can_multi_insert && - apply_api.can_multi_insert(rel)) - { - if (rel != last_insert_rel) - { - last_insert_rel = rel; - last_insert_rel_cnt = 0; - } - else if (last_insert_rel_cnt++ >= MIN_MULTI_INSERT_TUPLES) - { - use_multi_insert = true; - last_insert_rel_cnt = 0; - } - } - - /* Normal insert. */ - apply_api.do_insert(rel, &newtup); - - /* if INSERT was into our queue, process the message. */ - if (RelationGetRelid(rel->rel) == QueueRelid) - { - HeapTuple ht; - LockRelId lockid = rel->rel->rd_lockInfo.lockRelId; - Relation qrel; - - multi_insert_finish(); - - MemoryContextSwitchTo(MessageContext); - - ht = heap_form_tuple(RelationGetDescr(rel->rel), - newtup.values, newtup.nulls); - - LockRelationIdForSession(&lockid, RowExclusiveLock); - pglogical_relation_close(rel, NoLock); - - PopActiveSnapshot(); - CommandCounterIncrement(); - - apply_api.on_commit(); - - handle_queued_message(ht, started_tx); - - heap_freetuple(ht); - - qrel = table_open(QueueRelid, RowExclusiveLock); - - UnlockRelationIdForSession(&lockid, RowExclusiveLock); - - table_close(qrel, NoLock); - - apply_api.on_begin(); - MemoryContextSwitchTo(MessageContext); - -// if (oldxid != GetTopTransactionId()) -// CommitTransactionCommand(); - } - else - { - pglogical_relation_close(rel, NoLock); - - PopActiveSnapshot(); - CommandCounterIncrement(); - } -} - -static void -multi_insert_finish(void) -{ - if (use_multi_insert && last_insert_rel_cnt) - { - const char *old_action = errcallback_arg.action_name; - PGLogicalRelation *old_rel = errcallback_arg.rel; - errcallback_arg.action_name = "multi INSERT"; - errcallback_arg.rel = last_insert_rel; - - apply_api.multi_insert_finish(last_insert_rel); - pglogical_relation_close(last_insert_rel, NoLock); - use_multi_insert = false; - last_insert_rel = NULL; - last_insert_rel_cnt = 0; - - errcallback_arg.rel = old_rel; - errcallback_arg.action_name = old_action; - } -} - -static void -handle_update(StringInfo s) -{ - PGLogicalTupleData oldtup; - PGLogicalTupleData newtup; - PGLogicalRelation *rel; - bool hasoldtup; - - errcallback_arg.action_name = "UPDATE"; - xact_action_counter++; - - ensure_transaction(); - - multi_insert_finish(); - - PushActiveSnapshot(GetTransactionSnapshot()); - - rel = pglogical_read_update(s, RowExclusiveLock, &hasoldtup, &oldtup, - &newtup); - errcallback_arg.rel = rel; - - /* If in list of relations which are being synchronized, skip. */ - if (!should_apply_changes_for_rel(rel->nspname, rel->relname)) - { - pglogical_relation_close(rel, NoLock); - PopActiveSnapshot(); - CommandCounterIncrement(); - return; - } - - apply_api.do_update(rel, hasoldtup ? &oldtup : &newtup, &newtup); - - pglogical_relation_close(rel, NoLock); - - PopActiveSnapshot(); - CommandCounterIncrement(); -} - -static void -handle_delete(StringInfo s) -{ - PGLogicalTupleData oldtup; - PGLogicalRelation *rel; - - memset(&errcallback_arg, 0, sizeof(struct ActionErrCallbackArg)); - xact_action_counter++; - - ensure_transaction(); - - multi_insert_finish(); - - PushActiveSnapshot(GetTransactionSnapshot()); - - rel = pglogical_read_delete(s, RowExclusiveLock, &oldtup); - errcallback_arg.rel = rel; - - /* If in list of relations which are being synchronized, skip. */ - if (!should_apply_changes_for_rel(rel->nspname, rel->relname)) - { - pglogical_relation_close(rel, NoLock); - PopActiveSnapshot(); - CommandCounterIncrement(); - return; - } - - apply_api.do_delete(rel, &oldtup); - - pglogical_relation_close(rel, NoLock); - - PopActiveSnapshot(); - CommandCounterIncrement(); -} - -inline static bool -getmsgisend(StringInfo msg) -{ - return msg->cursor == msg->len; -} - -static void -handle_startup(StringInfo s) -{ - uint8 msgver = pq_getmsgbyte(s); - if (msgver != 1) - elog(ERROR, "Expected startup message version 1, but got %u", msgver); - - /* - * The startup message consists of null-terminated strings as key/value - * pairs. The first entry is always the format identifier. - */ - do { - const char *k, *v; - - k = pq_getmsgstring(s); - if (strlen(k) == 0) - ereport(ERROR, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("invalid startup message: key has zero length"))); - - if (getmsgisend(s)) - ereport(ERROR, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("invalid startup message: key '%s' has no following value", k))); - - /* It's OK to have a zero length value */ - v = pq_getmsgstring(s); - - handle_startup_param(k, v); - } while (!getmsgisend(s)); -} - -static bool -parse_bool_param(const char *key, const char *value) -{ - bool result; - - if (!parse_bool(value, &result)) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("couldn't parse value '%s' for key '%s' as boolean", - value, key))); - - return result; -} - -static void -handle_startup_param(const char *key, const char *value) -{ - elog(DEBUG2, "apply got pglogical startup msg param %s=%s", key, value); - - if (strcmp(key, "pg_version") == 0) - elog(DEBUG1, "upstream Pg version is %s", value); - - if (strcmp(key, "encoding") == 0) - { - int encoding = pg_char_to_encoding(value); - - if (encoding != GetDatabaseEncoding()) - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("expected encoding=%s from upstream but got %s", - GetDatabaseEncodingName(), value))); - } - - if (strcmp(key, "forward_changeset_origins") == 0) - { - bool fwd = parse_bool_param(key, value); - /* FIXME: Store this somewhere */ - elog(DEBUG1, "changeset origin forwarding enabled: %s", fwd ? "t" : "f"); - } - - /* - * We just ignore a bunch of parameters here because we specify what we - * require when we send our params to the upstream. It's required to ERROR - * if it can't match what we asked for. It may send the startup message - * first, but it'll be followed by an ERROR if it does. There's no need - * to check params we can't do anything about mismatches of, like protocol - * versions and type sizes. - */ -} - -static RangeVar * -parse_relation_message(Jsonb *message) -{ - JsonbIterator *it; - JsonbValue v; - int r; - int level = 0; - char *key = NULL; - char **parse_res = NULL; - char *nspname = NULL; - char *relname = NULL; - - /* Parse and validate the json message. */ - if (!JB_ROOT_IS_OBJECT(message)) - elog(ERROR, "malformed message in queued message tuple: root is not object"); - - it = JsonbIteratorInit(&message->root); - while ((r = JsonbIteratorNext(&it, &v, false)) != WJB_DONE) - { - if (level == 0 && r != WJB_BEGIN_OBJECT) - elog(ERROR, "root element needs to be an object"); - else if (level == 0 && r == WJB_BEGIN_OBJECT) - { - level++; - } - else if (level == 1 && r == WJB_KEY) - { - if (strncmp(v.val.string.val, "schema_name", v.val.string.len) == 0) - parse_res = &nspname; - else if (strncmp(v.val.string.val, "table_name", v.val.string.len) == 0) - parse_res = &relname; - else - elog(ERROR, "unexpected key: %s", - pnstrdup(v.val.string.val, v.val.string.len)); - - key = v.val.string.val; - } - else if (level == 1 && r == WJB_VALUE) - { - if (!key) - elog(ERROR, "in wrong state when parsing key"); - - if (v.type != jbvString) - elog(ERROR, "unexpected type for key '%s': %u", key, v.type); - - *parse_res = pnstrdup(v.val.string.val, v.val.string.len); - } - else if (level == 1 && r != WJB_END_OBJECT) - { - elog(ERROR, "unexpected content: %u at level %d", r, level); - } - else if (r == WJB_END_OBJECT) - { - level--; - parse_res = NULL; - key = NULL; - } - else - elog(ERROR, "unexpected content: %u at level %d", r, level); - - } - - /* Check if we got both schema and table names. */ - if (!nspname) - elog(ERROR, "missing schema_name in relation message"); - - if (!relname) - elog(ERROR, "missing table_name in relation message"); - - return makeRangeVar(nspname, relname, -1); -} - -/* - * Handle TRUNCATE message comming via queue table. - */ -static void -handle_truncate(QueuedMessage *queued_message) -{ - RangeVar *rv; - - /* - * If table doesn't exist locally, it can't be subscribed. - * - * TODO: should we error here? - */ - rv = parse_relation_message(queued_message->message); - - /* If in list of relations which are being synchronized, skip. */ - if (!should_apply_changes_for_rel(rv->schemaname, rv->relname)) - return; - - truncate_table(rv->schemaname, rv->relname); -} - -/* - * Handle TABLESYNC message comming via queue table. - */ -static void -handle_table_sync(QueuedMessage *queued_message) -{ - RangeVar *rv; - MemoryContext oldcontext; - PGLogicalSyncStatus *oldsync; - PGLogicalSyncStatus *newsync; - - rv = parse_relation_message(queued_message->message); - - oldsync = get_table_sync_status(MyApplyWorker->subid, rv->schemaname, - rv->relname, true); - - if (oldsync) - { - elog(INFO, - "table sync came from queue for table %s.%s which already being synchronized, skipping", - rv->schemaname, rv->relname); - - return; - } - - /* Keep the lists persistent. */ - oldcontext = MemoryContextSwitchTo(TopMemoryContext); - newsync = palloc0(sizeof(PGLogicalSyncStatus)); - MemoryContextSwitchTo(oldcontext); - - newsync->kind = SYNC_KIND_DATA; - newsync->subid = MyApplyWorker->subid; - newsync->status = SYNC_STATUS_INIT; - namestrcpy(&newsync->nspname, rv->schemaname); - namestrcpy(&newsync->relname, rv->relname); - create_local_sync_status(newsync); - - oldcontext = MemoryContextSwitchTo(TopMemoryContext); - MemoryContextSwitchTo(oldcontext); - - MyApplyWorker->sync_pending = true; -} - -/* - * Handle SEQUENCE message comming via queue table. - */ -static void -handle_sequence(QueuedMessage *queued_message) -{ - Jsonb *message = queued_message->message; - JsonbIterator *it; - JsonbValue v; - int r; - int level = 0; - char *key = NULL; - char **parse_res = NULL; - char *nspname = NULL; - char *relname = NULL; - char *last_value_raw = NULL; - int64 last_value; - Oid nspoid; - Oid reloid; - - /* Parse and validate the json message. */ - if (!JB_ROOT_IS_OBJECT(message)) - elog(ERROR, "malformed message in queued message tuple: root is not object"); - - it = JsonbIteratorInit(&message->root); - while ((r = JsonbIteratorNext(&it, &v, false)) != WJB_DONE) - { - if (level == 0 && r != WJB_BEGIN_OBJECT) - elog(ERROR, "root element needs to be an object"); - else if (level == 0 && r == WJB_BEGIN_OBJECT) - { - level++; - } - else if (level == 1 && r == WJB_KEY) - { - if (strncmp(v.val.string.val, "schema_name", v.val.string.len) == 0) - parse_res = &nspname; - else if (strncmp(v.val.string.val, "sequence_name", v.val.string.len) == 0) - parse_res = &relname; - else if (strncmp(v.val.string.val, "last_value", v.val.string.len) == 0) - parse_res = &last_value_raw; - else - elog(ERROR, "unexpected key: %s", - pnstrdup(v.val.string.val, v.val.string.len)); - - key = v.val.string.val; - } - else if (level == 1 && r == WJB_VALUE) - { - if (!key) - elog(ERROR, "in wrong state when parsing key"); - - if (v.type != jbvString) - elog(ERROR, "unexpected type for key '%s': %u", key, v.type); - - *parse_res = pnstrdup(v.val.string.val, v.val.string.len); - } - else if (level == 1 && r != WJB_END_OBJECT) - { - elog(ERROR, "unexpected content: %u at level %d", r, level); - } - else if (r == WJB_END_OBJECT) - { - level--; - parse_res = NULL; - key = NULL; - } - else - elog(ERROR, "unexpected content: %u at level %d", r, level); - - } - - /* Check if we got both schema and table names. */ - if (!nspname) - elog(ERROR, "missing schema_name in sequence message"); - - if (!relname) - elog(ERROR, "missing table_name in sequence message"); - - if (!last_value_raw) - elog(ERROR, "missing last_value in sequence message"); - - nspoid = get_namespace_oid(nspname, false); - reloid = get_relname_relid(relname, nspoid); - last_value = strtoll(last_value_raw, NULL, 10); - - DirectFunctionCall2(setval_oid, ObjectIdGetDatum(reloid), - Int64GetDatum(last_value)); -} -/* - * Handle SQL message comming via queue table. - */ -static void -handle_sql(QueuedMessage *queued_message, bool tx_just_started) -{ - JsonbIterator *it; - JsonbValue v; - int r; - char *sql; - - /* Validate the json and extract the SQL string from it. */ - if (!JB_ROOT_IS_SCALAR(queued_message->message)) - elog(ERROR, "malformed message in queued message tuple: root is not scalar"); - - it = JsonbIteratorInit(&queued_message->message->root); - r = JsonbIteratorNext(&it, &v, false); - if (r != WJB_BEGIN_ARRAY) - elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_BEGIN_ARRAY); - - r = JsonbIteratorNext(&it, &v, false); - if (r != WJB_ELEM) - elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_ELEM); - - if (v.type != jbvString) - elog(ERROR, "malformed message in queued message tuple, expected value type %d got %d", jbvString, v.type); - - sql = pnstrdup(v.val.string.val, v.val.string.len); - - r = JsonbIteratorNext(&it, &v, false); - if (r != WJB_END_ARRAY) - elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_END_ARRAY); - - r = JsonbIteratorNext(&it, &v, false); - if (r != WJB_DONE) - elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_DONE); - - /* Run the extracted SQL. */ - pglogical_execute_sql_command(sql, queued_message->role, tx_just_started); -} - -/* - * Handles messages comming from the queue. - */ -static void -handle_queued_message(HeapTuple msgtup, bool tx_just_started) -{ - QueuedMessage *queued_message; - const char *old_action_name; - - old_action_name = errcallback_arg.action_name; - errcallback_arg.is_ddl_or_drop = true; - - queued_message = queued_message_from_tuple(msgtup); - - switch (queued_message->message_type) - { - case QUEUE_COMMAND_TYPE_SQL: - errcallback_arg.action_name = "QUEUED_SQL"; - handle_sql(queued_message, tx_just_started); - break; - case QUEUE_COMMAND_TYPE_TRUNCATE: - errcallback_arg.action_name = "QUEUED_TRUNCATE"; - handle_truncate(queued_message); - break; - case QUEUE_COMMAND_TYPE_TABLESYNC: - errcallback_arg.action_name = "QUEUED_TABLESYNC"; - handle_table_sync(queued_message); - break; - case QUEUE_COMMAND_TYPE_SEQUENCE: - errcallback_arg.action_name = "QUEUED_SEQUENCE"; - handle_sequence(queued_message); - break; - default: - elog(ERROR, "unknown message type '%c'", - queued_message->message_type); - } - - errcallback_arg.action_name = old_action_name; - errcallback_arg.is_ddl_or_drop = false; -} - -static void -replication_handler(StringInfo s) -{ - ErrorContextCallback errcallback; - char action = pq_getmsgbyte(s); - - memset(&errcallback_arg, 0, sizeof(struct ActionErrCallbackArg)); - errcallback.callback = action_error_callback; - errcallback.arg = &errcallback_arg; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - Assert(CurrentMemoryContext == MessageContext); - - switch (action) - { - /* BEGIN */ - case 'B': - handle_begin(s); - break; - /* COMMIT */ - case 'C': - handle_commit(s); - break; - /* ORIGIN */ - case 'O': - handle_origin(s); - break; - /* RELATION */ - case 'R': - handle_relation(s); - break; - /* INSERT */ - case 'I': - handle_insert(s); - break; - /* UPDATE */ - case 'U': - handle_update(s); - break; - /* DELETE */ - case 'D': - handle_delete(s); - break; - /* STARTUP MESSAGE */ - case 'S': - handle_startup(s); - break; - default: - elog(ERROR, "unknown action of type %c", action); - } - - Assert(CurrentMemoryContext == MessageContext); - - if (error_context_stack == &errcallback) - error_context_stack = errcallback.previous; - - if (action == 'C') - { - /* - * We clobber MessageContext on commit. It doesn't matter much when we - * do it so long as we do so periodically, to prevent the context from - * growing too much. We might want to clean it up even 'n'th message - * too, but that adds testing burden and isn't done for now. - */ - MemoryContextReset(MessageContext); - } -} - -/* - * Figure out which write/flush positions to report to the walsender process. - * - * We can't simply report back the last LSN the walsender sent us because the - * local transaction might not yet be flushed to disk locally. Instead we - * build a list that associates local with remote LSNs for every commit. When - * reporting back the flush position to the sender we iterate that list and - * check which entries on it are already locally flushed. Those we can report - * as having been flushed. - * - * Returns true if there's no outstanding transactions that need to be - * flushed. - */ -static bool -get_flush_position(XLogRecPtr *write, XLogRecPtr *flush) -{ - dlist_mutable_iter iter; - XLogRecPtr local_flush = GetFlushRecPtr(); - - *write = InvalidXLogRecPtr; - *flush = InvalidXLogRecPtr; - - dlist_foreach_modify(iter, &lsn_mapping) - { - PGLFlushPosition *pos = - dlist_container(PGLFlushPosition, node, iter.cur); - - *write = pos->remote_end; - - if (pos->local_end <= local_flush) - { - *flush = pos->remote_end; - dlist_delete(iter.cur); - pfree(pos); - } - else - { - /* - * Don't want to uselessly iterate over the rest of the list which - * could potentially be long. Instead get the last element and - * grab the write position from there. - */ - pos = dlist_tail_element(PGLFlushPosition, node, - &lsn_mapping); - *write = pos->remote_end; - return false; - } - } - - return dlist_is_empty(&lsn_mapping); -} - -/* - * Send a Standby Status Update message to server. - * - * 'recvpos' is the latest LSN we've received data to, force is set if we need - * to send a response to avoid timeouts. - */ -static bool -send_feedback(PGconn *conn, XLogRecPtr recvpos, int64 now, bool force) -{ - static StringInfo reply_message = NULL; - - static XLogRecPtr last_recvpos = InvalidXLogRecPtr; - static XLogRecPtr last_writepos = InvalidXLogRecPtr; - static XLogRecPtr last_flushpos = InvalidXLogRecPtr; - - XLogRecPtr writepos; - XLogRecPtr flushpos; - - /* It's legal to not pass a recvpos */ - if (recvpos < last_recvpos) - recvpos = last_recvpos; - - if (get_flush_position(&writepos, &flushpos)) - { - /* - * No outstanding transactions to flush, we can report the latest - * received position. This is important for synchronous replication. - */ - flushpos = writepos = recvpos; - } - - if (writepos < last_writepos) - writepos = last_writepos; - - if (flushpos < last_flushpos) - flushpos = last_flushpos; - - /* if we've already reported everything we're good */ - if (!force && - writepos == last_writepos && - flushpos == last_flushpos) - return true; - - if (!reply_message) - { - MemoryContext oldcontext = MemoryContextSwitchTo(TopMemoryContext); - reply_message = makeStringInfo(); - MemoryContextSwitchTo(oldcontext); - } - else - resetStringInfo(reply_message); - - pq_sendbyte(reply_message, 'r'); - pq_sendint64(reply_message, recvpos); /* write */ - pq_sendint64(reply_message, flushpos); /* flush */ - pq_sendint64(reply_message, writepos); /* apply */ - pq_sendint64(reply_message, now); /* sendTime */ - pq_sendbyte(reply_message, false); /* replyRequested */ - - elog(DEBUG2, "sending feedback (force %d) to recv %X/%X, write %X/%X, flush %X/%X", - force, - (uint32) (recvpos >> 32), (uint32) recvpos, - (uint32) (writepos >> 32), (uint32) writepos, - (uint32) (flushpos >> 32), (uint32) flushpos - ); - - if (PQputCopyData(conn, reply_message->data, reply_message->len) <= 0 || - PQflush(conn)) - { - ereport(ERROR, - (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("could not send feedback packet: %s", - PQerrorMessage(conn)))); - return false; - } - - if (recvpos > last_recvpos) - last_recvpos = recvpos; - if (writepos > last_writepos) - last_writepos = writepos; - if (flushpos > last_flushpos) - last_flushpos = flushpos; - - return true; -} - -/* - * Apply main loop. - */ -void -apply_work(PGconn *streamConn) -{ - int fd; - char *copybuf = NULL; - XLogRecPtr last_received = InvalidXLogRecPtr; - - applyconn = streamConn; - fd = PQsocket(applyconn); - - /* Init the MessageContext which we use for easier cleanup. */ - MessageContext = AllocSetContextCreate(TopMemoryContext, - "MessageContext", - ALLOCSET_DEFAULT_SIZES); - - MemoryContextSwitchTo(MessageContext); - - /* mark as idle, before starting to loop */ - pgstat_report_activity(STATE_IDLE, NULL); - Assert(CurrentMemoryContext == MessageContext); - - while (!got_SIGTERM) - { - int rc; - int r; - - CHECK_FOR_INTERRUPTS(); - - /* - * Background workers mustn't call usleep() or any direct equivalent: - * instead, they may wait on their process latch, which sleeps as - * necessary, but is awakened if postmaster dies. That way the - * background process goes away immediately in an emergency. - */ - rc = WaitLatchOrSocket(&MyProc->procLatch, - WL_SOCKET_READABLE | WL_LATCH_SET | - WL_TIMEOUT | WL_POSTMASTER_DEATH, - fd, 1000L); - - ResetLatch(&MyProc->procLatch); - - Assert(CurrentMemoryContext == MessageContext); - - /* emergency bailout if postmaster has died */ - if (rc & WL_POSTMASTER_DEATH) - proc_exit(1); - - if (rc & WL_SOCKET_READABLE) - PQconsumeInput(applyconn); - - if (PQstatus(applyconn) == CONNECTION_BAD) - { - elog(ERROR, "connection to other side has died"); - } - - Assert(CurrentMemoryContext == MessageContext); - - for (;;) - { - if (got_SIGTERM) - break; - - /* We must not have fallen out of MessageContext by accident */ - Assert(CurrentMemoryContext == MessageContext); - - Assert(copybuf == NULL); - r = PQgetCopyData(applyconn, ©buf, 1); - - if (r == -1) - { - elog(ERROR, "data stream ended"); - } - else if (r == -2) - { - elog(ERROR, "could not read COPY data: %s", - PQerrorMessage(applyconn)); - } - else if (r < 0) - elog(ERROR, "invalid COPY status %d", r); - else if (r == 0) - { - /* need to wait for new data */ - break; - } - else - { - int c; - StringInfoData s; - - /* - * We're using a StringInfo to wrap existing data here, as a - * cursor. We init it manually to avoid a redundant allocation. - */ - memset(&s, 0, sizeof(StringInfoData)); - s.data = copybuf; - s.len = r; - s.maxlen = -1; - s.cursor = 0; - - c = pq_getmsgbyte(&s); - - if (c == 'w') - { - XLogRecPtr start_lsn; - XLogRecPtr end_lsn; - - start_lsn = pq_getmsgint64(&s); - end_lsn = pq_getmsgint64(&s); - pq_getmsgint64(&s); /* sendTime */ - - if (last_received < start_lsn) - last_received = start_lsn; - - if (last_received < end_lsn) - last_received = end_lsn; - - replication_handler(&s); - } - else if (c == 'k') - { - XLogRecPtr endpos; - bool reply_requested; - - endpos = pq_getmsgint64(&s); - /* timestamp = */ pq_getmsgint64(&s); - reply_requested = pq_getmsgbyte(&s); - - send_feedback(applyconn, endpos, - GetCurrentTimestamp(), - reply_requested); - - if (last_received < endpos) - last_received = endpos; - - /* - * For sync workers in catchup mode, check if we've reached - * the stop position based on the keepalive. This is needed - * because table-specific replication slots may not receive - * any actual commits if no changes are made to the table, - * but keepalives tell us the provider's current position. - */ - if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC && - MyApplyWorker->replay_stop_lsn != InvalidXLogRecPtr && - MyApplyWorker->replay_stop_lsn <= endpos) - { - ereport(LOG, - (errmsg("pglogical sync finished processing via keepalive; provider at %X/%X, required %X/%X", - (uint32)(endpos>>32), (uint32)endpos, - (uint32)(MyApplyWorker->replay_stop_lsn >>32), - (uint32)MyApplyWorker->replay_stop_lsn))); - - StartTransactionCommand(); - set_table_sync_status(MyApplyWorker->subid, - NameStr(MyPGLogicalWorker->worker.sync.nspname), - NameStr(MyPGLogicalWorker->worker.sync.relname), - SYNC_STATUS_SYNCDONE, endpos); - CommitTransactionCommand(); - - XLogFlush(GetXLogWriteRecPtr()); - PQfinish(applyconn); - pglogical_sync_worker_finish(); - proc_exit(0); - } - } - /* other message types are purposefully ignored */ - - /* copybuf is malloc'd not palloc'd */ - if (copybuf != NULL) - { - PQfreemem(copybuf); - copybuf = NULL; - } - } - - /* We must not have fallen out of MessageContext by accident */ - Assert(CurrentMemoryContext == MessageContext); - } - - /* confirm all writes at once */ - send_feedback(applyconn, last_received, GetCurrentTimestamp(), false); - - if (!in_remote_transaction) - process_syncing_tables(last_received); - - /* We must not have switched out of MessageContext by mistake */ - Assert(CurrentMemoryContext == MessageContext); - - /* Cleanup the memory. */ - MemoryContextResetAndDeleteChildren(MessageContext); - - /* - * Only do a leak check if we're between txns; we don't want lots of - * noise due to resources that only exist in a txn. - */ - if (!IsTransactionState()) - { - VALGRIND_DO_ADDED_LEAK_CHECK; - } - } -} - -/* - * Add context to the errors produced by pglogical_execute_sql_command(). - */ -static void -execute_sql_command_error_cb(void *arg) -{ - errcontext("during execution of queued SQL statement: %s", (char *) arg); -} - -/* - * Execute an SQL command. This can be multiple multiple queries. - */ -void -pglogical_execute_sql_command(char *cmdstr, char *role, bool isTopLevel) -{ - const char *save_debug_query_string = debug_query_string; - List *commands; - ListCell *command_i; -#ifdef PGXC - List *commandSourceQueries; - ListCell *commandSourceQuery_i; -#endif - MemoryContext oldcontext; - ErrorContextCallback errcallback; - - oldcontext = MemoryContextSwitchTo(MessageContext); - - errcallback.callback = execute_sql_command_error_cb; - errcallback.arg = cmdstr; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - debug_query_string = cmdstr; - - /* - * XL distributes individual statements using just executing them as plain - * SQL query and can't handle multistatements this way so we need to get - * individual statements using API provided by XL itself. - */ -#ifdef PGXC - commands = pg_parse_query_get_source(cmdstr, &commandSourceQueries); -#else - commands = pg_parse_query(cmdstr); -#endif - - MemoryContextSwitchTo(oldcontext); - - /* - * Do a limited amount of safety checking against CONCURRENTLY commands - * executed in situations where they aren't allowed. The sender side should - * provide protection, but better be safe than sorry. - */ - isTopLevel = isTopLevel && (list_length(commands) == 1); - -#ifdef PGXC - forboth(command_i, commands, commandSourceQuery_i, commandSourceQueries) -#else - foreach(command_i, commands) -#endif - { - List *plantree_list; - List *querytree_list; - RawStmt *command = (RawStmt *) lfirst(command_i); - CommandTag commandTag; - Portal portal; - int save_nestlevel; - DestReceiver *receiver; - -#ifdef PGXC - cmdstr = (char *) lfirst(commandSourceQuery_i); - errcallback.arg = cmdstr; -#endif - - /* temporarily push snapshot for parse analysis/planning */ - PushActiveSnapshot(GetTransactionSnapshot()); - - oldcontext = MemoryContextSwitchTo(MessageContext); - - /* - * Set the current role to the user that executed the command on the - * origin server. - */ - save_nestlevel = NewGUCNestLevel(); - SetConfigOption("role", role, PGC_INTERNAL, PGC_S_OVERRIDE); - - commandTag = CreateCommandTag(command); - - querytree_list = pg_analyze_and_rewrite( - command, - cmdstr, - NULL, 0); - - plantree_list = pg_plan_queries( - querytree_list, cmdstr, 0, NULL); - - PopActiveSnapshot(); - - portal = CreatePortal("pglogical", true, true); - PortalDefineQuery(portal, NULL, - cmdstr, - commandTag, - plantree_list, NULL); - PortalStart(portal, NULL, 0, InvalidSnapshot); - - receiver = CreateDestReceiver(DestNone); - - (void) PortalRun(portal, FETCH_ALL, - isTopLevel, - receiver, receiver, - NULL); - (*receiver->rDestroy) (receiver); - - PortalDrop(portal, false); - - CommandCounterIncrement(); - - /* - * Restore the GUC variables we set above. - */ - AtEOXact_GUC(true, save_nestlevel); - - MemoryContextSwitchTo(oldcontext); - } - - /* protect against stack resets during CONCURRENTLY processing */ - if (error_context_stack == &errcallback) - error_context_stack = errcallback.previous; - - debug_query_string = save_debug_query_string; -} - -/* - * Load list of tables currently pending sync. - * - * Must be inside transaction. - */ -static void -reread_unsynced_tables(Oid subid) -{ - MemoryContext saved_ctx; - List *unsynced_tables; - ListCell *lc; - - /* Cleanup first. */ - list_free_deep(SyncingTables); - SyncingTables = NIL; - - /* Read new state. */ - unsynced_tables = get_unsynced_tables(subid); - saved_ctx = MemoryContextSwitchTo(TopMemoryContext); - foreach (lc, unsynced_tables) - { - PGLogicalSyncStatus *sync = palloc(sizeof(PGLogicalSyncStatus)); - memcpy(sync, lfirst(lc), sizeof(PGLogicalSyncStatus)); - SyncingTables = lappend(SyncingTables, sync); - } - - MemoryContextSwitchTo(saved_ctx); -} - -static void -process_syncing_tables(XLogRecPtr end_lsn) -{ - ListCell *lc; - - Assert(CurrentMemoryContext == MessageContext); - Assert(!IsTransactionState()); - - /* First check if we need to update the cached information. */ - if (MyApplyWorker->sync_pending) - { - StartTransactionCommand(); - MyApplyWorker->sync_pending = false; - reread_unsynced_tables(MyApplyWorker->subid); - CommitTransactionCommand(); - MemoryContextSwitchTo(MessageContext); - } - - /* Process currently pending sync tables. */ - if (list_length(SyncingTables) > 0) - { -#if PG_VERSION_NUM < 130000 - ListCell *prev = NULL; - ListCell *next; -#endif - -#if PG_VERSION_NUM >= 130000 - foreach(lc, SyncingTables) -#else - for (lc = list_head(SyncingTables); lc; lc = next) -#endif - { - PGLogicalSyncStatus *sync = (PGLogicalSyncStatus *) lfirst(lc); - PGLogicalSyncStatus *newsync; - -#if PG_VERSION_NUM < 130000 - /* We might delete the cell so advance it now. */ - next = lnext(lc); -#endif - - StartTransactionCommand(); - newsync = get_table_sync_status(MyApplyWorker->subid, - NameStr(sync->nspname), - NameStr(sync->relname), true); - - /* - * TODO: what to do here? We don't really want to die, - * but this can mean many things, for now we just assume table is - * not relevant for us anymore and leave fixing to the user. - * - * The reason why this part happens in transaction is that the - * memory allocated for sync info will get automatically cleaned - * afterwards. - */ - if (!newsync) - { - sync->status = SYNC_STATUS_READY; - sync->statuslsn = InvalidXLogRecPtr; - } - else - memcpy(sync, newsync, sizeof(PGLogicalSyncStatus)); - CommitTransactionCommand(); - MemoryContextSwitchTo(MessageContext); - - if (sync->status == SYNC_STATUS_SYNCWAIT) - { - PGLogicalWorker *worker; - - LWLockAcquire(PGLogicalCtx->lock, LW_EXCLUSIVE); - worker = pglogical_sync_find(MyDatabaseId, - MyApplyWorker->subid, - NameStr(sync->nspname), - NameStr(sync->relname)); - - if (pglogical_worker_running(worker) && - end_lsn >= worker->worker.apply.replay_stop_lsn) - { - worker->worker.apply.replay_stop_lsn = end_lsn; - sync->status = SYNC_STATUS_CATCHUP; - - StartTransactionCommand(); - set_table_sync_status(MyApplyWorker->subid, - NameStr(sync->nspname), - NameStr(sync->relname), - sync->status, - sync->statuslsn); - CommitTransactionCommand(); - MemoryContextSwitchTo(MessageContext); - - if (pglogical_worker_running(worker)) - SetLatch(&worker->proc->procLatch); - LWLockRelease(PGLogicalCtx->lock); - - if (wait_for_sync_status_change(MyApplyWorker->subid, - NameStr(sync->nspname), - NameStr(sync->relname), - SYNC_STATUS_SYNCDONE, - &sync->statuslsn)) - sync->status = SYNC_STATUS_SYNCDONE; - } - else - LWLockRelease(PGLogicalCtx->lock); - } - - if (sync->status == SYNC_STATUS_SYNCDONE && - end_lsn >= sync->statuslsn) - { - sync->status = SYNC_STATUS_READY; - sync->statuslsn = end_lsn; - - StartTransactionCommand(); - set_table_sync_status(MyApplyWorker->subid, - NameStr(sync->nspname), - NameStr(sync->relname), - sync->status, - sync->statuslsn); - CommitTransactionCommand(); - MemoryContextSwitchTo(MessageContext); - } - - /* Ready? Remove it from local cache. */ - if (sync->status == SYNC_STATUS_READY) - { -#if PG_VERSION_NUM >= 130000 - SyncingTables = foreach_delete_current(SyncingTables, lc); -#else - SyncingTables = list_delete_cell(SyncingTables, lc, prev); -#endif - pfree(sync); - } - else - { -#if PG_VERSION_NUM < 130000 - prev = lc; -#endif - } - } - } - - /* - * If there are still pending tables for synchronization, launch the sync - * worker. - */ - foreach (lc, SyncingTables) - { - List *workers; - ListCell *wlc; - int nworkers = 0; - PGLogicalSyncStatus *sync = (PGLogicalSyncStatus *) lfirst(lc); - - if (sync->status == SYNC_STATUS_SYNCDONE || sync->status == SYNC_STATUS_READY) - continue; - - LWLockAcquire(PGLogicalCtx->lock, LW_EXCLUSIVE); - workers = pglogical_sync_find_all(MyDatabaseId, MyApplyWorker->subid); - foreach (wlc, workers) - { - PGLogicalWorker *worker = (PGLogicalWorker *) lfirst(wlc); - - if (pglogical_worker_running(worker)) - nworkers++; - } - LWLockRelease(PGLogicalCtx->lock); - - if (nworkers < 1) - { - start_sync_worker(&sync->nspname, &sync->relname); - break; - } - } - - Assert(CurrentMemoryContext == MessageContext); -} - -static void -start_sync_worker(Name nspname, Name relname) -{ - PGLogicalWorker worker; - - /* Start the sync worker. */ - memset(&worker, 0, sizeof(PGLogicalWorker)); - worker.worker_type = PGLOGICAL_WORKER_SYNC; - worker.dboid = MyPGLogicalWorker->dboid; - worker.worker.apply.subid = MyApplyWorker->subid; - worker.worker.apply.sync_pending = false; /* Makes no sense for sync worker. */ - - /* Tell the worker to stop at current position. */ - worker.worker.sync.apply.replay_stop_lsn = replorigin_session_origin_lsn; - memcpy(&worker.worker.sync.nspname, nspname, sizeof(NameData)); - memcpy(&worker.worker.sync.relname, relname, sizeof(NameData)); - - (void) pglogical_worker_register(&worker); -} - -static inline TimeOffset -interval_to_timeoffset(const Interval *interval) -{ - TimeOffset span; - - span = interval->time; - -#ifdef HAVE_INT64_TIMESTAMP - span += interval->month * INT64CONST(30) * USECS_PER_DAY; - span += interval->day * INT64CONST(24) * USECS_PER_HOUR; -#else - span += interval->month * ((double) DAYS_PER_MONTH * SECS_PER_DAY); - span += interval->day * ((double) HOURS_PER_DAY * SECS_PER_HOUR); -#endif - - return span; -} - -void -pglogical_apply_main(Datum main_arg) -{ - int slot = DatumGetInt32(main_arg); - PGconn *streamConn; - RepOriginId originid; - XLogRecPtr origin_startpos; - MemoryContext saved_ctx; - char *repsets; - char *origins; - - /* Setup shmem. */ - pglogical_worker_attach(slot, PGLOGICAL_WORKER_APPLY); - Assert(MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_APPLY); - MyApplyWorker = &MyPGLogicalWorker->worker.apply; - - /* Attach to dsm segment. */ - Assert(CurrentResourceOwner == NULL); - CurrentResourceOwner = ResourceOwnerCreate(NULL, "pglogical apply"); - - /* Load correct apply API. */ - if (pglogical_use_spi) - { - if (pglogical_conflict_resolver != PGLOGICAL_RESOLVE_ERROR) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("pglogical.use_spi can only be used when " - "pglogical.conflict_resolution is set to 'error'"))); - - apply_api.on_begin = pglogical_apply_spi_begin; - apply_api.on_commit = pglogical_apply_spi_commit; - apply_api.do_insert = pglogical_apply_spi_insert; - apply_api.do_update = pglogical_apply_spi_update; - apply_api.do_delete = pglogical_apply_spi_delete; - apply_api.can_multi_insert = pglogical_apply_spi_can_mi; - apply_api.multi_insert_add_tuple = pglogical_apply_spi_mi_add_tuple; - apply_api.multi_insert_finish = pglogical_apply_spi_mi_finish; - } - - /* Setup synchronous commit according to the user's wishes */ - SetConfigOption("synchronous_commit", - pglogical_synchronous_commit ? "local" : "off", - PGC_BACKEND, PGC_S_OVERRIDE); /* other context? */ - - /* Run as replica session replication role. */ - SetConfigOption("session_replication_role", "replica", - PGC_SUSET, PGC_S_OVERRIDE); /* other context? */ - - /* - * Disable function body checks during replay. That's necessary because a) - * the creator of the function might have had it disabled b) the function - * might be search_path dependant and we don't fix the contents of - * functions. - */ - SetConfigOption("check_function_bodies", "off", - PGC_INTERNAL, PGC_S_OVERRIDE); - - /* Load the subscription. */ - StartTransactionCommand(); - saved_ctx = MemoryContextSwitchTo(TopMemoryContext); - MySubscription = get_subscription(MyApplyWorker->subid); - MemoryContextSwitchTo(saved_ctx); - -#ifdef XCP - /* - * When runnin under XL, initialise the XL executor so that the datanode - * and coordinator information is initialised properly. - */ - InitMultinodeExecutor(false); -#endif - CommitTransactionCommand(); - - elog(LOG, "starting apply for subscription %s", MySubscription->name); - - /* Set apply delay if any. */ - if (MySubscription->apply_delay) - apply_delay = - interval_to_timeoffset(MySubscription->apply_delay) / 1000; - - /* If the subscription isn't initialized yet, initialize it. */ - pglogical_sync_subscription(MySubscription); - - elog(DEBUG1, "connecting to provider %s, dsn %s", - MySubscription->origin->name, MySubscription->origin_if->dsn); - - /* - * Cache the queue relation id. - * TODO: invalidation - */ - StartTransactionCommand(); - QueueRelid = get_queue_table_oid(); - - originid = replorigin_by_name(MySubscription->slot_name, false); - elog(DEBUG2, "setting up replication origin %s (oid %u)", - MySubscription->slot_name, originid); - PGLreplorigin_session_setup(originid); - replorigin_session_origin = originid; - origin_startpos = replorigin_session_get_progress(false); - - /* Start the replication. */ - streamConn = pglogical_connect_replica(MySubscription->origin_if->dsn, - MySubscription->name, NULL); - - repsets = stringlist_to_identifierstr(MySubscription->replication_sets); - origins = stringlist_to_identifierstr(MySubscription->forward_origins); - - /* - * IDENTIFY_SYSTEM sets up some internal state on walsender so call it even - * if we don't (yet) want to use any of the results. - */ - pglogical_identify_system(streamConn, NULL, NULL, NULL, NULL); - - pglogical_start_replication(streamConn, MySubscription->slot_name, - origin_startpos, origins, repsets, NULL, - MySubscription->force_text_transfer); - pfree(repsets); - - CommitTransactionCommand(); - - /* - * Do an initial leak check with reporting off; we don't want to see - * these results, just the later output from ADDED leak checks. - */ - VALGRIND_DISABLE_ERROR_REPORTING; - VALGRIND_DO_LEAK_CHECK; - VALGRIND_ENABLE_ERROR_REPORTING; - - apply_work(streamConn); - - PQfinish(streamConn); - - /* We should only get here if we received sigTERM */ - proc_exit(0); -} +/*------------------------------------------------------------------------- + * + * pglogical_apply.c + * pglogical apply logic + * + * Copyright (c) 2015, PostgreSQL Global Development Group + * + * IDENTIFICATION + * pglogical.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "miscadmin.h" +#include "libpq-fe.h" +#include "pgstat.h" + +#include "access/htup_details.h" +#include "access/xact.h" + +#include "catalog/namespace.h" + +#include "commands/async.h" +#include "commands/dbcommands.h" +#include "commands/sequence.h" +#include "commands/tablecmds.h" +#include "commands/trigger.h" + +#include "executor/executor.h" + +#include "libpq/pqformat.h" + +#include "mb/pg_wchar.h" + +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" + +#include "optimizer/planner.h" + +#ifdef XCP +#include "pgxc/pgxcnode.h" +#endif + +#include "replication/origin.h" +#include "replication/reorderbuffer.h" + +#include "rewrite/rewriteHandler.h" + +#include "storage/ipc.h" +#include "storage/lmgr.h" +#include "storage/proc.h" + +#include "tcop/pquery.h" +#include "tcop/utility.h" + +#include "utils/builtins.h" +#include "utils/jsonb.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/snapmgr.h" + +#include "pglogical_conflict.h" +#include "pglogical_executor.h" +#include "pglogical_node.h" +#include "pglogical_queue.h" +#include "pglogical_relcache.h" +#include "pglogical_repset.h" +#include "pglogical_rpc.h" +#include "pglogical_sync.h" +#include "pglogical_worker.h" +#include "pglogical_apply.h" +#include "pglogical_apply_heap.h" +#include "pglogical_apply_spi.h" +#include "pglogical.h" + + +void PGDLLEXPORT pglogical_apply_main(Datum main_arg); + +static bool in_remote_transaction = false; +static XLogRecPtr remote_origin_lsn = InvalidXLogRecPtr; +static RepOriginId remote_origin_id = InvalidRepOriginId; +static TimeOffset apply_delay = 0; + +static Oid QueueRelid = InvalidOid; + +static List *SyncingTables = NIL; + +PGLogicalApplyWorker *MyApplyWorker = NULL; +PGLogicalSubscription *MySubscription = NULL; + +static PGconn *applyconn = NULL; + +typedef struct PGLogicalApplyFunctions +{ + pglogical_apply_begin_fn on_begin; + pglogical_apply_commit_fn on_commit; + pglogical_apply_insert_fn do_insert; + pglogical_apply_update_fn do_update; + pglogical_apply_delete_fn do_delete; + pglogical_apply_can_mi_fn can_multi_insert; + pglogical_apply_mi_add_tuple_fn multi_insert_add_tuple; + pglogical_apply_mi_finish_fn multi_insert_finish; +} PGLogicalApplyFunctions; + +static PGLogicalApplyFunctions apply_api = +{ + .on_begin = pglogical_apply_heap_begin, + .on_commit = pglogical_apply_heap_commit, + .do_insert = pglogical_apply_heap_insert, + .do_update = pglogical_apply_heap_update, + .do_delete = pglogical_apply_heap_delete, + .can_multi_insert = pglogical_apply_heap_can_mi, + .multi_insert_add_tuple = pglogical_apply_heap_mi_add_tuple, + .multi_insert_finish = pglogical_apply_heap_mi_finish +}; + +/* Number of tuples inserted after which we switch to multi-insert. */ +#define MIN_MULTI_INSERT_TUPLES 5 +static PGLogicalRelation *last_insert_rel = NULL; +static int last_insert_rel_cnt = 0; +static bool use_multi_insert = false; + +/* + * A message counter for the xact, for debugging. We don't send + * the remote change LSN with messages, so this aids identification + * of which change causes an error. + */ +static uint32 xact_action_counter; + +typedef struct PGLFlushPosition +{ + dlist_node node; + XLogRecPtr local_end; + XLogRecPtr remote_end; +} PGLFlushPosition; + +static dlist_head lsn_mapping = DLIST_STATIC_INIT(lsn_mapping); + +typedef struct ApplyExecState +{ + EState *estate; + EPQState epqstate; + ResultRelInfo *resultRelInfo; + TupleTableSlot *slot; +} ApplyExecState; + +struct ActionErrCallbackArg +{ + const char * action_name; + PGLogicalRelation *rel; + bool is_ddl_or_drop; +}; + +static struct ActionErrCallbackArg errcallback_arg; +static TransactionId remote_xid; + +static void multi_insert_finish(void); + +static void handle_queued_message(HeapTuple msgtup, bool tx_just_started); +static void handle_startup_param(const char *key, const char *value); +static bool parse_bool_param(const char *key, const char *value); +static void process_syncing_tables(XLogRecPtr end_lsn); +static void start_sync_worker(Name nspname, Name relname); + +/* + * Check if given relation is in process of being synchronized. + * + * TODO: performance + */ +static bool +should_apply_changes_for_rel(const char *nspname, const char *relname) +{ + if (list_length(SyncingTables) > 0) + { + ListCell *lc; + + foreach (lc, SyncingTables) + { + PGLogicalSyncStatus *sync = (PGLogicalSyncStatus *) lfirst(lc); + + if (namestrcmp(&sync->nspname, nspname) == 0 && + namestrcmp(&sync->relname, relname) == 0 && + (sync->status != SYNC_STATUS_READY && + !(sync->status == SYNC_STATUS_SYNCDONE && + sync->statuslsn <= replorigin_session_origin_lsn))) + return false; + } + } + + return true; +} + +/* + * Prepare apply state details for errcontext or direct logging. + * + * This callback could be invoked at all sorts of weird times + * so it should assume as little as psosible about the invoking + * context. + */ +static void +format_action_description( + StringInfo si, + const char * action_name, + PGLogicalRelation *rel, + bool is_ddl_or_drop) +{ + appendStringInfoString(si, "apply "); + appendStringInfoString(si, + action_name == NULL ? "(unknown action)" : action_name); + + if (rel != NULL && + rel->nspname != NULL + && rel->relname != NULL + && !is_ddl_or_drop) + { + appendStringInfo(si, " from remote relation %s.%s", + rel->nspname, rel->relname); + } + + appendStringInfo(si, + " in commit before %X/%X, xid %u committed at %s (action #%u)", + (uint32)(replorigin_session_origin_lsn>>32), + (uint32)replorigin_session_origin_lsn, + remote_xid, + timestamptz_to_str(replorigin_session_origin_timestamp), + xact_action_counter); + + if (replorigin_session_origin != InvalidRepOriginId) + { + appendStringInfo(si, " from node replorigin %u", + replorigin_session_origin); + } + + if (remote_origin_id != InvalidRepOriginId) + { + appendStringInfo(si, " forwarded from commit %X/%X on node %u", + (uint32)(remote_origin_lsn>>32), + (uint32)remote_origin_lsn, + remote_origin_id); + } +} + +static void +action_error_callback(void *arg) +{ + StringInfoData si; + initStringInfo(&si); + + format_action_description(&si, + errcallback_arg.action_name, + errcallback_arg.rel, + errcallback_arg.is_ddl_or_drop); + + errcontext("%s", si.data); + pfree(si.data); +} + +static bool +ensure_transaction(void) +{ + if (IsTransactionState()) + { + if (CurrentMemoryContext != MessageContext) + MemoryContextSwitchTo(MessageContext); + return false; + } + + /* + * pglogical doesn't have "statements" as such, so we'll report one + * statement per applied transaction. We must set the statement start time + * because StartTransaction() uses it to initialize the transaction cached + * timestamp used by current_timestamp. If we don't set it, every xact will + * get the same current_timestamp. See 2ndQuadrant/pglogical_internal#148 + */ + SetCurrentStatementStartTimestamp(); + + StartTransactionCommand(); + apply_api.on_begin(); + MemoryContextSwitchTo(MessageContext); + + return true; +} + +static void +handle_begin(StringInfo s) +{ + XLogRecPtr commit_lsn; + TimestampTz commit_time; + + xact_action_counter = 1; + errcallback_arg.action_name = "BEGIN"; + + pglogical_read_begin(s, &commit_lsn, &commit_time, &remote_xid); + + replorigin_session_origin_timestamp = commit_time; + replorigin_session_origin_lsn = commit_lsn; + remote_origin_id = InvalidRepOriginId; + + VALGRIND_PRINTF("PGLOGICAL_APPLY: begin %u\n", remote_xid); + + /* don't want the overhead otherwise */ + if (apply_delay > 0) + { + TimestampTz current; + current = GetCurrentIntegerTimestamp(); + + /* ensure no weirdness due to clock drift */ + if (current > replorigin_session_origin_timestamp) + { + long sec; + int usec; + long sleep_usec; + + current = TimestampTzPlusMilliseconds(current, + -apply_delay); + + TimestampDifference(current, replorigin_session_origin_timestamp, + &sec, &usec); + + /* + * Calculate total sleep time in microseconds. + * Guard against negative values which can occur if we've already + * passed the target time (due to processing delays). + */ + sleep_usec = usec + (sec * USECS_PER_SEC); + if (sleep_usec > 0) + { +#ifdef WIN32 + /* + * On Windows, pg_usleep has lower precision (~15ms granularity) + * due to the default system timer resolution. Add a small buffer + * to ensure we meet the requested delay despite timer imprecision. + */ + sleep_usec += 100000; /* 100ms buffer */ +#endif + pg_usleep(sleep_usec); + } + } + } + + in_remote_transaction = true; + + pgstat_report_activity(STATE_RUNNING, NULL); +} + +/* + * Handle COMMIT message. + */ +static void +handle_commit(StringInfo s) +{ + XLogRecPtr commit_lsn; + XLogRecPtr end_lsn; + TimestampTz commit_time; + + errcallback_arg.action_name = "COMMIT"; + xact_action_counter++; + + pglogical_read_commit(s, &commit_lsn, &end_lsn, &commit_time); + + Assert(commit_time == replorigin_session_origin_timestamp); + + if (IsTransactionState()) + { + PGLFlushPosition *flushpos; + + multi_insert_finish(); + + apply_api.on_commit(); + + /* We need to write end_lsn to the commit record. */ + replorigin_session_origin_lsn = end_lsn; + + CommitTransactionCommand(); + MemoryContextSwitchTo(TopMemoryContext); + + /* Track commit lsn */ + flushpos = (PGLFlushPosition *) palloc(sizeof(PGLFlushPosition)); + flushpos->local_end = XactLastCommitEnd; + flushpos->remote_end = end_lsn; + + dlist_push_tail(&lsn_mapping, &flushpos->node); + MemoryContextSwitchTo(MessageContext); + } + + /* + * If the xact isn't from the immediate upstream, advance the slot of the + * node it originally came from so we start replay of that node's change + * data at the right place. + * + * This is only necessary when we're streaming data from one peer (A) that + * in turn receives from other peers (B, C), and we plan to later switch to + * replaying directly from B and/or C, no longer receiving forwarded xacts + * from A. When we do the switchover we need to know the right place at + * which to start replay from B and C. We don't actually do that yet, but + * we'll want to be able to do cascaded initialisation in future, so it's + * worth keeping track. + * + * A failure can occur here (see #79) if there's a cascading + * replication configuration like: + * + * X--> Y -> Z + * | ^ + * | | + * \---------/ + * + * where the direct and indirect connections from X to Z use different + * replication sets so as not to conflict, and where Y and Z are on the + * same PostgreSQL instance. In this case our attempt to advance the + * replication identifier here will ERROR because it's already in use + * for the direct connection from X to Z. So don't do that. + */ + if (remote_origin_id != InvalidRepOriginId && + remote_origin_id != replorigin_session_origin) + { +#if PG_VERSION_NUM >= 90500 + Relation replorigin_rel; +#endif + elog(DEBUG3, "advancing origin oid %u for forwarded row to %X/%X", + remote_origin_id, + (uint32)(XactLastCommitEnd>>32), (uint32)XactLastCommitEnd); + +#if PG_VERSION_NUM >= 90500 + replorigin_rel = table_open(ReplicationOriginRelationId, RowExclusiveLock); +#endif + replorigin_advance(remote_origin_id, remote_origin_lsn, + XactLastCommitEnd, false, false /* XXX ? */); +#if PG_VERSION_NUM >= 90500 + table_close(replorigin_rel, RowExclusiveLock); +#endif + } + + in_remote_transaction = false; + + /* + * Stop replay if we're doing limited replay and we've replayed up to the + * last record we're supposed to process. + */ + if (MyApplyWorker->replay_stop_lsn != InvalidXLogRecPtr + && MyApplyWorker->replay_stop_lsn <= end_lsn) + { + ereport(LOG, + (errmsg("pglogical %s finished processing; replayed to %X/%X of required %X/%X", + MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC ? "sync" : "apply", + (uint32)(end_lsn>>32), (uint32)end_lsn, + (uint32)(MyApplyWorker->replay_stop_lsn >>32), + (uint32)MyApplyWorker->replay_stop_lsn))); + + /* + * If this is sync worker, update syncing table state to done. + */ + if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC) + { + StartTransactionCommand(); + set_table_sync_status(MyApplyWorker->subid, + NameStr(MyPGLogicalWorker->worker.sync.nspname), + NameStr(MyPGLogicalWorker->worker.sync.relname), + SYNC_STATUS_SYNCDONE, end_lsn); + CommitTransactionCommand(); + } + + /* + * Flush all writes so the latest position can be reported back to the + * sender. + */ + XLogFlush(GetXLogWriteRecPtr()); + + /* + * Disconnect. + * + * This needs to happen before the pglogical_sync_worker_finish() + * call otherwise slot drop will fail. + */ + PQfinish(applyconn); + + /* + * If this is sync worker, finish it. + */ + if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC) + pglogical_sync_worker_finish(); + + /* Stop gracefully */ + proc_exit(0); + } + + VALGRIND_PRINTF("PGLOGICAL_APPLY: commit %u\n", remote_xid); + + xact_action_counter = 0; + remote_xid = InvalidTransactionId; + + process_syncing_tables(end_lsn); + +#if PG_VERSION_NUM < 150000 + /* + * Ensure any pending signals/self-notifies are sent out. + * + * (This is no longer needed in PG15 and also a no-op in later + * minor versions of PG13 and PG14. In those versions, + * notifications are sent at transaction commit.) + * + * Note that there is a possibility that this will result in an ERROR, + * which will result in the apply worker being killed and restarted. As + * the notification queues have already been flushed, the same error won't + * occur again, however if errors continue, they will dramatically slow + * down - but not stop - replication. + */ + ProcessCompletedNotifies(); +#endif + + pgstat_report_activity(STATE_IDLE, NULL); +} + +/* + * Handle ORIGIN message. + */ +static void +handle_origin(StringInfo s) +{ + char *origin; + + /* + * ORIGIN message can only come inside remote transaction and before + * any actual writes. + */ + if (!in_remote_transaction || IsTransactionState()) + elog(ERROR, "ORIGIN message sent out of order"); + + /* We have to start transaction here so that we can work with origins. */ + ensure_transaction(); + + origin = pglogical_read_origin(s, &remote_origin_lsn); + remote_origin_id = replorigin_by_name(origin, true); +} + +/* + * Handle RELATION message. + * + * Note we don't do validation against local schema here. The validation is + * posponed until first change for given relation comes. + */ +static void +handle_relation(StringInfo s) +{ + multi_insert_finish(); + + (void) pglogical_read_rel(s); +} + +static void +handle_insert(StringInfo s) +{ + PGLogicalTupleData newtup; + PGLogicalRelation *rel; + bool started_tx = ensure_transaction(); + + PushActiveSnapshot(GetTransactionSnapshot()); + + errcallback_arg.action_name = "INSERT"; + xact_action_counter++; + + rel = pglogical_read_insert(s, RowExclusiveLock, &newtup); + errcallback_arg.rel = rel; + + /* If in list of relations which are being synchronized, skip. */ + if (!should_apply_changes_for_rel(rel->nspname, rel->relname)) + { + pglogical_relation_close(rel, NoLock); + PopActiveSnapshot(); + CommandCounterIncrement(); + return; + } + + /* Handle multi_insert capabilities. */ + if (use_multi_insert) + { + if (rel != last_insert_rel) + { + multi_insert_finish(); + /* Fall through to normal insert. */ + } + else + { + apply_api.multi_insert_add_tuple(rel, &newtup); + last_insert_rel_cnt++; + return; + } + } + else if (pglogical_batch_inserts && + RelationGetRelid(rel->rel) != QueueRelid && + apply_api.can_multi_insert && + apply_api.can_multi_insert(rel)) + { + if (rel != last_insert_rel) + { + last_insert_rel = rel; + last_insert_rel_cnt = 0; + } + else if (last_insert_rel_cnt++ >= MIN_MULTI_INSERT_TUPLES) + { + use_multi_insert = true; + last_insert_rel_cnt = 0; + } + } + + /* Normal insert. */ + apply_api.do_insert(rel, &newtup); + + /* if INSERT was into our queue, process the message. */ + if (RelationGetRelid(rel->rel) == QueueRelid) + { + HeapTuple ht; + LockRelId lockid = rel->rel->rd_lockInfo.lockRelId; + Relation qrel; + + multi_insert_finish(); + + MemoryContextSwitchTo(MessageContext); + + ht = heap_form_tuple(RelationGetDescr(rel->rel), + newtup.values, newtup.nulls); + + LockRelationIdForSession(&lockid, RowExclusiveLock); + pglogical_relation_close(rel, NoLock); + + PopActiveSnapshot(); + CommandCounterIncrement(); + + apply_api.on_commit(); + + handle_queued_message(ht, started_tx); + + heap_freetuple(ht); + + qrel = table_open(QueueRelid, RowExclusiveLock); + + UnlockRelationIdForSession(&lockid, RowExclusiveLock); + + table_close(qrel, NoLock); + + apply_api.on_begin(); + MemoryContextSwitchTo(MessageContext); + +// if (oldxid != GetTopTransactionId()) +// CommitTransactionCommand(); + } + else + { + pglogical_relation_close(rel, NoLock); + + PopActiveSnapshot(); + CommandCounterIncrement(); + } +} + +static void +multi_insert_finish(void) +{ + if (use_multi_insert && last_insert_rel_cnt) + { + const char *old_action = errcallback_arg.action_name; + PGLogicalRelation *old_rel = errcallback_arg.rel; + errcallback_arg.action_name = "multi INSERT"; + errcallback_arg.rel = last_insert_rel; + + apply_api.multi_insert_finish(last_insert_rel); + pglogical_relation_close(last_insert_rel, NoLock); + use_multi_insert = false; + last_insert_rel = NULL; + last_insert_rel_cnt = 0; + + errcallback_arg.rel = old_rel; + errcallback_arg.action_name = old_action; + } +} + +static void +handle_update(StringInfo s) +{ + PGLogicalTupleData oldtup; + PGLogicalTupleData newtup; + PGLogicalRelation *rel; + bool hasoldtup; + + errcallback_arg.action_name = "UPDATE"; + xact_action_counter++; + + ensure_transaction(); + + multi_insert_finish(); + + PushActiveSnapshot(GetTransactionSnapshot()); + + rel = pglogical_read_update(s, RowExclusiveLock, &hasoldtup, &oldtup, + &newtup); + errcallback_arg.rel = rel; + + /* If in list of relations which are being synchronized, skip. */ + if (!should_apply_changes_for_rel(rel->nspname, rel->relname)) + { + pglogical_relation_close(rel, NoLock); + PopActiveSnapshot(); + CommandCounterIncrement(); + return; + } + + apply_api.do_update(rel, hasoldtup ? &oldtup : &newtup, &newtup); + + pglogical_relation_close(rel, NoLock); + + PopActiveSnapshot(); + CommandCounterIncrement(); +} + +static void +handle_delete(StringInfo s) +{ + PGLogicalTupleData oldtup; + PGLogicalRelation *rel; + + memset(&errcallback_arg, 0, sizeof(struct ActionErrCallbackArg)); + xact_action_counter++; + + ensure_transaction(); + + multi_insert_finish(); + + PushActiveSnapshot(GetTransactionSnapshot()); + + rel = pglogical_read_delete(s, RowExclusiveLock, &oldtup); + errcallback_arg.rel = rel; + + /* If in list of relations which are being synchronized, skip. */ + if (!should_apply_changes_for_rel(rel->nspname, rel->relname)) + { + pglogical_relation_close(rel, NoLock); + PopActiveSnapshot(); + CommandCounterIncrement(); + return; + } + + apply_api.do_delete(rel, &oldtup); + + pglogical_relation_close(rel, NoLock); + + PopActiveSnapshot(); + CommandCounterIncrement(); +} + +inline static bool +getmsgisend(StringInfo msg) +{ + return msg->cursor == msg->len; +} + +static void +handle_startup(StringInfo s) +{ + uint8 msgver = pq_getmsgbyte(s); + if (msgver != 1) + elog(ERROR, "Expected startup message version 1, but got %u", msgver); + + /* + * The startup message consists of null-terminated strings as key/value + * pairs. The first entry is always the format identifier. + */ + do { + const char *k, *v; + + k = pq_getmsgstring(s); + if (strlen(k) == 0) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid startup message: key has zero length"))); + + if (getmsgisend(s)) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid startup message: key '%s' has no following value", k))); + + /* It's OK to have a zero length value */ + v = pq_getmsgstring(s); + + handle_startup_param(k, v); + } while (!getmsgisend(s)); +} + +static bool +parse_bool_param(const char *key, const char *value) +{ + bool result; + + if (!parse_bool(value, &result)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("couldn't parse value '%s' for key '%s' as boolean", + value, key))); + + return result; +} + +static void +handle_startup_param(const char *key, const char *value) +{ + elog(DEBUG2, "apply got pglogical startup msg param %s=%s", key, value); + + if (strcmp(key, "pg_version") == 0) + elog(DEBUG1, "upstream Pg version is %s", value); + + if (strcmp(key, "encoding") == 0) + { + int encoding = pg_char_to_encoding(value); + + if (encoding != GetDatabaseEncoding()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("expected encoding=%s from upstream but got %s", + GetDatabaseEncodingName(), value))); + } + + if (strcmp(key, "forward_changeset_origins") == 0) + { + bool fwd = parse_bool_param(key, value); + /* FIXME: Store this somewhere */ + elog(DEBUG1, "changeset origin forwarding enabled: %s", fwd ? "t" : "f"); + } + + /* + * We just ignore a bunch of parameters here because we specify what we + * require when we send our params to the upstream. It's required to ERROR + * if it can't match what we asked for. It may send the startup message + * first, but it'll be followed by an ERROR if it does. There's no need + * to check params we can't do anything about mismatches of, like protocol + * versions and type sizes. + */ +} + +static RangeVar * +parse_relation_message(Jsonb *message) +{ + JsonbIterator *it; + JsonbValue v; + int r; + int level = 0; + char *key = NULL; + char **parse_res = NULL; + char *nspname = NULL; + char *relname = NULL; + + /* Parse and validate the json message. */ + if (!JB_ROOT_IS_OBJECT(message)) + elog(ERROR, "malformed message in queued message tuple: root is not object"); + + it = JsonbIteratorInit(&message->root); + while ((r = JsonbIteratorNext(&it, &v, false)) != WJB_DONE) + { + if (level == 0 && r != WJB_BEGIN_OBJECT) + elog(ERROR, "root element needs to be an object"); + else if (level == 0 && r == WJB_BEGIN_OBJECT) + { + level++; + } + else if (level == 1 && r == WJB_KEY) + { + if (strncmp(v.val.string.val, "schema_name", v.val.string.len) == 0) + parse_res = &nspname; + else if (strncmp(v.val.string.val, "table_name", v.val.string.len) == 0) + parse_res = &relname; + else + elog(ERROR, "unexpected key: %s", + pnstrdup(v.val.string.val, v.val.string.len)); + + key = v.val.string.val; + } + else if (level == 1 && r == WJB_VALUE) + { + if (!key) + elog(ERROR, "in wrong state when parsing key"); + + if (v.type != jbvString) + elog(ERROR, "unexpected type for key '%s': %u", key, v.type); + + *parse_res = pnstrdup(v.val.string.val, v.val.string.len); + } + else if (level == 1 && r != WJB_END_OBJECT) + { + elog(ERROR, "unexpected content: %u at level %d", r, level); + } + else if (r == WJB_END_OBJECT) + { + level--; + parse_res = NULL; + key = NULL; + } + else + elog(ERROR, "unexpected content: %u at level %d", r, level); + + } + + /* Check if we got both schema and table names. */ + if (!nspname) + elog(ERROR, "missing schema_name in relation message"); + + if (!relname) + elog(ERROR, "missing table_name in relation message"); + + return makeRangeVar(nspname, relname, -1); +} + +/* + * Handle TRUNCATE message comming via queue table. + */ +static void +handle_truncate(QueuedMessage *queued_message) +{ + RangeVar *rv; + + /* + * If table doesn't exist locally, it can't be subscribed. + * + * TODO: should we error here? + */ + rv = parse_relation_message(queued_message->message); + + /* If in list of relations which are being synchronized, skip. */ + if (!should_apply_changes_for_rel(rv->schemaname, rv->relname)) + return; + + truncate_table(rv->schemaname, rv->relname); +} + +/* + * Handle TABLESYNC message comming via queue table. + */ +static void +handle_table_sync(QueuedMessage *queued_message) +{ + RangeVar *rv; + MemoryContext oldcontext; + PGLogicalSyncStatus *oldsync; + PGLogicalSyncStatus *newsync; + + rv = parse_relation_message(queued_message->message); + + oldsync = get_table_sync_status(MyApplyWorker->subid, rv->schemaname, + rv->relname, true); + + if (oldsync) + { + elog(INFO, + "table sync came from queue for table %s.%s which already being synchronized, skipping", + rv->schemaname, rv->relname); + + return; + } + + /* Keep the lists persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + newsync = palloc0(sizeof(PGLogicalSyncStatus)); + MemoryContextSwitchTo(oldcontext); + + newsync->kind = SYNC_KIND_DATA; + newsync->subid = MyApplyWorker->subid; + newsync->status = SYNC_STATUS_INIT; + namestrcpy(&newsync->nspname, rv->schemaname); + namestrcpy(&newsync->relname, rv->relname); + create_local_sync_status(newsync); + + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + MemoryContextSwitchTo(oldcontext); + + MyApplyWorker->sync_pending = true; +} + +/* + * Handle SEQUENCE message comming via queue table. + */ +static void +handle_sequence(QueuedMessage *queued_message) +{ + Jsonb *message = queued_message->message; + JsonbIterator *it; + JsonbValue v; + int r; + int level = 0; + char *key = NULL; + char **parse_res = NULL; + char *nspname = NULL; + char *relname = NULL; + char *last_value_raw = NULL; + int64 last_value; + Oid nspoid; + Oid reloid; + + /* Parse and validate the json message. */ + if (!JB_ROOT_IS_OBJECT(message)) + elog(ERROR, "malformed message in queued message tuple: root is not object"); + + it = JsonbIteratorInit(&message->root); + while ((r = JsonbIteratorNext(&it, &v, false)) != WJB_DONE) + { + if (level == 0 && r != WJB_BEGIN_OBJECT) + elog(ERROR, "root element needs to be an object"); + else if (level == 0 && r == WJB_BEGIN_OBJECT) + { + level++; + } + else if (level == 1 && r == WJB_KEY) + { + if (strncmp(v.val.string.val, "schema_name", v.val.string.len) == 0) + parse_res = &nspname; + else if (strncmp(v.val.string.val, "sequence_name", v.val.string.len) == 0) + parse_res = &relname; + else if (strncmp(v.val.string.val, "last_value", v.val.string.len) == 0) + parse_res = &last_value_raw; + else + elog(ERROR, "unexpected key: %s", + pnstrdup(v.val.string.val, v.val.string.len)); + + key = v.val.string.val; + } + else if (level == 1 && r == WJB_VALUE) + { + if (!key) + elog(ERROR, "in wrong state when parsing key"); + + if (v.type != jbvString) + elog(ERROR, "unexpected type for key '%s': %u", key, v.type); + + *parse_res = pnstrdup(v.val.string.val, v.val.string.len); + } + else if (level == 1 && r != WJB_END_OBJECT) + { + elog(ERROR, "unexpected content: %u at level %d", r, level); + } + else if (r == WJB_END_OBJECT) + { + level--; + parse_res = NULL; + key = NULL; + } + else + elog(ERROR, "unexpected content: %u at level %d", r, level); + + } + + /* Check if we got both schema and table names. */ + if (!nspname) + elog(ERROR, "missing schema_name in sequence message"); + + if (!relname) + elog(ERROR, "missing table_name in sequence message"); + + if (!last_value_raw) + elog(ERROR, "missing last_value in sequence message"); + + nspoid = get_namespace_oid(nspname, false); + reloid = get_relname_relid(relname, nspoid); + last_value = strtoll(last_value_raw, NULL, 10); + + DirectFunctionCall2(setval_oid, ObjectIdGetDatum(reloid), + Int64GetDatum(last_value)); +} +/* + * Handle SQL message comming via queue table. + */ +static void +handle_sql(QueuedMessage *queued_message, bool tx_just_started) +{ + JsonbIterator *it; + JsonbValue v; + int r; + char *sql; + + /* Validate the json and extract the SQL string from it. */ + if (!JB_ROOT_IS_SCALAR(queued_message->message)) + elog(ERROR, "malformed message in queued message tuple: root is not scalar"); + + it = JsonbIteratorInit(&queued_message->message->root); + r = JsonbIteratorNext(&it, &v, false); + if (r != WJB_BEGIN_ARRAY) + elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_BEGIN_ARRAY); + + r = JsonbIteratorNext(&it, &v, false); + if (r != WJB_ELEM) + elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_ELEM); + + if (v.type != jbvString) + elog(ERROR, "malformed message in queued message tuple, expected value type %d got %d", jbvString, v.type); + + sql = pnstrdup(v.val.string.val, v.val.string.len); + + r = JsonbIteratorNext(&it, &v, false); + if (r != WJB_END_ARRAY) + elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_END_ARRAY); + + r = JsonbIteratorNext(&it, &v, false); + if (r != WJB_DONE) + elog(ERROR, "malformed message in queued message tuple, item type %d expected %d", r, WJB_DONE); + + /* Run the extracted SQL. */ + pglogical_execute_sql_command(sql, queued_message->role, tx_just_started); +} + +/* + * Handles messages comming from the queue. + */ +static void +handle_queued_message(HeapTuple msgtup, bool tx_just_started) +{ + QueuedMessage *queued_message; + const char *old_action_name; + + old_action_name = errcallback_arg.action_name; + errcallback_arg.is_ddl_or_drop = true; + + queued_message = queued_message_from_tuple(msgtup); + + switch (queued_message->message_type) + { + case QUEUE_COMMAND_TYPE_SQL: + errcallback_arg.action_name = "QUEUED_SQL"; + handle_sql(queued_message, tx_just_started); + break; + case QUEUE_COMMAND_TYPE_TRUNCATE: + errcallback_arg.action_name = "QUEUED_TRUNCATE"; + handle_truncate(queued_message); + break; + case QUEUE_COMMAND_TYPE_TABLESYNC: + errcallback_arg.action_name = "QUEUED_TABLESYNC"; + handle_table_sync(queued_message); + break; + case QUEUE_COMMAND_TYPE_SEQUENCE: + errcallback_arg.action_name = "QUEUED_SEQUENCE"; + handle_sequence(queued_message); + break; + default: + elog(ERROR, "unknown message type '%c'", + queued_message->message_type); + } + + errcallback_arg.action_name = old_action_name; + errcallback_arg.is_ddl_or_drop = false; +} + +static void +replication_handler(StringInfo s) +{ + ErrorContextCallback errcallback; + char action = pq_getmsgbyte(s); + + memset(&errcallback_arg, 0, sizeof(struct ActionErrCallbackArg)); + errcallback.callback = action_error_callback; + errcallback.arg = &errcallback_arg; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + Assert(CurrentMemoryContext == MessageContext); + + switch (action) + { + /* BEGIN */ + case 'B': + handle_begin(s); + break; + /* COMMIT */ + case 'C': + handle_commit(s); + break; + /* ORIGIN */ + case 'O': + handle_origin(s); + break; + /* RELATION */ + case 'R': + handle_relation(s); + break; + /* INSERT */ + case 'I': + handle_insert(s); + break; + /* UPDATE */ + case 'U': + handle_update(s); + break; + /* DELETE */ + case 'D': + handle_delete(s); + break; + /* STARTUP MESSAGE */ + case 'S': + handle_startup(s); + break; + default: + elog(ERROR, "unknown action of type %c", action); + } + + Assert(CurrentMemoryContext == MessageContext); + + if (error_context_stack == &errcallback) + error_context_stack = errcallback.previous; + + if (action == 'C') + { + /* + * We clobber MessageContext on commit. It doesn't matter much when we + * do it so long as we do so periodically, to prevent the context from + * growing too much. We might want to clean it up even 'n'th message + * too, but that adds testing burden and isn't done for now. + */ + MemoryContextReset(MessageContext); + } +} + +/* + * Figure out which write/flush positions to report to the walsender process. + * + * We can't simply report back the last LSN the walsender sent us because the + * local transaction might not yet be flushed to disk locally. Instead we + * build a list that associates local with remote LSNs for every commit. When + * reporting back the flush position to the sender we iterate that list and + * check which entries on it are already locally flushed. Those we can report + * as having been flushed. + * + * Returns true if there's no outstanding transactions that need to be + * flushed. + */ +static bool +get_flush_position(XLogRecPtr *write, XLogRecPtr *flush) +{ + dlist_mutable_iter iter; + XLogRecPtr local_flush = GetFlushRecPtr(); + + *write = InvalidXLogRecPtr; + *flush = InvalidXLogRecPtr; + + dlist_foreach_modify(iter, &lsn_mapping) + { + PGLFlushPosition *pos = + dlist_container(PGLFlushPosition, node, iter.cur); + + *write = pos->remote_end; + + if (pos->local_end <= local_flush) + { + *flush = pos->remote_end; + dlist_delete(iter.cur); + pfree(pos); + } + else + { + /* + * Don't want to uselessly iterate over the rest of the list which + * could potentially be long. Instead get the last element and + * grab the write position from there. + */ + pos = dlist_tail_element(PGLFlushPosition, node, + &lsn_mapping); + *write = pos->remote_end; + return false; + } + } + + return dlist_is_empty(&lsn_mapping); +} + +/* + * Send a Standby Status Update message to server. + * + * 'recvpos' is the latest LSN we've received data to, force is set if we need + * to send a response to avoid timeouts. + */ +static bool +send_feedback(PGconn *conn, XLogRecPtr recvpos, int64 now, bool force) +{ + static StringInfo reply_message = NULL; + + static XLogRecPtr last_recvpos = InvalidXLogRecPtr; + static XLogRecPtr last_writepos = InvalidXLogRecPtr; + static XLogRecPtr last_flushpos = InvalidXLogRecPtr; + + XLogRecPtr writepos; + XLogRecPtr flushpos; + + /* It's legal to not pass a recvpos */ + if (recvpos < last_recvpos) + recvpos = last_recvpos; + + if (get_flush_position(&writepos, &flushpos)) + { + /* + * No outstanding transactions to flush, we can report the latest + * received position. This is important for synchronous replication. + */ + flushpos = writepos = recvpos; + } + + if (writepos < last_writepos) + writepos = last_writepos; + + if (flushpos < last_flushpos) + flushpos = last_flushpos; + + /* if we've already reported everything we're good */ + if (!force && + writepos == last_writepos && + flushpos == last_flushpos) + return true; + + if (!reply_message) + { + MemoryContext oldcontext = MemoryContextSwitchTo(TopMemoryContext); + reply_message = makeStringInfo(); + MemoryContextSwitchTo(oldcontext); + } + else + resetStringInfo(reply_message); + + pq_sendbyte(reply_message, 'r'); + pq_sendint64(reply_message, recvpos); /* write */ + pq_sendint64(reply_message, flushpos); /* flush */ + pq_sendint64(reply_message, writepos); /* apply */ + pq_sendint64(reply_message, now); /* sendTime */ + pq_sendbyte(reply_message, false); /* replyRequested */ + + elog(DEBUG2, "sending feedback (force %d) to recv %X/%X, write %X/%X, flush %X/%X", + force, + (uint32) (recvpos >> 32), (uint32) recvpos, + (uint32) (writepos >> 32), (uint32) writepos, + (uint32) (flushpos >> 32), (uint32) flushpos + ); + + if (PQputCopyData(conn, reply_message->data, reply_message->len) <= 0 || + PQflush(conn)) + { + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("could not send feedback packet: %s", + PQerrorMessage(conn)))); + return false; + } + + if (recvpos > last_recvpos) + last_recvpos = recvpos; + if (writepos > last_writepos) + last_writepos = writepos; + if (flushpos > last_flushpos) + last_flushpos = flushpos; + + return true; +} + +/* + * Apply main loop. + */ +void +apply_work(PGconn *streamConn) +{ + int fd; + char *copybuf = NULL; + XLogRecPtr last_received = InvalidXLogRecPtr; + + applyconn = streamConn; + fd = PQsocket(applyconn); + + /* Init the MessageContext which we use for easier cleanup. */ + MessageContext = AllocSetContextCreate(TopMemoryContext, + "MessageContext", + ALLOCSET_DEFAULT_SIZES); + + MemoryContextSwitchTo(MessageContext); + + /* mark as idle, before starting to loop */ + pgstat_report_activity(STATE_IDLE, NULL); + Assert(CurrentMemoryContext == MessageContext); + + while (!got_SIGTERM) + { + int rc; + int r; + + CHECK_FOR_INTERRUPTS(); + + /* + * Background workers mustn't call usleep() or any direct equivalent: + * instead, they may wait on their process latch, which sleeps as + * necessary, but is awakened if postmaster dies. That way the + * background process goes away immediately in an emergency. + */ + rc = WaitLatchOrSocket(&MyProc->procLatch, + WL_SOCKET_READABLE | WL_LATCH_SET | + WL_TIMEOUT | WL_POSTMASTER_DEATH, + fd, 1000L); + + ResetLatch(&MyProc->procLatch); + + Assert(CurrentMemoryContext == MessageContext); + + /* emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + if (rc & WL_SOCKET_READABLE) + PQconsumeInput(applyconn); + + if (PQstatus(applyconn) == CONNECTION_BAD) + { + elog(ERROR, "connection to other side has died"); + } + + Assert(CurrentMemoryContext == MessageContext); + + for (;;) + { + if (got_SIGTERM) + break; + + /* We must not have fallen out of MessageContext by accident */ + Assert(CurrentMemoryContext == MessageContext); + + Assert(copybuf == NULL); + r = PQgetCopyData(applyconn, ©buf, 1); + + if (r == -1) + { + elog(ERROR, "data stream ended"); + } + else if (r == -2) + { + elog(ERROR, "could not read COPY data: %s", + PQerrorMessage(applyconn)); + } + else if (r < 0) + elog(ERROR, "invalid COPY status %d", r); + else if (r == 0) + { + /* need to wait for new data */ + break; + } + else + { + int c; + StringInfoData s; + + /* + * We're using a StringInfo to wrap existing data here, as a + * cursor. We init it manually to avoid a redundant allocation. + */ + memset(&s, 0, sizeof(StringInfoData)); + s.data = copybuf; + s.len = r; + s.maxlen = -1; + s.cursor = 0; + + c = pq_getmsgbyte(&s); + + if (c == 'w') + { + XLogRecPtr start_lsn; + XLogRecPtr end_lsn; + + start_lsn = pq_getmsgint64(&s); + end_lsn = pq_getmsgint64(&s); + pq_getmsgint64(&s); /* sendTime */ + + if (last_received < start_lsn) + last_received = start_lsn; + + if (last_received < end_lsn) + last_received = end_lsn; + + replication_handler(&s); + } + else if (c == 'k') + { + XLogRecPtr endpos; + bool reply_requested; + + endpos = pq_getmsgint64(&s); + /* timestamp = */ pq_getmsgint64(&s); + reply_requested = pq_getmsgbyte(&s); + + send_feedback(applyconn, endpos, + GetCurrentTimestamp(), + reply_requested); + + if (last_received < endpos) + last_received = endpos; + + /* + * For sync workers in catchup mode, check if we've reached + * the stop position based on the keepalive. This is needed + * because table-specific replication slots may not receive + * any actual commits if no changes are made to the table, + * but keepalives tell us the provider's current position. + */ + if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_SYNC && + MyApplyWorker->replay_stop_lsn != InvalidXLogRecPtr && + MyApplyWorker->replay_stop_lsn <= endpos) + { + ereport(LOG, + (errmsg("pglogical sync finished processing via keepalive; provider at %X/%X, required %X/%X", + (uint32)(endpos>>32), (uint32)endpos, + (uint32)(MyApplyWorker->replay_stop_lsn >>32), + (uint32)MyApplyWorker->replay_stop_lsn))); + + StartTransactionCommand(); + set_table_sync_status(MyApplyWorker->subid, + NameStr(MyPGLogicalWorker->worker.sync.nspname), + NameStr(MyPGLogicalWorker->worker.sync.relname), + SYNC_STATUS_SYNCDONE, endpos); + CommitTransactionCommand(); + + XLogFlush(GetXLogWriteRecPtr()); + PQfinish(applyconn); + pglogical_sync_worker_finish(); + proc_exit(0); + } + } + /* other message types are purposefully ignored */ + + /* copybuf is malloc'd not palloc'd */ + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + } + + /* We must not have fallen out of MessageContext by accident */ + Assert(CurrentMemoryContext == MessageContext); + } + + /* confirm all writes at once */ + send_feedback(applyconn, last_received, GetCurrentTimestamp(), false); + + if (!in_remote_transaction) + process_syncing_tables(last_received); + + /* We must not have switched out of MessageContext by mistake */ + Assert(CurrentMemoryContext == MessageContext); + + /* Cleanup the memory. */ + MemoryContextResetAndDeleteChildren(MessageContext); + + /* + * Only do a leak check if we're between txns; we don't want lots of + * noise due to resources that only exist in a txn. + */ + if (!IsTransactionState()) + { + VALGRIND_DO_ADDED_LEAK_CHECK; + } + } +} + +/* + * Add context to the errors produced by pglogical_execute_sql_command(). + */ +static void +execute_sql_command_error_cb(void *arg) +{ + errcontext("during execution of queued SQL statement: %s", (char *) arg); +} + +/* + * Execute an SQL command. This can be multiple multiple queries. + */ +void +pglogical_execute_sql_command(char *cmdstr, char *role, bool isTopLevel) +{ + const char *save_debug_query_string = debug_query_string; + List *commands; + ListCell *command_i; +#ifdef PGXC + List *commandSourceQueries; + ListCell *commandSourceQuery_i; +#endif + MemoryContext oldcontext; + ErrorContextCallback errcallback; + + oldcontext = MemoryContextSwitchTo(MessageContext); + + errcallback.callback = execute_sql_command_error_cb; + errcallback.arg = cmdstr; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + debug_query_string = cmdstr; + + /* + * XL distributes individual statements using just executing them as plain + * SQL query and can't handle multistatements this way so we need to get + * individual statements using API provided by XL itself. + */ +#ifdef PGXC + commands = pg_parse_query_get_source(cmdstr, &commandSourceQueries); +#else + commands = pg_parse_query(cmdstr); +#endif + + MemoryContextSwitchTo(oldcontext); + + /* + * Do a limited amount of safety checking against CONCURRENTLY commands + * executed in situations where they aren't allowed. The sender side should + * provide protection, but better be safe than sorry. + */ + isTopLevel = isTopLevel && (list_length(commands) == 1); + +#ifdef PGXC + forboth(command_i, commands, commandSourceQuery_i, commandSourceQueries) +#else + foreach(command_i, commands) +#endif + { + List *plantree_list; + List *querytree_list; + RawStmt *command = (RawStmt *) lfirst(command_i); + CommandTag commandTag; + Portal portal; + int save_nestlevel; + DestReceiver *receiver; + +#ifdef PGXC + cmdstr = (char *) lfirst(commandSourceQuery_i); + errcallback.arg = cmdstr; +#endif + + /* temporarily push snapshot for parse analysis/planning */ + PushActiveSnapshot(GetTransactionSnapshot()); + + oldcontext = MemoryContextSwitchTo(MessageContext); + + /* + * Set the current role to the user that executed the command on the + * origin server. + */ + save_nestlevel = NewGUCNestLevel(); + SetConfigOption("role", role, PGC_INTERNAL, PGC_S_OVERRIDE); + + commandTag = CreateCommandTag(command); + + querytree_list = pg_analyze_and_rewrite( + command, + cmdstr, + NULL, 0); + + plantree_list = pg_plan_queries( + querytree_list, cmdstr, 0, NULL); + + PopActiveSnapshot(); + + portal = CreatePortal("pglogical", true, true); + PortalDefineQuery(portal, NULL, + cmdstr, + commandTag, + plantree_list, NULL); + PortalStart(portal, NULL, 0, InvalidSnapshot); + + receiver = CreateDestReceiver(DestNone); + + (void) PortalRun(portal, FETCH_ALL, + isTopLevel, + receiver, receiver, + NULL); + (*receiver->rDestroy) (receiver); + + PortalDrop(portal, false); + + CommandCounterIncrement(); + + /* + * Restore the GUC variables we set above. + */ + AtEOXact_GUC(true, save_nestlevel); + + MemoryContextSwitchTo(oldcontext); + } + + /* protect against stack resets during CONCURRENTLY processing */ + if (error_context_stack == &errcallback) + error_context_stack = errcallback.previous; + + debug_query_string = save_debug_query_string; +} + +/* + * Load list of tables currently pending sync. + * + * Must be inside transaction. + */ +static void +reread_unsynced_tables(Oid subid) +{ + MemoryContext saved_ctx; + List *unsynced_tables; + ListCell *lc; + + /* Cleanup first. */ + list_free_deep(SyncingTables); + SyncingTables = NIL; + + /* Read new state. */ + unsynced_tables = get_unsynced_tables(subid); + saved_ctx = MemoryContextSwitchTo(TopMemoryContext); + foreach (lc, unsynced_tables) + { + PGLogicalSyncStatus *sync = palloc(sizeof(PGLogicalSyncStatus)); + memcpy(sync, lfirst(lc), sizeof(PGLogicalSyncStatus)); + SyncingTables = lappend(SyncingTables, sync); + } + + MemoryContextSwitchTo(saved_ctx); +} + +static void +process_syncing_tables(XLogRecPtr end_lsn) +{ + ListCell *lc; + + Assert(CurrentMemoryContext == MessageContext); + Assert(!IsTransactionState()); + + /* First check if we need to update the cached information. */ + if (MyApplyWorker->sync_pending) + { + StartTransactionCommand(); + MyApplyWorker->sync_pending = false; + reread_unsynced_tables(MyApplyWorker->subid); + CommitTransactionCommand(); + MemoryContextSwitchTo(MessageContext); + } + + /* Process currently pending sync tables. */ + if (list_length(SyncingTables) > 0) + { +#if PG_VERSION_NUM < 130000 + ListCell *prev = NULL; + ListCell *next; +#endif + +#if PG_VERSION_NUM >= 130000 + foreach(lc, SyncingTables) +#else + for (lc = list_head(SyncingTables); lc; lc = next) +#endif + { + PGLogicalSyncStatus *sync = (PGLogicalSyncStatus *) lfirst(lc); + PGLogicalSyncStatus *newsync; + +#if PG_VERSION_NUM < 130000 + /* We might delete the cell so advance it now. */ + next = lnext(lc); +#endif + + StartTransactionCommand(); + newsync = get_table_sync_status(MyApplyWorker->subid, + NameStr(sync->nspname), + NameStr(sync->relname), true); + + /* + * TODO: what to do here? We don't really want to die, + * but this can mean many things, for now we just assume table is + * not relevant for us anymore and leave fixing to the user. + * + * The reason why this part happens in transaction is that the + * memory allocated for sync info will get automatically cleaned + * afterwards. + */ + if (!newsync) + { + sync->status = SYNC_STATUS_READY; + sync->statuslsn = InvalidXLogRecPtr; + } + else + memcpy(sync, newsync, sizeof(PGLogicalSyncStatus)); + CommitTransactionCommand(); + MemoryContextSwitchTo(MessageContext); + + if (sync->status == SYNC_STATUS_SYNCWAIT) + { + PGLogicalWorker *worker; + + LWLockAcquire(PGLogicalCtx->lock, LW_EXCLUSIVE); + worker = pglogical_sync_find(MyDatabaseId, + MyApplyWorker->subid, + NameStr(sync->nspname), + NameStr(sync->relname)); + + if (pglogical_worker_running(worker) && + end_lsn >= worker->worker.apply.replay_stop_lsn) + { + worker->worker.apply.replay_stop_lsn = end_lsn; + sync->status = SYNC_STATUS_CATCHUP; + + StartTransactionCommand(); + set_table_sync_status(MyApplyWorker->subid, + NameStr(sync->nspname), + NameStr(sync->relname), + sync->status, + sync->statuslsn); + CommitTransactionCommand(); + MemoryContextSwitchTo(MessageContext); + + if (pglogical_worker_running(worker)) + SetLatch(&worker->proc->procLatch); + LWLockRelease(PGLogicalCtx->lock); + + if (wait_for_sync_status_change(MyApplyWorker->subid, + NameStr(sync->nspname), + NameStr(sync->relname), + SYNC_STATUS_SYNCDONE, + &sync->statuslsn)) + sync->status = SYNC_STATUS_SYNCDONE; + } + else + LWLockRelease(PGLogicalCtx->lock); + } + + if (sync->status == SYNC_STATUS_SYNCDONE && + end_lsn >= sync->statuslsn) + { + sync->status = SYNC_STATUS_READY; + sync->statuslsn = end_lsn; + + StartTransactionCommand(); + set_table_sync_status(MyApplyWorker->subid, + NameStr(sync->nspname), + NameStr(sync->relname), + sync->status, + sync->statuslsn); + CommitTransactionCommand(); + MemoryContextSwitchTo(MessageContext); + } + + /* Ready? Remove it from local cache. */ + if (sync->status == SYNC_STATUS_READY) + { +#if PG_VERSION_NUM >= 130000 + SyncingTables = foreach_delete_current(SyncingTables, lc); +#else + SyncingTables = list_delete_cell(SyncingTables, lc, prev); +#endif + pfree(sync); + } + else + { +#if PG_VERSION_NUM < 130000 + prev = lc; +#endif + } + } + } + + /* + * If there are still pending tables for synchronization, launch the sync + * worker. + */ + foreach (lc, SyncingTables) + { + List *workers; + ListCell *wlc; + int nworkers = 0; + PGLogicalSyncStatus *sync = (PGLogicalSyncStatus *) lfirst(lc); + + if (sync->status == SYNC_STATUS_SYNCDONE || sync->status == SYNC_STATUS_READY) + continue; + + LWLockAcquire(PGLogicalCtx->lock, LW_EXCLUSIVE); + workers = pglogical_sync_find_all(MyDatabaseId, MyApplyWorker->subid); + foreach (wlc, workers) + { + PGLogicalWorker *worker = (PGLogicalWorker *) lfirst(wlc); + + if (pglogical_worker_running(worker)) + nworkers++; + } + LWLockRelease(PGLogicalCtx->lock); + + if (nworkers < 1) + { + start_sync_worker(&sync->nspname, &sync->relname); + break; + } + } + + Assert(CurrentMemoryContext == MessageContext); +} + +static void +start_sync_worker(Name nspname, Name relname) +{ + PGLogicalWorker worker; + + /* Start the sync worker. */ + memset(&worker, 0, sizeof(PGLogicalWorker)); + worker.worker_type = PGLOGICAL_WORKER_SYNC; + worker.dboid = MyPGLogicalWorker->dboid; + worker.worker.apply.subid = MyApplyWorker->subid; + worker.worker.apply.sync_pending = false; /* Makes no sense for sync worker. */ + + /* Tell the worker to stop at current position. */ + worker.worker.sync.apply.replay_stop_lsn = replorigin_session_origin_lsn; + memcpy(&worker.worker.sync.nspname, nspname, sizeof(NameData)); + memcpy(&worker.worker.sync.relname, relname, sizeof(NameData)); + + (void) pglogical_worker_register(&worker); +} + +static inline TimeOffset +interval_to_timeoffset(const Interval *interval) +{ + TimeOffset span; + + span = interval->time; + +#ifdef HAVE_INT64_TIMESTAMP + span += interval->month * INT64CONST(30) * USECS_PER_DAY; + span += interval->day * INT64CONST(24) * USECS_PER_HOUR; +#else + span += interval->month * ((double) DAYS_PER_MONTH * SECS_PER_DAY); + span += interval->day * ((double) HOURS_PER_DAY * SECS_PER_HOUR); +#endif + + return span; +} + +void +pglogical_apply_main(Datum main_arg) +{ + int slot = DatumGetInt32(main_arg); + PGconn *streamConn; + RepOriginId originid; + XLogRecPtr origin_startpos; + MemoryContext saved_ctx; + char *repsets; + char *origins; + + /* Setup shmem. */ + pglogical_worker_attach(slot, PGLOGICAL_WORKER_APPLY); + Assert(MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_APPLY); + MyApplyWorker = &MyPGLogicalWorker->worker.apply; + + /* Attach to dsm segment. */ + Assert(CurrentResourceOwner == NULL); + CurrentResourceOwner = ResourceOwnerCreate(NULL, "pglogical apply"); + + /* Load correct apply API. */ + if (pglogical_use_spi) + { + if (pglogical_conflict_resolver != PGLOGICAL_RESOLVE_ERROR) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("pglogical.use_spi can only be used when " + "pglogical.conflict_resolution is set to 'error'"))); + + apply_api.on_begin = pglogical_apply_spi_begin; + apply_api.on_commit = pglogical_apply_spi_commit; + apply_api.do_insert = pglogical_apply_spi_insert; + apply_api.do_update = pglogical_apply_spi_update; + apply_api.do_delete = pglogical_apply_spi_delete; + apply_api.can_multi_insert = pglogical_apply_spi_can_mi; + apply_api.multi_insert_add_tuple = pglogical_apply_spi_mi_add_tuple; + apply_api.multi_insert_finish = pglogical_apply_spi_mi_finish; + } + + /* Setup synchronous commit according to the user's wishes */ + SetConfigOption("synchronous_commit", + pglogical_synchronous_commit ? "local" : "off", + PGC_BACKEND, PGC_S_OVERRIDE); /* other context? */ + + /* Run as replica session replication role. */ + SetConfigOption("session_replication_role", "replica", + PGC_SUSET, PGC_S_OVERRIDE); /* other context? */ + + /* + * Disable function body checks during replay. That's necessary because a) + * the creator of the function might have had it disabled b) the function + * might be search_path dependant and we don't fix the contents of + * functions. + */ + SetConfigOption("check_function_bodies", "off", + PGC_INTERNAL, PGC_S_OVERRIDE); + + /* Load the subscription. */ + StartTransactionCommand(); + saved_ctx = MemoryContextSwitchTo(TopMemoryContext); + MySubscription = get_subscription(MyApplyWorker->subid); + MemoryContextSwitchTo(saved_ctx); + +#ifdef XCP + /* + * When runnin under XL, initialise the XL executor so that the datanode + * and coordinator information is initialised properly. + */ + InitMultinodeExecutor(false); +#endif + CommitTransactionCommand(); + + elog(LOG, "starting apply for subscription %s", MySubscription->name); + + /* Set apply delay if any. */ + if (MySubscription->apply_delay) + apply_delay = + interval_to_timeoffset(MySubscription->apply_delay) / 1000; + + /* If the subscription isn't initialized yet, initialize it. */ + pglogical_sync_subscription(MySubscription); + + elog(DEBUG1, "connecting to provider %s, dsn %s", + MySubscription->origin->name, MySubscription->origin_if->dsn); + + /* + * Cache the queue relation id. + * TODO: invalidation + */ + StartTransactionCommand(); + QueueRelid = get_queue_table_oid(); + + originid = replorigin_by_name(MySubscription->slot_name, false); + elog(DEBUG2, "setting up replication origin %s (oid %u)", + MySubscription->slot_name, originid); + PGLreplorigin_session_setup(originid); + replorigin_session_origin = originid; + origin_startpos = replorigin_session_get_progress(false); + + /* Start the replication. */ + streamConn = pglogical_connect_replica(MySubscription->origin_if->dsn, + MySubscription->name, NULL); + + repsets = stringlist_to_identifierstr(MySubscription->replication_sets); + origins = stringlist_to_identifierstr(MySubscription->forward_origins); + + /* + * IDENTIFY_SYSTEM sets up some internal state on walsender so call it even + * if we don't (yet) want to use any of the results. + */ + pglogical_identify_system(streamConn, NULL, NULL, NULL, NULL); + + pglogical_start_replication(streamConn, MySubscription->slot_name, + origin_startpos, origins, repsets, NULL, + MySubscription->force_text_transfer); + pfree(repsets); + + CommitTransactionCommand(); + + /* + * Do an initial leak check with reporting off; we don't want to see + * these results, just the later output from ADDED leak checks. + */ + VALGRIND_DISABLE_ERROR_REPORTING; + VALGRIND_DO_LEAK_CHECK; + VALGRIND_ENABLE_ERROR_REPORTING; + + apply_work(streamConn); + + PQfinish(streamConn); + + /* We should only get here if we received sigTERM */ + proc_exit(0); +} diff --git a/pglogical_apply_heap.c b/pglogical_apply_heap.c index 83dc7ce1..9ea6602e 100644 --- a/pglogical_apply_heap.c +++ b/pglogical_apply_heap.c @@ -111,6 +111,11 @@ typedef struct ApplyMIState * Compute stored generated columns if the relation has any. * This must be called after storing the tuple in the slot but before * the actual insert/update operation. + * + * Note: ExecComputeStoredGenerated signature varies across PostgreSQL versions: + * - PG 12: 2 args (estate, slot) + * - PG 13: 3 args (estate, slot, cmdtype) - backported from later versions + * - PG 14+: 4 args (resultRelInfo, estate, slot, cmdtype) */ #if PG_VERSION_NUM >= 140000 #define PGL_COMPUTE_GENERATED(resultRelInfo, estate, slot, cmdtype) \ @@ -119,7 +124,16 @@ typedef struct ApplyMIState if (_tupdesc->constr && _tupdesc->constr->has_generated_stored) \ ExecComputeStoredGenerated(resultRelInfo, estate, slot, cmdtype); \ } while (0) +#elif PG_VERSION_NUM >= 130000 +/* PG 13 uses 3-argument version (cmdtype was backported) */ +#define PGL_COMPUTE_GENERATED(resultRelInfo, estate, slot, cmdtype) \ + do { \ + TupleDesc _tupdesc = RelationGetDescr((resultRelInfo)->ri_RelationDesc); \ + if (_tupdesc->constr && _tupdesc->constr->has_generated_stored) \ + ExecComputeStoredGenerated(estate, slot, cmdtype); \ + } while (0) #elif PG_VERSION_NUM >= 120000 +/* PG 12 uses 2-argument version */ #define PGL_COMPUTE_GENERATED(resultRelInfo, estate, slot, cmdtype) \ do { \ TupleDesc _tupdesc = RelationGetDescr((resultRelInfo)->ri_RelationDesc); \ diff --git a/pglogical_create_subscriber.c b/pglogical_create_subscriber.c index 088fc522..93e26e17 100644 --- a/pglogical_create_subscriber.c +++ b/pglogical_create_subscriber.c @@ -41,6 +41,7 @@ #include #include #include +#include #ifndef WIN32 #include #endif diff --git a/pglogical_dependency.c b/pglogical_dependency.c index 728a0dc4..dafe2415 100644 --- a/pglogical_dependency.c +++ b/pglogical_dependency.c @@ -1103,7 +1103,7 @@ find_expr_references_walker(Node *node, if (var->varlevelsup >= (Index) list_length(context->rtables)) elog(ERROR, "invalid varlevelsup %u", var->varlevelsup); rtable = (List *) list_nth(context->rtables, var->varlevelsup); - if (var->varno <= 0 || var->varno > list_length(rtable)) + if (var->varno == 0 || (Index) var->varno > (Index) list_length(rtable)) elog(ERROR, "invalid varno %d", var->varno); rte = rt_fetch(var->varno, rtable); diff --git a/pglogical_functions.c b/pglogical_functions.c index e644011e..5b934aa5 100644 --- a/pglogical_functions.c +++ b/pglogical_functions.c @@ -604,8 +604,17 @@ pglogical_drop_subscription(PG_FUNCTION_ARGS) CHECK_FOR_INTERRUPTS(); + /* + * Wait for worker termination. On Windows, use a shorter interval + * for faster cleanup since process termination can be slower. + */ +#ifdef WIN32 + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 200L); +#else rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 1000L); +#endif if (rc & WL_POSTMASTER_DEATH) proc_exit(1); @@ -1140,8 +1149,20 @@ pglogical_show_subscription_status(PG_FUNCTION_ARGS) } else if (!sub->enabled) status = "disabled"; + else if (apply != NULL) + { + /* Worker slot exists but worker not attached yet. */ + status = "down"; + } else + { + /* + * No worker slot at all. Either the subscription was just created + * and manager hasn't registered a worker yet, or the worker was + * killed (e.g., interface change) and slot was cleared. + */ status = "down"; + } LWLockRelease(PGLogicalCtx->lock); values[0] = CStringGetTextDatum(sub->name); @@ -2259,9 +2280,15 @@ pglogical_wait_for_sync_complete(char *subscription_name, char *relnamespace, ch */ if (relname != NULL) { - PGLogicalSyncStatus *table = get_table_sync_status(sub->id, relnamespace, relname, false); - isdone = table && table->status == SYNC_STATUS_READY; - free_sync_status(table); + /* + * Use missing_ok=true because the sync status entry may not + * exist yet if the apply worker hasn't processed the + * TABLESYNC queue message. In that case, we continue waiting. + */ + PGLogicalSyncStatus *table = get_table_sync_status(sub->id, relnamespace, relname, true); + isdone = table != NULL && table->status == SYNC_STATUS_READY; + if (table) + free_sync_status(table); } else { @@ -2284,7 +2311,18 @@ pglogical_wait_for_sync_complete(char *subscription_name, char *relnamespace, ch PopActiveSnapshot(); if (isdone) + { +#ifdef WIN32 + /* + * On Windows, add a small delay after sync completes to allow + * the streaming connection to be fully established. Without this, + * pg_replication_slots may show the slot as inactive when checked + * immediately after sync completes. + */ + pg_usleep(500000); /* 500ms */ +#endif break; + } CHECK_FOR_INTERRUPTS(); diff --git a/pglogical_manager.c b/pglogical_manager.c index dfe3ade3..bbf97d45 100644 --- a/pglogical_manager.c +++ b/pglogical_manager.c @@ -34,7 +34,16 @@ #define INITIAL_SLEEP 10000L #define MAX_SLEEP 180000L +/* + * MIN_SLEEP is the minimum time the manager sleeps between checking for + * subscription changes. On Windows, process startup is slower and we need + * more responsive checking to avoid test timeouts. + */ +#ifdef WIN32 +#define MIN_SLEEP 1000L +#else #define MIN_SLEEP 5000L +#endif void PGDLLEXPORT pglogical_manager_main(Datum main_arg); @@ -59,10 +68,17 @@ manage_apply_workers(void) StartTransactionCommand(); - /* Get local node, exit if no found. */ + /* + * Get local node. If not found, return false and let the main loop wait + * and retry. This handles the race condition where the extension is created + * but create_node() hasn't been called yet. + */ node = get_local_node(true, true); if (!node) - proc_exit(0); + { + CommitTransactionCommand(); + return false; + } /* Get list of subscribers. */ subscriptions = get_node_subscriptions(node->node->id, false); @@ -202,18 +218,55 @@ pglogical_manager_main(Datum main_arg) CurrentResourceOwner = ResourceOwnerCreate(NULL, "pglogical manager"); - StartTransactionCommand(); + /* + * Check if the extension is installed. We retry a few times with a short + * delay to handle the race condition where the extension is being dropped + * and recreated (e.g., between init_fail and init regression tests). + * Without this retry, we might exit just before the extension is created, + * causing the supervisor to not restart us until the next latch timeout. + */ + for (int retry = 0; retry < 5; retry++) + { + StartTransactionCommand(); + extoid = get_extension_oid(EXTENSION_NAME, true); + CommitTransactionCommand(); + + if (OidIsValid(extoid)) + break; - /* If the extension is not installed in this DB, exit. */ - extoid = get_extension_oid(EXTENSION_NAME, true); + /* Extension not found, wait and retry */ + if (retry < 4) + { + int rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 200L); /* 200ms */ + ResetLatch(&MyProc->procLatch); + + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + CHECK_FOR_INTERRUPTS(); + } + } + + /* If the extension is still not installed after retries, exit. */ if (!OidIsValid(extoid)) proc_exit(0); + StartTransactionCommand(); + elog(LOG, "starting pglogical database manager for database %s", get_database_name(MyDatabaseId)); CommitTransactionCommand(); + /* + * Brief delay before checking/upgrading extension version. + * This avoids a race condition where an explicit ALTER EXTENSION UPDATE + * runs concurrently with our auto-upgrade check on fast systems. + */ + pg_usleep(100000); /* 100ms */ + /* Use separate transaction to avoid lock escalation. */ StartTransactionCommand(); pglogical_manage_extension(); diff --git a/pglogical_monitoring.c b/pglogical_monitoring.c index fa4fe51c..80eac56c 100644 --- a/pglogical_monitoring.c +++ b/pglogical_monitoring.c @@ -98,9 +98,19 @@ pglogical_wait_slot_confirm_lsn(PG_FUNCTION_ARGS) if (oldest_confirmed_lsn >= target_lsn) break; + /* + * Poll for slot progress. On Windows, use a shorter interval for + * faster response times since IPC can be slower. + */ +#ifdef WIN32 + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 200); +#else rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 1000); +#endif ResetLatch(&MyProc->procLatch); diff --git a/pglogical_output.def b/pglogical_output.def new file mode 100644 index 00000000..7526c76a --- /dev/null +++ b/pglogical_output.def @@ -0,0 +1,4 @@ +; pglogical_output Windows DLL exports +EXPORTS + _PG_output_plugin_init + Pg_magic_func diff --git a/pglogical_sync.c b/pglogical_sync.c index 2a92bdb4..2dc00a56 100644 --- a/pglogical_sync.c +++ b/pglogical_sync.c @@ -796,9 +796,19 @@ pglogical_sync_worker_cleanup(PGLogicalSubscription *sub) if (!pglogical_remote_slot_active(origin_conn, sub->slot_name)) break; + /* + * Wait for remote slot to become inactive. On Windows, use a shorter + * interval for faster response times. + */ +#ifdef WIN32 + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 200L); +#else rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 1000L); +#endif ResetLatch(&MyProc->procLatch); @@ -1887,9 +1897,19 @@ wait_for_sync_status_change(Oid subid, const char *nspname, const char *relname, if (!worker) break; + /* + * Wait for table sync progress. On Windows, use a shorter interval + * for faster detection of worker failures. + */ +#ifdef WIN32 + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + 10000L); +#else rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 60000L); +#endif ResetLatch(&MyProc->procLatch); diff --git a/pglogical_worker.c b/pglogical_worker.c index ccee0b61..8ac47e45 100644 --- a/pglogical_worker.c +++ b/pglogical_worker.c @@ -277,8 +277,18 @@ wait_for_worker_startup(PGLogicalWorker *worker, Assert(status == BGWH_NOT_YET_STARTED || status == BGWH_STARTED); + /* + * Poll for worker startup. On Windows, use a shorter interval since + * process creation is slower and we want to detect attachment to + * shared memory as quickly as possible. + */ +#ifdef WIN32 + rc = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 200L); +#else rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 1000L); +#endif if (rc & WL_POSTMASTER_DEATH) proc_exit(1); @@ -448,11 +458,26 @@ pglogical_worker_detach(bool crash) } else { - /* Worker has finished work, clean up its state from shmem. */ + /* + * Worker has finished work, clean up its state from shmem. + * If this was a manager, notify the supervisor so it can restart + * the manager if needed (e.g., if the database still has subscriptions). + */ + if (MyPGLogicalWorker->worker_type == PGLOGICAL_WORKER_MANAGER) + PGLogicalCtx->subscriptions_changed = true; + MyPGLogicalWorker->worker_type = PGLOGICAL_WORKER_NONE; MyPGLogicalWorker->dboid = InvalidOid; } + /* + * If we notified that subscriptions changed (manager exit), wake up the + * supervisor so it can restart the manager immediately rather than waiting + * for the latch timeout. + */ + if (PGLogicalCtx->subscriptions_changed && PGLogicalCtx->supervisor) + SetLatch(&PGLogicalCtx->supervisor->procLatch); + MyPGLogicalWorker = NULL; LWLockRelease(PGLogicalCtx->lock); diff --git a/scripts/prepare-pg-binaries.ps1 b/scripts/prepare-pg-binaries.ps1 new file mode 100644 index 00000000..0ecfa419 --- /dev/null +++ b/scripts/prepare-pg-binaries.ps1 @@ -0,0 +1,59 @@ +# Script to download and repackage PostgreSQL binaries for CI +# Run this locally, then upload the resulting ZIPs to GitHub release "pg-binaries" + +$versions = @{ + "13" = "13.20-1" + "14" = "14.17-1" + "15" = "15.12-1" + "16" = "16.8-1" + "17" = "17.4-1" + "18" = "18.0-1" +} + +$outputDir = ".\pg-binaries" +New-Item -ItemType Directory -Force -Path $outputDir | Out-Null + +foreach ($pgMajor in $versions.Keys) { + $pgVersion = $versions[$pgMajor] + $edbUrl = "https://get.enterprisedb.com/postgresql/postgresql-$pgVersion-windows-x64-binaries.zip" + $downloadPath = "$env:TEMP\postgresql-$pgMajor-edb.zip" + $extractPath = "$env:TEMP\pg-extract-$pgMajor" + $outputZip = "$outputDir\postgresql-$pgMajor-windows-x64.zip" + + Write-Host "Processing PostgreSQL $pgMajor..." -ForegroundColor Cyan + + # Download from EDB + if (-not (Test-Path $downloadPath)) { + Write-Host " Downloading from EDB..." + Invoke-WebRequest -Uri $edbUrl -OutFile $downloadPath -UseBasicParsing + } else { + Write-Host " Using cached download" + } + + # Extract + Write-Host " Extracting..." + Remove-Item -Recurse -Force $extractPath -ErrorAction SilentlyContinue + Expand-Archive -Path $downloadPath -DestinationPath $extractPath -Force + + # EDB ZIP extracts to pgsql/ folder - perfect, just rezip + Write-Host " Creating $outputZip..." + Remove-Item $outputZip -ErrorAction SilentlyContinue + Compress-Archive -Path "$extractPath\pgsql" -DestinationPath $outputZip -CompressionLevel Optimal + + # Cleanup + Remove-Item -Recurse -Force $extractPath + + $size = [math]::Round((Get-Item $outputZip).Length / 1MB, 1) + Write-Host " Done: $outputZip ($size MB)" -ForegroundColor Green +} + +Write-Host "" +Write-Host "All binaries ready in $outputDir" -ForegroundColor Green +Write-Host "" +Write-Host "Next steps:" -ForegroundColor Yellow +Write-Host "1. Create GitHub release 'pg-binaries' at:" +Write-Host " https://github.com/willibrandon/pglogical/releases/new?tag=pg-binaries" +Write-Host "" +Write-Host "2. Upload all ZIP files from $outputDir" +Write-Host "" +Write-Host "3. Publish the release" diff --git a/scripts/prepare-pg-binaries.sh b/scripts/prepare-pg-binaries.sh new file mode 100644 index 00000000..1c0d4c71 --- /dev/null +++ b/scripts/prepare-pg-binaries.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Script to download and repackage PostgreSQL binaries for CI +# Run this locally, then upload the resulting ZIPs to GitHub release "pg-binaries" + +set -e + +declare -A versions=( + ["13"]="13.20-1" + ["14"]="14.17-1" + ["15"]="15.12-1" + ["16"]="16.8-1" + ["17"]="17.4-1" + ["18"]="18.0-1" +) + +OUTPUT_DIR="./pg-binaries" +TEMP_DIR="${TMPDIR:-/tmp}" + +mkdir -p "$OUTPUT_DIR" + +for pg_major in "${!versions[@]}"; do + pg_version="${versions[$pg_major]}" + edb_url="https://get.enterprisedb.com/postgresql/postgresql-${pg_version}-windows-x64-binaries.zip" + download_path="${TEMP_DIR}/postgresql-${pg_major}-edb.zip" + extract_path="${TEMP_DIR}/pg-extract-${pg_major}" + output_zip="${OUTPUT_DIR}/postgresql-${pg_major}-windows-x64.zip" + + echo -e "\033[36mProcessing PostgreSQL ${pg_major}...\033[0m" + + # Download from EDB + if [[ ! -f "$download_path" ]]; then + echo " Downloading from EDB..." + curl -L -o "$download_path" "$edb_url" + else + echo " Using cached download" + fi + + # Extract + echo " Extracting..." + rm -rf "$extract_path" + unzip -q "$download_path" -d "$extract_path" + + # EDB ZIP extracts to pgsql/ folder - perfect, just rezip + echo " Creating $output_zip..." + rm -f "$output_zip" + (cd "$extract_path" && zip -rq "../$(basename "$output_zip")" pgsql) + mv "${extract_path}/../$(basename "$output_zip")" "$output_zip" + + # Cleanup + rm -rf "$extract_path" + + size=$(du -m "$output_zip" | cut -f1) + echo -e " \033[32mDone: $output_zip (${size} MB)\033[0m" +done + +echo "" +echo -e "\033[32mAll binaries ready in $OUTPUT_DIR\033[0m" +echo "" +echo -e "\033[33mNext steps:\033[0m" +echo "1. Create GitHub release 'pg-binaries' at:" +echo " https://github.com/willibrandon/pglogical/releases/new?tag=pg-binaries" +echo "" +echo "2. Upload all ZIP files from $OUTPUT_DIR" +echo "" +echo "3. Publish the release" diff --git a/scripts/stress-tests.ps1 b/scripts/stress-tests.ps1 new file mode 100644 index 00000000..b5ab7a06 --- /dev/null +++ b/scripts/stress-tests.ps1 @@ -0,0 +1,178 @@ +<# +.SYNOPSIS + Stress test pglogical by running regression tests multiple times. + +.DESCRIPTION + This script runs the regression tests multiple times and tracks which tests + fail across runs. Useful for identifying intermittent/flaky test failures. + +.PARAMETER Runs + Number of times to run the test suite. Default is 10. + +.PARAMETER StopOnFailure + Stop running after the first failure. Default is false (continue all runs). + +.PARAMETER TestFilter + Run only specific tests (comma-separated). If not specified, runs all tests. + +.EXAMPLE + .\stress-tests.ps1 -Runs 5 + +.EXAMPLE + .\stress-tests.ps1 -Runs 10 -StopOnFailure + +.EXAMPLE + .\stress-tests.ps1 -TestFilter "init,basic,interfaces" +#> + +param( + [int]$Runs = 10, + [switch]$StopOnFailure, + [string]$TestFilter = "" +) + +$ErrorActionPreference = "Continue" +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$projectRoot = Split-Path -Parent $scriptDir +$buildDir = Join-Path $projectRoot "build" +$resultsDir = Join-Path $projectRoot "test-results" + +# Create results directory +if (-not (Test-Path $resultsDir)) { + New-Item -ItemType Directory -Path $resultsDir | Out-Null +} + +# Track results +$runResults = @() +$failedTests = @{} +$startTime = Get-Date + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "pglogical Stress Test Runner" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Runs planned: $Runs" +Write-Host "Stop on failure: $StopOnFailure" +Write-Host "Results dir: $resultsDir" +Write-Host "Started at: $startTime" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +for ($i = 1; $i -le $Runs; $i++) { + $runStart = Get-Date + Write-Host "Run $i of $Runs starting at $runStart..." -ForegroundColor Yellow + + # Clean previous results + $diffFile = Join-Path $projectRoot "regression.diffs" + $outFile = Join-Path $projectRoot "regression.out" + if (Test-Path $diffFile) { Remove-Item $diffFile -Force } + if (Test-Path $outFile) { Remove-Item $outFile -Force } + + # Run tests + $testOutput = & cmake --build $buildDir --target check 2>&1 | Out-String + $exitCode = $LASTEXITCODE + $runEnd = Get-Date + $duration = $runEnd - $runStart + + # Parse results + $passed = @() + $failed = @() + + foreach ($line in $testOutput -split "`n") { + if ($line -match "^ ok \d+\s+-\s+(\S+)") { + $passed += $matches[1] + } + elseif ($line -match "^ not ok \d+\s+-\s+(\S+)") { + $testName = $matches[1] + $failed += $testName + if (-not $failedTests.ContainsKey($testName)) { + $failedTests[$testName] = @() + } + $failedTests[$testName] += $i + } + } + + $result = @{ + Run = $i + Passed = $passed.Count + Failed = $failed.Count + FailedTests = $failed + Duration = $duration.TotalSeconds + ExitCode = $exitCode + } + $runResults += $result + + # Save artifacts for failed runs + if ($failed.Count -gt 0) { + $runDir = Join-Path $resultsDir "run-$i" + New-Item -ItemType Directory -Path $runDir -Force | Out-Null + + if (Test-Path $diffFile) { + Copy-Item $diffFile (Join-Path $runDir "regression.diffs") + } + if (Test-Path $outFile) { + Copy-Item $outFile (Join-Path $runDir "regression.out") + } + + # Copy individual result files + $resultsPath = Join-Path $projectRoot "results" + if (Test-Path $resultsPath) { + Copy-Item -Path $resultsPath -Destination (Join-Path $runDir "results") -Recurse -Force + } + + $testOutput | Out-File (Join-Path $runDir "output.txt") + } + + # Report + if ($failed.Count -eq 0) { + Write-Host " Run $i PASSED ($($passed.Count) tests in $([math]::Round($duration.TotalSeconds, 1))s)" -ForegroundColor Green + } else { + Write-Host " Run $i FAILED ($($failed.Count) failures: $($failed -join ', '))" -ForegroundColor Red + Write-Host " Artifacts saved to: $runDir" -ForegroundColor DarkGray + + if ($StopOnFailure) { + Write-Host "" + Write-Host "Stopping due to -StopOnFailure flag." -ForegroundColor Yellow + break + } + } +} + +$endTime = Get-Date +$totalDuration = $endTime - $startTime + +# Summary +Write-Host "" +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "SUMMARY" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Total runs: $($runResults.Count)" +Write-Host "Total time: $([math]::Round($totalDuration.TotalMinutes, 1)) minutes" +Write-Host "" + +$passedRuns = ($runResults | Where-Object { $_.Failed -eq 0 }).Count +$failedRuns = ($runResults | Where-Object { $_.Failed -gt 0 }).Count + +Write-Host "Passed runs: $passedRuns" -ForegroundColor Green +Write-Host "Failed runs: $failedRuns" -ForegroundColor $(if ($failedRuns -gt 0) { "Red" } else { "Green" }) +Write-Host "" + +if ($failedTests.Count -gt 0) { + Write-Host "Flaky tests detected:" -ForegroundColor Yellow + foreach ($test in $failedTests.Keys | Sort-Object) { + $runs = $failedTests[$test] -join ", " + $failRate = [math]::Round(($failedTests[$test].Count / $runResults.Count) * 100, 1) + Write-Host " $test - failed in runs: $runs ($failRate% failure rate)" -ForegroundColor Red + } +} else { + Write-Host "No flaky tests detected across $($runResults.Count) runs!" -ForegroundColor Green +} + +Write-Host "" +Write-Host "Results saved to: $resultsDir" -ForegroundColor DarkGray + +# Return exit code based on whether any tests failed +if ($failedRuns -gt 0) { + exit 1 +} else { + exit 0 +} diff --git a/scripts/stress-tests.sh b/scripts/stress-tests.sh new file mode 100644 index 00000000..46ba9511 --- /dev/null +++ b/scripts/stress-tests.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# +# stress-tests.sh - Run pglogical regression tests repeatedly to catch flaky tests +# +# Usage: +# ./stress-tests.sh [OPTIONS] +# +# Options: +# -n, --runs N Number of times to run the test suite (default: 10) +# -s, --stop-on-fail Stop running after the first failure +# -h, --help Show this help message +# +# Examples: +# ./stress-tests.sh -n 5 +# ./stress-tests.sh --runs 10 --stop-on-fail +# + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +GRAY='\033[0;90m' +NC='\033[0m' # No Color + +# Default values +RUNS=10 +STOP_ON_FAILURE=false + +# Script directory and project root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +RESULTS_DIR="$PROJECT_ROOT/test-results" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -n|--runs) + RUNS="$2" + shift 2 + ;; + -s|--stop-on-fail) + STOP_ON_FAILURE=true + shift + ;; + -h|--help) + head -20 "$0" | tail -18 + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Create results directory +mkdir -p "$RESULTS_DIR" + +# Track results +declare -A FAILED_TESTS +PASSED_RUNS=0 +FAILED_RUNS=0 +START_TIME=$(date +%s) + +echo -e "${CYAN}========================================${NC}" +echo -e "${CYAN}pglogical Stress Test Runner${NC}" +echo -e "${CYAN}========================================${NC}" +echo "Runs planned: $RUNS" +echo "Stop on failure: $STOP_ON_FAILURE" +echo "Results dir: $RESULTS_DIR" +echo "Started at: $(date)" +echo -e "${CYAN}========================================${NC}" +echo "" + +for ((i=1; i<=RUNS; i++)); do + RUN_START=$(date +%s) + echo -e "${YELLOW}Run $i of $RUNS starting at $(date)...${NC}" + + # Clean previous results + rm -f "$PROJECT_ROOT/regression.diffs" "$PROJECT_ROOT/regression.out" + + # Run tests and capture output + cd "$PROJECT_ROOT" + TEST_OUTPUT=$(make check 2>&1 || true) + RUN_END=$(date +%s) + DURATION=$((RUN_END - RUN_START)) + + # Parse results + PASSED=() + FAILED=() + + while IFS= read -r line; do + if [[ $line =~ ^[[:space:]]*ok[[:space:]]+[0-9]+[[:space:]]+-[[:space:]]+([^[:space:]]+) ]]; then + PASSED+=("${BASH_REMATCH[1]}") + elif [[ $line =~ ^[[:space:]]*not[[:space:]]+ok[[:space:]]+[0-9]+[[:space:]]+-[[:space:]]+([^[:space:]]+) ]]; then + TEST_NAME="${BASH_REMATCH[1]}" + FAILED+=("$TEST_NAME") + # Track which runs each test failed in + if [[ -z "${FAILED_TESTS[$TEST_NAME]}" ]]; then + FAILED_TESTS[$TEST_NAME]="$i" + else + FAILED_TESTS[$TEST_NAME]="${FAILED_TESTS[$TEST_NAME]}, $i" + fi + fi + done <<< "$TEST_OUTPUT" + + # Save artifacts for failed runs + if [[ ${#FAILED[@]} -gt 0 ]]; then + RUN_DIR="$RESULTS_DIR/run-$i" + mkdir -p "$RUN_DIR" + + [[ -f "$PROJECT_ROOT/regression.diffs" ]] && cp "$PROJECT_ROOT/regression.diffs" "$RUN_DIR/" + [[ -f "$PROJECT_ROOT/regression.out" ]] && cp "$PROJECT_ROOT/regression.out" "$RUN_DIR/" + [[ -d "$PROJECT_ROOT/results" ]] && cp -r "$PROJECT_ROOT/results" "$RUN_DIR/" + [[ -d "$PROJECT_ROOT/log" ]] && cp -r "$PROJECT_ROOT/log" "$RUN_DIR/" + + echo "$TEST_OUTPUT" > "$RUN_DIR/output.txt" + + FAILED_RUNS=$((FAILED_RUNS + 1)) + echo -e " ${RED}Run $i FAILED (${#FAILED[@]} failures: ${FAILED[*]})${NC}" + echo -e " ${GRAY}Artifacts saved to: $RUN_DIR${NC}" + + if [[ "$STOP_ON_FAILURE" == true ]]; then + echo "" + echo -e "${YELLOW}Stopping due to --stop-on-fail flag.${NC}" + break + fi + else + PASSED_RUNS=$((PASSED_RUNS + 1)) + echo -e " ${GREEN}Run $i PASSED (${#PASSED[@]} tests in ${DURATION}s)${NC}" + fi +done + +END_TIME=$(date +%s) +TOTAL_DURATION=$((END_TIME - START_TIME)) +TOTAL_MINUTES=$((TOTAL_DURATION / 60)) + +# Summary +echo "" +echo -e "${CYAN}========================================${NC}" +echo -e "${CYAN}SUMMARY${NC}" +echo -e "${CYAN}========================================${NC}" +echo "Total runs: $((PASSED_RUNS + FAILED_RUNS))" +echo "Total time: ${TOTAL_MINUTES} minutes" +echo "" + +echo -e "${GREEN}Passed runs: $PASSED_RUNS${NC}" +if [[ $FAILED_RUNS -gt 0 ]]; then + echo -e "${RED}Failed runs: $FAILED_RUNS${NC}" +else + echo -e "${GREEN}Failed runs: $FAILED_RUNS${NC}" +fi +echo "" + +if [[ ${#FAILED_TESTS[@]} -gt 0 ]]; then + echo -e "${YELLOW}Flaky tests detected:${NC}" + for test in "${!FAILED_TESTS[@]}"; do + RUNS_FAILED="${FAILED_TESTS[$test]}" + # Count failures + FAIL_COUNT=$(echo "$RUNS_FAILED" | tr ',' '\n' | wc -l) + TOTAL_RUNS=$((PASSED_RUNS + FAILED_RUNS)) + FAIL_RATE=$(echo "scale=1; $FAIL_COUNT * 100 / $TOTAL_RUNS" | bc) + echo -e " ${RED}$test - failed in runs: $RUNS_FAILED ($FAIL_RATE% failure rate)${NC}" + done | sort +else + echo -e "${GREEN}No flaky tests detected across $((PASSED_RUNS + FAILED_RUNS)) runs!${NC}" +fi + +echo "" +echo -e "${GRAY}Results saved to: $RESULTS_DIR${NC}" + +# Return exit code based on whether any tests failed +if [[ $FAILED_RUNS -gt 0 ]]; then + exit 1 +else + exit 0 +fi diff --git a/specs/002-github-releases/checklists/requirements.md b/specs/002-github-releases/checklists/requirements.md new file mode 100644 index 00000000..b6e72abe --- /dev/null +++ b/specs/002-github-releases/checklists/requirements.md @@ -0,0 +1,38 @@ +# Specification Quality Checklist: GitHub Releases Distribution + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-01-05 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- All items pass validation +- The specification is ready for `/speckit.clarify` or `/speckit.plan` +- The detailed input document provided comprehensive context, allowing all requirements to be derived without clarification markers +- Assumptions section documents reasonable defaults for package manager availability, runner configurations, etc. +- Out of Scope section clearly bounds what is not included in this release diff --git a/specs/002-github-releases/contracts/install-script.sh b/specs/002-github-releases/contracts/install-script.sh new file mode 100644 index 00000000..ac0dd925 --- /dev/null +++ b/specs/002-github-releases/contracts/install-script.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Contract: Linux/macOS Installation Script +# This contract defines the expected behavior of the install helper script. +# +# Requirements from spec: +# - FR-013: Linux and macOS packages MUST include an install.sh helper script +# - User Story 6: Install on Linux/macOS with Helper Script +# +# Expected Behavior: +# 1. Use pg_config from PATH or PG_CONFIG environment variable +# 2. Copy shared libraries to $(pg_config --pkglibdir) +# 3. Copy extension files to $(pg_config --sharedir)/extension +# 4. Use sudo if target directories require elevated permissions +# 5. Exit with clear error if pg_config not found + +set -e + +# Configuration (environment overrides) +# PG_CONFIG - path to pg_config if not in PATH + +# Expected outputs: +# - pglogical.so/dylib copied to pkglibdir +# - pglogical_output.so/dylib copied to pkglibdir +# - pglogical.control copied to sharedir/extension +# - pglogical--*.sql files copied to sharedir/extension + +# Expected error conditions: +# - pg_config not found (exit 1) +# - Copy permission denied without sudo (prompt for sudo) +# - Target directory doesn't exist (create it) + +# Contract validation: +# 1. Script must be executable (chmod +x) +# 2. Script must work on bash 3.2+ (macOS default) +# 3. Script must not require external dependencies beyond coreutils + +# Acceptance Scenarios (from spec): +# +# Scenario 1: ./install.sh with pg_config in PATH +# - Copies libraries to $(pg_config --pkglibdir) +# - Copies extension files to $(pg_config --sharedir)/extension +# +# Scenario 2: PG_CONFIG=/path/to/pg_config ./install.sh +# - Uses specified pg_config +# +# Scenario 3: Target directories require root +# - Uses sudo for copy operations +# - Prompts for password if needed diff --git a/specs/002-github-releases/contracts/msi-installer.wxs b/specs/002-github-releases/contracts/msi-installer.wxs new file mode 100644 index 00000000..b1497266 --- /dev/null +++ b/specs/002-github-releases/contracts/msi-installer.wxs @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + diff --git a/specs/002-github-releases/contracts/workflow-ci.yaml b/specs/002-github-releases/contracts/workflow-ci.yaml new file mode 100644 index 00000000..68746e1f --- /dev/null +++ b/specs/002-github-releases/contracts/workflow-ci.yaml @@ -0,0 +1,46 @@ +# Contract: CI Workflow Specification +# This contract defines the expected structure and behavior of the CI workflow. + +name: CI Tests + +on: + push: + branches: + - REL2_x_STABLE + - windows-build + pull_request: + branches: + - REL2_x_STABLE + +permissions: + contents: read + checks: write + +jobs: + build: + name: PG ${{ matrix.pg-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + pg-version: [13, 14, 15, 16, 17, 18] + os: [ubuntu-latest, windows-2022, macos-13, macos-14] + exclude: + # macos-14 (ARM64) may not have PG13 available + - os: macos-14 + pg-version: 13 + + # Expected steps (implementation details in tasks): + # 1. Checkout with submodules: recursive + # 2. Install PostgreSQL dev dependencies (platform-specific) + # 3. Build extension (make on Linux/macOS, cmake on Windows) + # 4. Run regression tests (Linux/macOS only per FR-006) + # 5. Upload test artifacts on failure + + # Expected outputs: + # - Build success/failure status per matrix entry + # - Test results in regression_output/ + + # Expected timing: + # - Complete within 30 minutes (SC-006) diff --git a/specs/002-github-releases/contracts/workflow-release.yaml b/specs/002-github-releases/contracts/workflow-release.yaml new file mode 100644 index 00000000..5f60d270 --- /dev/null +++ b/specs/002-github-releases/contracts/workflow-release.yaml @@ -0,0 +1,70 @@ +# Contract: Release Workflow Specification +# This contract defines the expected structure and behavior of the release workflow. + +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + build: + name: Build PG ${{ matrix.pg-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + pg-version: [13, 14, 15, 16, 17, 18] + os: [ubuntu-latest, windows-2022, macos-13, macos-14] + exclude: + - os: macos-14 + pg-version: 13 + + # Expected steps: + # 1. Checkout with submodules: recursive + # 2. Extract version from tag (strip 'v' prefix) + # 3. Install PostgreSQL dev dependencies + # 4. Build extension + # 5. Run regression tests + # 6. Package artifacts with correct naming + # 7. Upload workflow artifacts for release job + + outputs: + version: ${{ steps.version.outputs.version }} + + source: + name: Create Source Archives + runs-on: ubuntu-latest + + # Expected steps: + # 1. Checkout with submodules: recursive + # 2. Create tar.gz with submodule contents + # 3. Create zip with submodule contents + # 4. Upload workflow artifacts + + release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: [build, source] + + # Expected steps: + # 1. Download all workflow artifacts + # 2. Generate checksums.txt + # 3. Determine if prerelease (tag contains hyphen) + # 4. Create GitHub Release + # 5. Attach all artifacts + + # Expected outputs: + # - GitHub Release with: + # - All binary packages (30+ artifacts) + # - Source archives (2 artifacts) + # - checksums.txt + # - Auto-generated release notes + + # Expected timing: + # - Complete within 60 minutes of tag push (SC-002) diff --git a/specs/002-github-releases/data-model.md b/specs/002-github-releases/data-model.md new file mode 100644 index 00000000..e83ca06d --- /dev/null +++ b/specs/002-github-releases/data-model.md @@ -0,0 +1,158 @@ +# Data Model: GitHub Releases Distribution + +**Date**: 2026-01-05 +**Feature**: [spec.md](./spec.md) + +## Overview + +This feature is CI/CD infrastructure with no persistent application data model. The "entities" are conceptual workflow artifacts and configurations, not database entities. + +## Key Entities + +### 1. Release Artifact + +A downloadable file attached to a GitHub Release. + +**Attributes**: +| Attribute | Type | Description | +|-----------|------|-------------| +| filename | string | Full artifact name following naming convention | +| version | string | pglogical version (e.g., "2.5.0") | +| pg_version | integer | PostgreSQL major version (13-18) | +| platform | enum | "windows", "linux", "macos" | +| arch | enum | "x64", "arm64" | +| format | enum | "zip", "tar.gz", "msi" | +| sha256 | string | SHA256 checksum | + +**Naming Pattern**: `pglogical-{version}-pg{pg_version}-{platform}-{arch}.{format}` + +**Validation Rules**: +- Version must follow semantic versioning (major.minor.patch) +- pg_version must be in supported range (13-18) +- Platform/arch combinations must be valid (no Windows ARM64, no Linux ARM64) + +### 2. Build Matrix Entry + +A single build configuration in the CI/CD matrix. + +**Attributes**: +| Attribute | Type | Description | +|-----------|------|-------------| +| os | string | GitHub Actions runner (ubuntu-latest, windows-2022, macos-13, macos-14) | +| pg_version | integer | PostgreSQL major version | +| arch | string | Derived from runner (x64 or arm64) | +| compiler | string | Derived from platform (gcc, msvc, clang) | + +**Valid Combinations**: +| Runner | Architecture | PostgreSQL Versions | +|--------|--------------|---------------------| +| ubuntu-latest | x64 | 13, 14, 15, 16, 17, 18 | +| windows-2022 | x64 | 13, 14, 15, 16, 17, 18 | +| macos-13 | x64 | 13, 14, 15, 16, 17, 18 | +| macos-14 | arm64 | 14, 15, 16, 17, 18 | + +**Notes**: +- windows-2022 runner builds are compatible with Windows 10 and Windows 11 (builds on Windows Server 2022) +- macos-14 excludes PG13 as Homebrew may not provide it for ARM64 +- PG18 may have delayed availability on Homebrew for new releases; CI uses `fail-fast: false` to handle gracefully + +### 3. GitHub Release + +A tagged release on GitHub containing release notes and artifacts. + +**Attributes**: +| Attribute | Type | Description | +|-----------|------|-------------| +| tag_name | string | Git tag (e.g., "v2.5.0") | +| name | string | Release title (e.g., "pglogical 2.5.0") | +| body | string | Release notes with installation instructions | +| prerelease | boolean | True if tag contains hyphen (e.g., "-beta1") | +| draft | boolean | Always false (published immediately) | +| assets | array | List of Release Artifacts | + +**State Transitions**: +``` +[Tag Push] → [Builds Running] → [All Builds Complete] → [Release Created] → [Assets Attached] +``` + +### 4. MSI Installer Configuration + +WiX v5 installer configuration for Windows. + +**Attributes**: +| Attribute | Type | Description | +|-----------|------|-------------| +| product_name | string | "pglogical for PostgreSQL {pg_version}" | +| version | string | 4-part version (e.g., "2.5.0.0") | +| manufacturer | string | "2ndQuadrant" | +| upgrade_code | string | Human-readable ID per PG version | +| install_dir | path | Detected or user-selected PostgreSQL directory | + +**UpgradeCode Pattern**: +- PG 13: `com.2ndquadrant.pglogical.postgresql13` +- PG 14: `com.2ndquadrant.pglogical.postgresql14` +- PG 15: `com.2ndquadrant.pglogical.postgresql15` +- PG 16: `com.2ndquadrant.pglogical.postgresql16` +- PG 17: `com.2ndquadrant.pglogical.postgresql17` +- PG 18: `com.2ndquadrant.pglogical.postgresql18` + +**Validation Rules**: +- Same UpgradeCode = versions upgrade each other +- Different UpgradeCode = side-by-side installation allowed + +### 5. Workflow Configuration + +GitHub Actions workflow definition. + +**CI Workflow Triggers**: +| Event | Target | +|-------|--------| +| push | REL2_x_STABLE, windows-build branches | +| pull_request | REL2_x_STABLE branch | + +**Release Workflow Triggers**: +| Event | Pattern | +|-------|---------| +| push (tags) | v* | + +## Relationships + +``` +GitHub Release (1) ───────── (*) Release Artifact + │ + └──── contains checksums.txt with all artifact hashes + +Build Matrix Entry (*) ───── (1) Release Artifact + │ + └──── each matrix entry produces one or more artifacts + +MSI Installer Configuration (1) ───── (1) Release Artifact (MSI) + │ + └──── one MSI per PostgreSQL version +``` + +## File Outputs + +### Artifacts per Build + +| Platform | Artifacts Produced | +|----------|-------------------| +| Linux | `pglogical-{ver}-pg{pg}-linux-x64.tar.gz` | +| Windows | `pglogical-{ver}-pg{pg}-windows-x64.zip`, `pglogical-{ver}-pg{pg}-windows-x64.msi` | +| macOS x64 | `pglogical-{ver}-pg{pg}-macos-x64.tar.gz` | +| macOS ARM64 | `pglogical-{ver}-pg{pg}-macos-arm64.tar.gz` | + +### Source Archives + +- `pglogical-{version}-source.tar.gz` +- `pglogical-{version}-source.zip` + +### Checksum File + +`checksums.txt` containing: +``` + pglogical-2.5.0-pg17-linux-x64.tar.gz + pglogical-2.5.0-pg17-windows-x64.zip + pglogical-2.5.0-pg17-windows-x64.msi +... +``` diff --git a/specs/002-github-releases/plan.md b/specs/002-github-releases/plan.md new file mode 100644 index 00000000..4a1f31bf --- /dev/null +++ b/specs/002-github-releases/plan.md @@ -0,0 +1,93 @@ +# Implementation Plan: GitHub Releases Distribution + +**Branch**: `002-github-releases` | **Date**: 2026-01-05 | **Spec**: [spec.md](./spec.md) +**Input**: Feature specification from `/specs/002-github-releases/spec.md` + +## Summary + +This feature establishes automated GitHub Releases distribution for pglogical 2.5.0, producing pre-built binaries for Windows (MSI installers + ZIP archives), Linux (tar.gz), and macOS (tar.gz for both ARM64 and x64) across PostgreSQL versions 13-18. The implementation uses GitHub Actions workflows triggered by git tags matching `v*` pattern, with WiX Toolset v5 for MSI creation. The system also provides continuous integration for pull requests and branch pushes. + +## Technical Context + +**Language/Version**: YAML (GitHub Actions workflows), WiX v5 (MSI definitions), Bash (install scripts), PowerShell (Windows CI), Make (existing build system) +**Primary Dependencies**: GitHub Actions runners (ubuntu-latest, windows-2022, macos-13, macos-14), WiX Toolset v5 (.NET global tool), PostgreSQL development headers, Visual Studio 2022 Build Tools, Homebrew (macOS), apt/Chocolatey (package managers) +**Storage**: N/A (artifacts stored as GitHub Release assets) +**Testing**: Regression tests via `make check` on Linux/macOS (existing Makefile infrastructure) +**Target Platform**: GitHub Actions CI/CD (builds for Windows x64, Linux x64, macOS ARM64, macOS x64) +**Project Type**: CI/CD infrastructure (workflow definitions + installer packaging) +**Performance Goals**: Complete release build within 60 minutes of tag push (SC-002) +**Constraints**: CI provides build status within 30 minutes for any PR (SC-006); fail-fast: false to allow parallel job completion +**Scale/Scope**: 30+ artifacts per release (6 PG versions × 4 platforms × artifact types) + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +| Principle | Compliance | Notes | +|-----------|------------|-------| +| I. PostgreSQL Version Compatibility | ✅ PASS | Build matrix covers PG 13-18; compat directories exist for each version | +| II. Backward Compatibility | ✅ PASS | Distribution infrastructure only; does not modify extension behavior | +| III. Testing Discipline | ✅ PASS | CI runs regression tests on Linux/macOS builds; existing test suite unchanged | +| IV. Code Quality & Memory Safety | ✅ PASS | No C code changes; uses existing Makefile build system | +| V. Replication Integrity | ✅ PASS | Distribution only; replication code path unaffected | +| VI. Implementation Completeness | ✅ PASS | Plan delivers complete working workflows, no placeholders | +| C Extension Standards | ✅ N/A | No extension code changes | +| Development Workflow | ✅ PASS | CI validates PRs; branch protection enforced | + +**Gate Status**: PASS - No violations requiring justification + +## Project Structure + +### Documentation (this feature) + +```text +specs/002-github-releases/ +├── plan.md # This file (/speckit.plan command output) +├── research.md # Phase 0 output (/speckit.plan command) +├── data-model.md # Phase 1 output (/speckit.plan command) +├── quickstart.md # Phase 1 output (/speckit.plan command) +├── contracts/ # Phase 1 output (/speckit.plan command) +└── tasks.md # Phase 2 output (/speckit.tasks command) +``` + +### Source Code (repository root) + +```text +.github/ +└── workflows/ + ├── ci.yml # Continuous integration (PRs + branch pushes) + └── release.yml # Release pipeline (tag-triggered) + +packaging/ +├── windows/ +│ ├── pglogical.wxs # WiX v5 MSI installer definition +│ └── README.md # Windows installation instructions +├── unix/ +│ └── install.sh # Linux/macOS installation helper script +└── README-CI.md # CI/CD documentation and branch protection setup +``` + +**Structure Decision**: CI/CD infrastructure with platform-specific packaging. Workflows live in `.github/workflows/` per GitHub Actions convention. Packaging assets organized by platform under `packaging/` directory to keep installer definitions separate from source code. + +## Complexity Tracking + +> No violations requiring justification - all Constitution Check gates pass. + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| N/A | N/A | N/A | + +## Post-Design Constitution Re-Check + +*Re-evaluated after Phase 1 design completion.* + +| Principle | Status | Post-Design Notes | +|-----------|--------|-------------------| +| I. PostgreSQL Version Compatibility | ✅ PASS | Build matrix includes PG 13-18; macos-14 excludes PG13 due to Homebrew availability | +| II. Backward Compatibility | ✅ PASS | MSI installer uses MajorUpgrade for clean upgrades; side-by-side installation via unique UpgradeCodes | +| III. Testing Discipline | ✅ PASS | CI workflow runs `make check` on Linux/macOS and CMake check target on Windows; all platforms run regression tests | +| IV. Code Quality & Memory Safety | ✅ PASS | No C code changes; WiX v5 and YAML are declarative | +| V. Replication Integrity | ✅ PASS | No changes to replication code paths | +| VI. Implementation Completeness | ✅ PASS | All contracts fully specified; no placeholder implementations | + +**Post-Design Gate Status**: PASS - Design artifacts complete and constitution-compliant diff --git a/specs/002-github-releases/quickstart.md b/specs/002-github-releases/quickstart.md new file mode 100644 index 00000000..3f1b4075 --- /dev/null +++ b/specs/002-github-releases/quickstart.md @@ -0,0 +1,160 @@ +# Quickstart: GitHub Releases Distribution Implementation + +**Date**: 2026-01-05 +**Feature**: [spec.md](./spec.md) + +## Prerequisites + +- Git with access to pglogical repository +- GitHub repository with Actions enabled +- Understanding of GitHub Actions workflow syntax + +## Directory Structure + +After implementation, the repository will have: + +``` +pglogical/ +├── .github/ +│ └── workflows/ +│ ├── ci.yml # PR/push continuous integration +│ └── release.yml # Tag-triggered release pipeline +├── packaging/ +│ ├── windows/ +│ │ ├── pglogical.wxs # WiX v5 MSI installer definition +│ │ └── README.md # Windows installation instructions +│ └── unix/ +│ └── install.sh # Linux/macOS installation helper +└── ... (existing files) +``` + +## Implementation Order + +### Phase 1: CI Workflow (ci.yml) + +1. Create `.github/workflows/ci.yml` +2. Configure checkout with `submodules: recursive` +3. Set up build matrix (PG 13-18 × 4 platforms) +4. Add platform-specific PostgreSQL installation steps +5. Add build steps (make for Linux/macOS, cmake for Windows) +6. Add test steps (regression tests on Linux/macOS) +7. Test with a PR + +### Phase 2: Unix Packaging + +1. Create `packaging/unix/install.sh` +2. Implement pg_config detection +3. Implement sudo elevation for privileged directories +4. Test on local Linux/macOS machines + +### Phase 3: Windows Packaging + +1. Create `packaging/windows/pglogical.wxs` +2. Implement registry search for PostgreSQL path +3. Implement directory browse fallback +4. Configure MajorUpgrade for clean upgrades +5. Configure unique UpgradeCode per PG version +6. Create `packaging/windows/README.md` +7. Test MSI build locally with WiX v5 + +### Phase 4: Release Workflow (release.yml) + +1. Create `.github/workflows/release.yml` +2. Add build jobs (same as CI) +3. Add source archive creation job +4. Add release job with: + - Artifact download + - Checksum generation + - GitHub Release creation + - Asset attachment +5. Test with a test tag (e.g., `v2.5.0-test1`) + +### Phase 5: Integration Testing + +1. Create a prerelease tag +2. Verify all artifacts build +3. Verify GitHub Release creation +4. Verify artifact downloads work +5. Test installation on each platform + +## Key Commands + +### Local Development + +```bash +# Build extension +make clean all + +# Install locally +sudo make install + +# Run regression tests +make check +``` + +### Testing CI Locally (act) + +```bash +# Install act (GitHub Actions local runner) +brew install act # macOS + +# Run CI workflow locally +act push --job build +``` + +### Creating a Release + +```bash +# Create and push a release tag +git tag v2.5.0 +git push origin v2.5.0 + +# Create a prerelease tag +git tag v2.5.0-beta1 +git push origin v2.5.0-beta1 +``` + +### Building MSI Locally + +```powershell +# Install WiX v5 +dotnet tool install --global wix --version 5.0.0 + +# Build MSI +wix build -o pglogical-pg17.msi packaging/windows/pglogical.wxs -d PG_VERSION=17 -d VERSION=2.5.0 +``` + +## Verification Checklist + +After implementation, verify: + +- [ ] CI runs on PR creation +- [ ] CI runs on push to REL2_x_STABLE +- [ ] All 24 matrix jobs complete (6 PG versions × 4 platforms) +- [ ] Release workflow triggers on `v*` tag push +- [ ] All artifacts attach to GitHub Release +- [ ] checksums.txt contains all artifact hashes +- [ ] Prerelease tags create prerelease releases +- [ ] MSI installer detects PostgreSQL path +- [ ] install.sh works on Linux and macOS +- [ ] Source archives include submodule contents + +## Troubleshooting + +### CI Job Failures + +1. Check workflow logs in GitHub Actions tab +2. Look for PostgreSQL installation errors +3. Verify pg_config is in PATH for build steps + +### MSI Build Failures + +1. Verify WiX v5 is installed (`dotnet tool list -g`) +2. Check WiX build output for missing files +3. Verify registry search key matches installed PostgreSQL + +### Release Asset Upload Failures + +1. Check GITHUB_TOKEN permissions +2. Verify artifact names don't contain invalid characters +3. Check for duplicate asset names diff --git a/specs/002-github-releases/research.md b/specs/002-github-releases/research.md new file mode 100644 index 00000000..3ff69223 --- /dev/null +++ b/specs/002-github-releases/research.md @@ -0,0 +1,268 @@ +# Research: GitHub Releases Distribution + +**Date**: 2026-01-05 +**Feature**: [spec.md](./spec.md) +**Status**: Complete + +## Executive Summary + +Research covers three critical areas for implementing GitHub Releases distribution: +1. GitHub Actions CI/CD patterns for PostgreSQL extension builds +2. WiX Toolset v5 MSI installer configuration +3. Cross-platform PostgreSQL development environment setup + +All research questions resolved with specific, actionable recommendations. No NEEDS CLARIFICATION markers remain. + +--- + +## 1. GitHub Actions CI/CD Patterns + +### Decision: Dual-Workflow Architecture +**Rationale**: Separating CI (pull requests/pushes) from release (tag-triggered) workflows provides clear responsibility boundaries and prevents accidental releases. +**Alternatives Considered**: +- Single workflow with conditional jobs - rejected due to complexity and harder debugging +- Monorepo-style workflow dispatch - rejected as overkill for single project + +### Build Matrix Strategy + +**Decision**: Use `fail-fast: false` with explicit matrix configuration +```yaml +strategy: + fail-fast: false + matrix: + pg-version: [13, 14, 15, 16, 17, 18] + os: [ubuntu-latest, windows-2022, macos-13, macos-14] + exclude: + - os: macos-14 + pg-version: 13 # Homebrew may not have PG13 for ARM64 +``` + +**Rationale**: +- `fail-fast: false` ensures all platform/version combinations complete +- Visibility into which specific jobs fail +- PostgreSQL 18 experimental builds don't block stable version results + +### Submodule Configuration + +**Decision**: Use HTTPS URLs with `submodules: recursive` +```yaml +- uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 +``` + +**Rationale**: HTTPS URLs avoid SSH key configuration in CI; pglogical_dump submodule already uses HTTPS. + +### Artifact Management + +**Decision**: Use `softprops/action-gh-release@v2` for release asset upload +```yaml +- uses: softprops/action-gh-release@v2 + with: + files: | + pglogical-*.tar.gz + pglogical-*.zip + pglogical-*.msi + checksums.txt +``` + +**Rationale**: Simpler than `actions/upload-release-asset@v1`; supports glob patterns; actively maintained. + +--- + +## 2. WiX Toolset v5 MSI Configuration + +### Decision: Install WiX v5 as .NET Global Tool +```yaml +- name: Install WiX v5 + run: dotnet tool install --global wix --version 5.0.0 +``` + +**Rationale**: .NET global tool is the official distribution method for WiX v5; simplifies CI setup. + +### PostgreSQL Registry Detection + +**Decision**: Use `util:RegistrySearch` with bitness-aware searches +```xml + +``` + +**Registry Paths by Version**: +- PG 13: `SOFTWARE\PostgreSQL\Installations\postgresql-x64-13` +- PG 14: `SOFTWARE\PostgreSQL\Installations\postgresql-x64-14` +- PG 15: `SOFTWARE\PostgreSQL\Installations\postgresql-x64-15` +- PG 16: `SOFTWARE\PostgreSQL\Installations\postgresql-x64-16` +- PG 17: `SOFTWARE\PostgreSQL\Installations\postgresql-x64-17` +- PG 18: `SOFTWARE\PostgreSQL\Installations\postgresql-x64-18` + +### Directory Browse Fallback + +**Decision**: Use WixUI_InstallDir with custom property +```xml + + +``` + +**Rationale**: WiX v5 provides standard UI dialogs; avoids custom dialog authoring. + +### Side-by-Side Installation + +**Decision**: Use human-readable UpgradeCode per PostgreSQL version +```xml + + + + + +``` + +**Rationale**: WiX v5 converts human-readable strings to stable GUIDs internally; easier to track and version control than raw GUIDs. + +### Major Upgrade Configuration + +**Decision**: Use explicit MajorUpgrade element +```xml + +``` + +**Rationale**: WiX v5 provides defaults, but explicit configuration ensures predictable behavior and clear downgrade handling. + +--- + +## 3. Cross-Platform PostgreSQL Build Setup + +### Ubuntu (GitHub Actions ubuntu-latest) + +**Package Installation**: +```bash +sudo apt-get update +sudo apt-get install -y postgresql-server-dev-${PG_VERSION} +export PATH=/usr/lib/postgresql/${PG_VERSION}/bin:$PATH +``` + +**Packages by Version**: +- `postgresql-server-dev-13` through `postgresql-server-dev-18` +- Available from official PostgreSQL APT repository + +### Windows (GitHub Actions windows-2022, supports Windows 10/11) + +**Chocolatey Installation**: +```powershell +choco install postgresql${PG_VERSION} -y --params '/Password:postgres' +$env:PATH = "C:\Program Files\PostgreSQL\${PG_VERSION}\bin;$env:PATH" +``` + +**Build Approach**: Use CMake with Visual Studio 2022 +```powershell +cmake -G "Visual Studio 17 2022" -DPG_CONFIG="C:\Program Files\PostgreSQL\17\bin\pg_config.exe" .. +cmake --build . --config Release +``` + +**Rationale**: PGXS/Make doesn't work on Windows; CMakeLists.txt already exists in pglogical. + +### macOS (GitHub Actions macos-13, macos-14) + +**Runner Architecture** (runner names reflect image generation, not macOS version): +- `macos-13`: x86_64 (Intel) - for macOS x64 builds (runs current macOS) +- `macos-14`: ARM64 (Apple Silicon) - for macOS ARM64 builds (runs current macOS through 26 Tahoe) + +**Homebrew Installation**: +```bash +brew install postgresql@${PG_VERSION} +# Intel path +export PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" +# ARM64 path (macos-14) +export PATH="/opt/homebrew/opt/postgresql@${PG_VERSION}/bin:$PATH" +``` + +### pg_config Usage + +**Installation Script Pattern**: +```bash +#!/bin/bash +set -e +PKGLIBDIR=$(pg_config --pkglibdir) +SHAREDIR=$(pg_config --sharedir) + +install -m755 pglogical.so "$PKGLIBDIR/" +install -m755 pglogical_output.so "$PKGLIBDIR/" +install -m644 pglogical.control "$SHAREDIR/extension/" +install -m644 pglogical--*.sql "$SHAREDIR/extension/" +``` + +**Paths by Platform**: +| Platform | pkglibdir | sharedir/extension | +|----------|-----------|-------------------| +| Linux | `/usr/lib/postgresql/17/lib` | `/usr/share/postgresql/17/extension` | +| macOS Intel | `/usr/local/opt/postgresql@17/lib` | `/usr/local/opt/postgresql@17/share/extension` | +| macOS ARM64 | `/opt/homebrew/opt/postgresql@17/lib` | `/opt/homebrew/opt/postgresql@17/share/extension` | +| Windows | `C:\Program Files\PostgreSQL\17\lib` | `C:\Program Files\PostgreSQL\17\share\extension` | + +--- + +## 4. Artifact Naming Convention + +**Decision**: Follow pattern `pglogical-{version}-pg{pg_version}-{platform}-{arch}.{ext}` + +**Examples**: +- `pglogical-2.5.0-pg17-linux-x64.tar.gz` +- `pglogical-2.5.0-pg17-windows-x64.zip` +- `pglogical-2.5.0-pg17-windows-x64.msi` +- `pglogical-2.5.0-pg17-macos-arm64.tar.gz` +- `pglogical-2.5.0-pg17-macos-x64.tar.gz` +- `pglogical-2.5.0-source.tar.gz` + +**Version Extraction**: Strip `v` prefix from git tag +```bash +VERSION=${GITHUB_REF_NAME#v} # v2.5.0 -> 2.5.0 +``` + +--- + +## 5. Prerelease Detection + +**Decision**: Mark releases as prerelease if tag contains hyphen +```yaml +- name: Create Release + uses: softprops/action-gh-release@v2 + with: + prerelease: ${{ contains(github.ref_name, '-') }} +``` + +**Examples**: +- `v2.5.0` → release +- `v2.5.0-beta1` → prerelease +- `v2.5.0-rc1` → prerelease + +--- + +## Sources + +### GitHub Actions +- [adjust/pg-ext-actions](https://github.com/adjust/pg-ext-actions) +- [softprops/action-gh-release](https://github.com/softprops/action-gh-release) +- [GitHub Actions Matrix Strategy](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) +- [GitHub Actions checkout@v4](https://github.com/actions/checkout) + +### WiX Toolset v5 +- [WiX Toolset Documentation](https://wixtoolset.org/docs/) +- [FireGiant WiX Schema Reference](https://docs.firegiant.com/wix/schema/) +- [WiX v5 RegistrySearch](https://docs.firegiant.com/wix/schema/util/registrysearch/) +- [WiX v5 MajorUpgrade](https://docs.firegiant.com/wix/schema/wxs/majorupgrade/) + +### PostgreSQL +- [PostgreSQL APT Repository](https://wiki.postgresql.org/wiki/Apt) +- [Chocolatey PostgreSQL Packages](https://community.chocolatey.org/packages/postgresql) +- [PostgreSQL PGXS Documentation](https://www.postgresql.org/docs/current/extend-pgxs.html) +- [pg_config Reference](https://www.postgresql.org/docs/current/app-pgconfig.html) diff --git a/specs/002-github-releases/spec.md b/specs/002-github-releases/spec.md new file mode 100644 index 00000000..c1b499c2 --- /dev/null +++ b/specs/002-github-releases/spec.md @@ -0,0 +1,220 @@ +# Feature Specification: GitHub Releases Distribution + +**Feature Branch**: `002-github-releases` +**Created**: 2026-01-05 +**Status**: Draft +**Target**: pglogical 2.5.0 + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Download Pre-built Binary for My Platform (Priority: P1) + +As a pglogical user, I want to download a pre-built binary package for my specific platform and PostgreSQL version so that I can install pglogical without compiling from source. + +**Why this priority**: This is the core value proposition - users need working binaries to use the extension. Without downloadable artifacts, there's no distribution to speak of. + +**Independent Test**: Can be fully tested by navigating to a GitHub Release page, downloading a package for a specific platform/PostgreSQL version combination, and verifying the package contains the expected files (shared library + SQL files). + +**Acceptance Scenarios**: + +1. **Given** a published release v2.5.0 exists, **When** a Windows user with PostgreSQL 17 visits the release page, **Then** they can download `pglogical-2.5.0-pg17-windows-x64.zip` containing pglogical.dll, pglogical_output.dll, and extension SQL files +2. **Given** a published release v2.5.0 exists, **When** a Linux user with PostgreSQL 16 visits the release page, **Then** they can download `pglogical-2.5.0-pg16-linux-x64.tar.gz` containing pglogical.so, pglogical_output.so, and extension SQL files +3. **Given** a published release v2.5.0 exists, **When** a macOS Apple Silicon user with PostgreSQL 17 visits the release page, **Then** they can download `pglogical-2.5.0-pg17-macos-arm64.tar.gz` containing pglogical.dylib and extension SQL files +4. **Given** a published release v2.5.0 exists, **When** a macOS Intel user with PostgreSQL 15 visits the release page, **Then** they can download `pglogical-2.5.0-pg15-macos-x64.tar.gz` containing the appropriate dylib files + +--- + +### User Story 2 - Install via Windows MSI Installer (Priority: P2) + +As a Windows administrator, I want to use an MSI installer to deploy pglogical so that installation integrates with Windows standard software management practices and automatically detects my PostgreSQL installation directory. + +**Why this priority**: MSI installers provide the most seamless Windows experience with automatic path detection and proper uninstall support. This significantly reduces friction for Windows users who may be less comfortable with manual file copying. + +**Independent Test**: Can be fully tested by running the MSI installer on a Windows machine with PostgreSQL installed, verifying it detects the correct PostgreSQL directory, installs files to proper locations, and registers for clean uninstall. + +**Acceptance Scenarios**: + +1. **Given** PostgreSQL 17 is installed in the default location on Windows, **When** I run the MSI installer for pglogical-pg17, **Then** the installer automatically detects the PostgreSQL installation directory and copies files to the correct lib and share/extension directories +2. **Given** the MSI installer has been run, **When** I open Windows Add/Remove Programs, **Then** pglogical appears in the installed programs list with an uninstall option +3. **Given** the MSI installer detected a non-standard PostgreSQL path, **When** I browse to select a different directory, **Then** the installer allows me to specify the correct PostgreSQL root directory + +--- + +### User Story 3 - Automatic Release on Git Tag (Priority: P1) + +As a release manager, I want the release pipeline to automatically trigger when I push a version tag so that releases are consistent, reproducible, and require minimal manual intervention. + +**Why this priority**: Automation is critical for consistency and reducing human error. Without automated triggering, every release would require manual coordination across platforms. + +**Independent Test**: Can be fully tested by pushing a tag matching the pattern `v*` (e.g., `v2.5.0`) and verifying that GitHub Actions workflows start building for all platform/PostgreSQL version combinations. + +**Acceptance Scenarios**: + +1. **Given** I push a tag `v2.5.0` to the repository, **When** GitHub Actions processes the push event, **Then** build jobs start for all 6 PostgreSQL versions (13-18) across Windows, Linux, and macOS platforms +2. **Given** all platform builds complete successfully, **When** the release job runs, **Then** a GitHub Release is created with the tag name and all build artifacts attached +3. **Given** a tag contains a hyphen (e.g., `v2.5.0-beta1`), **When** the release is created, **Then** it is marked as a prerelease + +--- + +### User Story 4 - Download Source Package with Dependencies (Priority: P3) + +As a developer building from source, I want to download a complete source package that includes all submodule dependencies so that I can build pglogical without additional repository cloning steps. + +**Why this priority**: Source packages are essential for users who need to build for unsupported platforms or with custom configurations, but binary packages serve the majority of users. + +**Independent Test**: Can be fully tested by downloading the source tarball, extracting it, and running `make` without needing to initialize git submodules separately. + +**Acceptance Scenarios**: + +1. **Given** a release exists, **When** I download `pglogical-2.5.0-source.tar.gz`, **Then** the archive includes all submodule contents (not just submodule references) +2. **Given** I extract the source tarball, **When** I run `make` with a valid PostgreSQL installation, **Then** the build succeeds without git submodule commands + +--- + +### User Story 5 - Verify Download Integrity (Priority: P3) + +As a security-conscious user, I want SHA256 checksums for all release artifacts so that I can verify downloads haven't been corrupted or tampered with. + +**Why this priority**: Security verification is important but most users rely on HTTPS download integrity. This is a best practice but not blocking for basic functionality. + +**Independent Test**: Can be fully tested by downloading checksums.txt, downloading any artifact, computing its SHA256 hash locally, and comparing against the published checksum. + +**Acceptance Scenarios**: + +1. **Given** a release exists, **When** I download checksums.txt, **Then** it contains SHA256 hashes for every other artifact in the release +2. **Given** I download any artifact and its corresponding checksum, **When** I compute `sha256sum `, **Then** the hash matches the published value + +--- + +### User Story 6 - Install on Linux/macOS with Helper Script (Priority: P2) + +As a Linux or macOS user, I want a simple installation script included in the package so that I can install the extension without memorizing PostgreSQL directory locations. + +**Why this priority**: While experienced users can copy files manually, an install script significantly improves usability and reduces installation errors. + +**Independent Test**: Can be fully tested by extracting the package and running `./install.sh`, then verifying files are copied to the correct PostgreSQL directories. + +**Acceptance Scenarios**: + +1. **Given** I extract the Linux/macOS package, **When** I run `./install.sh`, **Then** shared libraries are copied to `$(pg_config --pkglibdir)` and extension files to `$(pg_config --sharedir)/extension` +2. **Given** pg_config is not in PATH, **When** I set `PG_CONFIG=/path/to/pg_config` and run `./install.sh`, **Then** the script uses my specified pg_config +3. **Given** the target directories require root access, **When** I run `./install.sh`, **Then** the script uses sudo for copy operations and prompts for password if needed + +--- + +### User Story 7 - CI Validation on Pull Requests (Priority: P2) + +As a contributor, I want continuous integration to validate my changes build correctly on all supported platforms so that I catch issues before they reach a release. + +**Why this priority**: CI prevents broken releases and gives contributors confidence their changes work. This is essential infrastructure that enables the release process. + +**Independent Test**: Can be fully tested by opening a pull request with a minor change and verifying build jobs run for all platform/PostgreSQL version combinations. + +**Acceptance Scenarios**: + +1. **Given** I open a pull request to REL2_x_STABLE, **When** GitHub Actions processes the PR, **Then** build and test jobs run for all PostgreSQL versions (13-18) on Linux, Windows, and macOS +2. **Given** a build fails for PostgreSQL 15 on Linux, **When** I view the Actions tab, **Then** I can see which specific job failed and access its logs +3. **Given** all CI checks pass on a PR, **When** a maintainer merges the PR, **Then** the merge is allowed (branch protection enforced) + +--- + +### Edge Cases + +- What happens when a PostgreSQL version is not available via package manager (e.g., PG18 not yet in Homebrew)? Build job fails gracefully with clear error message; matrix should allow individual job failures without stopping others (fail-fast: false) +- How does the system handle PostgreSQL registry entries missing on Windows? MSI installer provides a directory browse option as fallback +- What happens if submodule clone fails during CI? Submodules use HTTPS URLs (not SSH) to avoid authentication issues in CI +- How does the system handle simultaneous tag pushes? Each tag triggers its own independent workflow run +- What if a previous release with the same version exists? MSI installer handles upgrades via MajorUpgrade element; GitHub Release action uses existing release if present + +## Requirements *(mandatory)* + +### Functional Requirements + +#### Build Pipeline Requirements + +- **FR-001**: System MUST build pglogical for PostgreSQL versions 13, 14, 15, 16, 17, and 18 +- **FR-002**: System MUST build for Windows x64 using MSVC compiler via Visual Studio 2022 +- **FR-003**: System MUST build for Linux x64 using GCC compiler +- **FR-004**: System MUST build for macOS ARM64 (Apple Silicon) using Clang +- **FR-005**: System MUST build for macOS x64 (Intel) using Clang +- **FR-006**: System MUST run regression tests as part of CI for all platforms (Linux, macOS, and Windows) +- **FR-007**: Build matrix MUST allow individual job failures without stopping other jobs (fail-fast: false) + +#### Artifact Requirements + +- **FR-008**: System MUST produce Windows artifacts as both ZIP archive and MSI installer per PostgreSQL version +- **FR-009**: System MUST produce Linux artifacts as tar.gz archive per PostgreSQL version +- **FR-010**: System MUST produce macOS artifacts as tar.gz archive per PostgreSQL version per architecture (arm64, x64) +- **FR-011**: System MUST produce source archives as both tar.gz and zip including submodule contents +- **FR-012**: Each binary package MUST include shared library files (dll/so/dylib), control file, and SQL extension files +- **FR-013**: Linux and macOS packages MUST include an install.sh helper script +- **FR-014**: Windows packages MUST include README.md with installation instructions + +#### Naming Convention Requirements + +- **FR-015**: Artifact file names MUST follow pattern: `pglogical-{version}-pg{pg_version}-{platform}-{arch}.{ext}` +- **FR-016**: Source archives MUST follow pattern: `pglogical-{version}-source.{ext}` +- **FR-017**: Version in artifact names MUST exclude the 'v' prefix from the git tag + +#### Windows MSI Installer Requirements + +- **FR-018**: MSI installer MUST auto-detect PostgreSQL installation directory from Windows registry +- **FR-019**: MSI installer MUST allow user to browse and select PostgreSQL directory if auto-detection fails +- **FR-020**: MSI installer MUST support side-by-side installation for different PostgreSQL versions (unique UpgradeCode per PG version) +- **FR-021**: MSI installer MUST support clean uninstall via Windows Add/Remove Programs +- **FR-022**: MSI installer MUST support upgrade from previous pglogical versions (same PG version) + +#### GitHub Release Requirements + +- **FR-023**: System MUST automatically create GitHub Release when a tag matching `v*` is pushed +- **FR-024**: System MUST attach all build artifacts to the GitHub Release +- **FR-025**: System MUST generate release notes with installation instructions for each platform (template includes: version highlights, platform-specific install commands, link to checksums.txt, upgrade notes) +- **FR-026**: System MUST generate checksums.txt containing SHA256 hashes of all artifacts +- **FR-027**: Releases for tags containing a hyphen (e.g., `-beta`, `-rc`) MUST be marked as prerelease + +#### CI Requirements + +- **FR-028**: CI workflow MUST run on push to REL2_x_STABLE branch and windows-build branch +- **FR-029**: CI workflow MUST run on pull requests targeting REL2_x_STABLE +- **FR-030**: CI MUST configure git to use HTTPS for submodule URLs (not SSH) to avoid authentication issues +- **FR-031**: CI MUST check out repository with submodules recursively + +### Key Entities + +- **Release Artifact**: A downloadable file attached to a GitHub Release (binary package, installer, or source archive); identified by filename following naming convention +- **Build Matrix**: The combination of platform (Windows/Linux/macOS), architecture (x64/arm64), and PostgreSQL version (13-18) that defines all build targets +- **GitHub Release**: A tagged release on GitHub containing release notes, checksums, and all artifacts for a specific version +- **MSI Installer**: Windows Installer package built using WiX Toolset that provides automated installation with PostgreSQL path detection + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: Users can download and install pglogical on any supported platform in under 5 minutes (excluding download time); measured from package extraction to successful `CREATE EXTENSION pglogical` +- **SC-002**: A complete release (all 30+ artifacts) is published within 60 minutes of tag push +- **SC-003**: 100% of release artifacts pass SHA256 checksum verification +- **SC-004**: Windows MSI installer successfully detects PostgreSQL installation in 95% of standard installations (standard = default installation path with registry entries from official PostgreSQL installer or EDB installer) +- **SC-005**: Install helper scripts work without modification on standard Linux distributions (Ubuntu 20.04+, RHEL/CentOS 8+) and macOS versions (13 Ventura through 26 Tahoe) with Homebrew or standard PostgreSQL installations +- **SC-006**: CI provides build status within 30 minutes for any pull request +- **SC-007**: All 6 PostgreSQL versions build successfully for each supported platform before a release is published +- **SC-008**: Source package builds successfully when extracted and compiled with `make` (no git required after extraction) + +## Assumptions + +- PostgreSQL development headers are available via standard package managers (apt for Linux, Homebrew for macOS, Chocolatey for Windows) +- GitHub Actions runners provide sufficient resources for building PostgreSQL extensions +- WiX Toolset v5 is available as a .NET global tool in GitHub Actions Windows runners +- PostgreSQL versions 13-18 remain the supported range for pglogical 2.5.0 +- Repository submodules use HTTPS URLs to enable CI cloning without SSH key configuration +- GitHub Actions windows-2022 runner supports Windows 10 and Windows 11 compatible builds (builds on Windows Server 2022) +- GitHub Actions macos-14 runner provides ARM64 architecture; macos-13 runner provides x64 architecture (runner names reflect image generation, not macOS version - both run current macOS releases) + +## Out of Scope + +- Code signing for Windows binaries (noted as future enhancement) +- macOS notarization for Gatekeeper (noted as future enhancement) +- Windows ARM64 builds (PostgreSQL doesn't officially support Windows ARM64) +- Linux ARM64 builds (can be added in future if demand exists) +- Automated testing of installed packages (manual verification for now) +- RPM or DEB package creation (users install via helper script or manual copy) +- Homebrew formula or APT repository publishing diff --git a/specs/002-github-releases/tasks.md b/specs/002-github-releases/tasks.md new file mode 100644 index 00000000..c84c8f9c --- /dev/null +++ b/specs/002-github-releases/tasks.md @@ -0,0 +1,338 @@ +# Tasks: GitHub Releases Distribution + +**Input**: Design documents from `/specs/002-github-releases/` +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/ + +**Tests**: Not explicitly requested in specification. Tests are manual verification per acceptance scenarios. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing of each story. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US3, US7) +- Include exact file paths in descriptions + +## Path Conventions + +Based on plan.md structure: +- `.github/workflows/` - GitHub Actions workflow definitions +- `packaging/windows/` - Windows-specific packaging (WiX, README) +- `packaging/unix/` - Linux/macOS packaging (install script) + +--- + +## Phase 1: Setup (Directory Structure) + +**Purpose**: Create project directory structure for CI/CD infrastructure + +- [x] T001 Create `.github/workflows/` directory structure +- [x] T002 [P] Create `packaging/windows/` directory structure +- [x] T003 [P] Create `packaging/unix/` directory structure + +--- + +## Phase 2: Foundational (CI Workflow) + +**Purpose**: Core CI infrastructure that MUST be complete before release workflow can function + +**⚠️ CRITICAL**: Release workflow depends on CI workflow patterns; CI must be working first + +- [x] T004 Create CI workflow file skeleton in `.github/workflows/ci.yml` +- [x] T005 Add checkout step with `submodules: recursive` in `.github/workflows/ci.yml` +- [x] T006 Define build matrix (PG 13-18, ubuntu-latest, windows-2022, macos-13, macos-14) in `.github/workflows/ci.yml` +- [x] T007 Add Linux PostgreSQL installation steps (apt install postgresql-server-dev-XX) in `.github/workflows/ci.yml` +- [x] T008 [P] Add macOS PostgreSQL installation steps (brew install postgresql@XX) in `.github/workflows/ci.yml` +- [x] T009 [P] Add Windows PostgreSQL installation steps (choco install postgresqlXX) in `.github/workflows/ci.yml` +- [x] T010 Add Linux/macOS build steps (make clean all) in `.github/workflows/ci.yml` +- [x] T011 [P] Add Windows build steps (cmake with MSVC) in `.github/workflows/ci.yml` +- [x] T012 Add Linux/macOS regression test step (make check) in `.github/workflows/ci.yml` +- [x] T013 Configure fail-fast: false in build matrix in `.github/workflows/ci.yml` +- [x] T014 Add workflow triggers (push to REL2_x_STABLE, windows-build; PR to REL2_x_STABLE) in `.github/workflows/ci.yml` + +**Checkpoint**: CI workflow complete - PRs can be validated across all platforms + +--- + +## Phase 3: User Story 7 - CI Validation on Pull Requests (Priority: P2) 🎯 MVP + +**Goal**: Contributors can validate their changes build correctly on all supported platforms via CI + +**Independent Test**: Open a pull request with a minor change and verify build jobs run for all platform/PostgreSQL version combinations + +### Implementation for User Story 7 + +> Note: Most tasks completed in Phase 2. This phase adds PR-specific enhancements. + +- [x] T015 [US7] Add artifact upload for test results on failure in `.github/workflows/ci.yml` +- [x] T016 [US7] Add workflow run summary with build status matrix in `.github/workflows/ci.yml` +- [x] T017 [US7] Document CI workflow and branch protection requirements in `packaging/README-CI.md` + +**Checkpoint**: US7 complete - CI validates PRs and shows clear per-job status + +--- + +## Phase 4: User Story 3 - Automatic Release on Git Tag (Priority: P1) + +**Goal**: Release pipeline automatically triggers when pushing a version tag (v*) + +**Independent Test**: Push a tag matching `v*` pattern and verify GitHub Actions workflows start building for all platform/PostgreSQL version combinations + +### Implementation for User Story 3 + +- [x] T018 [US3] Create release workflow file skeleton in `.github/workflows/release.yml` +- [x] T019 [US3] Add tag trigger (on push tags: v*) in `.github/workflows/release.yml` +- [x] T020 [US3] Add version extraction step (strip 'v' prefix from tag) in `.github/workflows/release.yml` +- [x] T021 [US3] Copy build matrix and platform steps from ci.yml to `.github/workflows/release.yml` +- [x] T022 [P] [US3] Add artifact packaging step for Linux (tar.gz with naming convention) in `.github/workflows/release.yml` +- [x] T023 [P] [US3] Add artifact packaging step for Windows (zip with naming convention) in `.github/workflows/release.yml` +- [x] T024 [P] [US3] Add artifact packaging step for macOS (tar.gz with naming convention) in `.github/workflows/release.yml` +- [x] T025 [US3] Add workflow artifact upload for each platform in `.github/workflows/release.yml` +- [x] T026 [US3] Add prerelease detection (tag contains hyphen) in `.github/workflows/release.yml` +- [x] T027 [US3] Create release job with needs: [build] in `.github/workflows/release.yml` +- [x] T028 [US3] Add artifact download step in release job in `.github/workflows/release.yml` +- [x] T029 [US3] Add GitHub Release creation with softprops/action-gh-release in `.github/workflows/release.yml` + +**Checkpoint**: US3 complete - Pushing v* tag creates GitHub Release with all binary artifacts + +--- + +## Phase 5: User Story 1 - Download Pre-built Binary for My Platform (Priority: P1) + +**Goal**: Users can download pre-built binary packages for their specific platform and PostgreSQL version + +**Independent Test**: Navigate to a GitHub Release page, download a package for a specific platform/PostgreSQL version, and verify the package contains expected files + +### Implementation for User Story 1 + +> Note: Release workflow from US3 creates artifacts. This phase ensures correct contents. + +- [x] T030 [US1] Ensure Linux packages include pglogical.so, pglogical_output.so, pglogical.control, and SQL files in `.github/workflows/release.yml` +- [x] T031 [P] [US1] Ensure Windows packages include pglogical.dll, pglogical_output.dll, pglogical.control, SQL files, and README.md in `.github/workflows/release.yml` +- [x] T032 [P] [US1] Ensure macOS packages include pglogical.dylib, pglogical_output.dylib, pglogical.control, and SQL files in `.github/workflows/release.yml` +- [x] T033 [US1] Verify artifact naming follows pattern `pglogical-{version}-pg{pg_version}-{platform}-{arch}.{ext}` in `.github/workflows/release.yml` +- [x] T034 [US1] Add release notes template with platform-specific installation instructions in `.github/workflows/release.yml` + +**Checkpoint**: US1 complete - Users can download correctly named packages containing all required files + +--- + +## Phase 6: User Story 6 - Install on Linux/macOS with Helper Script (Priority: P2) + +**Goal**: Linux/macOS users can run install.sh to install the extension without memorizing PostgreSQL paths + +**Independent Test**: Extract package, run `./install.sh`, verify files are copied to correct PostgreSQL directories + +### Implementation for User Story 6 + +- [x] T035 [US6] Create install.sh skeleton with shebang and set -e in `packaging/unix/install.sh` +- [x] T036 [US6] Add pg_config detection (check PATH, then PG_CONFIG env var) in `packaging/unix/install.sh` +- [x] T037 [US6] Add error message if pg_config not found in `packaging/unix/install.sh` +- [x] T038 [US6] Implement library file copy to $(pg_config --pkglibdir) in `packaging/unix/install.sh` +- [x] T039 [US6] Implement extension file copy to $(pg_config --sharedir)/extension in `packaging/unix/install.sh` +- [x] T040 [US6] Add sudo detection for privileged directories in `packaging/unix/install.sh` +- [x] T041 [US6] Add success message with installed file locations in `packaging/unix/install.sh` +- [x] T042 [US6] Make script executable (chmod +x) and include in package in `.github/workflows/release.yml` + +**Checkpoint**: US6 complete - Users can install with `./install.sh` using pg_config auto-detection + +--- + +## Phase 7: User Story 2 - Install via Windows MSI Installer (Priority: P2) + +**Goal**: Windows administrators can use MSI installer with automatic PostgreSQL path detection + +**Independent Test**: Run MSI installer on Windows with PostgreSQL installed, verify it detects correct directory and registers for uninstall + +### Implementation for User Story 2 + +- [x] T043 [US2] Create WiX v5 project structure with Product element in `packaging/windows/pglogical.wxs` +- [x] T044 [US2] Add RegistrySearch for PostgreSQL installation path detection in `packaging/windows/pglogical.wxs` +- [x] T045 [US2] Add fallback directory search for common PostgreSQL paths in `packaging/windows/pglogical.wxs` +- [x] T046 [US2] Add WixUI_InstallDir for directory browse fallback in `packaging/windows/pglogical.wxs` +- [x] T047 [US2] Define ComponentGroup for library files (pglogical.dll, pglogical_output.dll) in `packaging/windows/pglogical.wxs` +- [x] T048 [US2] Define ComponentGroup for extension files (control, SQL) in `packaging/windows/pglogical.wxs` +- [x] T049 [US2] Add MajorUpgrade element for clean upgrades in `packaging/windows/pglogical.wxs` +- [x] T050 [US2] Configure unique UpgradeCode per PostgreSQL version (com.2ndquadrant.pglogical.postgresqlXX) in `packaging/windows/pglogical.wxs` +- [x] T051 [US2] Create Windows installation README with manual instructions in `packaging/windows/README.md` +- [x] T052 [US2] Add WiX v5 installation step (dotnet tool install wix) in `.github/workflows/release.yml` +- [x] T053 [US2] Add MSI build step with version and PG version parameters in `.github/workflows/release.yml` +- [x] T054 [US2] Add MSI artifact to release assets in `.github/workflows/release.yml` + +**Checkpoint**: US2 complete - Windows users can install via MSI with automatic path detection + +--- + +## Phase 8: User Story 4 - Download Source Package with Dependencies (Priority: P3) + +**Goal**: Developers can download complete source package including submodule contents + +**Independent Test**: Download source tarball, extract it, run `make` without needing git submodule commands + +### Implementation for User Story 4 + +- [x] T055 [US4] Create source archive job in `.github/workflows/release.yml` +- [x] T056 [US4] Checkout with submodules: recursive and fetch-depth: 0 for source job in `.github/workflows/release.yml` +- [x] T057 [US4] Create source tar.gz with submodule contents included in `.github/workflows/release.yml` +- [x] T058 [P] [US4] Create source zip with submodule contents included in `.github/workflows/release.yml` +- [x] T059 [US4] Name source archives as `pglogical-{version}-source.{ext}` in `.github/workflows/release.yml` +- [x] T060 [US4] Add source archives to release assets in `.github/workflows/release.yml` + +**Checkpoint**: US4 complete - Source packages build without git submodule initialization + +--- + +## Phase 9: User Story 5 - Verify Download Integrity (Priority: P3) + +**Goal**: Security-conscious users can verify downloads via SHA256 checksums + +**Independent Test**: Download checksums.txt, download any artifact, compute SHA256 hash locally, compare against published checksum + +### Implementation for User Story 5 + +- [x] T061 [US5] Generate SHA256 checksums for all artifacts in release job in `.github/workflows/release.yml` +- [x] T062 [US5] Create checksums.txt file with all artifact hashes in `.github/workflows/release.yml` +- [x] T063 [US5] Add checksums.txt to release assets in `.github/workflows/release.yml` + +**Checkpoint**: US5 complete - All release artifacts have verifiable SHA256 checksums + +--- + +## Phase 10: Polish & Cross-Cutting Concerns + +**Purpose**: Final improvements and documentation + +- [x] T064 [P] Add comments documenting workflow structure in `.github/workflows/ci.yml` +- [x] T065 [P] Add comments documenting workflow structure in `.github/workflows/release.yml` +- [x] T066 Verify all artifact naming follows FR-015 pattern in `.github/workflows/release.yml` +- [x] T067 [P] Add error handling for missing PostgreSQL versions in `.github/workflows/ci.yml` +- [x] T068 Test complete release workflow with a test tag (v2.5.0-rc16) +- [x] T069 Validate install.sh works on Ubuntu and macOS +- [x] T070 Validate MSI installer on Windows with PostgreSQL 17 +- [x] T071 Run quickstart.md verification checklist + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies - can start immediately +- **Foundational (Phase 2)**: Depends on Setup - establishes CI workflow +- **US7 (Phase 3)**: Depends on Phase 2 - enhances CI for PR validation +- **US3 (Phase 4)**: Depends on Phase 2 - creates release workflow using CI patterns +- **US1 (Phase 5)**: Depends on US3 - ensures artifact contents are correct +- **US6 (Phase 6)**: Can start after Phase 1 - independent install script development +- **US2 (Phase 7)**: Can start after Phase 1 - independent MSI development +- **US4 (Phase 8)**: Depends on US3 - adds source package to release workflow +- **US5 (Phase 9)**: Depends on US3 - adds checksums to release workflow +- **Polish (Phase 10)**: Depends on all user stories complete + +### User Story Dependencies + +``` +Phase 1 (Setup) ─────────┬──────────────────────────────────────────────────┐ + │ │ + ▼ │ +Phase 2 (Foundation) ────┼──────────────────────────────────────────────────┤ + │ │ + ┌───────────────┼───────────────┐ │ + │ │ │ │ + ▼ ▼ ▼ │ + US7 (Phase 3) US3 (Phase 4) US6 (Phase 6) ←── Can start parallel │ + │ │ │ + │ │ US2 (Phase 7) ←────────┘ + │ │ │ + ┌───────────────┴───────────┐ │ │ + │ │ │ │ │ + ▼ ▼ ▼ ▼ ▼ + US1 (Phase 5) US4 (Phase 8) US5 (Phase 9) │ + │ │ │ │ + └───────────────┴───────────┴───────────────────┘ + │ + ▼ + Phase 10 (Polish) +``` + +### Within Each Phase + +- Tasks marked [P] can run in parallel +- Non-parallel tasks should execute in listed order +- Each phase should be committed as a logical unit + +### Parallel Opportunities + +**Phase 1 (all parallel)**: +- T001, T002, T003 - different directories + +**Phase 2 (platform-specific parallel)**: +- T008, T009 - different platform installation steps +- T010, T011 - different platform build steps + +**Phase 4 (packaging parallel)**: +- T022, T023, T024 - different platform packaging steps + +**Phase 5 (platform contents parallel)**: +- T031, T032 - different platform artifact contents + +**Phase 6, 7 (can work in parallel with each other)**: +- US6 (install script) and US2 (MSI) are independent +- Can be developed by different team members simultaneously + +**Phase 8 (archive format parallel)**: +- T057, T058 - tar.gz and zip creation + +--- + +## Parallel Example: Phase 4 (Release Workflow) + +```bash +# Launch platform packaging steps together: +Task: "T022 [P] [US3] Add artifact packaging step for Linux in .github/workflows/release.yml" +Task: "T023 [P] [US3] Add artifact packaging step for Windows in .github/workflows/release.yml" +Task: "T024 [P] [US3] Add artifact packaging step for macOS in .github/workflows/release.yml" +``` + +--- + +## Implementation Strategy + +### MVP First (CI + Release + Binaries) + +1. Complete Phase 1: Setup +2. Complete Phase 2: Foundational (CI workflow) +3. Complete Phase 3: US7 (CI validation) +4. Complete Phase 4: US3 (release automation) +5. Complete Phase 5: US1 (binary packages) +6. **STOP and VALIDATE**: Push test tag, verify release is created with binaries +7. Deploy/demo if ready + +### Incremental Delivery + +1. Setup + Foundational + US7 → CI works for PRs +2. Add US3 + US1 → Tag-triggered releases with binaries (Core MVP!) +3. Add US6 → Linux/macOS install script +4. Add US2 → Windows MSI installer +5. Add US4 → Source packages +6. Add US5 → Checksums +7. Each addition enhances distribution without breaking existing functionality + +### Parallel Team Strategy + +With two developers: + +1. Team completes Setup + Foundational together +2. Developer A: US7 → US3 → US1 → US4 → US5 (workflow path) +3. Developer B: US6 → US2 (packaging path, can start immediately) +4. Polish phase after both paths complete + +--- + +## Notes + +- [P] tasks = different files, no dependencies +- [Story] label maps task to specific user story for traceability +- Each user story should be independently completable and testable +- Commit after each task or logical group +- Test workflows by pushing to a test branch before merging +- Verify MSI locally before including in release workflow diff --git a/sql/att_list.sql b/sql/att_list.sql index e7c1291a..f84d03f8 100644 --- a/sql/att_list.sql +++ b/sql/att_list.sql @@ -124,6 +124,8 @@ DELETE FROM basic_dml WHERE other = 2; SELECT * FROM basic_dml ORDER BY other; SELECT nspname, relname, att_list, has_row_filter FROM pglogical.show_repset_table_info('basic_dml', ARRAY['default']); +SELECT pglogical.wait_slot_confirm_lsn(NULL, NULL); + \c :subscriber_dsn -- verify that columns are not automatically added for filtering unless told so. SELECT * FROM pglogical.show_subscription_table('test_subscription', 'basic_dml');