From e1fcd68587314bd9afca6a7930d606d10a1546c6 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:15:24 +0000
Subject: [PATCH 001/103] remove unrelated workflows

---
 .github/workflows/db-migrations.yml    | 51 ------------------------
 .github/workflows/oidc-e2e.yml         | 49 -----------------------
 .github/workflows/oidc-integration.yml | 39 ------------------
 .github/workflows/s3-e2e.yml           | 39 ------------------
 .github/workflows/standard-e2e.yml     | 55 --------------------------
 .github/workflows/standard-suite.yml   | 37 -----------------
 6 files changed, 270 deletions(-)
 delete mode 100644 .github/workflows/db-migrations.yml
 delete mode 100644 .github/workflows/oidc-e2e.yml
 delete mode 100644 .github/workflows/oidc-integration.yml
 delete mode 100644 .github/workflows/s3-e2e.yml
 delete mode 100644 .github/workflows/standard-e2e.yml
 delete mode 100644 .github/workflows/standard-suite.yml

diff --git a/.github/workflows/db-migrations.yml b/.github/workflows/db-migrations.yml
deleted file mode 100644
index 1353620bf..000000000
--- a/.github/workflows/db-migrations.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-name: Database Migrations
-
-on:
-  pull_request:
-    paths:
-    - .github/workflows/db-migrations.yml
-    - lib/bin/create-docker-databases.js
-    - lib/model/migrations/**
-    - test/db-migrations/**
-    - package.json
-    - package-lock.json
-    - Makefile
-  push:
-    paths:
-    - .github/workflows/db-migrations.yml
-    - lib/bin/create-docker-databases.js
-    - lib/model/migrations/**
-    - test/db-migrations/**
-    - package.json
-    - package-lock.json
-    - Makefile
-
-jobs:
-  db-migration-tests:
-    timeout-minutes: 2
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - run: make test-db-migrations
diff --git a/.github/workflows/oidc-e2e.yml b/.github/workflows/oidc-e2e.yml
deleted file mode 100644
index 8dd0289ac..000000000
--- a/.github/workflows/oidc-e2e.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-name: OIDC e2e tests
-
-on:
-  push:
-  pull_request:
-
-env:
-  DEBUG: pw:api
-  ODK_PLAYWRIGHT_BROWSERS: chromium,firefox,webkit
-
-jobs:
-  oidc-e2e-test:
-    timeout-minutes: 6
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-
-    # This one weird trick speeds up every build!
-    # see: https://github.com/getodk/central-backend/pull/1642
-    # see: https://github.com/actions/runner/issues/4030
-    - run: sudo apt-get remove --purge man-db
-
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: make test-oidc-e2e
-    - name: Archive Playwright Test Report
-      if: failure()
-      uses: actions/upload-artifact@v4
-      with:
-        name: Playwright Test Report
-        path: test/e2e/oidc/playwright-tests/results/html-report/
diff --git a/.github/workflows/oidc-integration.yml b/.github/workflows/oidc-integration.yml
deleted file mode 100644
index 1ebea73c2..000000000
--- a/.github/workflows/oidc-integration.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: OIDC integration tests
-
-on:
-  push:
-  pull_request:
-
-jobs:
-  oidc-integration-test:
-    timeout-minutes: 6
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: FAKE_OIDC_ROOT_URL=http://localhost:9898 make fake-oidc-server-ci > fake-oidc-server.log &
-    - run: node lib/bin/create-docker-databases.js
-    - run: make test-oidc-integration
-    - name: Fake OIDC Server Logs
-      if: always()
-      run: "! [[ -f ./fake-oidc-server.log ]] || cat ./fake-oidc-server.log"
diff --git a/.github/workflows/s3-e2e.yml b/.github/workflows/s3-e2e.yml
deleted file mode 100644
index afbfc00e9..000000000
--- a/.github/workflows/s3-e2e.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: S3 E2E Tests
-
-on:
-  push:
-  pull_request:
-
-jobs:
-  s3-e2e:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - run: make fake-s3-server-persistent
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - run: sudo apt-get install wait-for-it
-    - name: E2E Test
-      timeout-minutes: 10
-      run: ./test/e2e/s3/run-tests.sh
diff --git a/.github/workflows/standard-e2e.yml b/.github/workflows/standard-e2e.yml
deleted file mode 100644
index 39aef161e..000000000
--- a/.github/workflows/standard-e2e.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: Standard E2E Tests
-
-on:
-  push:
-  pull_request:
-
-env:
-  LOG_LEVEL: DEBUG
-
-jobs:
-  standard-e2e:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-
-    - name: E2E Test
-      timeout-minutes: 10
-      run: ./test/e2e/standard/run-tests.sh
-
-    - uses: ./.github/actions/install-postgres-client
-      with:
-        postgres-version: 14
-    - name: Grant postgres superuser role
-      # As per #1368, this role is required to create the pgrowlocks extension
-      run: psql -c 'ALTER ROLE jubilant SUPERUSER' postgresql://postgres:odktest@localhost/postgres
-    - name: Backup/restore tests
-      timeout-minutes: 10
-      run: ./test/e2e/standard/backup-restore.sh
-
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./server.log ]] || cat ./server.log"
diff --git a/.github/workflows/standard-suite.yml b/.github/workflows/standard-suite.yml
deleted file mode 100644
index e57c139e9..000000000
--- a/.github/workflows/standard-suite.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-name: Full Standard Test Suite
-
-on:
-  push:
-  pull_request:
-
-jobs:
-  standard-tests:
-    timeout-minutes: 20
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: make check-for-large-files
-    - run: npm ci
-    - run: make api-docs-lint
-    - run: node lib/bin/create-docker-databases.js
-    - run: make test

From bb58322fb0e7d27046ea0f37b743152c26689332 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:16:49 +0000
Subject: [PATCH 002/103] run 10x

---
 .github/workflows/soak-test.yml | 333 +++++++++++++++++++++++++++++++-
 1 file changed, 331 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index d62427664..224c102a7 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -2,10 +2,339 @@ name: Soak Test
 
 on:
   push:
-    branches: master
 
 jobs:
-  soak-test:
+  soak-test-0:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-1:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-1:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-2:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-3:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-4:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-5:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-6:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-7:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-8:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-9:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest

From 60594c2d020a68b075f640aab59569ffee893fbf Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:17:44 +0000
Subject: [PATCH 003/103] 01

---
 .github/workflows/soak-test.yml | 33 ---------------------------------
 1 file changed, 33 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 224c102a7..b8abb8c62 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -70,39 +70,6 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-1:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
   soak-test-2:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?

From 56a0a56e4d80e5d1b03f7cee51f2e0ac277b6372 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:23:20 +0000
Subject: [PATCH 004/103] Show more info about open queries

---
 lib/bin/check-open-db-queries.js | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/lib/bin/check-open-db-queries.js b/lib/bin/check-open-db-queries.js
index a023924bf..f9983470b 100644
--- a/lib/bin/check-open-db-queries.js
+++ b/lib/bin/check-open-db-queries.js
@@ -12,10 +12,18 @@ const { withKnex } = require('../model/knex-migrator');
 (async () => {
   try {
     const { rows } = await withKnex(require('config').get('default.database'))((db) =>
-      db.raw('SELECT COUNT(*) FROM pg_stat_activity WHERE usename=CURRENT_USER'));
-    const queryCount = rows[0].count - 1; // the count query will appear as one of the open queries
+      db.raw(`
+        SELECT state
+             , query_start
+             , query
+          FROM pg_stat_activity
+          WHERE usename=CURRENT_USER
+      `)
+    );
 
-    console.log('Open query count:', queryCount);
+    console.log('Open queries:', JSON.stringify(rows, null, 2));
+
+    const queryCount = rows.length - 1; // the query above will appear as one of the open queries
 
     if (queryCount) process.exit(2);
   } catch (err) {

From a40ae55c3d561a43081d573f631b69c14b6024cc Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:25:24 +0000
Subject: [PATCH 005/103] simplify logic

---
 lib/bin/check-open-db-queries.js | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/bin/check-open-db-queries.js b/lib/bin/check-open-db-queries.js
index f9983470b..3000a4c29 100644
--- a/lib/bin/check-open-db-queries.js
+++ b/lib/bin/check-open-db-queries.js
@@ -11,8 +11,11 @@ const { withKnex } = require('../model/knex-migrator');
 
 (async () => {
   try {
-    const { rows } = await withKnex(require('config').get('default.database'))((db) =>
+    const markerComment = `-- marker: Math.random()`;
+
+    const { rows:unfiltered } = await withKnex(require('config').get('default.database'))((db) =>
       db.raw(`
+        ${markerComment}
         SELECT state
              , query_start
              , query
@@ -20,12 +23,11 @@ const { withKnex } = require('../model/knex-migrator');
           WHERE usename=CURRENT_USER
       `)
     );
+    const filtered = unfiltered.filter(({ query }) => !query.includes(markerComment));
 
-    console.log('Open queries:', JSON.stringify(rows, null, 2));
-
-    const queryCount = rows.length - 1; // the query above will appear as one of the open queries
+    console.log('Open queries:', JSON.stringify(filtered, null, 2));
 
-    if (queryCount) process.exit(2);
+    if (filtered.length) process.exit(2);
   } catch (err) {
     console.error('Error:', err.message);
     process.exit(1);

From 6d0340cffe98592ef56790978cf44cf448024671 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:25:57 +0000
Subject: [PATCH 006/103] lint

---
 lib/bin/check-open-db-queries.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/bin/check-open-db-queries.js b/lib/bin/check-open-db-queries.js
index 3000a4c29..ce538cc55 100644
--- a/lib/bin/check-open-db-queries.js
+++ b/lib/bin/check-open-db-queries.js
@@ -13,7 +13,7 @@ const { withKnex } = require('../model/knex-migrator');
   try {
     const markerComment = `-- marker: Math.random()`;
 
-    const { rows:unfiltered } = await withKnex(require('config').get('default.database'))((db) =>
+    const { rows: unfiltered } = await withKnex(require('config').get('default.database'))((db) =>
       db.raw(`
         ${markerComment}
         SELECT state

From 70caa46fdb43f1d23befc6090e828ed1ef932d31 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:40:26 +0000
Subject: [PATCH 007/103] reduce logging to one line

---
 lib/bin/check-open-db-queries.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/bin/check-open-db-queries.js b/lib/bin/check-open-db-queries.js
index ce538cc55..65d604c0c 100644
--- a/lib/bin/check-open-db-queries.js
+++ b/lib/bin/check-open-db-queries.js
@@ -25,7 +25,7 @@ const { withKnex } = require('../model/knex-migrator');
     );
     const filtered = unfiltered.filter(({ query }) => !query.includes(markerComment));
 
-    console.log('Open queries:', JSON.stringify(filtered, null, 2));
+    console.log('Open queries:', JSON.stringify(filtered));
 
     if (filtered.length) process.exit(2);
   } catch (err) {

From 119f58668932e1e8f587197ad21a498f286e5abc Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 10:56:49 +0000
Subject: [PATCH 008/103] store bigger queries?

---
 .github/workflows/soak-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index b8abb8c62..606c54d3f 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -22,6 +22,7 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          -c track_activity_query_size=1048576
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 42b6a383e1f096c15774b4ef255c56ba7ad3f27f Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 13:13:23 +0000
Subject: [PATCH 009/103] dump & analyse gha logs to check what's going on

---
 analyse-queries.js           | 36 ++++++++++++++++++++++++++++++++++++
 dump-last-github-job-logs.sh | 23 +++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100755 analyse-queries.js
 create mode 100755 dump-last-github-job-logs.sh

diff --git a/analyse-queries.js b/analyse-queries.js
new file mode 100755
index 000000000..ec73444c9
--- /dev/null
+++ b/analyse-queries.js
@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+
+const { execSync } = require('node:child_process');
+const { readdirSync, readFileSync } = require('node:fs');
+
+const log = (...args) => console.log('[analyse-queries]', ...args);
+
+const rootDir = './gha-logs';
+
+const [,,...args] = process.argv;
+const jobId = args.length ? args[0] : execSync(`ls -td ${rootDir}/* | head -1`, { encoding:'utf8' }).substring(rootDir.length + 1).trim();
+log('job id:', jobId);
+
+const allQueries = {};
+
+for(const _f of readdirSync(`./gha-logs/${jobId}`)) {
+  const f = `${rootDir}/${jobId}/${_f}`;
+  log('Processing:', f);
+
+  const queryPrefix = 'Open queries: ';
+  readFileSync(f, 'utf8')
+      .split('\n')
+      .filter(it => it.includes(queryPrefix))
+      .flatMap(it => {
+        const [ prefix, json ] = it.split(queryPrefix);
+        console.log(prefix);
+        const q = JSON.parse(json);
+        return q;
+      })
+      .forEach(processQuery)
+      ;
+}
+
+function processQuery({ state, query_start, query }) {
+  console.log(state);
+}
diff --git a/dump-last-github-job-logs.sh b/dump-last-github-job-logs.sh
new file mode 100755
index 000000000..e6e457fe3
--- /dev/null
+++ b/dump-last-github-job-logs.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+log() { echo >&2 "[$(basename "$0")] $*"; }
+
+runId=$(gh run list --branch $(git branch --show-current) --limit 1 --json databaseId --jq '.[0].databaseId')
+log "run id: $runId"
+
+logDir="gha-logs/$runId"
+log "log dir: $logDir"
+
+if [[ -d "$logDir" ]]; then
+  log "It looks like you already got the latest logs."
+  exit
+fi
+
+mkdir -p "$logDir"
+
+gh run view "$runId" --json jobs --jq '.jobs[] | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
+  safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
+  log "fetching logs for: $jobName..."
+  gh run view --job "$jobId" --log > "gha-logs/$runId/$safeName.log"
+done
+
+log "All done."

From 80665ada2b73ed350437ac36b63c76d83bde42b7 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:01:08 +0000
Subject: [PATCH 010/103] exclude idle queries

---
 lib/bin/check-open-db-queries.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/bin/check-open-db-queries.js b/lib/bin/check-open-db-queries.js
index 65d604c0c..c6e05e5ce 100644
--- a/lib/bin/check-open-db-queries.js
+++ b/lib/bin/check-open-db-queries.js
@@ -20,7 +20,8 @@ const { withKnex } = require('../model/knex-migrator');
              , query_start
              , query
           FROM pg_stat_activity
-          WHERE usename=CURRENT_USER
+          WHERE usename = CURRENT_USER
+            AND state = 'idle'
       `)
     );
     const filtered = unfiltered.filter(({ query }) => !query.includes(markerComment));

From 90f65b052f0bef83e636dd17cef523154712cb44 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:16:51 +0000
Subject: [PATCH 011/103] ignore logs dir

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 7fcff594c..dcf9969f1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ node_modules/
 .vscode
 /junit-reports/
 /config/development.json
+/gha-logs/

From a4ad01eb516b7c1d0f15706143dc1ed7a444b752 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:17:03 +0000
Subject: [PATCH 012/103] don't dump jobs in progress

---
 dump-last-github-job-logs.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dump-last-github-job-logs.sh b/dump-last-github-job-logs.sh
index e6e457fe3..857e0e412 100755
--- a/dump-last-github-job-logs.sh
+++ b/dump-last-github-job-logs.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -eu
 log() { echo >&2 "[$(basename "$0")] $*"; }
 
-runId=$(gh run list --branch $(git branch --show-current) --limit 1 --json databaseId --jq '.[0].databaseId')
+runId=$(gh run list --branch "$(git branch --show-current)" --limit 1 --json databaseId --jq '.[0].databaseId')
 log "run id: $runId"
 
 logDir="gha-logs/$runId"
@@ -14,10 +14,10 @@ fi
 
 mkdir -p "$logDir"
 
-gh run view "$runId" --json jobs --jq '.jobs[] | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
+gh run view "$runId" --json jobs --jq '.jobs[] | select(.status == "completed") | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
   safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
   log "fetching logs for: $jobName..."
-  gh run view --job "$jobId" --log > "gha-logs/$runId/$safeName.log"
+  gh run view --job "$jobId" --log > "$logDir/$safeName.log"
 done
 
 log "All done."

From 3e8670e2f488d829714481135c8b20140064c428 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:20:33 +0000
Subject: [PATCH 013/103] add list of jobs in progress

---
 dump-last-github-job-logs.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/dump-last-github-job-logs.sh b/dump-last-github-job-logs.sh
index 857e0e412..26c2097ec 100755
--- a/dump-last-github-job-logs.sh
+++ b/dump-last-github-job-logs.sh
@@ -1,6 +1,9 @@
 #!/bin/bash -eu
 log() { echo >&2 "[$(basename "$0")] $*"; }
 
+log "jobs in progress:"
+gh run list --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
+
 runId=$(gh run list --branch "$(git branch --show-current)" --limit 1 --json databaseId --jq '.[0].databaseId')
 log "run id: $runId"
 

From 677e526e27e63fe16bd5e6e7ae54dc794a27860d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:20:43 +0000
Subject: [PATCH 014/103] analyse usefully

---
 analyse-queries.js | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/analyse-queries.js b/analyse-queries.js
index ec73444c9..81e040a46 100755
--- a/analyse-queries.js
+++ b/analyse-queries.js
@@ -11,26 +11,45 @@ const [,,...args] = process.argv;
 const jobId = args.length ? args[0] : execSync(`ls -td ${rootDir}/* | head -1`, { encoding:'utf8' }).substring(rootDir.length + 1).trim();
 log('job id:', jobId);
 
-const allQueries = {};
+const allActiveQueries = {};
 
 for(const _f of readdirSync(`./gha-logs/${jobId}`)) {
   const f = `${rootDir}/${jobId}/${_f}`;
   log('Processing:', f);
 
   const queryPrefix = 'Open queries: ';
-  readFileSync(f, 'utf8')
+  const activeQueries = readFileSync(f, 'utf8')
       .split('\n')
       .filter(it => it.includes(queryPrefix))
-      .flatMap(it => {
+      .map(it => {
         const [ prefix, json ] = it.split(queryPrefix);
-        console.log(prefix);
-        const q = JSON.parse(json);
-        return q;
-      })
-      .forEach(processQuery)
-      ;
+        return JSON.parse(json)
+            .filter(it => it.state !== 'idle')
+            .map(q => {
+              const logged_at = new Date(prefix.split(/\s+/)[3]);;
+              const query_start = new Date(q.query_start);
+              return {
+                ...q,
+                logged_at,
+                query_start,
+                duration_so_far: logged_at.valueOf() - query_start.valueOf(),
+              };
+            });
+        });
+
+  const lastQueries = activeQueries.at(-1);
+  log('  last activeQueries:', lastQueries);
+
+  activeQueries
+      .flat()
+      .forEach(processQuery);
+
+  log('  Done.');
 }
 
-function processQuery({ state, query_start, query }) {
-  console.log(state);
+console.log('allActiveQueries:', JSON.stringify(allActiveQueries, null, 2));
+
+function processQuery({ state, duration_so_far, query }) {
+  if(!allActiveQueries[query]) allActiveQueries[query] = duration_so_far;
+  allActiveQueries[query] = Math.max(allActiveQueries[query], duration_so_far);
 }

From de9d72c979fd8b335a01eef5a1fd71bd1bfd095f Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:22:32 +0000
Subject: [PATCH 015/103] 10 more jobs

---
 .github/workflows/soak-test.yml | 351 +++++++++++++++++++++++++++++++-
 1 file changed, 341 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 606c54d3f..5de7bdb9f 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -4,7 +4,7 @@ on:
   push:
 
 jobs:
-  soak-test-0:
+  soak-test-00:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -38,7 +38,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-1:
+  soak-test-01:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -71,7 +71,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-2:
+  soak-test-02:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -104,7 +104,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-3:
+  soak-test-03:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -137,7 +137,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-4:
+  soak-test-04:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -170,7 +170,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-5:
+  soak-test-05:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -203,7 +203,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-6:
+  soak-test-06:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -236,7 +236,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-7:
+  soak-test-07:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -269,7 +269,7 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-8:
+  soak-test-08:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
@@ -302,7 +302,338 @@ jobs:
     - name: Backend Logs
       if: always()
       run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-9:
+  soak-test-09:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-10:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          -c track_activity_query_size=1048576
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-11:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-12:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-13:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-14:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-15:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-16:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-17:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-18:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-19:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest

From 3addc27a5c68e13942f1cdd01118395718bf50e3 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:24:52 +0000
Subject: [PATCH 016/103] dump for completed RUNs

---
 dump-last-github-job-logs.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dump-last-github-job-logs.sh b/dump-last-github-job-logs.sh
index 26c2097ec..a4975e4c0 100755
--- a/dump-last-github-job-logs.sh
+++ b/dump-last-github-job-logs.sh
@@ -4,7 +4,7 @@ log() { echo >&2 "[$(basename "$0")] $*"; }
 log "jobs in progress:"
 gh run list --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
 
-runId=$(gh run list --branch "$(git branch --show-current)" --limit 1 --json databaseId --jq '.[0].databaseId')
+runId="$(gh run list --branch "$(git branch --show-current)" --limit 1 --status completed --json databaseId --jq '.[0].databaseId')"
 log "run id: $runId"
 
 logDir="gha-logs/$runId"
@@ -17,7 +17,7 @@ fi
 
 mkdir -p "$logDir"
 
-gh run view "$runId" --json jobs --jq '.jobs[] | select(.status == "completed") | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
+gh run view "$runId" --json jobs --jq '.jobs[] | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
   safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
   log "fetching logs for: $jobName..."
   gh run view --job "$jobId" --log > "$logDir/$safeName.log"

From e551505a3914676ea46f6c3f833c0cc16fcacfd4 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:25:39 +0000
Subject: [PATCH 017/103] clearer output when no runs in progress

---
 dump-last-github-job-logs.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dump-last-github-job-logs.sh b/dump-last-github-job-logs.sh
index a4975e4c0..936dd252d 100755
--- a/dump-last-github-job-logs.sh
+++ b/dump-last-github-job-logs.sh
@@ -1,8 +1,9 @@
 #!/bin/bash -eu
 log() { echo >&2 "[$(basename "$0")] $*"; }
 
-log "jobs in progress:"
+log "--- runs in progress: ---"
 gh run list --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
+log "-------------------------"
 
 runId="$(gh run list --branch "$(git branch --show-current)" --limit 1 --status completed --json databaseId --jq '.[0].databaseId')"
 log "run id: $runId"

From 89b7f5952311e276369e53f3ebd1fb66eba5249f Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:39:46 +0000
Subject: [PATCH 018/103] rename script

---
 dump-last-github-job-logs.sh => dump-github-job-logs.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename dump-last-github-job-logs.sh => dump-github-job-logs.sh (100%)

diff --git a/dump-last-github-job-logs.sh b/dump-github-job-logs.sh
similarity index 100%
rename from dump-last-github-job-logs.sh
rename to dump-github-job-logs.sh

From 6ac0fd3dd2e692205c5995ecb7d3d2691c22150e Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:41:40 +0000
Subject: [PATCH 019/103] allow dump of specific run id

---
 dump-github-job-logs.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/dump-github-job-logs.sh b/dump-github-job-logs.sh
index 936dd252d..76315c92a 100755
--- a/dump-github-job-logs.sh
+++ b/dump-github-job-logs.sh
@@ -1,11 +1,18 @@
 #!/bin/bash -eu
 log() { echo >&2 "[$(basename "$0")] $*"; }
 
-log "--- runs in progress: ---"
-gh run list --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
-log "-------------------------"
 
-runId="$(gh run list --branch "$(git branch --show-current)" --limit 1 --status completed --json databaseId --jq '.[0].databaseId')"
+if [[ $# -gt 0 ]]; then
+  runId="$1"
+  shift
+else
+  log "--- runs in progress: ---"
+  gh run list --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
+  log "-------------------------"
+
+  log "Fetching last COMPLETED run id..."
+  runId="$(gh run list --branch "$(git branch --show-current)" --limit 1 --status completed --json databaseId --jq '.[0].databaseId')"
+fi
 log "run id: $runId"
 
 logDir="gha-logs/$runId"

From 2ecffcedc5edb3b82284d0d0d2fbdeaa02653ae4 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:43:31 +0000
Subject: [PATCH 020/103] larger queries: take 2

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 5de7bdb9f..dde066dda 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -14,6 +14,7 @@ jobs:
         image: postgres:14.20
         env:
           POSTGRES_PASSWORD: odktest
+          POSTGRES_INITDB_ARGS: "--track_activity_query_size=16384"
         ports:
           - 5432:5432
         # Set health checks to wait until postgres has started
@@ -22,7 +23,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-          -c track_activity_query_size=1048576
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 95d171efb6fb49f029d904906f2e967c3f9b3442 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:45:55 +0000
Subject: [PATCH 021/103] try something else

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index dde066dda..dfacd4a70 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -14,7 +14,6 @@ jobs:
         image: postgres:14.20
         env:
           POSTGRES_PASSWORD: odktest
-          POSTGRES_INITDB_ARGS: "--track_activity_query_size=16384"
         ports:
           - 5432:5432
         # Set health checks to wait until postgres has started
@@ -23,6 +22,7 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          -- postgres -c track_activity_query_size=16384
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 98ceffd872e4b3f30e6fb00c896d3b30287779d0 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Tue, 31 Mar 2026 16:47:25 +0000
Subject: [PATCH 022/103] try --t

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index dfacd4a70..d75ae10f9 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -22,7 +22,7 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-          -- postgres -c track_activity_query_size=16384
+          --track-activity-query-size=16384
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From d06db12b7fd686056420ae4b9b8d2c38dc14ebed Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 08:50:09 +0000
Subject: [PATCH 023/103] only list in-progress ON THIS BRANCH

---
 dump-github-job-logs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dump-github-job-logs.sh b/dump-github-job-logs.sh
index 76315c92a..adc7725e2 100755
--- a/dump-github-job-logs.sh
+++ b/dump-github-job-logs.sh
@@ -7,7 +7,7 @@ if [[ $# -gt 0 ]]; then
   shift
 else
   log "--- runs in progress: ---"
-  gh run list --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
+  gh run list  --branch "$(git branch --show-current)" --status in_progress --json headBranch,headSha,displayTitle,url --jq '.[] | "* \(.headBranch) \(.headSha[0:7]) \"\(.displayTitle)\": \(.url)"'
   log "-------------------------"
 
   log "Fetching last COMPLETED run id..."

From 81f6b9e18c23125ec2a439f65d5ae5f68f2ffc7a Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 08:57:01 +0000
Subject: [PATCH 024/103] PGOPTIONS

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index d75ae10f9..608fbe79c 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -14,6 +14,7 @@ jobs:
         image: postgres:14.20
         env:
           POSTGRES_PASSWORD: odktest
+          PGOPTIONS: "-c track_activity_query_size=16384"
         ports:
           - 5432:5432
         # Set health checks to wait until postgres has started
@@ -22,7 +23,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-          --track-activity-query-size=16384
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 0f4017c60c17eda62d11fe521f3203e84e8ffce8 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 08:59:27 +0000
Subject: [PATCH 025/103] entrypoint

---
 .github/workflows/soak-test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 608fbe79c..8c153079c 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -14,7 +14,6 @@ jobs:
         image: postgres:14.20
         env:
           POSTGRES_PASSWORD: odktest
-          PGOPTIONS: "-c track_activity_query_size=16384"
         ports:
           - 5432:5432
         # Set health checks to wait until postgres has started
@@ -23,6 +22,8 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --entrypoint docker-entrypoint.sh
+          postgres:14.20 postgres -c track_activity_query_size=16384
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 5a99f2a0241dceffd546a13c7b165ab56dc091a3 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:07:34 +0000
Subject: [PATCH 026/103] wat

---
 .github/workflows/soak-test.yml | 639 +-------------------------------
 1 file changed, 11 insertions(+), 628 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 8c153079c..4eab65d13 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -25,634 +25,17 @@ jobs:
           --entrypoint docker-entrypoint.sh
           postgres:14.20 postgres -c track_activity_query_size=16384
     steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-01:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-02:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-03:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-04:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-05:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-06:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-07:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-08:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-09:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-10:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-          -c track_activity_query_size=1048576
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-11:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-12:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-13:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-14:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-15:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-16:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-17:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-18:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-19:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
+    # Per https://github.com/orgs/community/discussions/126453
+    - name: show postgres track_activity_query_size
+      run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "SHOW track_activity_query_size"
+    - name: set postgres track_activity_query_size
+      run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "ALTER SYSTEM SET track_activity_query_size = 16384"
+    - name: Restart Postgres
+      run: docker restart $(docker ps -q --filter "ancestor=postgres")
+    - name: Print Services
+      run: service --status-all
+    - name: show postgres track_activity_query_size
+      run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "SHOW track_activity_query_size"
     - uses: actions/checkout@v6
     - name: Set node version
       uses: actions/setup-node@v6

From c72ba37cd75da9b6c68fcda2ae2fd7881278e3a2 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:25:34 +0000
Subject: [PATCH 027/103] tryagin

---
 .github/workflows/soak-test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 4eab65d13..bab0504da 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -22,8 +22,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-          --entrypoint docker-entrypoint.sh
-          postgres:14.20 postgres -c track_activity_query_size=16384
     steps:
     # Per https://github.com/orgs/community/discussions/126453
     - name: show postgres track_activity_query_size

From b92011fe96d285b0efc831f97818b232cd16c721 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:39:43 +0000
Subject: [PATCH 028/103] create db first

---
 .github/workflows/soak-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index bab0504da..eda905fc8 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -24,6 +24,8 @@ jobs:
           --health-retries 5
     steps:
     # Per https://github.com/orgs/community/discussions/126453
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
     - name: show postgres track_activity_query_size
       run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "SHOW track_activity_query_size"
     - name: set postgres track_activity_query_size
@@ -40,8 +42,6 @@ jobs:
       with:
         node-version-file: package.json
         cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
     - name: Soak Test
       timeout-minutes: 10
       run: ./test/e2e/soak/ci

From cce87b635b35a17b94a7b26e8508c18fc6c091cc Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:40:05 +0000
Subject: [PATCH 029/103] use postgres db

---
 .github/workflows/soak-test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index eda905fc8..a9fd537aa 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -24,24 +24,24 @@ jobs:
           --health-retries 5
     steps:
     # Per https://github.com/orgs/community/discussions/126453
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
     - name: show postgres track_activity_query_size
-      run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "SHOW track_activity_query_size"
+      run: PGPASSWORD=odktest psql -h localhost -U postgres -c "SHOW track_activity_query_size"
     - name: set postgres track_activity_query_size
-      run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "ALTER SYSTEM SET track_activity_query_size = 16384"
+      run: PGPASSWORD=odktest psql -h localhost -U postgres -c "ALTER SYSTEM SET track_activity_query_size = 16384"
     - name: Restart Postgres
       run: docker restart $(docker ps -q --filter "ancestor=postgres")
     - name: Print Services
       run: service --status-all
     - name: show postgres track_activity_query_size
-      run: PGPASSWORD=odktest psql -h localhost -U postgres -d jubilant_test -c "SHOW track_activity_query_size"
+      run: PGPASSWORD=odktest psql -h localhost -U postgres -c "SHOW track_activity_query_size"
     - uses: actions/checkout@v6
     - name: Set node version
       uses: actions/setup-node@v6
       with:
         node-version-file: package.json
         cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
     - name: Soak Test
       timeout-minutes: 10
       run: ./test/e2e/soak/ci

From 725e6f5f1b86eda16f84c74077be2991d621ba97 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:42:18 +0000
Subject: [PATCH 030/103] ps

---
 .github/workflows/soak-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index a9fd537aa..b0edd6cff 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -28,6 +28,7 @@ jobs:
       run: PGPASSWORD=odktest psql -h localhost -U postgres -c "SHOW track_activity_query_size"
     - name: set postgres track_activity_query_size
       run: PGPASSWORD=odktest psql -h localhost -U postgres -c "ALTER SYSTEM SET track_activity_query_size = 16384"
+    - run: docker ps
     - name: Restart Postgres
       run: docker restart $(docker ps -q --filter "ancestor=postgres")
     - name: Print Services

From 26cae440c310373fe709b61a8d2813ee5a607d51 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:49:37 +0000
Subject: [PATCH 031/103] name=

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index b0edd6cff..fcb48aba5 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -30,7 +30,7 @@ jobs:
       run: PGPASSWORD=odktest psql -h localhost -U postgres -c "ALTER SYSTEM SET track_activity_query_size = 16384"
     - run: docker ps
     - name: Restart Postgres
-      run: docker restart $(docker ps -q --filter "ancestor=postgres")
+      run: docker restart $(docker ps -q --filter "name=postgres")
     - name: Print Services
       run: service --status-all
     - name: show postgres track_activity_query_size

From ba99fbf4b1c11a6245dc0f64abc7fc2397895175 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:57:48 +0000
Subject: [PATCH 032/103] do it in bash

---
 .github/workflows/soak-test.yml | 12 ------------
 test/e2e/soak/ci                | 11 +++++++++++
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index fcb48aba5..2becc7084 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -23,18 +23,6 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-    # Per https://github.com/orgs/community/discussions/126453
-    - name: show postgres track_activity_query_size
-      run: PGPASSWORD=odktest psql -h localhost -U postgres -c "SHOW track_activity_query_size"
-    - name: set postgres track_activity_query_size
-      run: PGPASSWORD=odktest psql -h localhost -U postgres -c "ALTER SYSTEM SET track_activity_query_size = 16384"
-    - run: docker ps
-    - name: Restart Postgres
-      run: docker restart $(docker ps -q --filter "name=postgres")
-    - name: Print Services
-      run: service --status-all
-    - name: show postgres track_activity_query_size
-      run: PGPASSWORD=odktest psql -h localhost -U postgres -c "SHOW track_activity_query_size"
     - uses: actions/checkout@v6
     - name: Set node version
       uses: actions/setup-node@v6
diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index d06175598..b4ccabc2c 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -21,6 +21,17 @@ node ./lib/bin/cli.js user-promote -u "$userEmail"
 
 make run >backend.log 2>&1 &
 
+log "Increasing postgres query log lengths..."
+pg_exec() {
+  [[ $# = 1 ]] || fail_job
+  PGPASSWORD=odktest psql -h localhost -U postgres -c "$1"
+}
+log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
+pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
+log "  restarting postgres..."
+docker restart "$(docker ps -q --filter name=postgres)"
+log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
+
 log 'Waiting for backend to start...'
 timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done"
 log 'Backend started!'

From 84db8e115fdf95343fc83fa6cb705d9559386871 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 09:59:00 +0000
Subject: [PATCH 033/103] tuples

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index b4ccabc2c..08bfa0b89 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -24,7 +24,7 @@ make run >backend.log 2>&1 &
 log "Increasing postgres query log lengths..."
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql -h localhost -U postgres -c "$1"
+  PGPASSWORD=odktest psql -h localhost -U postgres --tuples-only -c "$1"
 }
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"

From 4f47c12cc35b1c3011a5c26e847a1c7631731f8a Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:00:00 +0000
Subject: [PATCH 034/103] slep

---
 test/e2e/soak/ci | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 08bfa0b89..a609f1e7e 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -30,6 +30,7 @@ log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 log "  restarting postgres..."
 docker restart "$(docker ps -q --filter name=postgres)"
+sleep 5
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 
 log 'Waiting for backend to start...'

From b2ddb9bf425bf5deac5a83888ce81ad51a15692d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:05:50 +0000
Subject: [PATCH 035/103] restart

---
 test/e2e/soak/ci | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index a609f1e7e..5923d1f4b 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -29,8 +29,12 @@ pg_exec() {
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 log "  restarting postgres..."
-docker restart "$(docker ps -q --filter name=postgres)"
-sleep 5
+pgImg="$(docker ps -q --filter name=postgres)"
+docker restart "$pgImg"
+while ! docker exec -it "$pgImg" pg_isready; do
+  sleep 1
+done
+log "  postgres restarted OK."
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 
 log 'Waiting for backend to start...'

From ca14a7b28d313f56fe6f19c8abdcba27a3808ab7 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:06:48 +0000
Subject: [PATCH 036/103] timeout

---
 test/e2e/soak/ci | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 5923d1f4b..5e3496a61 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -28,12 +28,10 @@ pg_exec() {
 }
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
-log "  restarting postgres..."
 pgImg="$(docker ps -q --filter name=postgres)"
+log "  restarting postgres..."
 docker restart "$pgImg"
-while ! docker exec -it "$pgImg" pg_isready; do
-  sleep 1
-done
+timeout 30 bash -c "while ! docker exec -t $pgImg pg_isready; do sleep 1; done"
 log "  postgres restarted OK."
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 

From adc8f23a709a753383f4a4b38aa88da59ec6a40f Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:06:58 +0000
Subject: [PATCH 037/103] smaller tmeout

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 5e3496a61..bcf97f233 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -31,7 +31,7 @@ pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 pgImg="$(docker ps -q --filter name=postgres)"
 log "  restarting postgres..."
 docker restart "$pgImg"
-timeout 30 bash -c "while ! docker exec -t $pgImg pg_isready; do sleep 1; done"
+timeout 10 bash -c "while ! docker exec -t $pgImg pg_isready; do sleep 1; done"
 log "  postgres restarted OK."
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 

From 238145c7a187d405dff77b96db836dc076e3230d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:07:46 +0000
Subject: [PATCH 038/103] no minust

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index bcf97f233..c2ae8eb46 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -31,7 +31,7 @@ pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 pgImg="$(docker ps -q --filter name=postgres)"
 log "  restarting postgres..."
 docker restart "$pgImg"
-timeout 10 bash -c "while ! docker exec -t $pgImg pg_isready; do sleep 1; done"
+timeout 10 bash -c "while ! docker exec $pgImg pg_isready; do sleep 1; done"
 log "  postgres restarted OK."
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 

From 0707e90c59db7f5c82e7f263a597faf726d7c5c6 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:17:32 +0000
Subject: [PATCH 039/103] add some slep

---
 test/e2e/soak/ci | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index c2ae8eb46..d4d6ad053 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -34,6 +34,7 @@ docker restart "$pgImg"
 timeout 10 bash -c "while ! docker exec $pgImg pg_isready; do sleep 1; done"
 log "  postgres restarted OK."
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
+sleep 20
 
 log 'Waiting for backend to start...'
 timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done"

From b6b8be69d48fdad1b5a0fde658ab857ac21fa8dc Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:21:53 +0000
Subject: [PATCH 040/103] make after

---
 test/e2e/soak/ci | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index d4d6ad053..e491ab524 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -14,13 +14,6 @@ fail_job() {
   exit 1
 }
 
-make base
-
-echo "$userPassword" | node ./lib/bin/cli.js user-create  -u "$userEmail"
-node ./lib/bin/cli.js user-promote -u "$userEmail"
-
-make run >backend.log 2>&1 &
-
 log "Increasing postgres query log lengths..."
 pg_exec() {
   [[ $# = 1 ]] || fail_job
@@ -36,6 +29,13 @@ log "  postgres restarted OK."
 log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 sleep 20
 
+make base
+
+echo "$userPassword" | node ./lib/bin/cli.js user-create  -u "$userEmail"
+node ./lib/bin/cli.js user-promote -u "$userEmail"
+
+make run >backend.log 2>&1 &
+
 log 'Waiting for backend to start...'
 timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done"
 log 'Backend started!'

From 6c471bc89470390eb2830e832ded3b792a998adb Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:23:46 +0000
Subject: [PATCH 041/103] nicer logging?

---
 test/e2e/soak/ci | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index e491ab524..4be993ad2 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -14,20 +14,21 @@ fail_job() {
   exit 1
 }
 
-log "Increasing postgres query log lengths..."
 pg_exec() {
   [[ $# = 1 ]] || fail_job
   PGPASSWORD=odktest psql -h localhost -U postgres --tuples-only -c "$1"
 }
-log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
+
+log "[postgres] Increasing query log lengths..."
+log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 pgImg="$(docker ps -q --filter name=postgres)"
-log "  restarting postgres..."
+log "[postgres]   restarting service..."
 docker restart "$pgImg"
 timeout 10 bash -c "while ! docker exec $pgImg pg_isready; do sleep 1; done"
-log "  postgres restarted OK."
-log "  track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
-sleep 20
+log "[postgres]   restarted OK."
+log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
+log "[postgres]   DONE."
 
 make base
 

From 4611a5bf1ce1b583f865cb9564670e02ec73a917 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:27:00 +0000
Subject: [PATCH 042/103] quieter

---
 test/e2e/soak/ci | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 4be993ad2..c7f811f11 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -24,8 +24,8 @@ log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_quer
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 pgImg="$(docker ps -q --filter name=postgres)"
 log "[postgres]   restarting service..."
-docker restart "$pgImg"
-timeout 10 bash -c "while ! docker exec $pgImg pg_isready; do sleep 1; done"
+docker restart "$pgImg" >/dev/null
+docker exec $pgImg pg_isready --quiet
 log "[postgres]   restarted OK."
 log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 log "[postgres]   DONE."

From fd81d0cc22c13360be94371d3774b5617b1bf275 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:28:29 +0000
Subject: [PATCH 043/103] quiet psql

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index c7f811f11..57f756a63 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -16,7 +16,7 @@ fail_job() {
 
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql -h localhost -U postgres --tuples-only -c "$1"
+  PGPASSWORD=odktest psql -h localhost -U postgres --quiet --tuples-only -c "$1"
 }
 
 log "[postgres] Increasing query log lengths..."

From 54a27ef2b0cd10500fd6cf7591c21ad8f8063d71 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:28:38 +0000
Subject: [PATCH 044/103] psql no host

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 57f756a63..9bae98eec 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -16,7 +16,7 @@ fail_job() {
 
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql -h localhost -U postgres --quiet --tuples-only -c "$1"
+  PGPASSWORD=odktest psql -U postgres --quiet --tuples-only -c "$1"
 }
 
 log "[postgres] Increasing query log lengths..."

From 73c98eb1eb50474d7e9a37badac6e054f0b7a0b7 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:28:50 +0000
Subject: [PATCH 045/103] psql no user

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 9bae98eec..cc4618a2f 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -16,7 +16,7 @@ fail_job() {
 
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql -U postgres --quiet --tuples-only -c "$1"
+  PGPASSWORD=odktest psql --quiet --tuples-only -c "$1"
 }
 
 log "[postgres] Increasing query log lengths..."

From 7df3884d23d952f70aa619ff9590470c5e11127c Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:29:13 +0000
Subject: [PATCH 046/103] psql --command

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index cc4618a2f..8bf207dcc 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -16,7 +16,7 @@ fail_job() {
 
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql --quiet --tuples-only -c "$1"
+  PGPASSWORD=odktest psql --quiet --tuples-only --command="$1"
 }
 
 log "[postgres] Increasing query log lengths..."

From c54dd6120af2d057b8f7e4bf464448183b01728e Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:30:00 +0000
Subject: [PATCH 047/103] bring back all the jobs

---
 .github/workflows/soak-test.yml | 629 ++++++++++++++++++++++++++++++++
 1 file changed, 629 insertions(+)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 2becc7084..5de7bdb9f 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -5,6 +5,635 @@ on:
 
 jobs:
   soak-test-00:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          -c track_activity_query_size=1048576
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-01:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-02:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-03:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-04:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-05:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-06:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-07:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-08:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-09:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-10:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          -c track_activity_query_size=1048576
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-11:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-12:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-13:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-14:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-15:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-16:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-17:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-18:
+    timeout-minutes: 15
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.20
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set node version
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: package.json
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - name: Soak Test
+      timeout-minutes: 10
+      run: ./test/e2e/soak/ci
+    - name: Backend Logs
+      if: always()
+      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
+  soak-test-19:
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest

From 4db9afa81408eb2b1b4deb93f2ac89b88e755572 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:30:57 +0000
Subject: [PATCH 048/103] re-add localhost

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 8bf207dcc..6ab501124 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -16,7 +16,7 @@ fail_job() {
 
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql --quiet --tuples-only --command="$1"
+  PGPASSWORD=odktest psql --host=localhost --quiet --tuples-only --command="$1"
 }
 
 log "[postgres] Increasing query log lengths..."

From 299fecc5532ce181fc364eb81c337865732b3c12 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:31:49 +0000
Subject: [PATCH 049/103] remove dodgy ar tpug

---
 .github/workflows/soak-test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 5de7bdb9f..f03a651af 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -22,7 +22,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-          -c track_activity_query_size=1048576
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 815ca617958b381e17e2c97818264de4058bd7fc Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:49:26 +0000
Subject: [PATCH 050/103] username

---
 test/e2e/soak/ci | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 6ab501124..3411a00a9 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -16,7 +16,12 @@ fail_job() {
 
 pg_exec() {
   [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql --host=localhost --quiet --tuples-only --command="$1"
+  PGPASSWORD=odktest psql \
+      --host=localhost \
+      --username=postgres \
+      --quiet \
+      --tuples-only \
+      --command="$1"
 }
 
 log "[postgres] Increasing query log lengths..."

From e4cb2a3823475eb8e1af8c71f176b2da13999d84 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:50:27 +0000
Subject: [PATCH 051/103] spacing

---
 test/e2e/soak/ci | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 3411a00a9..dd02e1661 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -25,13 +25,16 @@ pg_exec() {
 }
 
 log "[postgres] Increasing query log lengths..."
+
 log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 pgImg="$(docker ps -q --filter name=postgres)"
+
 log "[postgres]   restarting service..."
 docker restart "$pgImg" >/dev/null
 docker exec $pgImg pg_isready --quiet
 log "[postgres]   restarted OK."
+
 log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
 log "[postgres]   DONE."
 

From fb2a93a2f982daa992c709ef8fc8face1bd45012 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 10:51:10 +0000
Subject: [PATCH 052/103] remove duplicated old err

---
 .github/workflows/soak-test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index f03a651af..5d7242f08 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -352,7 +352,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-          -c track_activity_query_size=1048576
     steps:
     - uses: actions/checkout@v6
     - name: Set node version

From 07a75a5f868cda201dccb530e1ff432ee0b393c1 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:32:00 +0000
Subject: [PATCH 053/103] non-OK response: include method & URL

---
 test/e2e/util/api.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/util/api.js b/test/e2e/util/api.js
index 1c9c5c1f4..3fb9f9826 100644
--- a/test/e2e/util/api.js
+++ b/test/e2e/util/api.js
@@ -105,7 +105,7 @@ async function apiClient(suiteName, { serverUrl, userEmail, userPassword, logPat
       const responseStatus = res.status;
       const responseText = await res.text() || res.statusText;
 
-      const err = new Error(`${responseStatus}: ${responseText}`);
+      const err = new Error(`${method} ${url} ${responseStatus}: ${responseText}`);
       err.responseStatus = responseStatus;
       err.responseText = responseText;
 

From 47317e8305757c9fb592531ba5bb486ab08f5400 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:35:09 +0000
Subject: [PATCH 054/103] matrix

---
 .github/workflows/soak-test.yml | 634 +-------------------------------
 1 file changed, 6 insertions(+), 628 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 5d7242f08..de22df4bc 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -4,634 +4,12 @@ on:
   push:
 
 jobs:
-  soak-test-00:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-01:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-02:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-03:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-04:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-05:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-06:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-07:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-08:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-09:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-10:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-11:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-12:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-13:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-14:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-15:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-16:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-17:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-18:
-    timeout-minutes: 15
-    # TODO should we use the same container as circle & central?
-    runs-on: ubuntu-latest
-    services:
-      # see: https://docs.github.com/en/enterprise-server@3.5/actions/using-containerized-services/creating-postgresql-service-containers
-      postgres:
-        image: postgres:14.20
-        env:
-          POSTGRES_PASSWORD: odktest
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    steps:
-    - uses: actions/checkout@v6
-    - name: Set node version
-      uses: actions/setup-node@v6
-      with:
-        node-version-file: package.json
-        cache: 'npm'
-    - run: npm ci
-    - run: node lib/bin/create-docker-databases.js
-    - name: Soak Test
-      timeout-minutes: 10
-      run: ./test/e2e/soak/ci
-    - name: Backend Logs
-      if: always()
-      run: "! [[ -f ./backend.log ]] || cat ./backend.log"
-  soak-test-19:
+  soak-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+        y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
     timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest

From f0da12422a1601b1dc2ac7c4e92681ff19389926 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:50:34 +0000
Subject: [PATCH 055/103] restarting: don't suppress output

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index dd02e1661..bfeabb104 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -31,7 +31,7 @@ pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
 pgImg="$(docker ps -q --filter name=postgres)"
 
 log "[postgres]   restarting service..."
-docker restart "$pgImg" >/dev/null
+docker restart "$pgImg"
 docker exec $pgImg pg_isready --quiet
 log "[postgres]   restarted OK."
 

From f5c83ff47ee60ebcf1d65336ea2a71504b6a9856 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:50:52 +0000
Subject: [PATCH 056/103] pg_isready: don't suppress output

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index bfeabb104..a1b2e422d 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -32,7 +32,7 @@ pgImg="$(docker ps -q --filter name=postgres)"
 
 log "[postgres]   restarting service..."
 docker restart "$pgImg"
-docker exec $pgImg pg_isready --quiet
+docker exec $pgImg pg_isready
 log "[postgres]   restarted OK."
 
 log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"

From eaf5f2fac4b60514fbb0f1989984f4295d9f9d53 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:51:25 +0000
Subject: [PATCH 057/103] pg_isready: increase timeout

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index a1b2e422d..dcedc3571 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -32,7 +32,7 @@ pgImg="$(docker ps -q --filter name=postgres)"
 
 log "[postgres]   restarting service..."
 docker restart "$pgImg"
-docker exec $pgImg pg_isready
+docker exec $pgImg pg_isready --timeout=10
 log "[postgres]   restarted OK."
 
 log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"

From 866b5cbcfbe68b1f303c3a941d198386cc082d1b Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:53:57 +0000
Subject: [PATCH 058/103] clearer non-OK response logging

---
 test/e2e/util/api.js | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/test/e2e/util/api.js b/test/e2e/util/api.js
index 3fb9f9826..c5ead7410 100644
--- a/test/e2e/util/api.js
+++ b/test/e2e/util/api.js
@@ -103,7 +103,23 @@ async function apiClient(suiteName, { serverUrl, userEmail, userPassword, logPat
     if(isRedirected(res)) return new Redirect(res);
     if(!res.ok) {
       const responseStatus = res.status;
-      const responseText = await res.text() || res.statusText;
+      const responseBody = await res.text();
+      const responseText = responseBody || res.statusText;
+
+      console.log(`
+        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+        @
+        @ non-OK response:
+        @
+        @   method: ${method}
+        @      url: ${url}
+        @   status: ${responseStatus}
+        @     body: ${responseBody}
+        @
+        @ Should this be logged clearly somewhere else?
+        @
+        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+      `);
 
       const err = new Error(`${method} ${url} ${responseStatus}: ${responseText}`);
       err.responseStatus = responseStatus;

From 2c59e99eab0dbc4836f19faa907fe71a2a82a249 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:54:58 +0000
Subject: [PATCH 059/103] log at soak-test level

---
 test/e2e/soak/index.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index f6836f176..4e27537de 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -168,6 +168,7 @@ function doSoakTest(name, throughput, throughputPeriod, testDuration, minimumSuc
         resolve();
       }, +testDuration);
     } catch(err) {
+      log('Non-OK response:', err);
       reject(err);
     }
   });

From d62ac492cf307155aadc7eb5e15c11fdfb97774b Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 11:57:39 +0000
Subject: [PATCH 060/103] a bit more logging

---
 test/e2e/soak/index.js | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 4e27537de..3e21160a7 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -168,7 +168,7 @@ function doSoakTest(name, throughput, throughputPeriod, testDuration, minimumSuc
         resolve();
       }, +testDuration);
     } catch(err) {
-      log('Non-OK response:', err);
+      log('Err caught:', err);
       reject(err);
     }
   });
@@ -188,7 +188,7 @@ function reportWarning(message) {
   log.report('--------------------------');
 }
 
-function randomSubmission(n, projectId, formId) {
+async function randomSubmission(n, projectId, formId) {
   const headers = {
     'Content-Type': 'multipart/form-data; boundary=foo',
     'X-OpenRosa-Version': '1.0',
@@ -205,15 +205,25 @@ ${submissionTemplate
 \r
 --foo--`;
 
-  return api.apiPostAndDump('randomSubmission', n, `projects/${projectId}/forms/${formId}/submissions`, body, headers);
+  try {
+    return await api.apiPostAndDump('randomSubmission', n, `projects/${projectId}/forms/${formId}/submissions`, body, headers);
+  } catch(err) {
+    log('Err in randomSubmission()', err);
+    throw err;
+  }
 }
 
 function randInt() {
   return Math.floor(Math.random() * 9999);
 }
 
-function exportZipWithDataAndMedia(n, projectId, formId) {
-  return api.apiGetToFile('exportZipWithDataAndMedia', n, `projects/${projectId}/forms/${formId}/submissions.csv.zip?splitSelectMultiples=true&groupPaths=true&deletedFields=true`);
+async function exportZipWithDataAndMedia(n, projectId, formId) {
+  try {
+    return await api.apiGetToFile('exportZipWithDataAndMedia', n, `projects/${projectId}/forms/${formId}/submissions.csv.zip?splitSelectMultiples=true&groupPaths=true&deletedFields=true`);
+  } catch(err) {
+    log('Err in exportZipWithDataAndMedia()', err);
+    throw err;
+  }
 }
 
 function durationForHumans(ms) {

From 49d2be99f252e8aef5bb61360188d8dd45bea4e6 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 12:38:47 +0000
Subject: [PATCH 061/103] convert to PR

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index de22df4bc..2b660dbb6 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -1,7 +1,7 @@
 name: Soak Test
 
 on:
-  push:
+  pull_request:
 
 jobs:
   soak-test:

From aab836a33be3e26bfefc7586c1a13725d032f871 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 12:47:10 +0000
Subject: [PATCH 062/103] Inlclude error stack in response

---
 lib/http/endpoint.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/http/endpoint.js b/lib/http/endpoint.js
index 6e396b4f8..aa7f189b9 100644
--- a/lib/http/endpoint.js
+++ b/lib/http/endpoint.js
@@ -259,6 +259,7 @@ const defaultErrorWriter = (error, request, response) => {
     process.stderr.write(inspect(error) + '\n');
     response.status(500).type('application/json').send({
       message: 'Internal Server Error',
+      error: inspect(error),
     });
   }
 };

From 663430fce633b249f60f76b2fd23fbfe91ce2adc Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 14:24:11 +0000
Subject: [PATCH 063/103] more reliable?

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index dcedc3571..d55af0bf0 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -32,7 +32,7 @@ pgImg="$(docker ps -q --filter name=postgres)"
 
 log "[postgres]   restarting service..."
 docker restart "$pgImg"
-docker exec $pgImg pg_isready --timeout=10
+timeout 10 bash -c "while ! docker exec $pgImg pg_isready --timeout=1; do sleep 1; done"
 log "[postgres]   restarted OK."
 
 log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"

From c6e2732dfea5fb7c8f36983ed284d56da316dc13 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Wed, 1 Apr 2026 14:55:58 +0000
Subject: [PATCH 064/103] only fetch logs for failed jobs

---
 dump-github-job-logs.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dump-github-job-logs.sh b/dump-github-job-logs.sh
index adc7725e2..1364e5f85 100755
--- a/dump-github-job-logs.sh
+++ b/dump-github-job-logs.sh
@@ -25,7 +25,8 @@ fi
 
 mkdir -p "$logDir"
 
-gh run view "$runId" --json jobs --jq '.jobs[] | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
+log "Fetching logs for FAILED jobs..."
+gh run view "$runId" --json jobs --jq '.jobs[] | select(.conclusion=="failure") | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
   safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
   log "fetching logs for: $jobName..."
   gh run view --job "$jobId" --log > "$logDir/$safeName.log"

From 9363a5ba2840fa33bebc965f8fb4cfbf44d19aa2 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 12:40:20 +0000
Subject: [PATCH 065/103] dump: support attempts

---
 dump-github-job-logs.sh | 59 ++++++++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 12 deletions(-)

diff --git a/dump-github-job-logs.sh b/dump-github-job-logs.sh
index 1364e5f85..282f98ad0 100755
--- a/dump-github-job-logs.sh
+++ b/dump-github-job-logs.sh
@@ -1,6 +1,19 @@
 #!/bin/bash -eu
 log() { echo >&2 "[$(basename "$0")] $*"; }
 
+if [[ $# -gt 0 ]] && [[ $1 = --help ]]; then
+  cat <<EOF
+    USAGE
+      $(basename "$0") [--force] [run-id]
+EOF
+  exit
+fi
+
+forceDownload=
+if [[ $# -gt 0 ]] && [[ $1 = --force ]]; then
+  forceDownload=true
+  shift
+fi
 
 if [[ $# -gt 0 ]]; then
   runId="$1"
@@ -13,23 +26,45 @@ else
   log "Fetching last COMPLETED run id..."
   runId="$(gh run list --branch "$(git branch --show-current)" --limit 1 --status completed --json databaseId --jq '.[0].databaseId')"
 fi
-log "run id: $runId"
 
-logDir="gha-logs/$runId"
-log "log dir: $logDir"
+# Protect against injected run IDs.  With --force, e.g. runId=/../.. could cause some problems.
+if ! [[ $runId =~ ^[0-9]+$ ]]; then
+  log "Invalid run id: $runId"
+  exit 1
+fi
+log "run id: $runId"
 
-if [[ -d "$logDir" ]]; then
-  log "It looks like you already got the latest logs."
-  exit
+parentDir="gha-logs/run-$runId"
+if [[ $forceDownload = true ]] && [[ -d "$parentDir" ]]; then
+  log "Cleaning existing logs from $parentDir ..."
+  rm -rf ./$parentDir
 fi
 
-mkdir -p "$logDir"
+attempts="$(gh run view "$runId" --json attempt --jq .attempt)"
+log "attempts: $attempts"
+
+for ((attemptIdx=1; attemptIdx<=$attempts; ++attemptIdx)); do
+  logDir="$parentDir/attempt-$attemptIdx"
+  log "log dir: $logDir"
 
-log "Fetching logs for FAILED jobs..."
-gh run view "$runId" --json jobs --jq '.jobs[] | select(.conclusion=="failure") | "\(.databaseId) \(.name)"' | while read -r jobId jobName; do
-  safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
-  log "fetching logs for: $jobName..."
-  gh run view --job "$jobId" --log > "$logDir/$safeName.log"
+  if [[ -d "$logDir" ]]; then
+    log "  It looks like you already got the logs for this attempt."
+    continue
+  fi
+
+  mkdir -p "$logDir"
+
+  log "Fetching logs for FAILED jobs..."
+  gh run view "$runId" \
+      --attempt "$attemptIdx" \
+      --json jobs \
+      --jq '.jobs[] | select(.conclusion=="failure") | "\(.databaseId) \(.name)"' |
+  while read -r jobId jobName; do
+    safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
+    log "fetching logs for: $jobName..."
+    gh run view --job "$jobId" --attempt "$attemptIdx" --log > "$logDir/$safeName.log"
+  done
 done
 
+
 log "All done."

From 2ed2d2f67ad77e852cc690badfb8d146ec30ef88 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 12:49:42 +0000
Subject: [PATCH 066/103] Include web links

---
 dump-github-job-logs.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dump-github-job-logs.sh b/dump-github-job-logs.sh
index 282f98ad0..d569d7355 100755
--- a/dump-github-job-logs.sh
+++ b/dump-github-job-logs.sh
@@ -33,6 +33,7 @@ if ! [[ $runId =~ ^[0-9]+$ ]]; then
   exit 1
 fi
 log "run id: $runId"
+log "  web: $(gh run view "$runId" --json url --template '{{.url}}')"
 
 parentDir="gha-logs/run-$runId"
 if [[ $forceDownload = true ]] && [[ -d "$parentDir" ]]; then
@@ -44,6 +45,8 @@ attempts="$(gh run view "$runId" --json attempt --jq .attempt)"
 log "attempts: $attempts"
 
 for ((attemptIdx=1; attemptIdx<=$attempts; ++attemptIdx)); do
+  log "attempt: $attemptIdx"
+  log "  web: $(gh run view "$runId" --attempt "$attemptIdx" --json url --template '{{.url}}')"
   logDir="$parentDir/attempt-$attemptIdx"
   log "log dir: $logDir"
 
@@ -61,7 +64,7 @@ for ((attemptIdx=1; attemptIdx<=$attempts; ++attemptIdx)); do
       --jq '.jobs[] | select(.conclusion=="failure") | "\(.databaseId) \(.name)"' |
   while read -r jobId jobName; do
     safeName="$(echo "$jobName" | sed 's/[ /]/_/g')"
-    log "fetching logs for: $jobName..."
+    log "fetching logs for: $jobName ..."
     gh run view --job "$jobId" --attempt "$attemptIdx" --log > "$logDir/$safeName.log"
   done
 done

From 5f0872f48edbe644682b2c8ea1b39cd40f108ec9 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 12:57:58 +0000
Subject: [PATCH 067/103] fix analyse

---
 analyse-queries.js | 66 ++++++++++++++++++++++++----------------------
 1 file changed, 34 insertions(+), 32 deletions(-)

diff --git a/analyse-queries.js b/analyse-queries.js
index 81e040a46..d5c90ca5e 100755
--- a/analyse-queries.js
+++ b/analyse-queries.js
@@ -8,41 +8,43 @@ const log = (...args) => console.log('[analyse-queries]', ...args);
 const rootDir = './gha-logs';
 
 const [,,...args] = process.argv;
-const jobId = args.length ? args[0] : execSync(`ls -td ${rootDir}/* | head -1`, { encoding:'utf8' }).substring(rootDir.length + 1).trim();
-log('job id:', jobId);
+const runId = args.length ? args[0] : execSync(`ls -td ${rootDir}/* | head -1`, { encoding:'utf8' }).substring(rootDir.length + 1).trim();
+log('run id:', runId);
 
 const allActiveQueries = {};
 
-for(const _f of readdirSync(`./gha-logs/${jobId}`)) {
-  const f = `${rootDir}/${jobId}/${_f}`;
-  log('Processing:', f);
-
-  const queryPrefix = 'Open queries: ';
-  const activeQueries = readFileSync(f, 'utf8')
-      .split('\n')
-      .filter(it => it.includes(queryPrefix))
-      .map(it => {
-        const [ prefix, json ] = it.split(queryPrefix);
-        return JSON.parse(json)
-            .filter(it => it.state !== 'idle')
-            .map(q => {
-              const logged_at = new Date(prefix.split(/\s+/)[3]);;
-              const query_start = new Date(q.query_start);
-              return {
-                ...q,
-                logged_at,
-                query_start,
-                duration_so_far: logged_at.valueOf() - query_start.valueOf(),
-              };
-            });
-        });
-
-  const lastQueries = activeQueries.at(-1);
-  log('  last activeQueries:', lastQueries);
-
-  activeQueries
-      .flat()
-      .forEach(processQuery);
+for( const att of readdirSync(`${rootDir}/run-${runId}`)) {
+  for(const _f of readdirSync(`${rootDir}/run-${runId}/${att}`)) {
+    const f =                 `${rootDir}/run-${runId}/${att}/${_f}`;
+    log('Processing:', f);
+  
+    const queryPrefix = 'Open queries: ';
+    const activeQueries = readFileSync(f, 'utf8')
+        .split('\n')
+        .filter(it => it.includes(queryPrefix))
+        .map(it => {
+          const [ prefix, json ] = it.split(queryPrefix);
+          return JSON.parse(json)
+              .filter(it => it.state !== 'idle')
+              .map(q => {
+                const logged_at = new Date(prefix.match(/2026-\S*Z/)[0]);
+                const query_start = new Date(q.query_start);
+                return {
+                  ...q,
+                  logged_at,
+                  query_start,
+                  duration_so_far: logged_at.valueOf() - query_start.valueOf(),
+                };
+              });
+          });
+
+    const lastQueries = activeQueries.at(-1);
+    log('  last activeQueries:', lastQueries);
+
+    activeQueries
+        .flat()
+        .forEach(processQuery);
+  }
 
   log('  Done.');
 }

From 50827a7c804bb71843e3739425db232bf996cb3a Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 13:02:36 +0000
Subject: [PATCH 068/103] 10x exportZipWithDataAndMedia() to try to force a
 failure

---
 test/e2e/soak/ci       | 3 ---
 test/e2e/soak/index.js | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index d55af0bf0..26c2a578b 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -57,9 +57,6 @@ if ! curl -s -o /dev/null "$serverUrl"; then
   fail_job
 fi
 
-log 'Checking open DB query count...'
-timeout 120 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
-
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"
 requestBody='{"email":"'"$userEmail"'","password":"'"$userPassword"'"}'
diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 3e21160a7..7569685f3 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -64,7 +64,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 10, 3_000, 300_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 100, 30_000, 3000_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From 3c35f90b3413f6b3d412ebf1aee17721699b4afb Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 13:03:02 +0000
Subject: [PATCH 069/103] fail-fast

---
 .github/workflows/soak-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 2b660dbb6..c212680ee 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -6,7 +6,7 @@ on:
 jobs:
   soak-test:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
         y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]

From 0aea819e8e0ea2716ccae1ec065a2879af5556c7 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 13:10:40 +0000
Subject: [PATCH 070/103] Revert "10x exportZipWithDataAndMedia() to try to
 force a failure"

This reverts commit 50827a7c804bb71843e3739425db232bf996cb3a.
---
 test/e2e/soak/ci       | 3 +++
 test/e2e/soak/index.js | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 26c2a578b..d55af0bf0 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -57,6 +57,9 @@ if ! curl -s -o /dev/null "$serverUrl"; then
   fail_job
 fi
 
+log 'Checking open DB query count...'
+timeout 120 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
+
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"
 requestBody='{"email":"'"$userEmail"'","password":"'"$userPassword"'"}'
diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 7569685f3..3e21160a7 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -64,7 +64,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 100, 30_000, 3000_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 10, 3_000, 300_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From 91747cb36a1ebbd80e342cd0bb36824330d2bb62 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 13:11:10 +0000
Subject: [PATCH 071/103] allow 20 mins for connections to close

---
 .github/workflows/soak-test.yml | 2 +-
 test/e2e/soak/ci                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index c212680ee..73e2b933b 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
         y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
-    timeout-minutes: 15
+    timeout-minutes: 30
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
     services:
diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index d55af0bf0..d030f73d9 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -58,7 +58,7 @@ if ! curl -s -o /dev/null "$serverUrl"; then
 fi
 
 log 'Checking open DB query count...'
-timeout 120 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
+timeout 1200 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
 
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"

From 9241ebdf34dbfaa2d4d2f957f9329d92c75859f4 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Thu, 2 Apr 2026 13:59:50 +0000
Subject: [PATCH 072/103] try again long zipping, but only once

---
 .github/workflows/soak-test.yml | 7 +++----
 test/e2e/soak/ci                | 3 ---
 test/e2e/soak/index.js          | 2 +-
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 73e2b933b..2da1f01b8 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -8,9 +8,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
-        y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
-    timeout-minutes: 30
+        x: [ 1 ]
+    timeout-minutes: 120
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
     services:
@@ -37,7 +36,7 @@ jobs:
     - run: npm ci
     - run: node lib/bin/create-docker-databases.js
     - name: Soak Test
-      timeout-minutes: 10
+      timeout-minutes: 80
       run: ./test/e2e/soak/ci
     - name: Backend Logs
       if: always()
diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index d030f73d9..26c2a578b 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -57,9 +57,6 @@ if ! curl -s -o /dev/null "$serverUrl"; then
   fail_job
 fi
 
-log 'Checking open DB query count...'
-timeout 1200 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
-
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"
 requestBody='{"email":"'"$userEmail"'","password":"'"$userPassword"'"}'
diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 3e21160a7..7569685f3 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -64,7 +64,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 10, 3_000, 300_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 100, 30_000, 3000_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From 9333a9dcb9a16881790d1b103279bd004187778b Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:00:18 +0000
Subject: [PATCH 073/103] clearer error

---
 test/e2e/util/api.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/util/api.js b/test/e2e/util/api.js
index c5ead7410..d9a11f8cf 100644
--- a/test/e2e/util/api.js
+++ b/test/e2e/util/api.js
@@ -121,7 +121,7 @@ async function apiClient(suiteName, { serverUrl, userEmail, userPassword, logPat
         @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
       `);
 
-      const err = new Error(`${method} ${url} ${responseStatus}: ${responseText}`);
+      const err = new Error(`${method} ${url} returned ${responseStatus}: ${responseText}`);
       err.responseStatus = responseStatus;
       err.responseText = responseText;
 

From 1c9a1cd02ea3c1268d37fa178d105fe3f0c0c1a6 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:03:54 +0000
Subject: [PATCH 074/103] throw deliberately

---
 lib/resources/submissions.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/resources/submissions.js b/lib/resources/submissions.js
index a07b1dd3f..bab067c5e 100644
--- a/lib/resources/submissions.js
+++ b/lib/resources/submissions.js
@@ -102,7 +102,9 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
     };
 
     // Nonstandard REST; OpenRosa-specific API.
-    service.post(path, multipart.any(), multerUtf, multipartErrorHandler, _endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query, userAgent, headers }) =>
+    service.post(path, multipart.any(), multerUtf, multipartErrorHandler, _endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query, userAgent, headers }) => {
+      throw new Error('I am manufactured deliberately');
+      // eslint-disable-next-line no-unreachable
       Submission.fromXml(findMultipart(files).buffer)
         .then((partial) => getForm(auth, params, partial.xmlFormId, Forms, partial.def.version)
           .catch(forceAuthFailed)
@@ -154,7 +156,8 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
                     .catch(Problem.translate(Problem.user.uniquenessViolation, noargs(Problem.user.duplicateSubmission)));
                 });
           })
-          .then(always(createdMessage({ message: 'full submission upload was successful!' }))))));
+          .then(always(createdMessage({ message: 'full submission upload was successful!' }))));
+    }));
   };
 
   // default per-project submission path:

From 95f29da9a565df9233694165dadefbee8978216d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:04:29 +0000
Subject: [PATCH 075/103] execute quicker

---
 test/e2e/soak/index.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 7569685f3..27f0e0e8c 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -56,7 +56,7 @@ async function soakTest() {
 
   log.info('Setup complete.  Starting soak tests...');
 
-  await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
+  await doSoakTest('randomSubmission', 5, 1, 3, 100, n => randomSubmission(n, projectId, formId));
 
   // TODO work out a more scientific sleep duration
   const backgroundJobPause = 20_000;
@@ -64,7 +64,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 100, 30_000, 3000_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 1, 3, 3, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From b5652424d84a72794a023ce714d30585388fe985 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:07:05 +0000
Subject: [PATCH 076/103] add another failure

---
 lib/resources/submissions.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/resources/submissions.js b/lib/resources/submissions.js
index bab067c5e..080487fe4 100644
--- a/lib/resources/submissions.js
+++ b/lib/resources/submissions.js
@@ -103,7 +103,7 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
 
     // Nonstandard REST; OpenRosa-specific API.
     service.post(path, multipart.any(), multerUtf, multipartErrorHandler, _endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query, userAgent, headers }) => {
-      throw new Error('I am manufactured deliberately');
+      throw new Error('I am manufactured deliberately 1');
       // eslint-disable-next-line no-unreachable
       Submission.fromXml(findMultipart(files).buffer)
         .then((partial) => getForm(auth, params, partial.xmlFormId, Forms, partial.def.version)
@@ -205,6 +205,8 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
   // and repeated for draft/nondraft.
 
   const restSubmission = (base, draft, getForm) => {
+    throw new Error('I am manufactured deliberately 2');
+    // eslint-disable-next-line no-unreachable
     service.post(`${base}/submissions`, endpoint(({ Forms, Submissions, SubmissionAttachments }, { params, auth, query, userAgent, headers }, request) =>
       Submission.fromXml(request)
         .then((partial) => getForm(params, Forms, partial.def.version)

From 7af2cf3f54aec9bc171d380ec71c4b4b11c86b6d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:11:44 +0000
Subject: [PATCH 077/103] revert app code changes

---
 lib/resources/submissions.js | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/lib/resources/submissions.js b/lib/resources/submissions.js
index 080487fe4..a07b1dd3f 100644
--- a/lib/resources/submissions.js
+++ b/lib/resources/submissions.js
@@ -102,9 +102,7 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
     };
 
     // Nonstandard REST; OpenRosa-specific API.
-    service.post(path, multipart.any(), multerUtf, multipartErrorHandler, _endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query, userAgent, headers }) => {
-      throw new Error('I am manufactured deliberately 1');
-      // eslint-disable-next-line no-unreachable
+    service.post(path, multipart.any(), multerUtf, multipartErrorHandler, _endpoint.openRosa(({ Forms, Submissions, SubmissionAttachments }, { params, files, auth, query, userAgent, headers }) =>
       Submission.fromXml(findMultipart(files).buffer)
         .then((partial) => getForm(auth, params, partial.xmlFormId, Forms, partial.def.version)
           .catch(forceAuthFailed)
@@ -156,8 +154,7 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
                     .catch(Problem.translate(Problem.user.uniquenessViolation, noargs(Problem.user.duplicateSubmission)));
                 });
           })
-          .then(always(createdMessage({ message: 'full submission upload was successful!' }))));
-    }));
+          .then(always(createdMessage({ message: 'full submission upload was successful!' }))))));
   };
 
   // default per-project submission path:
@@ -205,8 +202,6 @@ module.exports = (service, endpoint, anonymousEndpoint) => {
   // and repeated for draft/nondraft.
 
   const restSubmission = (base, draft, getForm) => {
-    throw new Error('I am manufactured deliberately 2');
-    // eslint-disable-next-line no-unreachable
     service.post(`${base}/submissions`, endpoint(({ Forms, Submissions, SubmissionAttachments }, { params, auth, query, userAgent, headers }, request) =>
       Submission.fromXml(request)
         .then((partial) => getForm(params, Forms, partial.def.version)

From 4663227527a31151667c7f5f4ef90ce6c1c62764 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:13:02 +0000
Subject: [PATCH 078/103] bring back

---
 test/e2e/soak/index.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 27f0e0e8c..7569685f3 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -56,7 +56,7 @@ async function soakTest() {
 
   log.info('Setup complete.  Starting soak tests...');
 
-  await doSoakTest('randomSubmission', 5, 1, 3, 100, n => randomSubmission(n, projectId, formId));
+  await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
 
   // TODO work out a more scientific sleep duration
   const backgroundJobPause = 20_000;
@@ -64,7 +64,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 1, 3, 3, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 100, 30_000, 3000_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From e154f9ba98a9e948d764a7b96b0ea936f0db1f05 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:14:54 +0000
Subject: [PATCH 079/103] fail zip quicker?

---
 test/e2e/soak/index.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 7569685f3..c3c61b57b 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -56,7 +56,8 @@ async function soakTest() {
 
   log.info('Setup complete.  Starting soak tests...');
 
-  await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
+  // eslint-disable-next-line no-unused-expressions
+  false && await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
 
   // TODO work out a more scientific sleep duration
   const backgroundJobPause = 20_000;
@@ -64,7 +65,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 100, 30_000, 3000_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 100, 3_000, 300_000, 100, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From 496d5f7cd6b4781544731e7d7a3af01887f1f404 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:43:11 +0000
Subject: [PATCH 080/103] try reduced loggging

---
 test/e2e/util/api.js | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/test/e2e/util/api.js b/test/e2e/util/api.js
index d9a11f8cf..1c9c5c1f4 100644
--- a/test/e2e/util/api.js
+++ b/test/e2e/util/api.js
@@ -103,25 +103,9 @@ async function apiClient(suiteName, { serverUrl, userEmail, userPassword, logPat
     if(isRedirected(res)) return new Redirect(res);
     if(!res.ok) {
       const responseStatus = res.status;
-      const responseBody = await res.text();
-      const responseText = responseBody || res.statusText;
-
-      console.log(`
-        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
-        @
-        @ non-OK response:
-        @
-        @   method: ${method}
-        @      url: ${url}
-        @   status: ${responseStatus}
-        @     body: ${responseBody}
-        @
-        @ Should this be logged clearly somewhere else?
-        @
-        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
-      `);
-
-      const err = new Error(`${method} ${url} returned ${responseStatus}: ${responseText}`);
+      const responseText = await res.text() || res.statusText;
+
+      const err = new Error(`${responseStatus}: ${responseText}`);
       err.responseStatus = responseStatus;
       err.responseText = responseText;
 

From 6e344bfa78246e5b2b257b24a5e62af9f53fd124 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 4 Apr 2026 07:48:48 +0000
Subject: [PATCH 081/103] reinstate matrix and long check for danglers

---
 .github/workflows/soak-test.yml | 3 ++-
 test/e2e/soak/ci                | 3 +++
 test/e2e/soak/index.js          | 5 ++---
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 2da1f01b8..f89a2616b 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -8,7 +8,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        x: [ 1 ]
+        x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
+        y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
     timeout-minutes: 120
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 26c2a578b..d030f73d9 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -57,6 +57,9 @@ if ! curl -s -o /dev/null "$serverUrl"; then
   fail_job
 fi
 
+log 'Checking open DB query count...'
+timeout 1200 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
+
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"
 requestBody='{"email":"'"$userEmail"'","password":"'"$userPassword"'"}'
diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index c3c61b57b..3e21160a7 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -56,8 +56,7 @@ async function soakTest() {
 
   log.info('Setup complete.  Starting soak tests...');
 
-  // eslint-disable-next-line no-unused-expressions
-  false && await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
+  await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
 
   // TODO work out a more scientific sleep duration
   const backgroundJobPause = 20_000;
@@ -65,7 +64,7 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 100, 3_000, 300_000, 100, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest('exportZipWithDataAndMedia', 10, 3_000, 300_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
 
   log.info(`Check for extra logs at ${logPath}`);
 

From 6d16a55f0cdc1fe451384bf3a78ec6b938aea875 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 06:08:02 +0000
Subject: [PATCH 082/103] add description?

---
 .github/workflows/soak-test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index f89a2616b..8ac8f07b5 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -29,6 +29,8 @@ jobs:
           --health-retries 5
     steps:
     - uses: actions/checkout@v6
+    - run: echo "$(git log -1 --oneline)" >> $GITHUB_STEP_SUMMARY
+      if: github.event_name == 'pull_request'
     - name: Set node version
       uses: actions/setup-node@v6
       with:

From 34dcc651d2788670681da16e9a97696de1c1e341 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 06:21:02 +0000
Subject: [PATCH 083/103] add run-name

---
 .github/workflows/soak-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 8ac8f07b5..4284f579a 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -1,4 +1,5 @@
 name: Soak Test
+run-name: ${{ github.actor }} - ${{ github.event.head_commit.message || github.event.pull_request.title }}
 
 on:
   pull_request:

From 771f6f725f7b31c99bdd735a9db78f7349eadc5d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 06:24:03 +0000
Subject: [PATCH 084/103] more sensible timeouts?

---
 test/e2e/soak/ci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index d030f73d9..381593226 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -58,7 +58,7 @@ if ! curl -s -o /dev/null "$serverUrl"; then
 fi
 
 log 'Checking open DB query count...'
-timeout 1200 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
+timeout 240 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
 
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"

From 5edd34c58a54f364adb7a1eba25649d2d29e4d15 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 06:29:59 +0000
Subject: [PATCH 085/103] sha & title

---
 .github/workflows/soak-test.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 4284f579a..c10275399 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -1,5 +1,5 @@
 name: Soak Test
-run-name: ${{ github.actor }} - ${{ github.event.head_commit.message || github.event.pull_request.title }}
+run-name: ${{ github.event.head_commit.message || format('{0} {1}', github.event.pull_request.head.sha, github.event.pull_request.title) }}
 
 on:
   pull_request:
@@ -11,7 +11,7 @@ jobs:
       matrix:
         x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
         y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
-    timeout-minutes: 120
+    timeout-minutes: 20
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
     services:
@@ -29,6 +29,10 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      run: echo "$GITHUB_CONTEXT"
     - uses: actions/checkout@v6
     - run: echo "$(git log -1 --oneline)" >> $GITHUB_STEP_SUMMARY
       if: github.event_name == 'pull_request'
@@ -40,7 +44,7 @@ jobs:
     - run: npm ci
     - run: node lib/bin/create-docker-databases.js
     - name: Soak Test
-      timeout-minutes: 80
+      timeout-minutes: 15
       run: ./test/e2e/soak/ci
     - name: Backend Logs
       if: always()

From 5a8843f91d4bcce7ea4bb3527e2339c498145687 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 06:38:14 +0000
Subject: [PATCH 086/103] customise maxDrainDuration

---
 test/e2e/soak/index.js | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 3e21160a7..e78534f2b 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -56,7 +56,15 @@ async function soakTest() {
 
   log.info('Setup complete.  Starting soak tests...');
 
-  await doSoakTest('randomSubmission', 50, 1_000, 30_000, 100, n => randomSubmission(n, projectId, formId));
+  await doSoakTest({
+    name: 'randomSubmission',
+    throughput: 50,
+    throughputPeriod: 1_000,
+    testDuration: 30_000,
+    maxDrainDuration: 30_000,
+    minimumSuccessThreshold: 100,
+    fn: n => randomSubmission(n, projectId, formId),
+  });
 
   // TODO work out a more scientific sleep duration
   const backgroundJobPause = 20_000;
@@ -64,7 +72,15 @@ async function soakTest() {
   await new Promise(resolve => { setTimeout(resolve, backgroundJobPause); });
   log.info('Woke up.');
 
-  await doSoakTest('exportZipWithDataAndMedia', 10, 3_000, 300_000, 0, n => exportZipWithDataAndMedia(n, projectId, formId));
+  await doSoakTest({
+    name: 'exportZipWithDataAndMedia',
+    throughput: 10,
+    throughputPeriod: 3_000,
+    testDuration: 300_000,
+    maxDrainDuration: 300_000,
+    minimumSuccessThreshold: 0,
+    fn: n => exportZipWithDataAndMedia(n, projectId, formId),
+  });
 
   log.info(`Check for extra logs at ${logPath}`);
 
@@ -74,7 +90,7 @@ async function soakTest() {
   process.exit(0);
 }
 
-function doSoakTest(name, throughput, throughputPeriod, testDuration, minimumSuccessThreshold, fn) {
+function doSoakTest({ name, throughput, throughputPeriod, testDuration, maxDrainDuration, minimumSuccessThreshold, fn }) {
   log.info('Starting soak test:', name);
   log.info('        throughput:', throughput, 'per period');
   log.info('  throughputPeriod:', throughputPeriod, 'ms');
@@ -114,7 +130,6 @@ function doSoakTest(name, throughput, throughputPeriod, testDuration, minimumSuc
       setTimeout(async () => {
         clearTimeout(timerId);
 
-        const maxDrainDuration = 120_000;
         await new Promise(resolve => {
           log.info(`Waiting up to ${durationForHumans(maxDrainDuration)} for test drainage...`);
           const maxDrainTimeout = Date.now() + maxDrainDuration;

From 8e4cf740d1ba08f7eb5c8f9684bf11cdccb7adc3 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 06:42:43 +0000
Subject: [PATCH 087/103] expect success

---
 test/e2e/soak/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index e78534f2b..4da21a3e5 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -78,7 +78,7 @@ async function soakTest() {
     throughputPeriod: 3_000,
     testDuration: 300_000,
     maxDrainDuration: 300_000,
-    minimumSuccessThreshold: 0,
+    minimumSuccessThreshold: 100,
     fn: n => exportZipWithDataAndMedia(n, projectId, formId),
   });
 

From 9abc5386bafbec8c48f8b0817a6bba3f3a1f3c46 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:20:58 +0000
Subject: [PATCH 088/103] add check for initial respons

---
 test/e2e/soak/index.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 4da21a3e5..e4f14f5de 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -96,6 +96,10 @@ function doSoakTest({ name, throughput, throughputPeriod, testDuration, maxDrain
   log.info('  throughputPeriod:', throughputPeriod, 'ms');
   log.info('      testDuration:', durationForHumans(testDuration));
   log.info('-------------------------------');
+  log.info('Checking endpoint responding ok...');
+  const firstSize = await fn(n);
+  log.info(' 1st response size:', firstSize);
+  log.info('-------------------------------');
   return new Promise((resolve, reject) => {
     try {
       const successes = [];

From df5031628b1560034fb2f5c95f45c02489ed301c Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:22:57 +0000
Subject: [PATCH 089/103] add more logging

---
 test/e2e/soak/index.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index e4f14f5de..32cfd7a23 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -114,7 +114,10 @@ function doSoakTest({ name, throughput, throughputPeriod, testDuration, maxDrain
         const n = iterationCount++;
         const started = Date.now();
         try {
+          const nonce = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+          log.info(`iterate(${nonce}) requesting...`);
           const size = await fn(n);
+          log.info(`iterate(${nonce}) returned: ${size} bytes`);
           const finished = Date.now();
           const time = finished - started;
           successes.push(time);

From 686ff9e2e3cf520ab18857590fd91aee523940f4 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:26:16 +0000
Subject: [PATCH 090/103] await

---
 test/e2e/soak/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 32cfd7a23..496822846 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -90,7 +90,7 @@ async function soakTest() {
   process.exit(0);
 }
 
-function doSoakTest({ name, throughput, throughputPeriod, testDuration, maxDrainDuration, minimumSuccessThreshold, fn }) {
+async function doSoakTest({ name, throughput, throughputPeriod, testDuration, maxDrainDuration, minimumSuccessThreshold, fn }) {
   log.info('Starting soak test:', name);
   log.info('        throughput:', throughput, 'per period');
   log.info('  throughputPeriod:', throughputPeriod, 'ms');

From 57e5f794adf8b24f2ec677e6be3a08525f0ae8a2 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:28:51 +0000
Subject: [PATCH 091/103] n=-1

---
 test/e2e/soak/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 496822846..e954feb40 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -97,7 +97,7 @@ async function doSoakTest({ name, throughput, throughputPeriod, testDuration, ma
   log.info('      testDuration:', durationForHumans(testDuration));
   log.info('-------------------------------');
   log.info('Checking endpoint responding ok...');
-  const firstSize = await fn(n);
+  const firstSize = await fn(-1);
   log.info(' 1st response size:', firstSize);
   log.info('-------------------------------');
   return new Promise((resolve, reject) => {

From 6c1a733cc4ea2bbff694a3faca498c5578ecf778 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:35:58 +0000
Subject: [PATCH 092/103] mark selects

---
 lib/model/query/submissions.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/model/query/submissions.js b/lib/model/query/submissions.js
index 1e055974d..e7dc269be 100644
--- a/lib/model/query/submissions.js
+++ b/lib/model/query/submissions.js
@@ -190,6 +190,7 @@ const setSelectMultipleValues = (formId, submissionDefId, pairs) => ({ run }) =>
 };
 
 const getSelectMultipleValuesForExport = (formId, draft, options) => ({ all }) => all(sql`
+  -- Submissions.getSelectMultipleValuesForExport()
 select path, value from form_field_values
 inner join
   (select id, "submissionId" from submission_defs
@@ -422,9 +423,10 @@ const _exportFields = unjoiner(ExportSubmission, ExportSubmissionDef, Submission
   Actor.alias('actors', 'submitter'), Frame.define(table('attachments'), 'present', 'expected'),
   Frame.define(table('edits'), 'count'), Submission.Exports).fields;
 
-const _export = (formId, draft, keyIds = [], options) => {
+const _export = (caller, formId, draft, keyIds = [], options) => {
   const encrypted = keyIds.length !== 0;
   return sql`
+    -- ${sql.identifier(['Submissions', caller])}()
 select ${_exportFields} from submission_defs
 inner join (select * from submissions where draft=${draft}) as submissions
   on submissions.id=submission_defs."submissionId"
@@ -466,12 +468,12 @@ ${page(options)}`;
 };
 
 const streamForExport = (formId, draft, keyIds, options = QueryOptions.none) => ({ s3, stream }) =>
-  stream(_export(formId, draft, keyIds, options))
+  stream(_export('streamForExport', formId, draft, keyIds, options))
     .then(dbStream => streamEncBlobs(s3, dbStream))
     .then(stream.map(_exportUnjoiner));
 
 const getForExport = (formId, instanceId, draft, options = QueryOptions.none) => ({ maybeOne }) =>
-  maybeOne(_export(formId, draft, [], options.withCondition({ 'submissions.instanceId': instanceId })))
+  maybeOne(_export('getForExport', formId, draft, [], options.withCondition({ 'submissions.instanceId': instanceId })))
     .then(map(_exportUnjoiner));
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -546,6 +548,7 @@ select count(*) from deleted_submissions`);
 // Etag-deriving functions
 
 const getODataSelectionEtag = (formId, draft, options = QueryOptions.none) => ({ db }) => db.oneFirst(sql`
+    -- Submissios.getODataSelectionEtag()
     WITH selection AS (
       SELECT event
       FROM submissions

From 0c25e3b0c2d9979fe4acf83d026910e7f61426ce Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:37:52 +0000
Subject: [PATCH 093/103] expect less success

---
 test/e2e/soak/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index e954feb40..a1664dcae 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -78,7 +78,7 @@ async function soakTest() {
     throughputPeriod: 3_000,
     testDuration: 300_000,
     maxDrainDuration: 300_000,
-    minimumSuccessThreshold: 100,
+    minimumSuccessThreshold: 10,
     fn: n => exportZipWithDataAndMedia(n, projectId, formId),
   });
 

From 7fff26d1b1a1bca77bdbac79b8026897c437d8f1 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:44:23 +0000
Subject: [PATCH 094/103] expect nothing

---
 test/e2e/soak/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index a1664dcae..8b4a6a1a5 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -78,7 +78,7 @@ async function soakTest() {
     throughputPeriod: 3_000,
     testDuration: 300_000,
     maxDrainDuration: 300_000,
-    minimumSuccessThreshold: 10,
+    minimumSuccessThreshold: 0,
     fn: n => exportZipWithDataAndMedia(n, projectId, formId),
   });
 

From 785a6c77dd09a76d4ce80edc2205ab1bc323d92c Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:51:53 +0000
Subject: [PATCH 095/103] add more labvel

---
 lib/model/query/client-audits.js          | 1 +
 lib/model/query/submission-attachments.js | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lib/model/query/client-audits.js b/lib/model/query/client-audits.js
index c208d4eec..26a960b78 100644
--- a/lib/model/query/client-audits.js
+++ b/lib/model/query/client-audits.js
@@ -20,6 +20,7 @@ const existsForBlob = (blobId) => ({ maybeOne }) =>
     .then((x) => x.isDefined());
 
 const streamForExport = (formId, draft, keyIds, options = QueryOptions.none) => ({ s3, stream }) => stream(sql`
+  -- ClientAudits.streamForExport()
 select client_audits.*, blobs.id AS "blobId", blobs.s3_status, blobs.content, blobs.sha, submissions."instanceId", "localKey", "keyId", index, submissions."instanceId" from submission_defs
   inner join
     (select id, "submitterId", "createdAt", "updatedAt", "instanceId", "reviewState" from submissions
diff --git a/lib/model/query/submission-attachments.js b/lib/model/query/submission-attachments.js
index da7e41e2a..23b9aa47e 100644
--- a/lib/model/query/submission-attachments.js
+++ b/lib/model/query/submission-attachments.js
@@ -220,6 +220,7 @@ const getByFormAndInstanceIdAndName = (formId, instanceId, name, draft) => ({ ma
 // EXPORT
 
 const streamForExport = (formId, draft, keyIds, options = QueryOptions.none) => ({ s3, stream }) => stream(sql`
+  -- SubmissionAttachments.streamForExport()
 select submission_attachments.name, blobs.id AS "blobId", blobs.content, blobs.s3_status, blobs.sha, submission_attachments.index, form_defs."keyId", submissions."instanceId", submission_defs."localKey" from submission_defs
 inner join (select * from submissions where draft=${draft}) as submissions
   on submissions.id=submission_defs."submissionId"

From f68d2429e23234ec33ad4aa998a385e708cdbff6 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sun, 5 Apr 2026 07:55:27 +0000
Subject: [PATCH 096/103] count open requests

---
 test/e2e/soak/index.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 8b4a6a1a5..0ca57eee8 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -100,6 +100,7 @@ async function doSoakTest({ name, throughput, throughputPeriod, testDuration, ma
   const firstSize = await fn(-1);
   log.info(' 1st response size:', firstSize);
   log.info('-------------------------------');
+  let openRequests = 0;
   return new Promise((resolve, reject) => {
     try {
       const successes = [];
@@ -113,9 +114,9 @@ async function doSoakTest({ name, throughput, throughputPeriod, testDuration, ma
       const iterate = async () => {
         const n = iterationCount++;
         const started = Date.now();
+        const nonce = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
         try {
-          const nonce = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
-          log.info(`iterate(${nonce}) requesting...`);
+          log.info(`iterate(${nonce}) requesting; openRequests: ${++openRequests}`);
           const size = await fn(n);
           log.info(`iterate(${nonce}) returned: ${size} bytes`);
           const finished = Date.now();
@@ -127,6 +128,8 @@ async function doSoakTest({ name, throughput, throughputPeriod, testDuration, ma
           fails.push(err.message);
           results[n] = { success:false, started, finished:Date.now(), err:{ message:err.message, stack:err.stack } };
         } finally {
+          --openRequests;
+          log.info(`iterate(${nonce}) completed; openRequests: ${--openRequests}`);
           ++completedIterations;
         }
       };

From 72989836be337ee8ddb7e406c29e20910bfdd4b1 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 07:05:36 +0000
Subject: [PATCH 097/103] revert to original timeouts

---
 test/e2e/soak/ci       | 2 +-
 test/e2e/soak/index.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/soak/ci b/test/e2e/soak/ci
index 381593226..d55af0bf0 100755
--- a/test/e2e/soak/ci
+++ b/test/e2e/soak/ci
@@ -58,7 +58,7 @@ if ! curl -s -o /dev/null "$serverUrl"; then
 fi
 
 log 'Checking open DB query count...'
-timeout 240 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
+timeout 120 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
 
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"
diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index 0ca57eee8..d697e1b17 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -77,7 +77,7 @@ async function soakTest() {
     throughput: 10,
     throughputPeriod: 3_000,
     testDuration: 300_000,
-    maxDrainDuration: 300_000,
+    maxDrainDuration: 120_000,
     minimumSuccessThreshold: 0,
     fn: n => exportZipWithDataAndMedia(n, projectId, formId),
   });

From 39602abfdf4873ff95738e8afcc65367dea16f5d Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 07:55:51 +0000
Subject: [PATCH 098/103] rebuild

---
 rebuild | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 rebuild

diff --git a/rebuild b/rebuild
new file mode 100644
index 000000000..e69de29bb

From 42ca8c5e3afd504eec8a078b05fb5de30321670f Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 07:56:08 +0000
Subject: [PATCH 099/103] removed

---
 rebuild | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 rebuild

diff --git a/rebuild b/rebuild
deleted file mode 100644
index e69de29bb..000000000

From 126eb423fa885c9e52835ff20fbaca20b7e12590 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 07:57:22 +0000
Subject: [PATCH 100/103] remove job name stuff

---
 .github/workflows/soak-test.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index c10275399..03eb4e616 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -1,5 +1,4 @@
 name: Soak Test
-run-name: ${{ github.event.head_commit.message || format('{0} {1}', github.event.pull_request.head.sha, github.event.pull_request.title) }}
 
 on:
   pull_request:
@@ -34,8 +33,6 @@ jobs:
         GITHUB_CONTEXT: ${{ toJson(github) }}
       run: echo "$GITHUB_CONTEXT"
     - uses: actions/checkout@v6
-    - run: echo "$(git log -1 --oneline)" >> $GITHUB_STEP_SUMMARY
-      if: github.event_name == 'pull_request'
     - name: Set node version
       uses: actions/setup-node@v6
       with:

From 67726cf9a0a6fae0761e47804ce53f0aff5335ae Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 09:16:49 +0000
Subject: [PATCH 101/103] reduce changeset vs master

---
 .github/workflows/soak-test.yml |  6 +-----
 test/e2e/soak/ci/run-tests.sh   | 24 ------------------------
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/.github/workflows/soak-test.yml b/.github/workflows/soak-test.yml
index 5fc29768f..197596655 100644
--- a/.github/workflows/soak-test.yml
+++ b/.github/workflows/soak-test.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         x: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
         y: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]
-    timeout-minutes: 20
+    timeout-minutes: 15
     # TODO should we use the same container as circle & central?
     runs-on: ubuntu-latest
     services:
@@ -28,10 +28,6 @@ jobs:
           --health-timeout 5s
           --health-retries 50
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJson(github) }}
-      run: echo "$GITHUB_CONTEXT"
     - uses: actions/checkout@v6
     - run: ./test/e2e/soak/ci/prepare-postgres.sh
     - name: Set node version
diff --git a/test/e2e/soak/ci/run-tests.sh b/test/e2e/soak/ci/run-tests.sh
index 71625c32e..b83f18fc5 100755
--- a/test/e2e/soak/ci/run-tests.sh
+++ b/test/e2e/soak/ci/run-tests.sh
@@ -12,30 +12,6 @@ fail_job() {
   exit 1
 }
 
-pg_exec() {
-  [[ $# = 1 ]] || fail_job
-  PGPASSWORD=odktest psql \
-      --host=localhost \
-      --username=postgres \
-      --quiet \
-      --tuples-only \
-      --command="$1"
-}
-
-log "[postgres] Increasing query log lengths..."
-
-log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
-pg_exec "ALTER SYSTEM SET track_activity_query_size = 16384"
-pgImg="$(docker ps -q --filter name=postgres)"
-
-log "[postgres]   restarting service..."
-docker restart "$pgImg"
-timeout 10 bash -c "while ! docker exec $pgImg pg_isready --timeout=1; do sleep 1; done"
-log "[postgres]   restarted OK."
-
-log "[postgres]   track_activity_query_size: $(pg_exec 'SHOW track_activity_query_size')"
-log "[postgres]   DONE."
-
 make base
 
 echo "$userPassword" | node ./lib/bin/cli.js user-create  -u "$userEmail"

From a68d11004b22cc427e49d2e086e403c356180a5e Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 11:29:41 +0000
Subject: [PATCH 102/103] increase maxDrainDuration

---
 test/e2e/soak/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/index.js b/test/e2e/soak/index.js
index d697e1b17..f3d0247aa 100644
--- a/test/e2e/soak/index.js
+++ b/test/e2e/soak/index.js
@@ -77,7 +77,7 @@ async function soakTest() {
     throughput: 10,
     throughputPeriod: 3_000,
     testDuration: 300_000,
-    maxDrainDuration: 120_000,
+    maxDrainDuration: 240_000,
     minimumSuccessThreshold: 0,
     fn: n => exportZipWithDataAndMedia(n, projectId, formId),
   });

From 56bb5ca0d021c24a6f39ce50c3fe94c8d2b4b153 Mon Sep 17 00:00:00 2001
From: alxndrsn <alxndrsn>
Date: Sat, 25 Apr 2026 11:41:28 +0000
Subject: [PATCH 103/103] increase check for open queries timeout

---
 test/e2e/soak/ci/run-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/soak/ci/run-tests.sh b/test/e2e/soak/ci/run-tests.sh
index b83f18fc5..e3c66945a 100755
--- a/test/e2e/soak/ci/run-tests.sh
+++ b/test/e2e/soak/ci/run-tests.sh
@@ -32,7 +32,7 @@ if ! curl -s -o /dev/null "$serverUrl"; then
 fi
 
 log 'Checking open DB query count...'
-timeout 120 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
+timeout 600 bash -c "cd ../../..; while ! node lib/bin/check-open-db-queries.js; do sleep 1; done"
 
 log 'Checking backend is serving requests...'
 responseLog="$(mktemp)"