Dev/v0.5.0 resilience

.github/workflows/chaos.yml

@@ -0,0 +1,99 @@
+name: Chaos Tests
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Branch or tag to test"
+        required: false
+        default: "main"
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+jobs:
+  chaos:
+    name: Chaos Testing Suite
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      # ── Build proxy binary first (used by chaos_tests as CARGO_BIN_EXE_*) ──
+      - name: Build turbineproxy binary
+        run: cargo build --bin turbineproxy
+
+      # ── Start chaos stack ─────────────────────────────────────────────────
+      - name: Start chaos docker-compose stack
+        run: |
+          docker compose -f docker-compose.chaos.yml up -d \
+            mysql-primary mysql-replica1 mysql-replica2 toxiproxy
+
+      - name: Wait for MySQL primary
+        run: |
+          for i in $(seq 1 40); do
+            docker compose -f docker-compose.chaos.yml exec -T mysql-primary \
+              mysqladmin ping -h 127.0.0.1 -uroot -proot --silent 2>/dev/null \
+              && echo "MySQL primary ready" && break
+            echo "waiting ($i/40)..."
+            sleep 3
+          done
+
+      - name: Wait for Toxiproxy
+        run: |
+          for i in $(seq 1 20); do
+            curl -sf http://127.0.0.1:8474/proxies && break
+            echo "waiting for toxiproxy ($i/20)..."
+            sleep 2
+          done
+
+      - name: Wire MySQL replication
+        run: |
+          docker compose -f docker-compose.chaos.yml run --rm chaos-setup
+
+      - name: Create Toxiproxy proxies
+        run: |
+          docker compose -f docker-compose.chaos.yml run --rm toxiproxy-init
+
+      - name: Verify Toxiproxy routes MySQL
+        run: |
+          mysql -h 127.0.0.1 -P 13306 -uroot -proot -e "SELECT 1" turbineproxy_test
+
+      # ── Run chaos tests ───────────────────────────────────────────────────
+      - name: Run chaos tests
+        run: |
+          cargo test --test chaos_tests -- --test-threads=1 --nocapture
+        env:
+          TOXIPROXY_API: http://127.0.0.1:8474
+          RUST_LOG: turbineproxy=info
+
+      # ── Collect logs on failure ────────────────────────────────────────────
+      - name: Collect container logs on failure
+        if: failure()
+        run: |
+          docker compose -f docker-compose.chaos.yml logs mysql-primary   > /tmp/primary.log 2>&1 || true
+          docker compose -f docker-compose.chaos.yml logs mysql-replica1  > /tmp/replica1.log 2>&1 || true
+          docker compose -f docker-compose.chaos.yml logs mysql-replica2  > /tmp/replica2.log 2>&1 || true
+          docker compose -f docker-compose.chaos.yml logs toxiproxy       > /tmp/toxiproxy.log 2>&1 || true
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: chaos-logs
+          path: /tmp/*.log
+          retention-days: 7
+
+      # ── Teardown ──────────────────────────────────────────────────────────
+      - name: Teardown chaos stack
+        if: always()
+        run: docker compose -f docker-compose.chaos.yml down -v

.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  # Skip the `npm run build` step inside build.rs for all Rust jobs.
+  # The dashboard is built independently in the frontend-tests job.
+  TURBINEPROXY_SKIP_DASHBOARD_BUILD: "1"
 
 jobs:
   # ── Fast checks (no MySQL needed) ──────────────────────────────────────────
@@ -24,6 +27,17 @@ jobs:
         run: cargo fmt --all -- --check
       - name: cargo clippy
         run: cargo clippy --all-targets -- -D warnings
+      - name: cargo clippy (unwrap lint on critical paths)
+        run: |
+          cargo clippy \
+            --all-targets \
+            -- \
+            -D clippy::unwrap_used \
+            --allow clippy::all \
+            2>&1 | grep -E "^error\[clippy::unwrap_used\]" | \
+            grep -E "src/(dashboard|analytics)/" | \
+            (! grep .) || \
+            (echo "clippy::unwrap_used found in critical paths" && exit 1)
 
   # ── Security audit ─────────────────────────────────────────────────────────
   security:
@@ -52,8 +66,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - name: cargo test (lib + bins)
-        run: cargo test --lib --bins
+      - name: cargo test (bin unit tests)
+        run: cargo test --bins
 
   # ── Code coverage ──────────────────────────────────────────────────────────
   coverage:
@@ -107,19 +121,37 @@ jobs:
           - db-label: MySQL 8.0
             db-image: mysql:8.0
             db-port: 3306
+            db-cmd: "--default-authentication-plugin=mysql_native_password"
+            health-cmd: "mysqladmin ping -h 127.0.0.1 -uroot -proot"
           - db-label: MySQL 8.4
             db-image: mysql:8.4
             db-port: 3306
+            # In 8.4 the plugin is disabled by default and
+            # --default-authentication-plugin was removed. We must:
+            #   1. Load the plugin: --mysql-native-password=ON
+            #   2. Set it as the default: --authentication-policy=mysql_native_password
+            # This ensures the docker init script creates root with native
+            # password, so no ALTER USER is needed afterwards.
+            db-cmd: "--mysql-native-password=ON --authentication-policy=mysql_native_password,,"
+            health-cmd: "mysqladmin ping -h 127.0.0.1 -uroot -proot"
           - db-label: MariaDB 10.11
             db-image: mariadb:10.11
             db-port: 3306
+            db-cmd: "mariadbd"
+            health-cmd: "healthcheck.sh --connect --innodb_initialized"
           - db-label: MariaDB 11.4
             db-image: mariadb:11.4
             db-port: 3306
+            db-cmd: "mariadbd"
+            health-cmd: "healthcheck.sh --connect --innodb_initialized"
 
     services:
       mysql:
         image: ${{ matrix.db-image }}
+        # Pass mysqld startup flags (e.g. --mysql-native-passwords=ON for mysql 8.4).
+        # MariaDB entries use 'mariadbd' (the image default CMD) to avoid an empty
+        # command override breaking the container.
+        command: ${{ matrix.db-cmd }}
         env:
           # MySQL uses MYSQL_ROOT_PASSWORD; MariaDB accepts both.
           MYSQL_ROOT_PASSWORD: root
@@ -129,10 +161,12 @@ jobs:
         ports:
           - ${{ matrix.db-port }}:3306
         options: >-
-          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -proot"
+          --name mysql-service
+          --health-cmd="${{ matrix.health-cmd }}"
           --health-interval=5s
           --health-timeout=5s
-          --health-retries=15
+          --health-retries=20
+          --health-start-period=30s
 
     steps:
       - uses: actions/checkout@v4

CHANGELOG.md

@@ -7,6 +7,109 @@ Versioning follows [Semantic Versioning](https://semver.org/).
 
 ---
 
+## [0.5.0] - 2026-05-14
+
+### Security & Hardening
+
+- **No lock poisoning** — All internal mutexes and RW-locks migrated from
+  `std::sync::{Mutex,RwLock}` to `parking_lot::{Mutex,RwLock}`. Lock poisoning is
+  impossible: if a thread panics while holding a lock, the lock is released cleanly
+  and subsequent acquires succeed without any `PoisonError` handling. Covered by
+  dedicated panic-recovery unit tests.
+
+- **SCRAM-SHA-256 hardening** — PostgreSQL SCRAM-SHA-256 authentication passes a
+  full fuzz corpus (15 invariant assertions, randomised per-run) covering malformed
+  first/final server messages, short/empty nonce, missing fields, and replay
+  variants. No panics or incorrect accept/reject decisions observed.
+
+- **Dashboard auth failure counter** — A new atomic counter
+  (`turbineproxy_dashboard_auth_failures_total`) is incremented on every failed
+  authentication event: wrong password at login, invalid/expired token in the
+  `auth_middleware`, and invalid/expired token presented to `/api/auth/refresh`.
+  Exposed via Prometheus.
+
+### Dashboard Auth
+
+- **`POST /api/auth/refresh`** — Endpoint for renewing an authentication token
+  without re-entering credentials. The old token is atomically revoked and a new
+  UUID token is issued with a fresh TTL. Readonly and admin tokens are both
+  supported. Invalid or expired tokens increment the auth failure counter and
+  return `401 Unauthorized`.
+
+- **`POST /api/auth/logout`** — Explicitly revoke the current session token.
+  The token is removed from the in-memory store immediately. Readonly tokens can
+  call this endpoint (previously blocked by the `is_mutating` guard — now
+  explicitly exempted alongside `/api/auth/refresh`).
+
+### Reliability
+
+- **Failover flap protection** — Failover events are guarded by a configurable
+  cooldown (`failover_cooldown_secs`) and a minimum number of consecutive health
+  check passes before a recovered backend is re-admitted
+  (`failover_min_recovery_checks`). Prevents oscillation in unstable network
+  conditions.
+
+- **Per-backend circuit breaker** — Each backend gets an independent circuit breaker
+  (Closed → Open → Half-Open state machine). Once a backend accumulates
+  `circuit_breaker_threshold` consecutive failures the circuit opens and requests
+  are rejected immediately without hitting the network. After
+  `circuit_breaker_timeout_secs` the circuit enters Half-Open and probes with one
+  request before fully closing.
+
+- **Bounded connection wait queue** — Connection acquisition from the pool is now
+  queued rather than immediately failing. `pool_wait_timeout_ms` caps how long a
+  query waits for a connection before returning an error to the client. Prevents
+  thundering-herd during traffic spikes.
+
+### Performance
+
+- **Connection multiplexing (Phase A)** — Multiple client sessions can share a
+  single backend connection during idle phases. A multiplexing ratio gauge
+  (`turbineproxy_multiplex_ratio`) is exported via Prometheus. Values > 1 indicate
+  effective connection reuse.
+
+- **PostgreSQL HA parity** — All HA features available for MySQL (health checks,
+  replica lag monitoring, weighted read routing, automatic failover, flap
+  protection, circuit breakers, bounded wait queue, multiplexing) now apply equally
+  to the PostgreSQL listener.
+
+### Observability
+
+- **`turbineproxy_dashboard_auth_failures_total`** — New Prometheus counter for
+  monitoring brute-force activity and token lifecycle issues.
+
+- **`turbineproxy_multiplex_ratio`** — New Prometheus gauge for connection
+  multiplexing efficiency.
+
+- **`turbineproxy_pg_replica_lag_seconds`** — Prometheus gauge per PostgreSQL
+  replica, sourced from `pg_last_xact_replay_timestamp`.
+
+- **`turbineproxy_sessions_pinned_total`** — Counter of sessions pinned to a
+  specific backend (user variables, prepared statements, open transactions).
+
+### CI / Quality
+
+- **`clippy::unwrap_used` lint on critical paths** — A dedicated CI step runs
+  `clippy` with `-D clippy::unwrap_used` scoped to `src/dashboard/` and
+  `src/analytics/`. Any unguarded `.unwrap()` in those paths fails the build.
+
+- **Panic recovery tests** — Two `#[test]` functions in `src/proxy/server.rs`
+  assert that `parking_lot::Mutex` and `parking_lot::RwLock` remain usable after
+  a thread panics while holding the lock. Run with
+  `cargo test --bin turbineproxy parking_lot`.
+
+- **PostgreSQL TLS integration test** — `tests/pg_integration_tests.rs` now
+  includes a `pg_tls_connection_with_psql` test that connects with
+  `sslmode=require` using the `psql` CLI and asserts `ssl=t` in `pg_stat_ssl`.
+  Auto-skips when `psql` is not in `PATH` or when `TEST_PG_SKIP_TLS` is set.
+
+### Dashboard Configuration (breaking)
+
+- **`token_ttl_secs = 0`** no longer means "default 24 h" — it means "tokens
+  never expire". Set an explicit positive value to enforce expiry.
+
+---
+
 ## [0.3.1] - 2026-05-09
 
 ### Fixed

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "turbineproxy"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 description = "Intelligent MySQL proxy with read/write splitting, query analytics, and automatic index advice"
 license = "Apache-2.0"
@@ -64,6 +64,9 @@ regex = "1"
 # Memory allocator
 mimalloc = "0.1"
 
+# Non-poisoning Mutex/RwLock (eliminates panic cascade from lock poisoning)
+parking_lot = "0.12"
+
 # HTTP client — used for cluster config sync (push to peer nodes)
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
 
@@ -74,6 +77,8 @@ tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] }
 tempfile = "3"
 criterion = { version = "0.5", features = ["async_tokio"] }
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
+libc = "0.2"
+hex = "0.4"
 
 [[bench]]
 name = "proxy_bench"