Make DB2 a safe automated failover target

deploy/compose.observability.yml

@@ -76,6 +76,14 @@ services:
     environment:
       GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:?GRAFANA_ADMIN_USER is required}
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD is required}
+      # Slack destination for the provisioned MaxScale-down alert. Deliberately
+      # NOT `:?required` -- the alert rule and its Grafana UI state are useful
+      # on their own, and a missing webhook must not stop the whole stack from
+      # starting. The default is a syntactically valid but non-functional URL:
+      # Grafana's provisioning rejects an empty one, so a placeholder is what
+      # lets this deploy before the webhook exists. Delivery then fails and is
+      # logged until it is replaced, which is visible rather than silent.
+      SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL:-https://hooks.slack.com/services/REPLACE-ME}
     ports:
       - "127.0.0.1:3000:3000"
     volumes:
@@ -84,6 +92,29 @@ services:
       # datasource or dashboard is a file drop rather than a Compose edit.
       - ../observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro,z
       - ../observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro,z
+      - ../observability/grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro,z
+    networks:
+      - observability_net
+
+  # Probes MaxScale's client listener from Delta. This is the only thing in the
+  # stack that can report the database tier being unreachable: MaxScale's own
+  # alert script cannot, because a dead MaxScale sends nothing. See
+  # ../observability/blackbox/blackbox.yml for why it is a TCP connect on :4006
+  # rather than the admin API.
+  blackbox:
+    image: prom/blackbox-exporter:v0.25.0
+    restart: unless-stopped
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    command: ["--config.file=/etc/blackbox/blackbox.yml"]
+    volumes:
+      - ../observability/blackbox/blackbox.yml:/etc/blackbox/blackbox.yml:ro,z
+    # No ports published: only Prometheus talks to it, over the internal
+    # network. It probes OUTWARD to the MaxScale host, which needs no inbound
+    # exposure here.
     networks:
       - observability_net
 

deploy/mariadb/primary.cnf

@@ -74,3 +74,38 @@ binlog_expire_logs_seconds       = 604800  # keep 7 days of binlogs for replica
 gtid_domain_id                   = 1
 gtid_strict_mode                 = ON
 log_slave_updates                = ON      # lets the replica be chained / used for backups
+
+# --- Semi-synchronous replication: MASTER side --------------------------------
+# This is the setting that makes MaxScale's automated failover safe to turn on.
+# Without it, replication is async: DB1 acknowledges a commit to the CMS before
+# DB2 has seen it, so promoting DB2 silently discards the tail of the write
+# stream. With it, a commit is not acknowledged until DB2 has the event.
+rpl_semi_sync_master_enabled     = ON
+# AFTER_SYNC (not the AFTER_COMMIT default): wait for the replica's ack BEFORE
+# committing to the storage engine, so a write is never visible to other
+# sessions on DB1 until it is safe on DB2. AFTER_COMMIT makes the write visible
+# first, which is precisely the window that loses data on promotion.
+rpl_semi_sync_master_wait_point  = AFTER_SYNC
+# How long a commit waits for an ack before degrading to async, in ms.
+rpl_semi_sync_master_timeout     = 1000
+# OFF is load-bearing for availability. With the ON default, DB1 pays the full
+# timeout above on EVERY commit while no replica is connected — so planned DB2
+# maintenance would add a second of latency to every write. OFF drops straight
+# to async the moment there is no semi-sync replica, and back to semi-sync when
+# DB2 reconnects. The tradeoff is real and must be understood: while DB2 is
+# down, the zero-loss guarantee is NOT in force.
+#
+# MONITOR THIS WITH Rpl_semi_sync_master_clients, NOT ..._status. Verified by
+# stopping DB2 on 2026-08-05: status stayed **ON** through seven unacknowledged
+# commits while clients sat at 0 and no_tx climbed 0 -> 7. With
+# wait_no_slave=OFF the master never enters the "off" state it would otherwise
+# fall into, so status is not evidence the guarantee is holding.
+#   Rpl_semi_sync_master_clients  == 0  -> nothing is acknowledging: ALERT
+#   Rpl_semi_sync_master_no_tx    rising -> commits completing unacknowledged
+#   Rpl_semi_sync_master_yes_tx   rising -> the guarantee is actually in force
+rpl_semi_sync_master_wait_no_slave = OFF
+# Slave side, inert while this node is the primary. Present so that after a
+# failover and auto_rejoin — when this node comes back as a REPLICA of DB2 — it
+# registers as a semi-sync client instead of silently leaving the new primary
+# running asynchronously. Mirrors replica.cnf; both nodes carry both roles.
+rpl_semi_sync_slave_enabled      = ON

deploy/mariadb/provision-db2.sh

@@ -0,0 +1,108 @@
+#!/bin/sh
+# One-time: install MariaDB 11.8 on DB2 and put the replica config in place.
+#
+# Run this ON DB2 (THETRIANGLE-DB2-LXC, 10.248.40.155) as root:
+#
+#   sudo sh provision-db2.sh
+#
+# It does NOT start replication and does NOT touch MaxScale — it only gets a
+# correctly-configured, correctly-bound MariaDB running. Run setup-replica.sh
+# afterwards, then follow "Bringing up DB2" in README.md from step 5.
+#
+# Idempotent: safe to re-run. Existing repo/key/config are refreshed in place
+# and apt skips packages already at the right version.
+#
+# NOTE ON ACCESS: tadmin has no NOPASSWD sudo on DB2 (unlike DB1 and MaxScale),
+# so this cannot be driven over ssh non-interactively until such a rule exists:
+#   echo 'tadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-tadmin
+#   chmod 440 /etc/sudoers.d/90-tadmin
+set -eu
+
+EXPECT_HOST=THETRIANGLE-DB2-LXC
+EXPECT_ADDR=10.248.40.155
+SERIES=11.8                      # LTS. deb.mariadb.org carries ONLY LTS lines.
+CNF_SRC="$(dirname "$0")/replica.cnf"
+CNF_DST=/etc/mysql/mariadb.conf.d/70-triangle-replica.cnf
+
+# --- Guards ------------------------------------------------------------------
+# DB2 was briefly live on DB1's address (10.248.40.154) and ssh gives no
+# host-key warning when the ARP winner changes underneath you. Never let this
+# script run against the primary: it would overwrite the primary's config with
+# a read_only replica config.
+[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 1; }
+if [ "$(hostname)" != "$EXPECT_HOST" ]; then
+  echo "REFUSING: hostname is '$(hostname)', expected '$EXPECT_HOST'." >&2
+  echo "You are not on DB2. Check which host you actually reached." >&2
+  exit 1
+fi
+if ! ip -4 addr show | grep -q "inet ${EXPECT_ADDR}/"; then
+  echo "REFUSING: ${EXPECT_ADDR} is not configured on this host." >&2
+  exit 1
+fi
+[ -f "$CNF_SRC" ] || { echo "cannot find replica.cnf next to this script" >&2; exit 1; }
+
+echo "==> Host verified: $(hostname) / ${EXPECT_ADDR}"
+
+# --- MariaDB apt repo ---------------------------------------------------------
+# Note this is the SERVER key. MaxScale uses a different key entirely and is not
+# installed here — see README.md.
+echo "==> Adding MariaDB ${SERIES} repository"
+apt-get update -qq
+apt-get install -y -qq curl gpg apt-transport-https ca-certificates
+
+install -d -m 0755 /etc/apt/keyrings
+curl -fsSL https://supplychain.mariadb.com/MariaDB-Server-GPG-KEY \
+  | gpg --dearmor --yes -o /etc/apt/keyrings/mariadb.gpg
+chmod 0644 /etc/apt/keyrings/mariadb.gpg
+
+. /etc/os-release
+cat > /etc/apt/sources.list.d/mariadb.list <<EOF
+deb [signed-by=/etc/apt/keyrings/mariadb.gpg] https://deb.mariadb.org/${SERIES}/ubuntu ${VERSION_CODENAME} main
+EOF
+
+apt-get update -qq
+
+# --- Install ------------------------------------------------------------------
+# Pinned to the series, not "latest", so DB2 does not drift ahead of DB1 — a
+# promotion should not also be a version change.
+echo "==> Installing mariadb-server"
+DEBIAN_FRONTEND=noninteractive apt-get install -y -qq mariadb-server mariadb-client
+mariadbd --version
+
+# --- Config -------------------------------------------------------------------
+# The 70- prefix is load-bearing: Ubuntu's stock 50-server.cnf sets
+# bind-address = 127.0.0.1 and mariadb.conf.d is read in lexical order, so a
+# file sorting before it cannot override the bind and the replica would be
+# unreachable from both MaxScale and the primary.
+echo "==> Installing ${CNF_DST}"
+install -o root -g root -m 0644 "$CNF_SRC" "$CNF_DST"
+
+echo "==> Restarting mariadb"
+systemctl enable --now mariadb
+systemctl restart mariadb
+
+# --- Verify -------------------------------------------------------------------
+echo "==> Verifying"
+mariadb -N -B -e "SELECT @@hostname, @@server_id, @@read_only, @@gtid_domain_id, @@gtid_strict_mode"
+
+# server_id must differ from DB1's (1) or replication refuses to start.
+SID=$(mariadb -N -B -e "SELECT @@server_id")
+[ "$SID" = "2" ] || { echo "FAIL: server_id is ${SID}, expected 2" >&2; exit 1; }
+
+# The whole point of the 70- prefix. If this shows 127.0.0.1, the config did not
+# take and nothing downstream will work.
+echo "--- listening sockets ---"
+ss -ltnp 2>/dev/null | grep 3306 || echo "WARNING: nothing listening on 3306"
+if ! ss -ltn 2>/dev/null | grep -q "${EXPECT_ADDR}:3306\|0.0.0.0:3306\|\*:3306"; then
+  echo "FAIL: not bound to ${EXPECT_ADDR}:3306 — check ${CNF_DST} ordering" >&2
+  exit 1
+fi
+
+# Durability must match the primary: DB2 is a failover target, not a read cache.
+echo "--- durability (must be 1 / 1) ---"
+mariadb -N -B -e "SELECT @@innodb_flush_log_at_trx_commit, @@sync_binlog"
+
+echo
+echo "OK. MariaDB ${SERIES} is installed, bound to ${EXPECT_ADDR}, and read_only."
+echo "NEXT: run setup-replica.sh on this host to seed from DB1 and start"
+echo "replication, then continue at README.md 'Bringing up DB2' step 5."

deploy/mariadb/replica.cnf

@@ -1,19 +1,19 @@
 [mysqld]
 # =============================================================================
-# Triangle CMS — MariaDB READ REPLICA (DB2) config. NOT YET DEPLOYED: DB2 does
-# not exist as of 2026-07-30, so this is the config to install when it lands.
-# Runs on its OWN host (separate from DB1), replicating asynchronously from the
-# primary via GTID. Serves READ traffic only.
+# Triangle CMS — MariaDB REPLICA + FAILOVER TARGET (DB2) config.
+# Host: THETRIANGLE-DB2-LXC, 10.248.40.155, CT 111 (4 vCPU / 4 GB / 63 GB).
+# Runs on its OWN host (separate from DB1), replicating from the primary via
+# GTID with semi-synchronous acknowledgement. Serves READ traffic, and is
+# promoted to primary by MaxScale's mariadbmon on failover.
 # Installed natively at /etc/mysql/mariadb.conf.d/70-triangle-replica.cnf — the
 # 70- prefix is load-bearing, see bind-address below.
 # =============================================================================
 
 # --- Network ------------------------------------------------------------------
-# UNCOMMENT AND SET to DB2's internal NIC address before starting MariaDB.
 # Ubuntu's stock 50-server.cnf binds 127.0.0.1, which would leave the replica
 # unreachable from both MaxScale and the primary; this file must sort after it.
 # Firewall 3306 to the MaxScale and primary hosts only.
-# bind-address                   = 10.248.40.xxx
+bind-address                     = 10.248.40.155
 
 # --- InnoDB memory -----------------------------------------------------------
 # Sized to match the primary; see the rationale in primary.cnf. Adjust the two
@@ -24,13 +24,21 @@ innodb_buffer_pool_size          = 1G
 innodb_log_file_size             = 256M
 innodb_log_buffer_size           = 32M
 
-# --- Durability: relaxed on the replica ---------------------------------------
-# A read replica can re-fetch anything it loses on crash from the primary via
-# GTID, so we trade per-commit fsyncs for throughput on apply. Never do this on
-# the primary.
-innodb_flush_log_at_trx_commit   = 2
-sync_binlog                      = 0
+# --- Durability: FULL, same as the primary ------------------------------------
+# Deliberately NOT the relaxed (2 / 0) setting usually given to a read replica.
+# Two reasons, both consequences of DB2 being a failover target rather than a
+# pure read cache:
+#   1. On promotion DB2 *becomes* the primary. Relaxed settings would silently
+#      leave production running without per-commit fsyncs until someone noticed.
+#   2. Semi-sync's guarantee is only as strong as the ack. With sync_binlog=0
+#      the replica acks once the event is in the OS page cache, so a power loss
+#      on DB2 discards writes the primary already told the client were durable.
+# The CMS write volume is trivial (a newsroom, not a transaction processor), so
+# the throughput this costs is not measurable here.
+innodb_flush_log_at_trx_commit   = 1       # fsync redo on every commit
+sync_binlog                      = 1       # fsync binlog on every commit
 innodb_flush_method              = O_DIRECT
+innodb_doublewrite               = ON
 
 # --- Concurrency / caches ----------------------------------------------------
 max_connections                  = 300     # replica typically fields more read conns
@@ -70,3 +78,22 @@ binlog_expire_logs_seconds       = 604800
 # Parallel apply keeps replication lag low under write bursts from the primary.
 slave_parallel_threads           = 4
 slave_parallel_mode              = optimistic
+
+# --- Semi-synchronous replication ---------------------------------------------
+# BOTH sides are enabled on BOTH nodes, because either node can hold either role
+# after a failover. MariaDB only acts on the side matching its current role, so
+# the master settings sit inert here until this node is promoted.
+#
+# Slave side: acknowledges each binlog event back to the primary, which is what
+# lets automated failover promote DB2 without losing acknowledged commits. Must
+# be ON here for the primary's AFTER_SYNC wait to ever be satisfied — if this is
+# OFF, the primary just times out and degrades to async on every commit.
+rpl_semi_sync_slave_enabled      = ON
+# Master side: inert while this node is a replica, load-bearing the moment it is
+# promoted. Without it a failover silently drops to asynchronous replication —
+# losing the zero-data-loss guarantee at exactly the moment you have just proven
+# you need it. Values must match primary.cnf; see the rationale there.
+rpl_semi_sync_master_enabled     = ON
+rpl_semi_sync_master_wait_point  = AFTER_SYNC
+rpl_semi_sync_master_timeout     = 1000
+rpl_semi_sync_master_wait_no_slave = OFF