From 59e24e27d60f6620a7c32540a4220b4e108e32c9 Mon Sep 17 00:00:00 2001 From: MaXiMo000 Date: Sun, 27 Sep 2026 02:52:34 +0530 Subject: [PATCH 1/3] Drill pgBackRest and WAL-G backups; webhook alerts (0.5.0) `firedrill physical pgbackrest|walg` restores with the tool itself into a container of the backup's own major, replays WAL (to --target for PITR) and runs smoke, semantics and integrity. The major version is read from backup.info / the WAL-G stop sentinel; remote repositories use the tools' own PGBACKREST_*/WALG_*/AWS_* variables, passed by name. The restored server is started with -c archive_mode=off and then asked, so production's archive_command can never push WAL into the repository being checked -- tested with a backup carrying archive_mode=on. Tested against repositories made by pgBackRest 2.59.1 and WAL-G v3.0.9 (tests/make_physical.py): full replay, exact PITR boundary, target past the archive, target before every backup (pgBackRest [075] and Postgres's own message), a corrupted file, and --max-age staleness. A restore that ran and broke is FAIL, not COULD NOT VERIFY. --notify ENV_VAR posts the verdict to a Slack/Discord/any webhook on failure or could-not-verify; the Action gains physical and notify inputs. Co-Authored-By: Claude Opus 5.5 --- README.md | 92 +++++++++++++ action.yml | 51 +++++++ firedrill/cli.py | 56 ++++++++ firedrill/drill.py | 177 +++++++++++++++++++++++- firedrill/notify.py | 64 +++++++++ firedrill/physical.py | 299 ++++++++++++++++++++++++++++++++++++++++ firedrill/report.py | 14 +- pyproject.toml | 2 +- tests/make_physical.py | 87 ++++++++++++ tests/test_firedrill.py | 150 +++++++++++++++++++- 10 files changed, 988 insertions(+), 4 deletions(-) create mode 100644 firedrill/notify.py create mode 100644 firedrill/physical.py create mode 100644 tests/make_physical.py diff --git a/README.md b/README.md index 71da0ac..ec5b049 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ firedrill run path/to/dump.dump # restore and report firedrill run s3://backups/pg/daily/ # the newest object under a prefix firedrill run --live DATABASE_URL --keep backups/ # dump a running database now, prove it restores firedrill run dump.dump --image 'pgvector/pgvector:pg{major}' # extensions the stock image lacks +firedrill physical pgbackrest --repo DIR --target "..." # pgBackRest / WAL-G, with PITR -- see below firedrill run dump.dump --json report.json # machine-readable firedrill run dump.dump --rto 45m # exceeding the budget is a finding firedrill run dump.dump --tier fast # schema only, for every commit @@ -404,6 +405,97 @@ catalog facts. It has no field that could hold a row, which is pinned by a test, because it is the artefact of this tool most likely to be committed to a repo by accident. +## pgBackRest and WAL-G + +Most production Postgres isn't backed up with `pg_dump` at all. It's +backed up with [pgBackRest](https://pgbackrest.org) or +[WAL-G](https://github.com/wal-g/wal-g): a base backup plus a WAL archive. +firedrill restores those **with the tool itself**, replays the WAL, and puts +the recovered server through the same checks: + +``` +$ firedrill physical pgbackrest --repo /backups/pgbackrest --target "2026-09-26 20:28:59+00" --config boundary.yml + backup pgBackRest 20260926-202837F stanza demo finished 2026-09-26T20:28:56+00:00 + source PostgreSQL 16 -> recovered to 2026-09-26 20:28:59+00 on postgres:16 + + [ok ] recover 23.81s 20260926-202837F -> 2026-09-26 20:28:59+00 + [ok ] smoke 0.84s shop: 2 table(s) + [ok ] semantics 0.72s 2 check(s) in shop + [ok ] integrity 2.39s 1 sequence(s) + + PASS -- recovered to the target, and the boundary is where it should be. +``` + +```bash +firedrill physical pgbackrest --repo DIR # newest backup, all archived WAL +firedrill physical walg --repo DIR --target "2026-09-26 20:30:23+00" +firedrill physical pgbackrest --stanza main --postgres 16 # a remote (S3/GCS/Azure) repository +firedrill physical walg --repo DIR --max-age 26h # and the newest backup must be recent +``` + +- The major version comes from the repository (`backup.info`, the WAL-G stop + sentinel), and the restore runs in `postgres:` with the tool added: + pgBackRest from the image's own PGDG repository, WAL-G's release binary + pinned by sha256. `--image` takes a `{major}` template for extension images. +- **A remote repository** is configured the way the tools are always + configured -- `PGBACKREST_REPO1_TYPE=s3`, `WALG_S3_PREFIX`, `AWS_*`, and so + on. Those variables are passed to the container **by name**, never value. +- **It never archives into the repository it checks.** A restored data + directory carries production's own `archive_command`; started as it is, the + drill would push WAL into production's archive. The server is started with + `-c archive_mode=off` (which outranks every file), and then *asked*: if it + reports anything but `off`, the drill stops with `ARCHIVING_ENABLED`. + Tested with a backup that carries `archive_mode = 'on'` from `ALTER SYSTEM`: + the repository held the same WAL files before and after. A local + repository is also mounted read-only. + +Measured against repositories made by pgBackRest 2.59.1 and WAL-G v3.0.9 +themselves (built by `tests/make_physical.py`, in CI): + +| case | result | +|---|---| +| replay all archived WAL | PASS: all 3 tables back, including the 2 written after the backup | +| recover to the timestamp between `before_target` and `after_target` | PASS: exactly one of them | +| a target later than the archive | FAIL `PITR_TARGET_UNREACHED` | +| a target before every backup (retention already pruned what you'd need) | FAIL `PITR_TARGET_BEFORE_BACKUP` | +| 64 bytes zeroed in one pgBackRest file | FAIL `RESTORE_FAILED`: *zlib threw error: [-3] data error* | +| newest backup older than `--max-age` | FAIL `BACKUP_STALE` | + +A restore that ran and broke says **FAIL**: the backup was tested, and it is +broken. COULD NOT VERIFY stays reserved for a check that could not happen. + +## Alerts, and running it on a schedule + +```bash +export BACKUP_ALERTS=https://hooks.slack.com/services/... +firedrill physical pgbackrest --repo /backups --max-age 26h --notify BACKUP_ALERTS +``` + +`--notify ENV_VAR` POSTs one JSON message when a drill fails **or could not +run**. It carries `text` (Slack, Mattermost) and `content` (Discord) with a +one-line verdict, plus `ok`, `verified`, `worst` and the finding rules for +anything else. `--notify-on always` reports passes too. The argument is the +variable's *name*: a webhook URL is a credential. A webhook that refuses the +message fails the run rather than being skipped quietly, and its URL never +appears in the error. + +Without GitHub Actions, a systemd timer on any host with Docker does it: + +```ini +# /etc/systemd/system/firedrill.service +[Service] +Type=oneshot +EnvironmentFile=/etc/firedrill.env # BACKUP_ALERTS=..., PGBACKREST_REPO1_* ... +ExecStart=/usr/local/bin/firedrill physical pgbackrest --stanza main --postgres 16 --max-age 26h --notify BACKUP_ALERTS + +# /etc/systemd/system/firedrill.timer +[Timer] +OnCalendar=*-*-* 04:17:00 +Persistent=true +[Install] +WantedBy=timers.target +``` + ## Point-in-time recovery A dump proves you can get *a* database back. PITR proves you can get it back to diff --git a/action.yml b/action.yml index bf23e38..246b6b8 100644 --- a/action.yml +++ b/action.yml @@ -35,6 +35,40 @@ inputs: for databases using extensions the stock postgres image lacks. required: false default: "" + physical: + description: >- + pgbackrest or walg: restore a physical backup with that tool instead of a + dump. Configure a remote repository with PGBACKREST_* / WALG_* / AWS_* + variables in the step's `env:`, or give a local one in `repo`. + required: false + default: "" + repo: + description: With `physical`, a local repository directory. + required: false + default: "" + stanza: + description: "With `physical: pgbackrest`, the stanza." + required: false + default: "" + target: + description: With `physical`, recover to this timestamp instead of the end of the archive. + required: false + default: "" + max-age: + description: With `physical`, fail when the newest backup is older than this, e.g. 26h. + required: false + default: "" + postgres: + description: With `physical` and a remote repository, the Postgres major version. + required: false + default: "" + notify: + description: >- + Name of an environment variable holding a Slack/Discord/any webhook URL + (set it from a secret with `env:`). Posts the verdict when the drill fails + or could not run. + required: false + default: "" config: description: Path to firedrill.yml. Defaults to one beside the checkout. required: false @@ -118,9 +152,25 @@ runs: # becomes an unactionable one. run: | set -uo pipefail + if [ -n "${{ inputs.physical }}" ]; then + args=(physical "${{ inputs.physical }}") + [ -n "${{ inputs.repo }}" ] && args+=(--repo "${{ inputs.repo }}") + [ -n "${{ inputs.stanza }}" ] && args+=(--stanza "${{ inputs.stanza }}") + [ -n "${{ inputs.target }}" ] && args+=(--target "${{ inputs.target }}") + [ -n "${{ inputs.max-age }}" ] && args+=(--max-age "${{ inputs.max-age }}") + [ -n "${{ inputs.postgres }}" ] && args+=(--postgres "${{ inputs.postgres }}") + [ -n "${{ inputs.notify }}" ] && args+=(--notify "${{ inputs.notify }}") + [ -n "${{ inputs.image }}" ] && args+=(--image "${{ inputs.image }}") + [ -n "${{ inputs.config }}" ] && args+=(--config "${{ inputs.config }}") + [ -n "${{ inputs.junit }}" ] && args+=(--junit "${{ inputs.junit }}") + [ -n "${{ inputs.json }}" ] && args+=(--json "${{ inputs.json }}") + args+=(--fail-on "${{ inputs.fail-on }}") + firedrill "${args[@]}" | tee firedrill-output.txt || true + else args=(run) [ -n "${{ inputs.dump }}" ] && args+=("${{ inputs.dump }}") [ -n "${{ inputs.live }}" ] && args+=(--live "${{ inputs.live }}") + [ -n "${{ inputs.notify }}" ] && args+=(--notify "${{ inputs.notify }}") [ -n "${{ inputs.keep }}" ] && args+=(--keep "${{ inputs.keep }}") [ -n "${{ inputs.image }}" ] && args+=(--image "${{ inputs.image }}") [ -n "${{ inputs.config }}" ] && args+=(--config "${{ inputs.config }}") @@ -132,6 +182,7 @@ runs: args+=(--fail-on "${{ inputs.fail-on }}") firedrill "${args[@]}" | tee firedrill-output.txt || true + fi report="${{ inputs.json }}" if [ -n "$report" ] && [ -f "$report" ]; then diff --git a/firedrill/cli.py b/firedrill/cli.py index bd1e77a..807fe27 100644 --- a/firedrill/cli.py +++ b/firedrill/cli.py @@ -102,6 +102,43 @@ def build_parser() -> argparse.ArgumentParser: drill_pitr.add_argument("--junit", metavar="PATH") drill_pitr.add_argument("--quiet", action="store_true") + phys = sub.add_parser( + "physical", help="restore a pgBackRest or WAL-G backup with the tool itself, and report") + phys.add_argument("tool", choices=("pgbackrest", "walg")) + phys.add_argument("--repo", metavar="DIR", + help="a local repository (mounted read-only). Omit it for a remote " + "one configured through PGBACKREST_* / WALG_* / AWS_* variables") + phys.add_argument("--stanza", help="pgBackRest stanza (found automatically if there is one)") + phys.add_argument("--backup", metavar="NAME", + help="a specific backup set or base backup (default: the newest)") + phys.add_argument("--target", metavar="TIMESTAMP", + help="recover to this moment instead of replaying all archived WAL") + phys.add_argument("--database", metavar="NAME", + help="where semantics checks run (default: the only database there is)") + phys.add_argument("--postgres", metavar="MAJOR", + help="the major version; needed for a remote repository") + phys.add_argument("--max-age", metavar="DURATION", + help="the newest backup must be younger than this, e.g. 26h") + phys.add_argument("--image", metavar="TEMPLATE", + help="base image with a {major} slot, e.g. 'pgvector/pgvector:pg{major}'") + phys.add_argument("--config", metavar="PATH") + phys.add_argument("--no-config", action="store_true") + phys.add_argument("--fail-on", choices=SEVERITIES, default="high") + phys.add_argument("--ready-timeout", type=int, default=600, + help="seconds to wait for restore and WAL replay (default: %(default)s)") + phys.add_argument("--json", metavar="PATH") + phys.add_argument("--junit", metavar="PATH") + phys.add_argument("--quiet", action="store_true") + + for parser_ in (run, drill_pitr, phys): + parser_.add_argument( + "--notify", metavar="ENV_VAR", + help="POST the verdict to the webhook URL held in this variable (Slack, " + "Discord, Mattermost, or any endpoint). The name, never the URL: a " + "webhook URL is a credential") + parser_.add_argument("--notify-on", choices=("fail", "always"), default="fail", + help="when to notify (default: fail -- including COULD NOT VERIFY)") + sub.add_parser("clean", help="remove containers left behind by a crash") sub.add_parser("doctor", help="check the environment before trusting a run") return parser @@ -143,6 +180,16 @@ def main(argv: list[str] | None = None) -> int: # for and still print a pass. try: cfg = _load_config(args) + if args.command == "physical": + if args.image and "{major}" not in args.image: + raise config.ConfigError("--image needs a {major} slot") + result = drill.run_physical( + args.tool, repo=args.repo, stanza=args.stanza, backup=args.backup, + target=args.target, database=args.database, cfg=cfg, + flavour=args.image or cfg.image or "", pin_major=args.postgres, + max_age=_duration(args.max_age) if args.max_age else None, + fail_on=args.fail_on, ready_timeout=args.ready_timeout) + return _emit(result, args) if args.command == "pitr": result = drill.run_pitr( args.base, args.wal, args.target, cfg=cfg, @@ -228,6 +275,15 @@ def _emit(result, args) -> int: else: with open(args.json, "w", encoding="utf-8") as handle: handle.write(blob + "\n") + if getattr(args, "notify", None): + from . import notify + try: + notify.send(result, args.notify, args.notify_on) + except notify.NotifyError as exc: + # The drill's verdict stands; an alert that did not go out is said + # loudly rather than swallowed, and fails the run. + print(f"firedrill: could not notify: {exc}", file=sys.stderr) + return result.exit_code or 1 return result.exit_code diff --git a/firedrill/drill.py b/firedrill/drill.py index f79beac..1e773f3 100644 --- a/firedrill/drill.py +++ b/firedrill/drill.py @@ -14,7 +14,7 @@ import tempfile import time -from . import archive, docker, history as history_module, ladder, pitr, \ +from . import archive, docker, history as history_module, ladder, physical, pitr, \ restore as restore_stage, sources from . import config as _config_module from .config import DEFAULT as DEFAULT_CONFIG @@ -27,6 +27,9 @@ # to run, and the rung that matters is the boundary assertion. PITR_STAGES = ("recover", "boundary", "integrity") +# pgBackRest / WAL-G: the tool restores, WAL replays, then the usual rungs. +PHYSICAL_STAGES = ("recover", "smoke", "semantics", "integrity") + OK = "ok" FAILED = "failed" NOT_RUN = "not run" @@ -276,6 +279,178 @@ def stage(name): return _suppress(report, cfg) +def run_physical(tool: str, *, repo=None, stanza=None, backup=None, target=None, + database=None, cfg=None, flavour: str = "", pin_major=None, + max_age: float | None = None, fail_on: str = DEFAULT_FAIL_ON, + ready_timeout: int = docker.DEFAULT_READY_TIMEOUT) -> Report: + """Restore a pgBackRest or WAL-G backup with the tool itself, replay WAL + (to `target`, if given), and put the recovered server through the ladder.""" + cfg = cfg if cfg is not None else DEFAULT_CONFIG + stages = [Stage(name) for name in PHYSICAL_STAGES] + report = Report(dump=f"{tool} {repo or '(remote repository)'}", stages=stages, + findings=[], archive={"tool": tool}, fail_on=fail_on, tier="full") + began = time.monotonic() + + def stage(name): + return next(s for s in stages if s.name == name) + + def stop(rule, message, fix, evidence="", status=FAILED): + # A restore that ran and broke has *tested* the backup: FAIL. Only a + # check that could not happen at all (NOT_RUN) is COULD NOT VERIFY. + report.verified = status == FAILED + stage("recover").status = status + stage("recover").detail = rule + report.findings.append(Finding(stage="recover", rule=rule, severity="critical", + message=message, fix=fix, evidence=evidence)) + report.total_seconds = time.monotonic() - began + return _suppress(report, cfg) + + if target is not None: + try: + target = pitr.check_target(target) + except pitr.InvalidTarget as exc: + return stop("PITR_TARGET_INVALID", str(exc), "Nothing was started.", status=NOT_RUN) + try: + meta = physical.describe(tool, pathlib.Path(repo) if repo else None, + stanza, backup, pin_major) + script = physical.script_for(tool, repo=repo is not None, stanza=meta.get("stanza"), + backup=backup, target=target) + except physical.PhysicalError as exc: + return stop("BACKUP_UNREADABLE", f"the backup could NOT be verified: {exc}", + "Nothing was restored.", status=NOT_RUN) + report.archive.update({k: (v.isoformat() if hasattr(v, "isoformat") else v) + for k, v in meta.items()}) + report.archive["server_major"] = meta["major"] + if target: + report.archive["recovery_target_time"] = target + report.findings.extend(physical.age_finding(meta.get("finished"), max_age)) + if meta.get("failed"): + report.findings.append(Finding( + stage="recover", rule="BACKUP_FAILED", severity="high", + message=f"the repository marks backup {meta['label']} as having errors", + fix="pgBackRest recorded a problem while taking it. Read its log for that run.", + evidence="")) + + usable, why = docker.docker_available() + if not usable: + return stop("TARGET_UNAVAILABLE", f"the restore could NOT be verified: {why}", + "Start Docker and run again.", status=NOT_RUN) + + started = time.monotonic() + try: + image = physical.tool_image(tool, docker.image_for(meta["major"], flavour)) + except docker.TargetError as exc: + return stop("TARGET_UNAVAILABLE", f"the restore could NOT be verified: {exc}", + "The restore image could not be prepared; nothing was restored.", + status=NOT_RUN) + container = physical.PhysicalContainer(meta["major"], image, script, + pathlib.Path(repo) if repo else None, + ready_timeout=ready_timeout) + try: + try: + container.start() + container.wait_ready() + except docker.TargetError as exc: + logs = container.logs(tail=200) + stage("recover").seconds = time.monotonic() - started + if "[075]" in logs or "before consistent recovery point" in logs: + return stop("PITR_TARGET_BEFORE_BACKUP", + f"no backup finished before {target}, so that moment cannot be " + "recovered from this repository", + "Retention has already removed the backup you would need for " + "this moment -- or the target is older than any backup ever taken.", + physical.tool_error(logs)) + if pitr._TARGET_UNREACHED in logs: + return stop("PITR_TARGET_UNREACHED", + f"recovery could not reach {target}: the WAL archive ends before it", + "The base backup and the archived WAL cannot reconstruct that " + "moment -- the failure you would meet during the incident.", + pitr._recovery_trace(logs)) + return stop("RESTORE_FAILED", f"the {tool} restore failed: {physical.tool_error(logs)}", + "This backup cannot be restored as it stands -- a corrupt or " + "incomplete file, or WAL missing from the archive.", + pitr._last_error(logs)) + + promoted = pitr.confirm_promoted(container) + stage("recover").seconds = time.monotonic() - started + if promoted: + report.findings.extend(promoted) + stage("recover").status = FAILED + report.total_seconds = time.monotonic() - began + return _suppress(report, cfg) + # Asked, not assumed: `-c archive_mode=off` should outrank every file, + # but a drill that archived into production's repository would be the + # worst thing this tool could do, so the server confirms it. + archiving = container.sql("show archive_mode").stdout.strip() + if archiving != "off": + container.teardown() + return stop("ARCHIVING_ENABLED", + f"the restored server reports archive_mode={archiving or '?'}", + "Stopped immediately: it could push WAL into the repository " + "being checked.") + report.verified = True + stage("recover").status = OK + stage("recover").detail = (f"{meta['label']} -> {target}" if target + else f"{meta['label']}, WAL replayed to the end") + report.archive["restored_into_major"] = _served_major(container) + + started = time.monotonic() + names = physical.databases(container) + counts = {n: physical.user_tables(container, n) for n in names} + stage("smoke").seconds = time.monotonic() - started + if all(c in (None, 0) for c in counts.values()): + stage("smoke").status = FAILED + report.findings.append(Finding( + stage="smoke", rule="SMOKE_EMPTY", severity="high", + message="the recovered server holds no user tables in any database", + fix="The restore produced a server, not your data.", evidence=str(counts))) + else: + stage("smoke").status = OK + stage("smoke").detail = ", ".join(f"{n}: {c} table(s)" for n, c in counts.items()) + + db = database or (names[0] if len(names) == 1 else None) + if cfg.semantics: + if db is None: + stage("semantics").status = NOT_RUN + report.findings.append(Finding( + stage="semantics", rule="DATABASE_AMBIGUOUS", severity="high", + message=f"semantics checks need one database; the backup has {', '.join(names)}", + fix="Pass --database NAME.", evidence="")) + else: + started = time.monotonic() + sem, _ = ladder.semantics(container, cfg, db) + report.findings.extend(sem) + stage("semantics").seconds = time.monotonic() - started + stage("semantics").status = FAILED if sem else OK + stage("semantics").detail = f"{len(cfg.semantics)} check(s) in {db}" + else: + stage("semantics").status = NOT_CONFIGURED + stage("semantics").detail = "no semantics checks in the config" + if target: + report.findings.append(Finding( + stage="semantics", rule="PITR_UNASSERTED", severity="high", + message="recovery reached the target, and nothing checked what the " + "database then contained", + fix="Add two semantics checks: one row written before the target " + "that must exist, one written after that must not.", + evidence="")) + + started = time.monotonic() + checked = 0 + for name in ([db] if db else names): + integ, info = ladder.integrity(container, cfg, name) + report.findings.extend(integ) + checked += info.get("sequences", 0) + stage("integrity").seconds = time.monotonic() - started + stage("integrity").status = FAILED if any(f.stage == "integrity" for f in report.findings) else OK + stage("integrity").detail = f"{checked} sequence(s)" + finally: + container.teardown() + + report.total_seconds = time.monotonic() - began + return _suppress(report, cfg) + + def _sequence_detail(info: dict) -> str: """`9 of 13 sequence(s)` when some could not be linked to a column. diff --git a/firedrill/notify.py b/firedrill/notify.py new file mode 100644 index 0000000..cb0d89c --- /dev/null +++ b/firedrill/notify.py @@ -0,0 +1,64 @@ +"""One webhook POST with the verdict. + +The payload carries `text` (what Slack and Mattermost read) and `content` +(what Discord reads) with the same one-line summary, plus the structured +fields for anything else. The URL comes from an environment variable named +on the command line: a webhook URL is a credential, and argv is not a place +for one. +""" +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request + +from .finding import worst + + +class NotifyError(Exception): + pass + + +def summary(report) -> str: + if not report.verified: + head = "COULD NOT VERIFY" + elif report.ok: + head = "PASS" + else: + head = "FAIL" + line = f"firedrill {head}: {report.dump}" + if report.findings: + top = sorted(report.findings, key=lambda f: ("critical", "high", "medium", "low", "info") + .index(f.severity))[0] + line += f" -- {len(report.findings)} finding(s), worst {top.rule}: {top.message}" + return line[:1900] # Discord's content limit is 2000 + + +def payload(report) -> dict: + text = summary(report) + return {"text": text, "content": text, "ok": report.ok, "verified": report.verified, + "worst": worst(report.findings), "findings": [f.rule for f in report.findings], + "seconds": round(report.total_seconds, 2)} + + +def send(report, env_var: str, when: str = "fail", opener=urllib.request.urlopen) -> bool: + """True if a notification went out. `when="fail"` also covers COULD NOT + VERIFY -- a drill that did not happen is exactly what someone must hear.""" + if when == "fail" and report.ok: + return False + url = os.environ.get(env_var) + if not url: + raise NotifyError(f"${env_var} is not set") + if not url.startswith("https://") and not url.startswith(("http://localhost", "http://127.0.0.1")): + raise NotifyError(f"${env_var} must be an https:// URL") + request = urllib.request.Request(url, data=json.dumps(payload(report)).encode(), + headers={"Content-Type": "application/json"}, method="POST") + try: + with opener(request, timeout=30) as response: + response.read() + except (urllib.error.URLError, TimeoutError, OSError) as exc: + # The URL itself is never in the message. + raise NotifyError(f"the webhook in ${env_var} did not accept it: " + f"{getattr(exc, 'code', '') or type(exc).__name__}") from None + return True diff --git a/firedrill/physical.py b/firedrill/physical.py new file mode 100644 index 0000000..272f93c --- /dev/null +++ b/firedrill/physical.py @@ -0,0 +1,299 @@ +"""Physical backups: pgBackRest and WAL-G. + +Most production Postgres is not backed up with pg_dump. It is backed up +with pgBackRest or WAL-G -- a base backup plus a WAL archive -- and those are +restored with the tool itself, into a data directory, followed by WAL +replay. This module does exactly that, into a disposable container of the +backup's own major version, and hands the recovered server to the same +ladder every other drill uses. + +Safety, because a physical restore carries production's own configuration: + +* **archive_mode is forced off.** A restored data directory keeps the + source's postgresql.conf, and production's `archive_command` pushes WAL + into production's backup repository. Started as-is, the drill would write + into the very archive it is checking. `-c archive_mode=off` on the server's + command line outranks every file. +* A local repository is mounted read-only. A remote one (S3, GCS, Azure) is + reached with credentials passed to the container by *name*, never value. +* Nothing is published: no port, same as every other target. +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +import pathlib +import re +import subprocess + +from . import docker +from .finding import Finding + +TOOLS = ("pgbackrest", "walg") + +# WAL-G has no distro package, so its release binary is fetched at image +# build time and checked against the digest published with the release. +WALG_VERSION = "v3.0.9" +WALG_URL = f"https://github.com/wal-g/wal-g/releases/download/{WALG_VERSION}/wal-g-pg-22.04-amd64" +WALG_SHA256 = "d6795c663894d836ba20840bffe8970aed81bbe25d3151bd4951f6f9c362def9" + +# Environment the tools read their repository settings and cloud +# credentials from. Passed through by name only. +_PASS_THROUGH = ("PGBACKREST_", "WALG_", "WALE_", "AWS_", "GOOGLE_", "AZURE_", "GCS_") + +REPO_MOUNT = "/firedrill/repo" + +_NAME = re.compile(r"^[A-Za-z0-9._-]+$") + + +class PhysicalError(Exception): + """The backup could not be located or read before anything started.""" + + +# -- reading the repository ------------------------------------------------ + +def _ini_section(text: str, section: str) -> dict[str, str]: + out, current = {}, None + for line in text.splitlines(): + line = line.strip() + if line.startswith("[") and line.endswith("]"): + current = line[1:-1] + elif current == section and "=" in line: + key, _, value = line.partition("=") + out[key.strip()] = value.strip() + return out + + +def pgbackrest_backup(repo: pathlib.Path, stanza: str | None) -> dict: + """{stanza, major, label, finished} for the newest backup, read from the + repository's own backup.info -- the same principle as reading a dump + header: ask the artefact, not the host.""" + stanzas = sorted(p.name for p in (repo / "backup").glob("*") if (p / "backup.info").exists()) + if stanza is None: + if len(stanzas) != 1: + raise PhysicalError( + f"{repo} holds {len(stanzas)} stanzas ({', '.join(stanzas) or 'none'}); " + "name one with --stanza") + stanza = stanzas[0] + info_path = repo / "backup" / stanza / "backup.info" + try: + text = info_path.read_text(encoding="utf-8") + except OSError as exc: + raise PhysicalError(f"no readable backup.info for stanza {stanza!r}: {exc}") from None + major = _ini_section(text, "db").get("db-version", "").strip('"') + current = _ini_section(text, "backup:current") + if not major or not current: + raise PhysicalError(f"{info_path} lists no backups") + label, raw = max(current.items(), key=lambda kv: json.loads(kv[1]).get("backup-timestamp-stop", 0)) + meta = json.loads(raw) + return {"stanza": stanza, "major": major, "label": label, + "failed": bool(meta.get("backup-error")), + "finished": dt.datetime.fromtimestamp(meta["backup-timestamp-stop"], dt.timezone.utc)} + + +def walg_backup(repo: pathlib.Path, name: str | None) -> dict: + """{major, label, finished} for the named (or newest) base backup, from + its stop sentinel.""" + sentinels = list((repo / "basebackups_005").glob("*_backup_stop_sentinel.json")) + if not sentinels: + raise PhysicalError(f"{repo} has no basebackups_005/*_backup_stop_sentinel.json") + found = [] + for path in sentinels: + try: + meta = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + found.append((path.name[: -len("_backup_stop_sentinel.json")], meta)) + if name and name != "LATEST": + found = [f for f in found if f[0] == name] + if not found: + raise PhysicalError(f"no base backup named {name!r} in {repo}") + label, meta = max(found, key=lambda f: f[1].get("FinishTime", "")) + version = int(meta["PgVersion"]) + major = str(version // 10000) if version >= 100000 else f"{version // 10000}.{version // 100 % 100}" + finished = dt.datetime.fromisoformat(meta["FinishTime"].replace("Z", "+00:00")) + return {"major": major, "label": label, "failed": False, "finished": finished} + + +# -- the container --------------------------------------------------------- + +def tool_image(tool: str, base: str) -> str: + """A postgres image with the backup tool in it, built once and reused.""" + tag = f"firedrill-{tool}:{base.replace('/', '_').replace(':', '-')}" + have = docker._run(["docker", "image", "inspect", tag], timeout=60) + if have.returncode == 0: + return tag + if tool == "pgbackrest": + # The official images carry the PGDG apt repository, which ships + # pgBackRest built for exactly this distribution. + dockerfile = (f"FROM {base}\nRUN apt-get update && apt-get install -y " + "--no-install-recommends pgbackrest && rm -rf /var/lib/apt/lists/*\n") + else: + dockerfile = (f"FROM {base}\nADD --checksum=sha256:{WALG_SHA256} {WALG_URL} /usr/local/bin/wal-g\n" + "RUN chmod 755 /usr/local/bin/wal-g\n") + try: + built = subprocess.run(["docker", "build", "-q", "-t", tag, "-"], input=dockerfile, + capture_output=True, text=True, timeout=900, check=False) + except subprocess.TimeoutExpired: + raise docker.TargetError(f"building {tag} timed out") from None + if built.returncode != 0: + raise docker.TargetError(f"could not build {tag}: {(built.stderr or '').strip()[-500:]}") + return tag + + +_PGBACKREST_SCRIPT = """ +set -e +mkdir -p "$PGDATA" /tmp/pgbr/lock /tmp/pgbr/log +chown -R postgres:postgres "$PGDATA" /tmp/pgbr +chmod 700 "$PGDATA" +export PGBACKREST_STANZA={stanza} PGBACKREST_PG1_PATH="$PGDATA" +export PGBACKREST_LOCK_PATH=/tmp/pgbr/lock PGBACKREST_LOG_PATH=/tmp/pgbr/log +{repo_env} +gosu postgres pgbackrest restore --log-level-console=warn {options} +exec docker-entrypoint.sh postgres -c archive_mode=off +""" + +_WALG_SCRIPT = """ +set -e +mkdir -p "$PGDATA" +chown -R postgres:postgres "$PGDATA" +chmod 700 "$PGDATA" +{repo_env} +gosu postgres wal-g backup-fetch "$PGDATA" {backup} +gosu postgres touch "$PGDATA/recovery.signal" +cat >> "$PGDATA/postgresql.auto.conf" <<'FIREDRILL' +restore_command = 'wal-g wal-fetch %f %p' +{target} +FIREDRILL +exec docker-entrypoint.sh postgres -c archive_mode=off +""" + + +class PhysicalContainer(docker.Container): + def __init__(self, major: str, image: str, script: str, repo: pathlib.Path | None, **kw): + super().__init__(major, **kw) + self.image = image + self.script = script + self.repo = pathlib.Path(repo).resolve() if repo else None + + def _run_argv(self) -> list[str]: + args = ["docker", "run", "-d", "--name", self.name, "--label", "firedrill=1", + "-e", "POSTGRES_PASSWORD"] + for name in sorted(os.environ): + if name.startswith(_PASS_THROUGH): + args += ["-e", name] # by name: the value never enters argv + if self.repo is not None: + args += ["-v", f"{self.repo}:{REPO_MOUNT}:ro"] + return args + ["--entrypoint", "bash", self.image, "-c", self.script] + + def start(self) -> None: + # The base class hands docker a minimal environment; the pass-through + # variables have to be in it for `-e NAME` to carry them. + extra = {k: v for k, v in os.environ.items() if k.startswith(_PASS_THROUGH)} + env = {"PATH": docker._path(), "POSTGRES_PASSWORD": self.password, + "HOME": docker._home(), **extra} + result = docker._run(self._run_argv(), env=env) + if result.returncode != 0: + raise docker.TargetError(f"could not start {self.image}: {(result.stderr or '').strip()}") + self.started = True + + +def script_for(tool: str, *, repo: bool, stanza: str | None, backup: str | None, + target: str | None) -> str: + for label, value in (("stanza", stanza), ("backup", backup)): + if value is not None and not _NAME.match(value): + raise PhysicalError(f"{label} {value!r} is not a plain name") + if target is not None: + # Interpolated into a shell script and the server's config, so it is + # held to the one shape it can have -- here, not only at the caller. + from .pitr import InvalidTarget, check_target + try: + target = check_target(target) + except InvalidTarget as exc: + raise PhysicalError(str(exc)) from None + if tool == "pgbackrest": + options = [] + if backup: + options.append(f"--set={backup}") + if target: + options += ["--type=time", f"--target='{target}'", "--target-action=promote"] + return _PGBACKREST_SCRIPT.format( + stanza=stanza, options=" ".join(options), + repo_env=f"export PGBACKREST_REPO1_PATH={REPO_MOUNT}" if repo else "") + lines = "" + if target: + lines = f"recovery_target_time = '{target}'\nrecovery_target_action = 'promote'" + return _WALG_SCRIPT.format( + backup=backup or "LATEST", target=lines, + repo_env=f"export WALG_FILE_PREFIX={REPO_MOUNT}" if repo else "") + + +def describe(tool: str, repo: pathlib.Path | None, stanza: str | None, + backup: str | None, major: str | None) -> dict: + """What is about to be restored. A local repository says for itself; a + remote one needs --postgres, since firedrill will not guess a major.""" + if repo is not None: + if not repo.is_dir(): + raise PhysicalError(f"{repo} is not a directory") + meta = pgbackrest_backup(repo, stanza) if tool == "pgbackrest" else walg_backup(repo, backup) + if major and major != meta["major"]: + meta["pinned_from"] = meta["major"] + meta["major"] = major + return meta + if not major: + raise PhysicalError("a remote repository cannot be read from here; pass --postgres MAJOR") + if tool == "pgbackrest" and not stanza: + raise PhysicalError("a remote pgBackRest repository needs --stanza") + return {"stanza": stanza, "major": major, "label": backup or "latest", + "failed": False, "finished": None} + + +def databases(container) -> list[str]: + listed = container.sql("select datname from pg_database where not datistemplate order by 1") + names = [n for n in (listed.stdout or "").split() if n] + return [n for n in names if n != "postgres"] or ["postgres"] + + +def user_tables(container, database: str) -> int | None: + counted = container.sql("select count(*) from pg_tables where schemaname not in " + "('pg_catalog', 'information_schema')", database=database) + try: + return int(counted.stdout.strip()) + except (ValueError, AttributeError): + return None + + +def tool_error(logs: str) -> str: + """The backup tool's own reason, which is what anyone reads first: + pgBackRest prints `ERROR: [nnn]: ...`, WAL-G `ERROR: ...`.""" + for line in reversed(logs.splitlines()): + if "ERROR:" in line and "LOG:" not in line: + return line.split("ERROR:", 1)[1].strip()[:300] + return "the server exited before accepting connections" + + +def _span(seconds: float) -> str: + if seconds < 3600: + return f"{seconds / 60:.0f}m" if seconds >= 60 else f"{seconds:.0f}s" + return f"{seconds / 3600:.1f}h" if seconds < 172800 else f"{seconds / 86400:.1f}d" + + +def age_finding(finished, max_age: float | None, now=None) -> list[Finding]: + if finished is None or max_age is None: + return [] + now = now or dt.datetime.now(dt.timezone.utc) + age = (now - finished).total_seconds() + if age <= max_age: + return [] + return [Finding( + stage="recover", rule="BACKUP_STALE", severity="high", + message=f"the newest backup finished {_span(age)} ago, past the " + f"{_span(max_age)} allowed", + fix="The backup job has stopped, or is failing before it finishes. A " + "restore of an old backup can still pass; this is the part that is broken.", + evidence=finished.isoformat(), + )] + + diff --git a/firedrill/report.py b/firedrill/report.py index 8c46759..4bdbbc5 100644 --- a/firedrill/report.py +++ b/firedrill/report.py @@ -31,7 +31,17 @@ def human(report: Report, colour: bool = False) -> str: lines = [] a = report.archive lines.append(f"firedrill {report.dump}") - if a.get("recovery_target_time"): + if a.get("tool"): + # pgBackRest / WAL-G: a backup set in a repository, not an archive file. + name = {"pgbackrest": "pgBackRest", "walg": "WAL-G"}[a["tool"]] + stanza = f" stanza {a['stanza']}" if a.get("stanza") else "" + finished = f" finished {a['finished']}" if a.get("finished") else "" + lines.append(f" backup {name} {a.get('label')}{stanza}{finished}") + towards = (f"recovered to {a['recovery_target_time']}" if a.get("recovery_target_time") + else "WAL replayed to the end of the archive") + lines.append(f" source PostgreSQL {a.get('server_major')} -> {towards} " + f"on postgres:{a.get('restored_into_major')}") + elif a.get("recovery_target_time"): # A PITR run has no archive to describe. Printing the dump fields here # rendered "archive None vNone 0B from None", which is worse than # printing nothing: it looks like a parse that went wrong. @@ -118,6 +128,8 @@ def _verdict(report: Report) -> str: # run proved the schema comes back and nothing more. return ("PASS (fast tier) -- the schema restored. Whether the DATA is " "there was not checked.") + if report.ok and report.archive.get("tool") and not report.archive.get("recovery_target_time"): + return "PASS -- restored with the backup tool, WAL replayed, and it answered queries." if report.ok and report.stages and report.stages[0].name == "recover": return ("PASS -- recovered to the target, and the boundary is where it " "should be.") diff --git a/pyproject.toml b/pyproject.toml index 6b7c044..042dc06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "firedrill" -version = "0.4.0" +version = "0.5.0" description = "A backup you have not restored is not a backup. firedrill restores it and proves it is usable." requires-python = ">=3.10" readme = "README.md" diff --git a/tests/make_physical.py b/tests/make_physical.py new file mode 100644 index 0000000..485989d --- /dev/null +++ b/tests/make_physical.py @@ -0,0 +1,87 @@ +"""Real pgBackRest and WAL-G repositories, made by the tools themselves. + +Each: a PostgreSQL 16 server archiving WAL, a table `orders`, a full base +backup, then `before_target` created, a timestamp taken, and `after_target` +created -- so a restore to that timestamp has exactly one right answer. + +Returns {"pgbackrest": (repo, target), "walg": (repo, target)}. Needs Docker +and the network (apt for pgBackRest, the WAL-G release binary). +""" +from __future__ import annotations + +import pathlib +import subprocess +import time +import uuid + +from firedrill import physical + +_SETUP = { + "pgbackrest": ( + "apt-get update -qq && apt-get install -y -qq pgbackrest >/dev/null && " + "mkdir -p /etc/pgbackrest && printf '[global]\\nrepo1-path=/repo\\nstart-fast=y\\n" + "log-level-console=warn\\n[demo]\\npg1-path=/var/lib/postgresql/data\\n' " + "> /etc/pgbackrest/pgbackrest.conf && chown -R postgres /repo", + "pgbackrest --stanza=demo archive-push %p", + "pgbackrest --stanza=demo stanza-create && pgbackrest --stanza=demo --type=full backup", + ), + "walg": ( + "apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null && " + f"curl -sfL -o /usr/local/bin/wal-g {physical.WALG_URL} && " + f"echo '{physical.WALG_SHA256} /usr/local/bin/wal-g' | sha256sum -c - && " + "chmod 755 /usr/local/bin/wal-g && chown postgres /repo", + "/usr/local/bin/wal-g wal-push %p", + "wal-g backup-push \"$PGDATA\"", + ), +} + +_WORKLOAD = """ +set -e +export WALG_FILE_PREFIX=/repo PGHOST=/var/run/postgresql +psql -q -c "create database shop" +psql -q -d shop -c "create table orders(id serial primary key, amount int); insert into orders(amount) select g from generate_series(1,5000) g;" +{backup} +psql -q -d shop -c "insert into orders(amount) select g from generate_series(1,1000) g; create table before_target(x int);" +sleep 2; date -u +"%Y-%m-%d %H:%M:%S+00"; sleep 2 +psql -q -d shop -c "create table after_target(x int);" +psql -q -c "select pg_switch_wal()" >/dev/null +sleep 12 +""" + + +def _sh(*args, **kw): + return subprocess.run(list(args), capture_output=True, text=True, check=True, timeout=900, **kw) + + +def build(root: pathlib.Path) -> dict: + out = {} + for tool, (install, archive_command, backup) in _SETUP.items(): + repo = root / f"{tool}-repo" + target_file = root / f"{tool}-target" + if target_file.exists(): + out[tool] = (repo, target_file.read_text().strip()) + continue + repo.mkdir(parents=True, exist_ok=True) + repo.chmod(0o777) # the container's postgres uid writes here + name = f"firedrill-src-{uuid.uuid4().hex[:8]}" + # The server's archive_command reads its storage from the environment. + _sh("docker", "run", "-d", "--name", name, "-e", "POSTGRES_PASSWORD=x", + "-e", "WALG_FILE_PREFIX=/repo", + "-v", f"{repo.resolve()}:/repo", "postgres:16", + "-c", "wal_level=replica", "-c", "archive_mode=on", + "-c", f"archive_command={archive_command}", "-c", "archive_timeout=10") + try: + for _ in range(60): + if subprocess.run(["docker", "exec", name, "pg_isready", "-q", "-h", "127.0.0.1"], + capture_output=True).returncode == 0: + break + time.sleep(1) + _sh("docker", "exec", name, "bash", "-c", install) + ran = _sh("docker", "exec", "-u", "postgres", name, "bash", "-c", + _WORKLOAD.format(backup=backup)) + target = ran.stdout.strip().splitlines()[-1] + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + target_file.write_text(target) + out[tool] = (repo, target) + return out diff --git a/tests/test_firedrill.py b/tests/test_firedrill.py index 0e44b73..2c9b3c6 100644 --- a/tests/test_firedrill.py +++ b/tests/test_firedrill.py @@ -245,6 +245,154 @@ def test_image_template_for_extension_databases(): check("config carries the template", cfg.image, "postgis/postgis:{major}-3.5") +# ---------------------------------------------------------------- physical -- + +# Copied from a real pgBackRest 2.59.1 repository (checksum line trimmed). +PGBACKREST_INFO = """[backrest] +backrest-format=5 +backrest-version="2.59.1" + +[backup:current] +20260926-202837F={"backrest-format":5,"backup-error":false,"backup-timestamp-start":1790454517,"backup-timestamp-stop":1790454536,"backup-type":"full"} + +[db] +db-catalog-version=202307071 +db-id=1 +db-version="16" +""" + + +def test_physical_reads_the_backup_from_the_repository(): + """Major version and newest backup come from the repository itself, as + they come from a dump header -- never from the host.""" + import tempfile + from firedrill import physical + with tempfile.TemporaryDirectory() as tmp: + repo = pathlib.Path(tmp) + (repo / "backup" / "demo").mkdir(parents=True) + (repo / "backup" / "demo" / "backup.info").write_text(PGBACKREST_INFO) + meta = physical.pgbackrest_backup(repo, None) + check("stanza found", meta["stanza"], "demo") + check("major from db-version", meta["major"], "16") + check("newest set", meta["label"], "20260926-202837F") + + walg = repo / "basebackups_005" + walg.mkdir() + (walg / "base_000000010000000000000003_backup_stop_sentinel.json").write_text( + '{"PgVersion":160015,"FinishTime":"2026-09-26T20:30:21.644694Z"}') + meta = physical.walg_backup(repo, None) + check("walg major from PgVersion", (meta["major"], meta["label"]), + ("16", "base_000000010000000000000003")) + + +def test_physical_never_archives_into_the_repository_it_checks(): + """A restored data directory carries production's archive_command.""" + from firedrill import physical + for tool in physical.TOOLS: + script = physical.script_for(tool, repo=True, stanza="demo", backup=None, target=None) + check(f"{tool}: server started with archive_mode=off", "postgres -c archive_mode=off" in script, True) + try: + physical.script_for("pgbackrest", repo=True, stanza="demo; rm -rf /", backup=None, target=None) + check("an injected stanza is refused", "accepted", "PhysicalError") + except physical.PhysicalError: + pass + try: + physical.script_for("walg", repo=True, stanza=None, backup=None, + target="2026-01-01 00:00:00'; archive_command='x") + check("an injected target is refused", "accepted", "PhysicalError") + except physical.PhysicalError: + pass + + +def test_physical_staleness_and_tool_errors_read_plainly(): + import datetime as dt + from firedrill import physical + now = dt.datetime(2026, 9, 27, tzinfo=dt.timezone.utc) + stale = physical.age_finding(now - dt.timedelta(hours=30), 26 * 3600, now=now) + check("stale", [f.rule for f in stale], ["BACKUP_STALE"]) + check("readable span", "30.0h" in stale[0].message and "26.0h" in stale[0].message, True) + check("fresh", physical.age_finding(now - dt.timedelta(hours=2), 26 * 3600, now=now), []) + check("pgBackRest's own reason", physical.tool_error( + "2026 P00 ERROR: [029]: raised from local-1 protocol: zlib threw error: [-3] data error"), + "[029]: raised from local-1 protocol: zlib threw error: [-3] data error") + + +def test_physical_restores_real_pgbackrest_and_walg_repositories(): + """Repositories made by pgBackRest and WAL-G themselves. Replaying all WAL + brings back all three tables; recovering to the timestamp brings back the + one before it and not the one after; a timestamp past the archive fails.""" + needs_docker() + import make_physical + built = make_physical.build(CORPUS / "physical") + cfg = config.loads( + "version: 1\nsemantics:\n" + " - name: before\n sql: select count(*) from pg_class where relname = 'before_target'\n" + " expect: \"== 1\"\n" + " - name: after\n sql: select count(*) from pg_class where relname = 'after_target'\n" + " expect: \"== 0\"\n") + for tool, (repo, target) in built.items(): + full = drill.run_physical(tool, repo=repo) + check(f"{tool}: full replay passes", full.ok, True) + check(f"{tool}: all tables back", "shop: 3 table(s)" in full.stages[1].detail, True) + pitr_run = drill.run_physical(tool, repo=repo, target=target, cfg=cfg) + check(f"{tool}: recovered exactly to the boundary", pitr_run.ok, True) + # A day past the archive's end -- not 2099: pgBackRest 2.59 answers a + # post-2038 target with [075], as if no backup preceded it. + import datetime as dt + after = (dt.datetime.fromisoformat(target.replace("+00", "+00:00")) + + dt.timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S+00") + late = drill.run_physical(tool, repo=repo, target=after) + check(f"{tool}: a target past the archive fails", [f.rule for f in late.findings], + ["PITR_TARGET_UNREACHED"]) + early = drill.run_physical(tool, repo=repo, target="2020-01-01 00:00:00+00") + check(f"{tool}: a target before every backup says so", [f.rule for f in early.findings], + ["PITR_TARGET_BEFORE_BACKUP"]) + check(f"{tool}: and that is a FAIL, not could-not-verify", early.verified, True) + + +def test_notify_posts_the_verdict_to_a_real_webhook(): + """A real HTTP receiver on localhost. Failing and could-not-verify runs + notify; a pass does not unless asked; the URL never reaches an error.""" + import http.server + import os + import threading + from firedrill import notify + got = [] + + class Hook(http.server.BaseHTTPRequestHandler): + def do_POST(self): + got.append(json.loads(self.rfile.read(int(self.headers["Content-Length"])))) + self.send_response(204 if self.path == "/ok" else 500) + self.end_headers() + + def log_message(self, *a): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Hook) + threading.Thread(target=server.serve_forever, daemon=True).start() + base = f"http://127.0.0.1:{server.server_port}" + try: + os.environ["FD_HOOK"] = base + "/ok" + failed = _blank_report(verified=True, findings=[Finding("restore", "ARCHIVE_TRUNCATED", "critical", "cut short")]) + check("a failure notifies", notify.send(failed, "FD_HOOK"), True) + check("Slack and Discord both read it", (got[0]["text"] == got[0]["content"], + "ARCHIVE_TRUNCATED" in got[0]["text"]), (True, True)) + check("could-not-verify notifies", notify.send(_blank_report(verified=False), "FD_HOOK"), True) + check("COULD NOT VERIFY is named", got[1]["text"].startswith("firedrill COULD NOT VERIFY"), True) + passed = _blank_report(verified=True) + check("a pass is quiet by default", notify.send(passed, "FD_HOOK"), False) + check("unless asked", notify.send(passed, "FD_HOOK", "always"), True) + os.environ["FD_HOOK"] = base + "/broken" + try: + notify.send(failed, "FD_HOOK") + check("a refused webhook is an error", "sent", "NotifyError") + except notify.NotifyError as exc: + check("and the URL stays out of the message", base in str(exc), False) + finally: + server.shutdown() + os.environ.pop("FD_HOOK", None) + + def test_live_dsn_never_prints_the_password(): import os from firedrill import finding, sources @@ -2620,7 +2768,7 @@ def main() -> int: # A floor, not a target. Edits that replace a range of lines have silently # swallowed whole blocks of tests before; the suite then goes green with # fewer tests and says nothing. - FLOOR = 151 + FLOOR = 155 if len(tests) < FLOOR: raise SystemExit( f"test suite shrank: {len(tests)} < {FLOOR}. An edit probably deleted " From e3eb9760d9d57b74b7fba12c33d68e139244ee59 Mon Sep 17 00:00:00 2001 From: MaXiMo000 Date: Sun, 27 Sep 2026 03:28:33 +0530 Subject: [PATCH 2/3] PITR target before a backup is decided from the repository CI showed Postgres recovering to a LATER moment than asked: with no commit during the backup it cannot stop before the backup's end, so it promotes there. Backup finish times come from backup.info / the WAL-G sentinel, so a target before every backup is PITR_TARGET_BEFORE_BACKUP up front, and WAL-G now fetches the newest base backup that finished before the target. A server that looked ready (Postgres serves reads during recovery) and then exited because the archive ran out is PITR_TARGET_UNREACHED, not PITR_FAILED. The action-flags test knows the action calls `physical`. Co-Authored-By: Claude Opus 5.5 --- firedrill/drill.py | 23 ++++++++++++++++--- firedrill/physical.py | 51 ++++++++++++++++++++++++++++++++++------- tests/test_firedrill.py | 30 ++++++++++++++++++++---- 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/firedrill/drill.py b/firedrill/drill.py index 1e773f3..6875411 100644 --- a/firedrill/drill.py +++ b/firedrill/drill.py @@ -312,9 +312,18 @@ def stop(rule, message, fix, evidence="", status=FAILED): return stop("PITR_TARGET_INVALID", str(exc), "Nothing was started.", status=NOT_RUN) try: meta = physical.describe(tool, pathlib.Path(repo) if repo else None, - stanza, backup, pin_major) - script = physical.script_for(tool, repo=repo is not None, stanza=meta.get("stanza"), - backup=backup, target=target) + stanza, backup, pin_major, target) + # WAL-G fetches exactly the base backup chosen here -- for a PITR run, + # the newest one that finished before the target. + script = physical.script_for( + tool, repo=repo is not None, stanza=meta.get("stanza"), + backup=meta["label"] if tool == "walg" and repo else backup, target=target) + except physical.TargetBeforeBackup as exc: + return stop("PITR_TARGET_BEFORE_BACKUP", f"{exc}, so {target} cannot be recovered", + "Postgres cannot stop recovery before a backup's end: asked to, it " + "refuses -- or, when nothing committed during the backup, silently " + "recovers to a LATER moment. Retention has removed the backup this " + "moment needs, or none was ever taken before it.") except physical.PhysicalError as exc: return stop("BACKUP_UNREADABLE", f"the backup could NOT be verified: {exc}", "Nothing was restored.", status=NOT_RUN) @@ -373,6 +382,14 @@ def stop(rule, message, fix, evidence="", status=FAILED): promoted = pitr.confirm_promoted(container) stage("recover").seconds = time.monotonic() - started + if promoted and pitr._TARGET_UNREACHED in container.logs(tail=200): + # Postgres serves read-only queries during recovery, so the server + # can look ready and then exit because the archive ran out. + return stop("PITR_TARGET_UNREACHED", + f"recovery could not reach {target}: the WAL archive ends before it", + "The base backup and the archived WAL cannot reconstruct that " + "moment -- the failure you would meet during the incident.", + pitr._recovery_trace(container.logs(tail=200))) if promoted: report.findings.extend(promoted) stage("recover").status = FAILED diff --git a/firedrill/physical.py b/firedrill/physical.py index 272f93c..a848da4 100644 --- a/firedrill/physical.py +++ b/firedrill/physical.py @@ -52,6 +52,25 @@ class PhysicalError(Exception): """The backup could not be located or read before anything started.""" +class TargetBeforeBackup(PhysicalError): + """No backup finished at or before the recovery target. Postgres cannot + stop recovery before a backup's end: asked to, it either refuses or -- + when nothing committed during the backup -- quietly recovers to a LATER + moment than the one asked for. Decided here, from the repository.""" + + +def parse_target(target: str) -> dt.datetime: + """A validated target as an aware datetime; no offset means UTC, which + is the restore container's zone.""" + text = target.strip().replace(" ", "T", 1) + if re.search(r"[+-]\d\d$", text): + text += ":00" + elif text.endswith("Z"): + text = text[:-1] + "+00:00" + moment = dt.datetime.fromisoformat(text) + return moment if moment.tzinfo else moment.replace(tzinfo=dt.timezone.utc) + + # -- reading the repository ------------------------------------------------ def _ini_section(text: str, section: str) -> dict[str, str]: @@ -66,7 +85,7 @@ def _ini_section(text: str, section: str) -> dict[str, str]: return out -def pgbackrest_backup(repo: pathlib.Path, stanza: str | None) -> dict: +def pgbackrest_backup(repo: pathlib.Path, stanza: str | None, before=None) -> dict: """{stanza, major, label, finished} for the newest backup, read from the repository's own backup.info -- the same principle as reading a dump header: ask the artefact, not the host.""" @@ -86,14 +105,21 @@ def pgbackrest_backup(repo: pathlib.Path, stanza: str | None) -> dict: current = _ini_section(text, "backup:current") if not major or not current: raise PhysicalError(f"{info_path} lists no backups") - label, raw = max(current.items(), key=lambda kv: json.loads(kv[1]).get("backup-timestamp-stop", 0)) + sets = current.items() + if before is not None: + # pgBackRest restores from the newest set that stopped before the + # target; report that one, and refuse when there is none. + sets = [kv for kv in sets if json.loads(kv[1]).get("backup-timestamp-stop", 0) <= before.timestamp()] + if not sets: + raise TargetBeforeBackup(f"no backup in stanza {stanza!r} finished before {before.isoformat()}") + label, raw = max(sets, key=lambda kv: json.loads(kv[1]).get("backup-timestamp-stop", 0)) meta = json.loads(raw) return {"stanza": stanza, "major": major, "label": label, "failed": bool(meta.get("backup-error")), "finished": dt.datetime.fromtimestamp(meta["backup-timestamp-stop"], dt.timezone.utc)} -def walg_backup(repo: pathlib.Path, name: str | None) -> dict: +def walg_backup(repo: pathlib.Path, name: str | None, before=None) -> dict: """{major, label, finished} for the named (or newest) base backup, from its stop sentinel.""" sentinels = list((repo / "basebackups_005").glob("*_backup_stop_sentinel.json")) @@ -110,11 +136,18 @@ def walg_backup(repo: pathlib.Path, name: str | None) -> dict: found = [f for f in found if f[0] == name] if not found: raise PhysicalError(f"no base backup named {name!r} in {repo}") - label, meta = max(found, key=lambda f: f[1].get("FinishTime", "")) + if before is not None: + found = [f for f in found if _finish(f[1]) <= before] + if not found: + raise TargetBeforeBackup(f"no base backup in {repo} finished before {before.isoformat()}") + label, meta = max(found, key=lambda f: _finish(f[1])) version = int(meta["PgVersion"]) major = str(version // 10000) if version >= 100000 else f"{version // 10000}.{version // 100 % 100}" - finished = dt.datetime.fromisoformat(meta["FinishTime"].replace("Z", "+00:00")) - return {"major": major, "label": label, "failed": False, "finished": finished} + return {"major": major, "label": label, "failed": False, "finished": _finish(meta)} + + +def _finish(meta: dict) -> dt.datetime: + return dt.datetime.fromisoformat(meta["FinishTime"].replace("Z", "+00:00")) # -- the container --------------------------------------------------------- @@ -231,13 +264,15 @@ def script_for(tool: str, *, repo: bool, stanza: str | None, backup: str | None, def describe(tool: str, repo: pathlib.Path | None, stanza: str | None, - backup: str | None, major: str | None) -> dict: + backup: str | None, major: str | None, target: str | None = None) -> dict: """What is about to be restored. A local repository says for itself; a remote one needs --postgres, since firedrill will not guess a major.""" if repo is not None: if not repo.is_dir(): raise PhysicalError(f"{repo} is not a directory") - meta = pgbackrest_backup(repo, stanza) if tool == "pgbackrest" else walg_backup(repo, backup) + before = parse_target(target) if target else None + meta = (pgbackrest_backup(repo, stanza, before) if tool == "pgbackrest" + else walg_backup(repo, backup, before)) if major and major != meta["major"]: meta["pinned_from"] = meta["major"] meta["major"] = major diff --git a/tests/test_firedrill.py b/tests/test_firedrill.py index 2c9b3c6..8d131d3 100644 --- a/tests/test_firedrill.py +++ b/tests/test_firedrill.py @@ -284,6 +284,24 @@ def test_physical_reads_the_backup_from_the_repository(): check("walg major from PgVersion", (meta["major"], meta["label"]), ("16", "base_000000010000000000000003")) + # PITR picks the newest backup that finished before the target, and a + # target before every backup is refused up front. CI showed why: with + # no commit during the backup, Postgres recovered to a LATER moment. + (walg / "base_000000010000000000000009_backup_stop_sentinel.json").write_text( + '{"PgVersion":160015,"FinishTime":"2026-09-27T08:00:00Z"}') + between = physical.parse_target("2026-09-27 01:00:00+00") + check("the backup before the target", physical.walg_backup(repo, None, between)["label"], + "base_000000010000000000000003") + check("no target: the newest", physical.walg_backup(repo, None)["label"], + "base_000000010000000000000009") + for tool, fn in (("walg", lambda b: physical.walg_backup(repo, None, b)), + ("pgbackrest", lambda b: physical.pgbackrest_backup(repo, None, b))): + try: + fn(physical.parse_target("2020-01-01 00:00:00")) + check(f"{tool}: a target before every backup is refused", "accepted", "TargetBeforeBackup") + except physical.TargetBeforeBackup: + pass + def test_physical_never_archives_into_the_repository_it_checks(): """A restored data directory carries production's archive_command.""" @@ -336,8 +354,8 @@ def test_physical_restores_real_pgbackrest_and_walg_repositories(): check(f"{tool}: all tables back", "shop: 3 table(s)" in full.stages[1].detail, True) pitr_run = drill.run_physical(tool, repo=repo, target=target, cfg=cfg) check(f"{tool}: recovered exactly to the boundary", pitr_run.ok, True) - # A day past the archive's end -- not 2099: pgBackRest 2.59 answers a - # post-2038 target with [075], as if no backup preceded it. + # A day past the archive's end -- not 2099: pgBackRest 2.59 answered a + # 2099 target with [075], as if no backup preceded it (cause unconfirmed). import datetime as dt after = (dt.datetime.fromisoformat(target.replace("+00", "+00:00")) + dt.timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S+00") @@ -1016,11 +1034,13 @@ def test_action_yml_passes_only_flags_the_cli_actually_has(): used.update(re.findall(r"--[a-z][a-z-]*", line)) check("the action passes some flags", len(used) >= 5, True) + # The action calls `run` or `physical`; each flag must exist in one. known = set() - for action in build_parser()._subparsers._group_actions[0].choices["run"]._actions: - known.update(action.option_strings) + for command in ("run", "physical"): + for action in build_parser()._subparsers._group_actions[0].choices[command]._actions: + known.update(action.option_strings) missing = sorted(used - known) - check(f"every flag exists in `firedrill run` (missing: {missing})", missing, []) + check(f"every flag exists in `firedrill run` or `physical` (missing: {missing})", missing, []) def test_action_shell_scripts_are_valid_bash(): From 9501275d43d46b2f37f9ba7f86b8ffbcb2b6a26a Mon Sep 17 00:00:00 2001 From: Ritish Saini <109451965+MaXiMo000@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:34:28 +0530 Subject: [PATCH 3/3] MySQL and MariaDB dumps (#7) `firedrill run` recognises a mysqldump / mariadb-dump file (plain or .gz) by its header, restores it into mysql: / mariadb: from that header, and runs smoke, semantics and CHECK TABLE. A missing `-- Dump completed` trailer is ARCHIVE_TRUNCATED before anything is restored; the client's `ERROR nnnn at line N` becomes a finding. MariaDB 10.5.25+/11.x dumps open with a sandbox-mode line before the header; it is read past. The password reaches the client via MYSQL_PWD inside the container -- exported, since a prefix on a pipeline applies only to its first command and `cat dump | mysql` logged in without one. Tested on datacharmer/test_db's employees database (300,024 employees, 2,844,047 salaries) dumped by MySQL 8.4 and MariaDB 11.4: both pass, gzipped passes, a copy cut at 80 MB fails with the trailer, the client's error line, 7 of 8 tables and both counts wrong. Co-authored-by: Claude Opus 5.5 --- README.md | 32 ++++++ firedrill/drill.py | 133 +++++++++++++++++++++++- firedrill/mysql.py | 222 ++++++++++++++++++++++++++++++++++++++++ firedrill/report.py | 6 ++ tests/test_firedrill.py | 67 +++++++++++- 5 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 firedrill/mysql.py diff --git a/README.md b/README.md index ec5b049..e03ba91 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,38 @@ supplies. A plain dump can only be restored whole, so the `fast` and `sample` tiers refuse it (`TIER_UNSUPPORTED`) instead of quietly running a full restore. +## MySQL and MariaDB + +`firedrill run backup.sql` recognises a `mysqldump` or `mariadb-dump` file +(plain or `.gz`) by its own header and takes the same path Postgres does: +the server version comes from the dump, the restore runs in `mysql:` +or `mariadb:`, and then it runs smoke, your `semantics:` checks, and +`CHECK TABLE` on every restored table. + +- **Truncation.** Both tools end a finished dump with `-- Dump completed on + ...`. The `mysql` client restores a dump cut at a statement boundary with + exit 0, so a missing trailer is `ARCHIVE_TRUNCATED` (critical) before + anything is restored. +- **Errors.** The client stops at the first failing statement; its + `ERROR nnnn (state) at line N` becomes a finding naming the line, since + everything after it was never restored. +- **MariaDB 10.5.25+/11.x** dumps open with a `/*M!999999\- enable the + sandbox mode */` line before the header -- the reason they won't load into + MySQL. firedrill reads past it and restores them into MariaDB. +- The password reaches the client through `MYSQL_PWD` inside the container, + never argv. + +Tested on [datacharmer/test_db](https://github.com/datacharmer/test_db)'s +employees database (300,024 employees, 2,844,047 salary rows), dumped by +each engine's own tool: + +| dump | result | +|---|---| +| MySQL 8.4, 168 MB | PASS: 8 tables, both exact counts, `CHECK TABLE` clean (restore 63 s) | +| MariaDB 11.4, 172 MB, with the sandbox-mode line | PASS (restore 25 s) | +| the MySQL dump gzipped, 36 MB | PASS | +| the MySQL dump cut off at 80 MB | FAIL: `ARCHIVE_TRUNCATED`, the client's `ERROR 1064 at line 265`, 7 of 8 tables, both counts wrong | + ## Which PostgreSQL versions Tested end to end against **13, 14, 15, 16, 17 and 18** — dump on any of them, diff --git a/firedrill/drill.py b/firedrill/drill.py index 6875411..81e49df 100644 --- a/firedrill/drill.py +++ b/firedrill/drill.py @@ -14,7 +14,7 @@ import tempfile import time -from . import archive, docker, history as history_module, ladder, physical, pitr, \ +from . import archive, docker, history as history_module, ladder, mysql, physical, pitr, \ restore as restore_stage, sources from . import config as _config_module from .config import DEFAULT as DEFAULT_CONFIG @@ -468,6 +468,131 @@ def stop(rule, message, fix, evidence="", status=FAILED): return _suppress(report, cfg) +def _run_mysql(report: Report, found: dict, dump_path, *, cfg, flavour: str, + pin_major, ready_timeout: int, began: float) -> Report: + """A mysqldump / mariadb-dump: version from its header, restored into that + same engine and version, then smoke, semantics and CHECK TABLE.""" + def stage(name): + return next(s for s in report.stages if s.name == name) + + engine, version = found["engine"], pin_major or found["version"] + label = "MariaDB" if engine == "mariadb" else "MySQL" + report.archive.update({"format": f"{engine} dump", "server_version": found["version"], + "engine": engine, "gzipped": found["gzipped"]}) + for name in ("structure", "volume"): + stage(name).status = NOT_CONFIGURED + stage(name).detail = "PostgreSQL only" + + stage("inspect").status = OK + stage("inspect").detail = f"{label} {found['version'] or '?'}" + if not version: + stage("inspect").status = FAILED + report.findings.append(Finding( + stage="inspect", rule="VERSION_UNKNOWN", severity="critical", + message=f"the {label} dump names no server version", + fix="Pass --postgres with the server version (e.g. 8.4) -- the flag " + "pins whichever engine the dump is from. firedrill will not guess.", + evidence="")) + report.total_seconds = time.monotonic() - began + return report + if not found["complete"]: + # The trailer is written only when the dump finished. The mysql client + # restores a dump cut at a statement boundary with exit 0. + stage("inspect").status = FAILED + report.findings.append(Finding( + stage="inspect", rule="ARCHIVE_TRUNCATED", severity="critical", + message=f"the dump has no `-- Dump completed` trailer: {label}'s dump " + "tool did not finish writing it", + fix="The backup job ran out of disk, was killed, or its output was " + "cut off. Check the writer's exit status; the file is incomplete.", + evidence="")) + + usable, why = docker.docker_available() + if not usable: + stage("target").status = NOT_RUN + report.findings.append(Finding( + stage="target", rule="TARGET_UNAVAILABLE", severity="critical", + message=f"the restore could NOT be verified: {why}", + fix="Start Docker and run again.", evidence="")) + report.total_seconds = time.monotonic() - began + return report + + container = mysql.MySQLContainer(engine, version, dump_path, flavour=flavour, + ready_timeout=ready_timeout) + started = time.monotonic() + try: + try: + container.start() + container.wait_ready() + except docker.TargetError as exc: + stage("target").status = FAILED + stage("target").seconds = time.monotonic() - started + report.findings.append(Finding( + stage="target", rule="TARGET_UNAVAILABLE", severity="critical", + message=f"the restore could NOT be verified: {str(exc).splitlines()[0]}", + fix="The target never came up, so nothing about this dump was proved.", + evidence="")) + report.total_seconds = time.monotonic() - began + return report + stage("target").status = OK + stage("target").seconds = time.monotonic() - started + stage("target").detail = container.image + report.archive["restored_into_major"] = version + + started = time.monotonic() + code, found_errors, _ = mysql.restore(container, found["gzipped"]) + report.verified = True + report.findings.extend(found_errors) + stage("restore").seconds = time.monotonic() - started + stage("restore").status = FAILED if found_errors else OK + stage("restore").detail = f"exit {code}" + + started = time.monotonic() + restored = mysql.schemas(container) + stage("smoke").seconds = time.monotonic() - started + stage("smoke").detail = ", ".join(f"{n}: {c} table(s)" for n, c in restored.items()) or "nothing" + if not restored: + stage("smoke").status = FAILED + report.findings.append(Finding( + stage="smoke", rule="SMOKE_EMPTY", severity="high", + message="the restore left no user tables in any database", + fix="The dump restored nothing you would miss.", evidence="")) + else: + stage("smoke").status = OK + + if cfg.semantics: + db = next(iter(restored), None) if len(restored) == 1 else None + if db is None: + stage("semantics").status = NOT_RUN + report.findings.append(Finding( + stage="semantics", rule="DATABASE_AMBIGUOUS", severity="high", + message="semantics checks need one database; the dump restored " + f"{', '.join(restored) or 'none'}", + fix="Qualify table names in the checks (db.table).", evidence="")) + else: + started = time.monotonic() + sem, _ = ladder.semantics(container, cfg, db) + report.findings.extend(sem) + stage("semantics").seconds = time.monotonic() - started + stage("semantics").status = FAILED if sem else OK + stage("semantics").detail = f"{len(cfg.semantics)} check(s) in {db}" + else: + stage("semantics").status = NOT_CONFIGURED + stage("semantics").detail = "no semantics checks in the config" + + started = time.monotonic() + checked = mysql.check_tables(container, list(restored)) + report.findings.extend(checked) + stage("integrity").seconds = time.monotonic() - started + stage("integrity").status = FAILED if checked else OK + stage("integrity").detail = f"CHECK TABLE on {sum(restored.values())} table(s)" + finally: + container.teardown() + + report.total_seconds = time.monotonic() - began + return report + + def _sequence_detail(info: dict) -> str: """`9 of 13 sequence(s)` when some could not be linked to a column. @@ -615,6 +740,12 @@ def stage(name: str) -> Stage: evidence=f"{artifact.origin}", )) + # A MySQL or MariaDB dump takes its own path from here. + found = mysql.detect(dump_path) + if found is not None: + return _run_mysql(report, found, dump_path, cfg=cfg, flavour=flavour, + pin_major=pin_major, ready_timeout=ready_timeout, began=began) + # -- inspect ----------------------------------------------------------- started = time.monotonic() try: diff --git a/firedrill/mysql.py b/firedrill/mysql.py new file mode 100644 index 0000000..78d7d87 --- /dev/null +++ b/firedrill/mysql.py @@ -0,0 +1,222 @@ +"""MySQL and MariaDB dumps: the same promise, a second engine. + +`mysqldump` output has the two properties firedrill's plain-SQL Postgres +support rests on: a header naming the server version, and a trailer +(`-- Dump completed on ...`) written only when the dump finished. So the +version is read from the file, the restore runs in a container of that +same version, and a dump without its trailer is truncated before anything +is restored. + +MariaDB's `mariadb-dump` writes the same shape and says so in its header; +its 11.x images ship `mariadb*` clients only, so the client name follows the +engine. +""" +from __future__ import annotations + +import gzip +import pathlib +import re + +from . import docker +from .finding import Finding + +_HEAD = re.compile(r"^-- (MySQL|MariaDB) dump ") +_SERVER = re.compile(r"^-- Server version\s+(\d+)\.(\d+)\.\d+(-MariaDB)?", re.M) +_TRAILER = "-- Dump completed" +# ERROR 1064 (42000) at line 12: You have an error in your SQL syntax ... +_CLIENT_ERROR = re.compile(r"^ERROR (\d+) \(([0-9A-Z]{5})\)(?: at line (\d+))?: (.*)$", re.M) +_SYSTEM_SCHEMAS = ("mysql", "sys", "information_schema", "performance_schema") + + +def _open(path: pathlib.Path): + with open(path, "rb") as handle: + magic = handle.read(2) + return gzip.open(path, "rb") if magic == b"\x1f\x8b" else open(path, "rb") + + +def detect(path) -> dict | None: + """{engine, version, gzipped, complete} for a mysqldump/mariadb-dump + file, or None when it is not one.""" + path = pathlib.Path(path) + if not path.is_file(): + return None + try: + with _open(path) as handle: + head = handle.read(4096).decode("utf-8", errors="replace") + except (OSError, EOFError): + return None + # MariaDB 10.5.25+/11.x dumps open with `/*M!999999\- enable the sandbox + # mode */` before the header -- the line that makes them unloadable into + # MySQL. Skipped here, so every modern MariaDB dump is still recognised. + lines = head.splitlines() + if lines and lines[0].startswith("/*M!999999"): + head = "\n".join(lines[1:]) + if not _HEAD.match(head): + return None + server = _SERVER.search(head) + engine = "mariadb" if (server and server.group(3)) or head.startswith("-- MariaDB") else "mysql" + version = f"{server.group(1)}.{server.group(2)}" if server else None + gzipped = _is_gz(path) + return {"engine": engine, "version": version, "gzipped": gzipped, + "complete": _TRAILER in _tail(path, gzipped)} + + +def _is_gz(path) -> bool: + with open(path, "rb") as handle: + return handle.read(2) == b"\x1f\x8b" + + +def _tail(path: pathlib.Path, gzipped: bool, size: int = 512) -> str: + """The last bytes of the (decompressed) dump. A .gz is streamed through, + and a .gz that is itself cut short reads as incomplete.""" + if not gzipped: + with open(path, "rb") as handle: + handle.seek(max(0, path.stat().st_size - size)) + return handle.read().decode("utf-8", errors="replace") + last = b"" + try: + with gzip.open(path, "rb") as handle: + while chunk := handle.read(1 << 20): + last = (last + chunk)[-size:] + except (OSError, EOFError): + return "" + return last.decode("utf-8", errors="replace") + + +def image_for(engine: str, version: str, flavour: str = "") -> str: + if "{major}" in flavour: + return flavour.format(major=version) + return f"{engine}:{version}{flavour}" + + +def _client(engine: str, version: str, tool: str) -> str: + """mysql, mysqladmin, mysqlcheck -- or mariadb, mariadb-admin, mariadb-check, + which is all MariaDB 11 images still ship.""" + if engine == "mariadb": + return {"mysql": "mariadb", "mysqladmin": "mariadb-admin", "mysqlcheck": "mariadb-check"}[tool] + return tool + + +class MySQLContainer(docker.Container): + """A disposable MySQL/MariaDB of the dump's own version.""" + + def __init__(self, engine: str, version: str, dump, flavour: str = "", + ready_timeout: int = docker.DEFAULT_READY_TIMEOUT): + super().__init__(version, dump=dump, ready_timeout=ready_timeout) + self.engine = engine + self.image = image_for(engine, version, flavour) + + def _cmd(self, tool: str) -> str: + return _client(self.engine, self.major, tool) + + def _run_argv(self) -> list[str]: + # By name, never value -- same rule as the Postgres target. + return ["docker", "run", "-d", "--name", self.name, "--label", f"{docker.LABEL}=1", + "-e", "MYSQL_ROOT_PASSWORD", "-e", "MARIADB_ROOT_PASSWORD", + "-v", f"{self.dump}:{docker.DUMP_PATH}:ro", self.image] + + def start(self) -> None: + env = {"PATH": docker._path(), "HOME": docker._home(), + "MYSQL_ROOT_PASSWORD": self.password, "MARIADB_ROOT_PASSWORD": self.password} + result = docker._run(self._run_argv(), env=env) + if result.returncode != 0: + raise docker.TargetError(f"could not start {self.image}: {(result.stderr or '').strip()}") + self.started = True + + def shell(self, script: str, timeout: int = 3600): + """A command with the client's password in MYSQL_PWD, read inside the + container from its own environment -- never through argv.""" + # Exported, not prefixed: a prefix applies only to the first command + # of a pipeline, so `cat dump | mysql` logged in with no password. + return self.exec(["sh", "-c", f'export MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; {script}'], + timeout=timeout, user="root") + + def wait_ready(self) -> None: + # Over TCP: like the Postgres image, the entrypoint runs a temporary + # server with networking off while it initialises. + import time + deadline = time.monotonic() + self.ready_timeout + while time.monotonic() < deadline: + probe = self.shell(f"{self._cmd('mysqladmin')} -h127.0.0.1 -uroot ping", timeout=30) + if probe.returncode == 0: + return + if not self.running(): + raise docker.TargetError(f"{self.image} exited before accepting connections.\n{self.logs()}") + time.sleep(1) + raise docker.TargetError(f"{self.image} did not accept connections within " + f"{self.ready_timeout}s\n{self.logs()}") + + def sql(self, statement: str, database: str = "mysql", timeout: int = 300): + quoted = statement.replace("'", "'\"'\"'") + return self.shell(f"{self._cmd('mysql')} -h127.0.0.1 -uroot -N -B -D {database} -e '{quoted}'", + timeout=timeout) + + +def restore(container: MySQLContainer, gzipped: bool) -> tuple[int, list[Finding], str]: + reader = "gunzip -c" if gzipped else "cat" + result = container.shell( + f"{reader} {docker.DUMP_PATH} | {container._cmd('mysql')} -h127.0.0.1 -uroot") + stderr = "\n".join(l for l in (result.stderr or "").splitlines() + if "Using a password on the command line" not in l) + findings = [] + for code, state, line, message in _CLIENT_ERROR.findall(stderr): + findings.append(Finding( + stage="restore", rule="RESTORE_ERROR", severity="high", + message=f"ERROR {code} ({state}){f' at line {line}' if line else ''}: {message.strip()}", + fix="The mysql client stops at the first failing statement, so " + "everything after this line of the dump was never restored.", + evidence="")) + if result.returncode != 0 and not findings: + findings.append(Finding( + stage="restore", rule="RESTORE_FAILED", severity="critical", + message=f"the restore exited {result.returncode}", + fix="An unexplained non-zero exit is not evidence of success.", + evidence=stderr.strip()[-500:] or "(no stderr)")) + return result.returncode, findings, stderr + + +def schemas(container: MySQLContainer) -> dict[str, int]: + listed = container.sql( + "select table_schema, count(*) from information_schema.tables where table_schema " + f"not in {tuple(_SYSTEM_SCHEMAS)} group by table_schema order by 1") + out = {} + for line in (listed.stdout or "").splitlines(): + name, _, count = line.partition("\t") + if count.strip().isdigit(): + out[name] = int(count) + return out + + +def check_tables(container: MySQLContainer, databases: list[str]) -> list[Finding]: + """CHECK TABLE on every restored table: the engine's own verdict on each. + Plain SQL rather than mysqlcheck, which the mysql:8.4 image doesn't ship.""" + if not databases: + return [] + listed = container.sql( + "select concat('`', table_schema, '`.`', table_name, '`') from information_schema.tables " + f"where table_type = 'BASE TABLE' and table_schema in {tuple(databases) + ('',)}") + tables = [t for t in (listed.stdout or "").split() if t] + if listed.returncode != 0: + return [Finding(stage="integrity", rule="TABLE_CHECK_UNRUNNABLE", severity="high", + message="could not list the restored tables", + fix="Nothing about table integrity has been proved.", + evidence=(listed.stderr or "").strip()[-300:])] + findings = [] + for i in range(0, len(tables), 50): + ran = container.sql("check table " + ", ".join(tables[i:i + 50])) + if ran.returncode != 0: + findings.append(Finding(stage="integrity", rule="TABLE_CHECK_UNRUNNABLE", severity="high", + message="CHECK TABLE could not run", + fix="Nothing about table integrity has been proved.", + evidence=(ran.stderr or "").strip()[-300:])) + continue + # Table Op Msg_type Msg_text -- anything but "status OK" is the + # engine saying the table is not sound. + for row in (ran.stdout or "").splitlines(): + cols = row.split("\t") + if len(cols) == 4 and not (cols[2] == "status" and cols[3] == "OK") and cols[2] != "note": + findings.append(Finding(stage="integrity", rule="TABLE_CHECK_FAILED", severity="high", + message=f"CHECK TABLE {cols[0]}: {cols[2]} {cols[3]}", + fix="The engine itself reports this table as damaged.", + evidence="")) + return findings diff --git a/firedrill/report.py b/firedrill/report.py index 4bdbbc5..45e9ce4 100644 --- a/firedrill/report.py +++ b/firedrill/report.py @@ -41,6 +41,12 @@ def human(report: Report, colour: bool = False) -> str: else "WAL replayed to the end of the archive") lines.append(f" source PostgreSQL {a.get('server_major')} -> {towards} " f"on postgres:{a.get('restored_into_major')}") + elif a.get("engine"): + name = "MariaDB" if a["engine"] == "mariadb" else "MySQL" + packed = ", gzipped" if a.get("gzipped") else "" + lines.append(f" archive {name} dump{packed}") + lines.append(f" source {name} {a.get('server_version')} -> restored into " + f"{a['engine']}:{a.get('restored_into_major')}") elif a.get("recovery_target_time"): # A PITR run has no archive to describe. Printing the dump fields here # rendered "archive None vNone 0B from None", which is worse than diff --git a/tests/test_firedrill.py b/tests/test_firedrill.py index 8d131d3..aeb9a93 100644 --- a/tests/test_firedrill.py +++ b/tests/test_firedrill.py @@ -368,6 +368,71 @@ def test_physical_restores_real_pgbackrest_and_walg_repositories(): check(f"{tool}: and that is a FAIL, not could-not-verify", early.verified, True) +# ------------------------------------------------------------------- mysql -- + +MYSQL_DUMP = """-- MySQL dump 10.13 Distrib 8.4.11, for Linux (x86_64) +-- +-- Host: localhost Database: shop +-- ------------------------------------------------------ +-- Server version\t8.4.11 + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `shop`; +USE `shop`; +CREATE TABLE `orders` (`id` int NOT NULL AUTO_INCREMENT, `amount` int, PRIMARY KEY (`id`)) ENGINE=InnoDB; +INSERT INTO `orders` VALUES (1,10),(2,20),(3,30); +CREATE TABLE `customers` (`id` int NOT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB; +INSERT INTO `customers` VALUES (1),(2); + +-- Dump completed on 2026-09-26 21:12:51 +""" + + +def test_mysql_and_mariadb_dumps_are_recognised(): + import gzip + import tempfile + from firedrill import mysql + with tempfile.TemporaryDirectory() as tmp: + tmp = pathlib.Path(tmp) + (tmp / "m.sql").write_text(MYSQL_DUMP) + check("mysql", mysql.detect(tmp / "m.sql"), + {"engine": "mysql", "version": "8.4", "gzipped": False, "complete": True}) + # MariaDB 11.x opens with a sandbox-mode line before its header. + maria = ("/*M!999999\\- enable the sandbox mode */ \n-- MariaDB dump 10.19-11.4.13-MariaDB, " + "for debian-linux-gnu (x86_64)\n--\n-- Server version\t11.4.13-MariaDB-ubu2404\n" + "-- Dump completed on 2026-09-26 21:33:06\n") + (tmp / "mdb.sql").write_text(maria) + check("mariadb behind the sandbox line", (mysql.detect(tmp / "mdb.sql") or {}).get("engine"), "mariadb") + check("its version", mysql.detect(tmp / "mdb.sql")["version"], "11.4") + (tmp / "m.sql.gz").write_bytes(gzip.compress(MYSQL_DUMP.encode())) + check("gzipped", mysql.detect(tmp / "m.sql.gz")["gzipped"], True) + (tmp / "cut.sql").write_text(MYSQL_DUMP[:len(MYSQL_DUMP) // 2]) + check("no trailer is incomplete", mysql.detect(tmp / "cut.sql")["complete"], False) + (tmp / "pg.sql").write_text("--\n-- PostgreSQL database dump\n--\n") + check("a Postgres dump is not MySQL", mysql.detect(tmp / "pg.sql"), None) + + +def test_integration_mysql_dump_restores_and_truncation_is_caught(): + """Against a real mysql:8.4. The truncated copy ends on a statement + boundary, which the mysql client restores with exit 0 -- only the + missing trailer says it is incomplete.""" + import tempfile + needs_docker() + with tempfile.TemporaryDirectory() as tmp: + good = pathlib.Path(tmp) / "shop.sql" + good.write_text(MYSQL_DUMP) + cfg = config.loads("version: 1\nsemantics:\n - name: orders\n" + " sql: select count(*) from orders\n expect: \"== 3\"\n") + report = drill.run(good, cfg=cfg) + check("mysql dump passes", (report.ok, [f.rule for f in report.findings]), (True, [])) + check("both tables back", "shop: 2 table(s)" in next( + s for s in report.stages if s.name == "smoke").detail, True) + cut = pathlib.Path(tmp) / "cut.sql" + cut.write_text(MYSQL_DUMP.split("CREATE TABLE `customers`")[0]) + report = drill.run(cut) + check("truncation caught", [f.rule for f in report.findings], ["ARCHIVE_TRUNCATED"]) + check("and it is a FAIL", (report.verified, report.ok), (True, False)) + + def test_notify_posts_the_verdict_to_a_real_webhook(): """A real HTTP receiver on localhost. Failing and could-not-verify runs notify; a pass does not unless asked; the URL never reaches an error.""" @@ -2788,7 +2853,7 @@ def main() -> int: # A floor, not a target. Edits that replace a range of lines have silently # swallowed whole blocks of tests before; the suite then goes green with # fewer tests and says nothing. - FLOOR = 155 + FLOOR = 157 if len(tests) < FLOOR: raise SystemExit( f"test suite shrank: {len(tests)} < {FLOOR}. An edit probably deleted "