firedrill: MySQL and MariaDB dumps

README.md

@@ -201,6 +201,38 @@ supplies.
 A plain dump can only be restored whole, so the `fast` and `sample` tiers
 refuse it (`TIER_UNSUPPORTED`) instead of quietly running a full restore.
 
+## MySQL and MariaDB
+
+`firedrill run backup.sql` recognises a `mysqldump` or `mariadb-dump` file
+(plain or `.gz`) by its own header and takes the same path Postgres does:
+the server version comes from the dump, the restore runs in `mysql:<version>`
+or `mariadb:<version>`, and then it runs smoke, your `semantics:` checks, and
+`CHECK TABLE` on every restored table.
+
+- **Truncation.** Both tools end a finished dump with `-- Dump completed on
+  ...`. The `mysql` client restores a dump cut at a statement boundary with
+  exit 0, so a missing trailer is `ARCHIVE_TRUNCATED` (critical) before
+  anything is restored.
+- **Errors.** The client stops at the first failing statement; its
+  `ERROR nnnn (state) at line N` becomes a finding naming the line, since
+  everything after it was never restored.
+- **MariaDB 10.5.25+/11.x** dumps open with a `/*M!999999\- enable the
+  sandbox mode */` line before the header -- the reason they won't load into
+  MySQL. firedrill reads past it and restores them into MariaDB.
+- The password reaches the client through `MYSQL_PWD` inside the container,
+  never argv.
+
+Tested on [datacharmer/test_db](https://github.com/datacharmer/test_db)'s
+employees database (300,024 employees, 2,844,047 salary rows), dumped by
+each engine's own tool:
+
+| dump | result |
+|---|---|
+| MySQL 8.4, 168 MB | PASS: 8 tables, both exact counts, `CHECK TABLE` clean (restore 63 s) |
+| MariaDB 11.4, 172 MB, with the sandbox-mode line | PASS (restore 25 s) |
+| the MySQL dump gzipped, 36 MB | PASS |
+| the MySQL dump cut off at 80 MB | FAIL: `ARCHIVE_TRUNCATED`, the client's `ERROR 1064 at line 265`, 7 of 8 tables, both counts wrong |
+
 ## Which PostgreSQL versions
 
 Tested end to end against **13, 14, 15, 16, 17 and 18** — dump on any of them,

firedrill/drill.py

@@ -14,7 +14,7 @@
 import tempfile
 import time
 
-from . import archive, docker, history as history_module, ladder, physical, pitr, \
+from . import archive, docker, history as history_module, ladder, mysql, physical, pitr, \
     restore as restore_stage, sources
 from . import config as _config_module
 from .config import DEFAULT as DEFAULT_CONFIG
@@ -468,6 +468,131 @@ def stop(rule, message, fix, evidence="", status=FAILED):
     return _suppress(report, cfg)
 
 
+def _run_mysql(report: Report, found: dict, dump_path, *, cfg, flavour: str,
+               pin_major, ready_timeout: int, began: float) -> Report:
+    """A mysqldump / mariadb-dump: version from its header, restored into that
+    same engine and version, then smoke, semantics and CHECK TABLE."""
+    def stage(name):
+        return next(s for s in report.stages if s.name == name)
+
+    engine, version = found["engine"], pin_major or found["version"]
+    label = "MariaDB" if engine == "mariadb" else "MySQL"
+    report.archive.update({"format": f"{engine} dump", "server_version": found["version"],
+                           "engine": engine, "gzipped": found["gzipped"]})
+    for name in ("structure", "volume"):
+        stage(name).status = NOT_CONFIGURED
+        stage(name).detail = "PostgreSQL only"
+
+    stage("inspect").status = OK
+    stage("inspect").detail = f"{label} {found['version'] or '?'}"
+    if not version:
+        stage("inspect").status = FAILED
+        report.findings.append(Finding(
+            stage="inspect", rule="VERSION_UNKNOWN", severity="critical",
+            message=f"the {label} dump names no server version",
+            fix="Pass --postgres with the server version (e.g. 8.4) -- the flag "
+                "pins whichever engine the dump is from. firedrill will not guess.",
+            evidence=""))
+        report.total_seconds = time.monotonic() - began
+        return report
+    if not found["complete"]:
+        # The trailer is written only when the dump finished. The mysql client
+        # restores a dump cut at a statement boundary with exit 0.
+        stage("inspect").status = FAILED
+        report.findings.append(Finding(
+            stage="inspect", rule="ARCHIVE_TRUNCATED", severity="critical",
+            message=f"the dump has no `-- Dump completed` trailer: {label}'s dump "
+                    "tool did not finish writing it",
+            fix="The backup job ran out of disk, was killed, or its output was "
+                "cut off. Check the writer's exit status; the file is incomplete.",
+            evidence=""))
+
+    usable, why = docker.docker_available()
+    if not usable:
+        stage("target").status = NOT_RUN
+        report.findings.append(Finding(
+            stage="target", rule="TARGET_UNAVAILABLE", severity="critical",
+            message=f"the restore could NOT be verified: {why}",
+            fix="Start Docker and run again.", evidence=""))
+        report.total_seconds = time.monotonic() - began
+        return report
+
+    container = mysql.MySQLContainer(engine, version, dump_path, flavour=flavour,
+                                     ready_timeout=ready_timeout)
+    started = time.monotonic()
+    try:
+        try:
+            container.start()
+            container.wait_ready()
+        except docker.TargetError as exc:
+            stage("target").status = FAILED
+            stage("target").seconds = time.monotonic() - started
+            report.findings.append(Finding(
+                stage="target", rule="TARGET_UNAVAILABLE", severity="critical",
+                message=f"the restore could NOT be verified: {str(exc).splitlines()[0]}",
+                fix="The target never came up, so nothing about this dump was proved.",
+                evidence=""))
+            report.total_seconds = time.monotonic() - began
+            return report
+        stage("target").status = OK
+        stage("target").seconds = time.monotonic() - started
+        stage("target").detail = container.image
+        report.archive["restored_into_major"] = version
+
+        started = time.monotonic()
+        code, found_errors, _ = mysql.restore(container, found["gzipped"])
+        report.verified = True
+        report.findings.extend(found_errors)
+        stage("restore").seconds = time.monotonic() - started
+        stage("restore").status = FAILED if found_errors else OK
+        stage("restore").detail = f"exit {code}"
+
+        started = time.monotonic()
+        restored = mysql.schemas(container)
+        stage("smoke").seconds = time.monotonic() - started
+        stage("smoke").detail = ", ".join(f"{n}: {c} table(s)" for n, c in restored.items()) or "nothing"
+        if not restored:
+            stage("smoke").status = FAILED
+            report.findings.append(Finding(
+                stage="smoke", rule="SMOKE_EMPTY", severity="high",
+                message="the restore left no user tables in any database",
+                fix="The dump restored nothing you would miss.", evidence=""))
+        else:
+            stage("smoke").status = OK
+
+        if cfg.semantics:
+            db = next(iter(restored), None) if len(restored) == 1 else None
+            if db is None:
+                stage("semantics").status = NOT_RUN
+                report.findings.append(Finding(
+                    stage="semantics", rule="DATABASE_AMBIGUOUS", severity="high",
+                    message="semantics checks need one database; the dump restored "
+                            f"{', '.join(restored) or 'none'}",
+                    fix="Qualify table names in the checks (db.table).", evidence=""))
+            else:
+                started = time.monotonic()
+                sem, _ = ladder.semantics(container, cfg, db)
+                report.findings.extend(sem)
+                stage("semantics").seconds = time.monotonic() - started
+                stage("semantics").status = FAILED if sem else OK
+                stage("semantics").detail = f"{len(cfg.semantics)} check(s) in {db}"
+        else:
+            stage("semantics").status = NOT_CONFIGURED
+            stage("semantics").detail = "no semantics checks in the config"
+
+        started = time.monotonic()
+        checked = mysql.check_tables(container, list(restored))
+        report.findings.extend(checked)
+        stage("integrity").seconds = time.monotonic() - started
+        stage("integrity").status = FAILED if checked else OK
+        stage("integrity").detail = f"CHECK TABLE on {sum(restored.values())} table(s)"
+    finally:
+        container.teardown()
+
+    report.total_seconds = time.monotonic() - began
+    return report
+
+
 def _sequence_detail(info: dict) -> str:
     """`9 of 13 sequence(s)` when some could not be linked to a column.
 
@@ -615,6 +740,12 @@ def stage(name: str) -> Stage:
             evidence=f"{artifact.origin}",
         ))
 
+    # A MySQL or MariaDB dump takes its own path from here.
+    found = mysql.detect(dump_path)
+    if found is not None:
+        return _run_mysql(report, found, dump_path, cfg=cfg, flavour=flavour,
+                          pin_major=pin_major, ready_timeout=ready_timeout, began=began)
+
     # -- inspect -----------------------------------------------------------
     started = time.monotonic()
     try: