From 775a152e76ccb3ba06c84de038b4cb7115e1db4f Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:08:26 +0000 Subject: [PATCH 01/21] docs: add ClickHouse migration driver design spec Two interchangeable version-table strategies (Mutation / AppendOnly) selected via a Mode enum, native phpClickHouse configuration, no-op transactions, and real-ClickHouse integration testing wired into make tests / CI. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../2026-05-28-clickhouse-driver-design.md | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md diff --git a/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md b/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md new file mode 100644 index 0000000..0d7e0cd --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md @@ -0,0 +1,290 @@ +# ClickHouse migration driver for dbschemix — design + +- **Date:** 2026-05-28 +- **Package:** `dbschemix/clickhouse` +- **Depends on:** `dbschemix/core` ^1.1, `smi2/phpclickhouse` ^1.26, PHP ^8.3 +- **Reference:** the PDO driver in `runtime/src` (structural model only — no API parity required) + +## Goal + +Provide a ClickHouse driver for the dbschemix migration core, following the +*structure* of the existing PDO driver but adapting freely to ClickHouse +semantics (no DSN, no real transactions, append/mutation-based bookkeeping). + +The driver ships **two interchangeable strategies** for the migration version +table, selected by the user via a `Mode` enum, so both can be exercised and +compared in practice. + +## Constraints and key facts + +- `smi2/phpclickhouse` `ClickHouseDB\Client` is configured with an array + (`host`, `port`, `username`, `password`, ...), **not** a DSN. `username` and + `password` are required by the client constructor. Database is selected with + `$client->database($db)`. +- The client's `Bindings` degeneration substitutes both `:key` and `{key}` + placeholders client-side (with value quoting via `ValueFormatter`), so the + core's `:name`/`:version`/`:atime` placeholder style is compatible. +- ClickHouse has **no transactions** over the HTTP interface. +- The core's default `Command` (`dbschemix\core\command\Command`) wraps each + migration in `beginTransaction → exec → commit/rollback` and uses plain + `INSERT`/`DELETE`/`SELECT`. We do **not** reuse it; each mode gets a + ClickHouse-specific `Command`. +- `DriverInterface` (core) requires: `getName(): non-empty-lowercase-string`, + `getSourceName(): non-empty-lowercase-string`, `getSetupPath(): non-empty-string`, + `makeCommand(Config): CommandInterface`. +- `Setup` (core) globs `*.sql` in `getSetupPath()` and replaces the + `%SYSTEM_TABLE%` token with the configured table name. +- `path('dbschemix/clickhouse')` resolves the package install path (in dev = + `/app`), so `getSetupPath()` can point inside this package. +- The database itself is assumed to already exist (same as PDO). The driver's + `setup.sql` creates only the version table, never the database. + +## Public API + +### `dbschemix\clickhouse\Mode` (enum, `@api`) + +```php +enum Mode +{ + /** ReplacingMergeTree version table; reads use SELECT ... FINAL for dedup, + * rollback removes the row via ALTER TABLE ... DELETE (synchronous mutation). */ + case Mutation; + + /** Append-only journal; every up/down appends a row, current state is + * derived with argMax over insert time. Nothing is physically deleted. */ + case AppendOnly; +} +``` + +### `dbschemix\clickhouse\Driver` (`@api`, implements `DriverInterface`) + +```php +public function __construct( + string $host, + int $port, + string $database, // private readonly; getSourceName() + client->database() + Mode $mode, // private readonly; selects Command + setup path + ?string $username = null, // -> 'default' when null + ?string $password = null, // -> '' when null + array $options = [], // phpClickHouse passthrough only: + // settings, https, auth_method, readonly, sslCA, curl_options +); +``` + +- `database` and `mode` are mandatory and become first-class readonly + properties (not buried in `$options`). They are placed before the optional + parameters because PHP requires mandatory parameters first. +- `$options` carries only values forwarded to `ClickHouseDB\Client` + (`connectParams` extras and `settings`). +- `getName()` → `'clickhouse'` (constant; only one dialect, so no `Type` enum). +- `getSourceName()` → `strtolower($database)`. +- `getSetupPath()` → `path('dbschemix/clickhouse') . '/src/connection/clickhouse/' . . '/migration/'` + where `` is `mutation` or `appendonly` per `Mode`. +- `makeCommand(Config)` → `MutationCommand` or `JournalCommand` per `Mode` + (composition root, analogous to PDO's `makeFactoryTransaction()`). +- Connection reuse with a TTL timer, mirroring PDO's `Driver::makeConnection()` + (the timer branch carries `@infection-ignore-all`). +- Client exceptions (`ClickHouseDB\Exception\*`) are **not** pre-wrapped; the + core `Workflow` catches `Throwable` and wraps into `ActionException` / + `InitializationException`. + +## Internal structure + +``` +src/ + Driver.php + Mode.php + internal/ + Connection.php // implements ConnectionInterface (wraps ClickHouseDB\Client) + Transaction.php // implements TransactionInterface (no-op) + command/ + AbstractCommand.php // shared: connection+config, dryRun guard, exec(Context), helpers + MutationCommand.php // implements CommandInterface (Mode::Mutation) + JournalCommand.php // implements CommandInterface (Mode::AppendOnly) + connection/clickhouse/ + mutation/migration/setup.sql + appendonly/migration/setup.sql +``` + +All `internal\*` types carry `@psalm-internal dbschemix\clickhouse`. The driver +depends directly on the concrete `ClickHouseDB\Client` (as PDO depends on +concrete `PDO`); no extra client port is introduced because tests run against a +real ClickHouse. + +### `internal\Connection implements ConnectionInterface` + +Thin adapter over an already-configured `ClickHouseDB\Client` (database set, +settings applied): + +- `fetchRecord(string $query, array $params = []): array` + → `$client->select($query, $params)`, then map `rows()` to + `name => (int) version` using the `name`/`version` columns (the core + contract: key = filename, value = version). +- `exec(string $query, array $params = []): void` → `$client->write($query, $params)` + (default `exception: true`, so client errors throw). +- `beginTransaction(): TransactionInterface` → returns `internal\Transaction` + (required by `ConnectionInterface`). + +### `internal\Transaction implements TransactionInterface` + +No-op wrapper (ClickHouse has no transactions over HTTP): + +- `exec(...)` forwards immediately to the client (same as `Connection::exec`). +- `fetchRecord(...)` forwards to the client. +- `isActive()` → `false`. +- `commit()` → `true`. +- `rollback()` → `false` (cannot undo; documented; `@infection-ignore-all`). + +The driver's own `Command` implementations run statements **directly** through +`Connection` (no transactional ceremony — variant A). `Transaction` exists for +`ConnectionInterface` compliance and for anyone using the core default +`Command`; it is covered by tests. + +### Commands + +`AbstractCommand` holds `ConnectionInterface` + `Config`, the `dryRun` guard, +the shared `exec(Context)` (used for fixtures/repeatable, identical in both +modes), binding/param helpers, and an `atime()` helper that returns the current +UTC time at millisecond precision (`(new DateTimeImmutable())->format('Y-m-d H:i:s.v')`) +— used as the `:atime` binding for every bookkeeping `INSERT` (both modes, +both `up` and `down`). Subclasses override `fetchApplied`, `up`, `down`. `fetchApplied` builds mode-specific SQL and delegates to +`$connection->fetchRecord($sql, $params)`. All bookkeeping placeholders use the +core's `:name`/`:version`/`:atime` style. + +`%TABLE%` below denotes `Config::$table` (validated `^\w+$` by core). + +#### `MutationCommand` (`Mode::Mutation`) + +`setup.sql` — `src/connection/clickhouse/mutation/migration/setup.sql`: + +```sql +CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% +( + name String, + version UInt64 DEFAULT 0, + atime DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(atime) +ORDER BY name; +``` + +- `fetchApplied(Options)`: + ```sql + SELECT name, version FROM %TABLE% FINAL + [WHERE version = :version] -- when options->version > 0 + ORDER BY atime DESC, name DESC + [LIMIT n] -- when options->limit > 0 + ``` +- `up(Context)`: `dryRun → false`; else `exec($context->query)` then + `INSERT INTO %TABLE% (name, version, atime) VALUES (:name, :version, :atime)`; + return `true`. +- `down(Context)`: `dryRun → false`; else `exec($context->query)` then + `ALTER TABLE %TABLE% DELETE WHERE name = :name SETTINGS mutations_sync = 2` + (synchronous mutation so the delete is immediately visible; `mutations_sync` + value kept as a named constant); return `true`. + +#### `JournalCommand` (`Mode::AppendOnly`) + +`setup.sql` — `src/connection/clickhouse/appendonly/migration/setup.sql`: + +```sql +CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% +( + name String, + version UInt64 DEFAULT 0, + atime DateTime64(3) DEFAULT now64(3), + active UInt8 DEFAULT 1 -- 1 = applied (up), 0 = rolled back (down) +) +ENGINE = MergeTree +ORDER BY (name, atime); +``` + +- `fetchApplied(Options)` (latest state per name via argMax): + ```sql + SELECT name, version FROM ( + SELECT name, + argMax(version, atime) AS version, + argMax(active, atime) AS active, + max(atime) AS atime + FROM %TABLE% + GROUP BY name + ) + WHERE active = 1 + [AND version = :version] + ORDER BY atime DESC, name DESC + [LIMIT n] + ``` +- `up(Context)`: `dryRun → false`; else `exec($context->query)` then + `INSERT INTO %TABLE% (name, version, atime, active) VALUES (:name, :version, :atime, 1)`; + return `true`. +- `down(Context)`: `dryRun → false`; else `exec($context->query)` then + `INSERT INTO %TABLE% (name, version, atime, active) VALUES (:name, :version, :atime, 0)`, + where `:atime` is the current time from `atime()` (millisecond precision) so + `argMax(atime)` selects the rollback as the latest event; return `true`. + +## Known limitations (documented) + +- **No atomicity.** `up`/`down` run two statements (the migration plus the + version bookkeeping) without a transaction. If the bookkeeping statement fails + after a successful migration, it cannot be rolled back. This is inherent to + ClickHouse. +- **AppendOnly ordering** relies on millisecond `atime` precision to order + up/down of the same file. If two events collide within the same millisecond + the derived state may be wrong; precision can be raised to `DateTime64(6)` if + this proves a problem in practice. +- **Mutation latency.** `ALTER TABLE ... DELETE` is a mutation; we force + synchronous execution with `SETTINGS mutations_sync = 2`. + +## Testing and delivery + +### Testing — real ClickHouse, locally and in CI + +- Framework: **Testo** (suite `integration`, location `tests/`). Mutation + testing: **Infection** on `src/`, `minCoveredMsi: 99`. Tests organized under + `tests/` including `tests/workflow/` for end-to-end `up/down/redo/verify/ + fixtures` per mode, plus `fetchApplied` with limit/version filters and + `dryRun` paths. +- Existing infra to wire up: `.docker/ClickHouse/` already provides a + `clickhouse/clickhouse-server:26.3-alpine` image whose `init.sh` creates + database `main`. It is **not** yet connected to `make tests` or CI. + - **Local:** add a ClickHouse container alongside the php-cli `tests` image + (docker-compose or a shared docker network) and extend `make tests` so the + test container can reach ClickHouse. Tests connect via host (service name / + `127.0.0.1`), HTTP port `8123`, database `main`, credentials from env. + - **CI:** add a ClickHouse `services:` container (or build `.docker/ClickHouse`) + to `.github/workflows/tests.yml`, expose `8123`, and point the tests at it. +- Test connection conventions: `database = main`, `port = 8123`, + `username`/`password` from env (`CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD`, + defaulting to `default` / empty). +- `@infection-ignore-all` applied narrowly, mirroring PDO (no-op `rollback`, + connection TTL timer). + +### Static analysis + +`make check` (phpcs + psalm + phpstan) must stay green. Strict typing, +`declare(strict_types=1)`, `@api` on the public surface, `@psalm-internal +dbschemix\clickhouse` on internals — consistent with the PDO reference and the +core package. + +### Deliverables + +1. `src/Driver.php`, `src/Mode.php`. +2. `src/internal/`: `Connection.php`, `Transaction.php`, + `command/AbstractCommand.php`, `command/MutationCommand.php`, + `command/JournalCommand.php`. +3. `src/connection/clickhouse/mutation/migration/setup.sql` and + `src/connection/clickhouse/appendonly/migration/setup.sql`. +4. `tests/` (Testo), covering both modes and the workflow scenarios above. +5. `example/` — a runnable sample (Migrator + ClickHouse `Driver`, both modes), + usable via `make app`. +6. Wire `.docker/ClickHouse` into local `make tests` and into + `.github/workflows/tests.yml`. +7. Update `README.md`; add a `CHANGELOG.md` entry. +8. `make check` and `make tests` (Testo + Infection MSI ≥ 99) green. + +## Out of scope + +- Database creation (`CREATE DATABASE`) — the database is assumed to exist. +- Reusing the core default `Command`. +- Any DSN-based configuration or full API parity with the PDO driver. +- ClickHouse experimental native transactions. From 247b6f9dc80bf2d15b92b1656cda30e19c177cf0 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:19:12 +0000 Subject: [PATCH 02/21] docs: add ClickHouse driver implementation plan Task-by-task TDD plan covering Mode enum, Driver, Connection/Transaction, both Command strategies, setup.sql, Testo integration tests against real ClickHouse, docker compose / CI wiring, and Infection MSI >= 99. Aligns both version tables on DateTime64(6) for lossless, reliably ordered atime bookkeeping. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../plans/2026-05-28-clickhouse-driver.md | 1930 +++++++++++++++++ .../2026-05-28-clickhouse-driver-design.md | 21 +- 2 files changed, 1941 insertions(+), 10 deletions(-) create mode 100644 docs/superpowers/plans/2026-05-28-clickhouse-driver.md diff --git a/docs/superpowers/plans/2026-05-28-clickhouse-driver.md b/docs/superpowers/plans/2026-05-28-clickhouse-driver.md new file mode 100644 index 0000000..b7bc561 --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-clickhouse-driver.md @@ -0,0 +1,1930 @@ +# ClickHouse Migration Driver Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the `dbschemix/clickhouse` migration driver with two interchangeable version-table strategies (`Mode::Mutation` and `Mode::AppendOnly`), tested against a real ClickHouse server locally and in CI. + +**Architecture:** A `Driver` (implements `dbschemix\core\connection\DriverInterface`) is configured natively (`host`, `port`, `database`, `Mode`, optional `username`/`password`/`options`) and builds a `ClickHouseDB\Client`. An internal `Connection` adapts the client to the core `ConnectionInterface`; a no-op `Transaction` satisfies the interface (ClickHouse has no HTTP transactions). Two `CommandInterface` implementations (`MutationCommand`, `JournalCommand`) extend a shared `AbstractCommand` and own the version-table SQL; the `Driver` picks one per `Mode`. Each mode ships its own `setup.sql`. + +**Tech Stack:** PHP 8.3+, `dbschemix/core` ^1.1, `smi2/phpclickhouse` ^1.26, Testo (tests), Infection (mutation testing, `minCoveredMsi: 99`), Psalm/PHPStan/PHP_CodeSniffer (static analysis), Docker / docker compose (ClickHouse for tests). + +**Spec:** `docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md` + +--- + +## File Structure + +**Production code (`src/`):** +- `src/Mode.php` — `@api` enum `Mode { Mutation, AppendOnly }` + `setupDir()` helper. +- `src/Driver.php` — `@api` `final class Driver implements DriverInterface`. Native constructor, builds the client factory, picks the `Command` and `setup.sql` directory per `Mode`, caches the connection with a TTL. +- `src/internal/ClientStatement.php` — trait with shared `fetchRecord()`/`exec()` over a `ClickHouseDB\Client` (used by `Connection` and `Transaction`). +- `src/internal/Connection.php` — `final readonly class Connection implements ConnectionInterface`. Uses `ClientStatement`; `beginTransaction()` returns a `Transaction`. +- `src/internal/Transaction.php` — `final readonly class Transaction implements TransactionInterface`. Uses `ClientStatement`; `isActive()`/`commit()`/`rollback()` are no-ops. +- `src/internal/command/AbstractCommand.php` — `abstract class AbstractCommand implements CommandInterface`. Holds `ConnectionInterface` + `Config`; implements `exec(Context)` and shared helpers (`atime()`, `filters()`); declares `fetchApplied`/`up`/`down` abstract. +- `src/internal/command/MutationCommand.php` — `Mode::Mutation` SQL (ReplacingMergeTree + FINAL + ALTER DELETE). +- `src/internal/command/JournalCommand.php` — `Mode::AppendOnly` SQL (append + argMax). +- `src/connection/clickhouse/mutation/migration/setup.sql` — version table DDL for `Mutation`. +- `src/connection/clickhouse/appendonly/migration/setup.sql` — version table DDL for `AppendOnly`. + +**Tests (`tests/`):** +- `tests/Support/ClickHouse.php` — test helper: reads connection env, builds `Client`/`Driver`, runs a mode's `setup.sql`, drops the version table. +- `tests/SmokeTest.php` — connectivity check. +- `tests/ConnectionTest.php` — `Connection` + `Transaction` behavior. +- `tests/command/MutationCommandTest.php` — `MutationCommand` behavior. +- `tests/command/JournalCommandTest.php` — `JournalCommand` behavior. +- `tests/DriverTest.php` — `Driver` pure methods + `makeCommand`. +- `tests/workflow/MigratorMutationTest.php` — end-to-end `Migrator` for `Mutation`. +- `tests/workflow/MigratorAppendOnlyTest.php` — end-to-end `Migrator` for `AppendOnly`. + +**Example (`example/`):** +- `example/run.php` — runnable sample wiring `Migrator` + ClickHouse `Driver`. +- `example/migration/main/0001_demo.sql` — sample migration with `-- @up`/`-- @down`. + +**Infra / docs:** +- `compose.yaml` — `clickhouse` + `cli` services for local `make tests`. +- `Makefile` — `tests`/`infection` retargeted to docker compose. +- `.github/workflows/tests.yml` — add a ClickHouse service + connection env. +- `README.md`, `CHANGELOG.md` — usage + changelog entry. + +**Connection env convention (all tests + CI + example):** +`CLICKHOUSE_HOST` (default `127.0.0.1`), `CLICKHOUSE_PORT` (default `8123`), `CLICKHOUSE_DB` (default `main`), `CLICKHOUSE_USER` (default `default`), `CLICKHOUSE_PASSWORD` (default empty). + +--- + +## Task 1: ClickHouse test harness + connectivity smoke test + +Stand up a real ClickHouse for tests (local `make tests` via docker compose; CI via a service container) and a test helper. This must work before any integration test. + +**Files:** +- Create: `compose.yaml` +- Modify: `Makefile:90-111` (the `tests:` target) and `Makefile:73-88` (the `infection:` target) +- Modify: `.github/workflows/tests.yml` +- Create: `tests/Support/ClickHouse.php` +- Create: `tests/SmokeTest.php` + +- [ ] **Step 1: Write `compose.yaml`** + +```yaml +services: + clickhouse: + build: .docker/ClickHouse + environment: + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8123/ping"] + interval: 2s + timeout: 5s + retries: 30 + + cli: + build: + context: .docker/php/cli + target: tests + args: + PHP_VERSION: "${PHP_VERSION:-8.3}" + UID: "${UID:-10001}" + WORKDIR: /app + volumes: + - ".:/app" + working_dir: /app + environment: + CLICKHOUSE_HOST: clickhouse + CLICKHOUSE_PORT: "8123" + CLICKHOUSE_DB: main + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + depends_on: + clickhouse: + condition: service_healthy +``` + +- [ ] **Step 2: Create the test helper `tests/Support/ClickHouse.php`** + +```php + getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', + 'port' => (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + 'username' => getenv('CLICKHOUSE_USER') ?: 'default', + 'password' => getenv('CLICKHOUSE_PASSWORD') ?: '', + ]); + $client->database(self::database()); + + return $client; + } + + public static function driver(Mode $mode): Driver + { + return new Driver( + host: getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', + port: (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + database: self::database(), + mode: $mode, + username: getenv('CLICKHOUSE_USER') ?: 'default', + password: getenv('CLICKHOUSE_PASSWORD') ?: '', + ); + } + + /** + * Drops the version table, then runs the mode's setup.sql for the given table. + */ + public static function resetTable(Mode $mode, string $table = 'migration'): void + { + $client = self::client(); + $client->write("DROP TABLE IF EXISTS {$table}"); + + $file = __DIR__ . '/../../src/connection/clickhouse/' . $mode->setupDir() . '/migration/setup.sql'; + $sql = (string) file_get_contents($file); + foreach (array_filter(array_map('trim', explode(';', $sql))) as $statement) { + $client->write(str_replace('%SYSTEM_TABLE%', $table, $statement)); + } + } + + public static function dropTable(string $table = 'migration'): void + { + self::client()->write("DROP TABLE IF EXISTS {$table}"); + } + + private static function database(): string + { + return getenv('CLICKHOUSE_DB') ?: 'main'; + } +} +``` + +Note: `resetTable()` and `driver()` reference `Mode` / `Driver` which are created in later tasks. That is fine — they are not exercised until those tasks. `SmokeTest` below uses only `client()`. + +- [ ] **Step 3: Write the smoke test `tests/SmokeTest.php`** + +```php +ping(true)); + } + + public function selectOneReturnsOne(): void + { + $rows = ClickHouse::client()->select('SELECT 1 AS one')->rows(); + + Assert::same((int) $rows[0]['one'], 1); + } +} +``` + +- [ ] **Step 4: Retarget the `Makefile` `tests:` target to docker compose** + +Replace the body of `tests:` (currently `Makefile:90-111`) with: + +```make +tests: + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose up -d --build --wait clickhouse + - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/testo \ + --coverage --log-junit=/app/runtime/coverage/junit.xml + - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/infection \ + --coverage=/app/runtime/coverage \ + --threads=max \ + --skip-initial-tests + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose down -v +``` + +And replace the body of `infection:` (currently `Makefile:73-88`) with: + +```make +infection: + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose up -d --build --wait clickhouse + - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/infection \ + --coverage=/app/runtime/coverage \ + --threads=max \ + --skip-initial-tests + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose down -v +``` + +- [ ] **Step 5: Add a ClickHouse service to CI `.github/workflows/tests.yml`** + +Under `jobs.tests`, add a `services:` block and pass connection env to the test step. The full file becomes: + +```yaml +name: PHPUnit + +on: [ pull_request ] + +jobs: + tests: + name: unit tests + runs-on: ubuntu-latest + + services: + clickhouse: + image: clickhouse/clickhouse-server:26.3-alpine + ports: + - 8123:8123 + env: + CLICKHOUSE_DB: main + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + options: >- + --health-cmd "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + strategy: + fail-fast: false + matrix: + php-version: + - "8.3" + - "8.4" + - "8.5" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-version }} + extensions: curl xdebug + coverage: xdebug + env: + fail-fast: true + COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Composer install dependencies + uses: ramsey/composer-install@v3 + with: + dependency-versions: "highest" + composer-options: "--optimize-autoloader" + + - name: PHPUnit + run: vendor/bin/testo + env: + XDEBUG_MODE: coverage + CLICKHOUSE_HOST: 127.0.0.1 + CLICKHOUSE_PORT: "8123" + CLICKHOUSE_DB: main + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" +``` + +- [ ] **Step 6: Run the smoke test to verify the harness** + +Run: `make tests` (or, if Docker is available directly: `docker compose up -d --build --wait clickhouse && docker compose run --rm cli ./vendor/bin/testo`) +Expected: `SmokeTest` PASS (both methods). If Testo reports "no tests found", verify `testo.php` suite `integration` points at `tests` (it does). + +- [ ] **Step 7: Commit** + +```bash +git add compose.yaml Makefile .github/workflows/tests.yml tests/Support/ClickHouse.php tests/SmokeTest.php +git commit -m "test: add ClickHouse test harness (docker compose, CI service, smoke test)" +``` + +--- + +## Task 2: `Mode` enum + +**Files:** +- Create: `src/Mode.php` +- Create: `tests/ModeTest.php` + +- [ ] **Step 1: Write the failing test `tests/ModeTest.php`** + +```php +setupDir(), 'mutation'); + } + + public function appendOnlySetupDir(): void + { + Assert::same(Mode::AppendOnly->setupDir(), 'appendonly'); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=ModeTest` +Expected: FAIL — class `dbschemix\clickhouse\Mode` not found. + +- [ ] **Step 3: Write `src/Mode.php`** + +```php + 'mutation', + self::AppendOnly => 'appendonly', + }; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=ModeTest` +Expected: PASS (both methods). + +- [ ] **Step 5: Commit** + +```bash +git add src/Mode.php tests/ModeTest.php +git commit -m "feat: add Mode enum (Mutation, AppendOnly)" +``` + +--- + +## Task 3: `Connection` + `Transaction` (client adapter) + +**Files:** +- Create: `src/internal/ClientStatement.php` +- Create: `src/internal/Connection.php` +- Create: `src/internal/Transaction.php` +- Create: `tests/ConnectionTest.php` + +- [ ] **Step 1: Write the failing test `tests/ConnectionTest.php`** + +```php +write('DROP TABLE IF EXISTS conn_test'); + $client->write('CREATE TABLE conn_test (name String, version UInt64) ENGINE = Memory'); + } + + public function execInsertsAndFetchRecordMapsNameToVersion(): void + { + $connection = new Connection(ClickHouse::client()); + + $connection->exec( + 'INSERT INTO conn_test (name, version) VALUES (:name, :version)', + ['name' => 'a.sql', 'version' => 7], + ); + $connection->exec( + 'INSERT INTO conn_test (name, version) VALUES (:name, :version)', + ['name' => 'b.sql', 'version' => 9], + ); + + $applied = $connection->fetchRecord('SELECT name, version FROM conn_test ORDER BY name'); + + Assert::same($applied, ['a.sql' => 7, 'b.sql' => 9]); + } + + public function fetchRecordReturnsEmptyArrayWhenNoRows(): void + { + $connection = new Connection(ClickHouse::client()); + + Assert::same($connection->fetchRecord('SELECT name, version FROM conn_test'), []); + } + + public function beginTransactionReturnsTransaction(): void + { + $connection = new Connection(ClickHouse::client()); + + Assert::instanceOf($connection->beginTransaction(), TransactionInterface::class); + } + + public function transactionIsNoOp(): void + { + $transaction = new Transaction(ClickHouse::client()); + + Assert::false($transaction->isActive()); + Assert::true($transaction->commit()); + Assert::false($transaction->rollback()); + } + + public function transactionExecRunsImmediately(): void + { + $transaction = new Transaction(ClickHouse::client()); + + $transaction->exec( + 'INSERT INTO conn_test (name, version) VALUES (:name, :version)', + ['name' => 'c.sql', 'version' => 3], + ); + + Assert::same( + $transaction->fetchRecord('SELECT name, version FROM conn_test'), + ['c.sql' => 3], + ); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=ConnectionTest` +Expected: FAIL — `dbschemix\clickhouse\internal\Connection` not found. + +- [ ] **Step 3: Write `src/internal/ClientStatement.php`** + +```php + $params + * @return array + */ + #[Override] + public function fetchRecord(string $query, array $params = []): array + { + /** @var list $rows */ + $rows = $this->client->select($query, $params)->rows(); + + $result = []; + foreach ($rows as $row) { + $result[$row['name']] = (int) $row['version']; + } + + /** @var array */ + return $result; + } + + /** + * @param non-empty-string $query + * @param array $params + */ + #[Override] + public function exec(string $query, array $params = []): void + { + $this->client->write($query, $params); + } +} +``` + +- [ ] **Step 4: Write `src/internal/Connection.php`** + +```php +client); + } +} +``` + +- [ ] **Step 5: Write `src/internal/Transaction.php`** + +```php +command()->fetchApplied(), []); + } + + public function upRecordsVersionAndFetchAppliedReturnsIt(): void + { + $command = $this->command(); + + $applied = $command->up(new Context( + dbName: 'clickhouse/main', + filename: '0001_a.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_a (id UInt64) ENGINE = Memory', + version: 100, + )); + + Assert::true($applied); + Assert::same($command->fetchApplied(), ['0001_a.sql' => 100]); + } + + public function downRemovesTheVersionRow(): void + { + $command = $this->command(); + $context = new Context( + dbName: 'clickhouse/main', + filename: '0001_a.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_a (id UInt64) ENGINE = Memory', + version: 100, + ); + $command->up($context); + + $applied = $command->down(new Context( + dbName: 'clickhouse/main', + filename: '0001_a.sql', + query: 'DROP TABLE IF EXISTS demo_a', + version: 100, + )); + + Assert::true($applied); + Assert::same($command->fetchApplied(), []); + } + + public function fetchAppliedRespectsVersionAndLimit(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + + Assert::same($command->fetchApplied(new Options(version: 20)), ['b.sql' => 20]); + Assert::count($command->fetchApplied(new Options(limit: 1)), 1); + } + + public function dryRunReturnsFalseAndRecordsNothing(): void + { + $command = $this->command(); + + $result = $command->up(new Context( + dbName: 'd', + filename: 'a.sql', + query: 'SELECT 1', + version: 10, + dryRun: true, + )); + + Assert::false($result); + Assert::same($command->fetchApplied(), []); + } + + public function execRunsQueryWithoutBookkeeping(): void + { + $command = $this->command(); + + $result = $command->exec(new Context( + dbName: 'd', + filename: 'fixture.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_fx (id UInt64) ENGINE = Memory', + )); + + Assert::true($result); + Assert::same($command->fetchApplied(), []); + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=MutationCommandTest` +Expected: FAIL — `MutationCommand` not found. + +- [ ] **Step 4: Write `src/internal/command/AbstractCommand.php`** + +```php +dryRun) { + return false; + } + + $this->connection->exec($context->query); + + return true; + } + + /** + * @return non-empty-string current UTC time, microsecond precision + */ + final protected function atime(): string + { + return (new DateTimeImmutable())->format('Y-m-d H:i:s.u'); + } + + /** + * Shared version/limit fragments for fetchApplied. + * + * @return array{0: string, 1: string, 2: array} + * [0] = "version = :version" or "", [1] = "LIMIT n" or "", [2] = params + */ + final protected function filters(Options $options): array + { + $where = ''; + $params = []; + if ($options->version > 0) { + $where = 'version = :version'; + $params['version'] = $options->version; + } + + $limit = $options->limit > 0 ? 'LIMIT ' . $options->limit : ''; + + return [$where, $limit, $params]; + } +} +``` + +- [ ] **Step 5: Write `src/internal/command/MutationCommand.php`** + +```php +filters($options); + + $query = 'SELECT name, version FROM ' . $this->config->table . ' FINAL' + . ($where !== '' ? ' WHERE ' . $where : '') + . ' ORDER BY atime DESC, name DESC' + . ($limit !== '' ? ' ' . $limit : ''); + + return $this->connection->fetchRecord($query, $params); + } + + #[Override] + public function up(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->connection->exec( + 'INSERT INTO ' . $this->config->table + . ' (name, version, atime) VALUES (:name, :version, :atime)', + [ + 'name' => $context->filename, + 'version' => $context->version, + 'atime' => $this->atime(), + ], + ); + + return true; + } + + #[Override] + public function down(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->connection->exec( + 'ALTER TABLE ' . $this->config->table + . ' DELETE WHERE name = :name SETTINGS mutations_sync = ' . self::MUTATIONS_SYNC, + ['name' => $context->filename], + ); + + return true; + } +} +``` + +- [ ] **Step 6: Run test to verify it passes** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=MutationCommandTest` +Expected: PASS (all methods). + +- [ ] **Step 7: Commit** + +```bash +git add src/connection/clickhouse/mutation/migration/setup.sql src/internal/command/AbstractCommand.php src/internal/command/MutationCommand.php tests/command/MutationCommandTest.php +git commit -m "feat: add Mutation mode (ReplacingMergeTree command + setup.sql)" +``` + +--- + +## Task 5: `AppendOnly` mode — setup.sql + JournalCommand + +**Files:** +- Create: `src/connection/clickhouse/appendonly/migration/setup.sql` +- Create: `src/internal/command/JournalCommand.php` +- Create: `tests/command/JournalCommandTest.php` + +- [ ] **Step 1: Write `src/connection/clickhouse/appendonly/migration/setup.sql`** + +```sql +CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% +( + name String, + version UInt64 DEFAULT 0, + atime DateTime64(6) DEFAULT now64(6), + active UInt8 DEFAULT 1 +) +ENGINE = MergeTree +ORDER BY (name, atime) +``` + +- [ ] **Step 2: Write the failing test `tests/command/JournalCommandTest.php`** + +```php +command()->fetchApplied(), []); + } + + public function upAppendsAndFetchAppliedReturnsLatestActive(): void + { + $command = $this->command(); + + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + Assert::same($command->fetchApplied(), ['a.sql' => 100]); + } + + public function downAppendsTombstoneAndFetchAppliedExcludesIt(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + $command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + Assert::same($command->fetchApplied(), []); + } + + public function reUpAfterDownIsActiveAgain(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + $command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 200)); + + Assert::same($command->fetchApplied(), ['a.sql' => 200]); + } + + public function fetchAppliedRespectsVersionAndLimit(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + + Assert::same($command->fetchApplied(new Options(version: 20)), ['b.sql' => 20]); + Assert::count($command->fetchApplied(new Options(limit: 1)), 1); + } + + public function dryRunReturnsFalseAndRecordsNothing(): void + { + $command = $this->command(); + + $result = $command->up(new Context( + dbName: 'd', + filename: 'a.sql', + query: 'SELECT 1', + version: 10, + dryRun: true, + )); + + Assert::false($result); + Assert::same($command->fetchApplied(), []); + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=JournalCommandTest` +Expected: FAIL — `JournalCommand` not found. + +- [ ] **Step 4: Write `src/internal/command/JournalCommand.php`** + +```php +filters($options); + + $query = 'SELECT name, version FROM (' + . 'SELECT name,' + . ' argMax(version, atime) AS version,' + . ' argMax(active, atime) AS active,' + . ' max(atime) AS atime' + . ' FROM ' . $this->config->table + . ' GROUP BY name' + . ') WHERE active = 1' + . ($where !== '' ? ' AND ' . $where : '') + . ' ORDER BY atime DESC, name DESC' + . ($limit !== '' ? ' ' . $limit : ''); + + return $this->connection->fetchRecord($query, $params); + } + + #[Override] + public function up(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->insertJournal($context->filename, $context->version, 1); + + return true; + } + + #[Override] + public function down(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->insertJournal($context->filename, $context->version, 0); + + return true; + } + + /** + * @param non-empty-string $name + * @param non-negative-int $version + */ + private function insertJournal(string $name, int $version, int $active): void + { + $this->connection->exec( + 'INSERT INTO ' . $this->config->table + . ' (name, version, atime, active) VALUES (:name, :version, :atime, :active)', + [ + 'name' => $name, + 'version' => $version, + 'atime' => $this->atime(), + 'active' => $active, + ], + ); + } +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=JournalCommandTest` +Expected: PASS (all methods). + +- [ ] **Step 6: Commit** + +```bash +git add src/connection/clickhouse/appendonly/migration/setup.sql src/internal/command/JournalCommand.php tests/command/JournalCommandTest.php +git commit -m "feat: add AppendOnly mode (journal command + setup.sql)" +``` + +--- + +## Task 6: `Driver` + +**Files:** +- Create: `src/Driver.php` +- Create: `tests/DriverTest.php` + +- [ ] **Step 1: Write the failing test `tests/DriverTest.php`** + +```php +getName(), 'clickhouse'); + } + + public function sourceNameIsLowercasedDatabase(): void + { + $driver = new \dbschemix\clickhouse\Driver( + host: '127.0.0.1', + port: 8123, + database: 'MainDB', + mode: Mode::Mutation, + ); + + Assert::same($driver->getSourceName(), 'maindb'); + } + + public function emptyDatabaseThrows(): void + { + try { + new \dbschemix\clickhouse\Driver(host: '127.0.0.1', port: 8123, database: '', mode: Mode::Mutation); + Assert::fail('expected ConfigurationException'); + } catch (ConfigurationException) { + Assert::true(true); + } + } + + public function setupPathMatchesMutationMode(): void + { + $path = ClickHouse::driver(Mode::Mutation)->getSetupPath(); + + Assert::true(str_ends_with($path, '/src/connection/clickhouse/mutation/migration/')); + Assert::true(is_file($path . 'setup.sql')); + } + + public function setupPathMatchesAppendOnlyMode(): void + { + $path = ClickHouse::driver(Mode::AppendOnly)->getSetupPath(); + + Assert::true(str_ends_with($path, '/src/connection/clickhouse/appendonly/migration/')); + Assert::true(is_file($path . 'setup.sql')); + } + + public function makeCommandReturnsMutationCommandForMutationMode(): void + { + Assert::instanceOf( + ClickHouse::driver(Mode::Mutation)->makeCommand(new Config()), + MutationCommand::class, + ); + } + + public function makeCommandReturnsJournalCommandForAppendOnlyMode(): void + { + Assert::instanceOf( + ClickHouse::driver(Mode::AppendOnly)->makeCommand(new Config()), + JournalCommand::class, + ); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=DriverTest` +Expected: FAIL — `dbschemix\clickhouse\Driver` not found. + +- [ ] **Step 3: Write `src/Driver.php`** + +```php + $options phpClickHouse passthrough (settings + connectParams extras) + * @throws ConfigurationException + */ + public function __construct( + string $host, + int $port, + string $database, + private readonly Mode $mode, + ?string $username = null, + ?string $password = null, + array $options = [], + ) { + if ($database === '') { + throw new ConfigurationException('ClickHouseDriver: database must not be empty.'); + } + + $this->dbname = strtolower($database); + + /** @var array $settings */ + $settings = $options['settings'] ?? []; + unset($options['settings']); + + $connectParams = [ + 'host' => $host, + 'port' => $port, + 'username' => $username ?? 'default', + 'password' => $password ?? '', + ] + $options; + + $db = $database; + $this->clientFactory = static function () use ($connectParams, $settings, $db): Client { + $client = new Client($connectParams, $settings); + $client->database($db); + + return $client; + }; + } + + #[Override] + public function getName(): string + { + return 'clickhouse'; + } + + #[Override] + public function getSourceName(): string + { + return $this->dbname; + } + + #[Override] + public function getSetupPath(): string + { + return path('dbschemix/clickhouse') + . '/src/connection/clickhouse/' . $this->mode->setupDir() . '/migration/'; + } + + #[Override] + public function makeCommand(Config $config): CommandInterface + { + return $this->makeConcreteCommand($this->makeConnection(), $config); + } + + private function makeConcreteCommand(ConnectionInterface $connection, Config $config): AbstractCommand + { + return match ($this->mode) { + Mode::Mutation => new MutationCommand($connection, $config), + Mode::AppendOnly => new JournalCommand($connection, $config), + }; + } + + /** + * @infection-ignore-all + * @throws ConnectionException + */ + private function makeConnection(): ConnectionInterface + { + $timeout = 300; + + if (!$this->connectionInstance instanceof Connection || $this->connectionTimer < time()) { + $this->connectionTimer = time() + $timeout; + try { + return $this->connectionInstance = new Connection(($this->clientFactory)()); + } catch (Throwable $exception) { + throw new ConnectionException($this, $exception); + } + } + + return $this->connectionInstance; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=DriverTest` +Expected: PASS (all methods). + +- [ ] **Step 5: Commit** + +```bash +git add src/Driver.php tests/DriverTest.php +git commit -m "feat: add ClickHouse Driver wiring Mode to Command and setup path" +``` + +--- + +## Task 7: End-to-end Migrator workflow tests (both modes) + +Verify the full core `Migrator` flow against real ClickHouse for each mode: `init` (runs setup.sql), `up`, `fetchApplied`, `down`, `redo`. Both test classes share one parameterizable body; they differ only in the `Mode` and migration directory name. + +**Files:** +- Create: `tests/workflow/MigratorMutationTest.php` +- Create: `tests/workflow/MigratorAppendOnlyTest.php` + +- [ ] **Step 1: Write `tests/workflow/MigratorMutationTest.php`** + +```php +write('DROP TABLE IF EXISTS demo_orders'); + } + + #[AfterClass] + public static function tearDownClass(): void + { + ClickHouse::dropTable(); + ClickHouse::client()->write('DROP TABLE IF EXISTS demo_orders'); + array_map('unlink', glob(self::MIGRATION_DIR . '/*.sql') ?: []); + @rmdir(self::MIGRATION_DIR); + } + + private function migrator(): Migrator + { + $migration = new Migration( + path: self::MIGRATION_DIR, + driver: ClickHouse::driver(Mode::Mutation), + config: new Config(), + ); + + return new Migrator([$migration]); + } + + public function upThenFetchAppliedThenDown(): void + { + $migrator = $this->migrator(); + $migrator->init(); + + $migrator->up(); + $applied = $this->command()->fetchApplied(); + Assert::count($applied, 1); + Assert::true($this->tableExists('demo_orders')); + + $migrator->down(new InputOptions(version: $this->onlyVersion($applied))); + Assert::same($this->command()->fetchApplied(), []); + Assert::false($this->tableExists('demo_orders')); + } + + public function redoReappliesMigrations(): void + { + $migrator = $this->migrator(); + $migrator->init(); + $migrator->up(); + + $migrator->redo(new InputOptions(version: $this->onlyVersion($this->command()->fetchApplied()))); + + Assert::count($this->command()->fetchApplied(), 1); + Assert::true($this->tableExists('demo_orders')); + } + + private function command(): \dbschemix\core\command\CommandInterface + { + return ClickHouse::driver(Mode::Mutation)->makeCommand(new Config()); + } + + /** + * @param array $applied + * @return non-negative-int + */ + private function onlyVersion(array $applied): int + { + return (int) current($applied); + } + + private function tableExists(string $table): bool + { + $rows = ClickHouse::client() + ->select("EXISTS TABLE {$table}") + ->rows(); + + return (int) $rows[0]['result'] === 1; + } + + private static function writeMigrations(): void + { + if (!is_dir(self::MIGRATION_DIR)) { + mkdir(self::MIGRATION_DIR, 0o775, true); + } + + file_put_contents( + self::MIGRATION_DIR . '/0001_orders.sql', + "-- @up\nCREATE TABLE IF NOT EXISTS demo_orders (id UInt64) ENGINE = MergeTree ORDER BY id\n\n" + . "-- @down\nDROP TABLE IF EXISTS demo_orders\n", + ); + } +} +``` + +Note: `EXISTS TABLE x` returns a single column named `result` (0/1) in ClickHouse. + +- [ ] **Step 2: Run test to verify it fails (then passes)** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=MigratorMutationTest` +Expected: PASS — all production code it needs already exists (Tasks 2-6). If `init()` fails with "directory does not exist", confirm `getSetupPath()` returns an absolute path ending in `mutation/migration/` and that `setup.sql` is present. + +- [ ] **Step 3: Write `tests/workflow/MigratorAppendOnlyTest.php`** + +Identical to `MigratorMutationTest` with these substitutions: class name `MigratorAppendOnlyTest`; `MIGRATION_DIR = '/tmp/dbschemix-ch-appendonly'`; every `Mode::Mutation` → `Mode::AppendOnly`. + +```php +write('DROP TABLE IF EXISTS demo_orders'); + } + + #[AfterClass] + public static function tearDownClass(): void + { + ClickHouse::dropTable(); + ClickHouse::client()->write('DROP TABLE IF EXISTS demo_orders'); + array_map('unlink', glob(self::MIGRATION_DIR . '/*.sql') ?: []); + @rmdir(self::MIGRATION_DIR); + } + + private function migrator(): Migrator + { + $migration = new Migration( + path: self::MIGRATION_DIR, + driver: ClickHouse::driver(Mode::AppendOnly), + config: new Config(), + ); + + return new Migrator([$migration]); + } + + public function upThenFetchAppliedThenDown(): void + { + $migrator = $this->migrator(); + $migrator->init(); + + $migrator->up(); + $applied = $this->command()->fetchApplied(); + Assert::count($applied, 1); + Assert::true($this->tableExists('demo_orders')); + + $migrator->down(new InputOptions(version: $this->onlyVersion($applied))); + Assert::same($this->command()->fetchApplied(), []); + Assert::false($this->tableExists('demo_orders')); + } + + public function redoReappliesMigrations(): void + { + $migrator = $this->migrator(); + $migrator->init(); + $migrator->up(); + + $migrator->redo(new InputOptions(version: $this->onlyVersion($this->command()->fetchApplied()))); + + Assert::count($this->command()->fetchApplied(), 1); + Assert::true($this->tableExists('demo_orders')); + } + + private function command(): \dbschemix\core\command\CommandInterface + { + return ClickHouse::driver(Mode::AppendOnly)->makeCommand(new Config()); + } + + /** + * @param array $applied + * @return non-negative-int + */ + private function onlyVersion(array $applied): int + { + return (int) current($applied); + } + + private function tableExists(string $table): bool + { + $rows = ClickHouse::client() + ->select("EXISTS TABLE {$table}") + ->rows(); + + return (int) $rows[0]['result'] === 1; + } + + private static function writeMigrations(): void + { + if (!is_dir(self::MIGRATION_DIR)) { + mkdir(self::MIGRATION_DIR, 0o775, true); + } + + file_put_contents( + self::MIGRATION_DIR . '/0001_orders.sql', + "-- @up\nCREATE TABLE IF NOT EXISTS demo_orders (id UInt64) ENGINE = MergeTree ORDER BY id\n\n" + . "-- @down\nDROP TABLE IF EXISTS demo_orders\n", + ); + } +} +``` + +- [ ] **Step 4: Run both workflow tests** + +Run: `docker compose run --rm cli ./vendor/bin/testo --filter=Migrator` +Expected: PASS — `MigratorMutationTest` and `MigratorAppendOnlyTest`. + +- [ ] **Step 5: Commit** + +```bash +git add tests/workflow/MigratorMutationTest.php tests/workflow/MigratorAppendOnlyTest.php +git commit -m "test: add end-to-end Migrator workflow tests for both modes" +``` + +--- + +## Task 8: Runnable example + +**Files:** +- Create: `example/migration/main/0001_demo.sql` +- Create: `example/run.php` + +- [ ] **Step 1: Write the sample migration `example/migration/main/0001_demo.sql`** + +```sql +-- @up +CREATE TABLE IF NOT EXISTS demo_events +( + id UInt64, + name String, + ts DateTime DEFAULT now() +) +ENGINE = MergeTree +ORDER BY id + +-- @down +DROP TABLE IF EXISTS demo_events +``` + +- [ ] **Step 2: Write `example/run.php`** + +```php +init(); +$migrator->up(); + +$command = $driver->makeCommand(new Config()); +echo "Applied migrations (mode={$mode->name}):\n"; +foreach ($command->fetchApplied() as $name => $version) { + echo " {$name} => {$version}\n"; +} +``` + +- [ ] **Step 3: Verify the example runs** + +Run: `docker compose run --rm -e CLICKHOUSE_HOST=clickhouse cli php example/run.php mutation` +Expected output (version is a timestamp): +``` +Applied migrations (mode=Mutation): + 0001_demo.sql => <12-digit-version> +``` +Run again with `appendonly` to confirm the second mode: +Run: `docker compose run --rm -e CLICKHOUSE_HOST=clickhouse cli php example/run.php appendonly` +Expected: `Applied migrations (mode=AppendOnly):` with one entry. + +- [ ] **Step 4: Commit** + +```bash +git add example/run.php example/migration/main/0001_demo.sql +git commit -m "docs: add runnable ClickHouse migration example" +``` + +--- + +## Task 9: README + CHANGELOG + +**Files:** +- Modify: `README.md` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Add a usage section to `README.md`** + +Insert after the title line (`# Database Migrator: ClickHouse`): + +```markdown + +A [dbschemix](https://dbschemix.github.io/) migration driver for ClickHouse, +built on [smi2/phpclickhouse](https://github.com/smi2/phpClickHouse). + +## Usage + +```php +use dbschemix\clickhouse\Driver; +use dbschemix\clickhouse\Mode; +use dbschemix\core\Config; +use dbschemix\core\Migration; +use dbschemix\core\Migrator; + +$driver = new Driver( + host: '127.0.0.1', + port: 8123, + database: 'main', + mode: Mode::Mutation, // or Mode::AppendOnly + username: 'default', + password: '', + // options: ['settings' => [...], 'https' => true, ...] // phpClickHouse passthrough +); + +$migrator = new Migrator([ + new Migration(path: __DIR__ . '/migration/main', driver: $driver, config: new Config()), +]); + +$migrator->init(); // creates the version table +$migrator->up(); // applies pending migrations +``` + +### Modes + +- `Mode::Mutation` — version table is a `ReplacingMergeTree`; reads use `FINAL`, + rollback removes the row via a synchronous `ALTER TABLE ... DELETE`. +- `Mode::AppendOnly` — append-only journal; every `up`/`down` appends a row and + the current state is derived with `argMax`. Nothing is physically deleted. + +> ClickHouse has no transactions: `up`/`down` run the migration and the version +> bookkeeping as separate statements, so a failure between them cannot be rolled +> back. The target database must already exist. +``` + +- [ ] **Step 2: Add a `CHANGELOG.md` entry** + +Replace the contents of `CHANGELOG.md` with: + +```markdown +# Changelog + +## [Unreleased] + +### Added +- ClickHouse migration driver with two version-table strategies selectable via + `Mode` (`Mutation`, `AppendOnly`). +- Native `Driver(host, port, database, mode, username?, password?, options?)` + configuration over `smi2/phpclickhouse`. +- End-to-end Migrator workflow tests against a real ClickHouse (local docker + compose + CI service). +``` + +- [ ] **Step 3: Commit** + +```bash +git add README.md CHANGELOG.md +git commit -m "docs: document ClickHouse driver usage and modes" +``` + +--- + +## Task 10: Static analysis + full test + Infection MSI ≥ 99 + +**Files:** (fixes only, as needed) + +- [ ] **Step 1: Run static analysis** + +Run: `make check` (runs `phpcs`, `psalm`, `phpstan`) +Expected: no errors. Common fixes if any appear: +- Run `make fix` (phpcbf + rector) to auto-correct code style. +- If Psalm flags `non-empty-lowercase-string` on `getName()`/`getSourceName()`, ensure `'clickhouse'` is returned literally and `$this->dbname` is annotated `non-empty-lowercase-string` (it is, in `Driver`). +- If Psalm flags the `rows()` shape, confirm the `@var list` annotation in `ClientStatement`. + +- [ ] **Step 2: Run the full suite + Infection** + +Run: `make tests` +Expected: all Testo tests PASS; Infection reports `Mutation Score Indicator (MSI): >= 99%`. + +- [ ] **Step 3: Triage Infection survivors** + +If MSI < 99, use the project's `infection-survivor-analyzer` agent (see `.claude/agents/infection-survivor-analyzer.md`) to read `runtime/coverage/`/`runtime/logs/infection.log`, group survivors by file, and for each either add a precise killing test or a justified `@infection-ignore-all` annotation consistent with the project convention. + +Likely-justified `@infection-ignore-all` (only if a survivor actually appears, with a one-line reason): +- `Driver::makeConnection` connection TTL timer (already annotated). +- `Transaction::rollback` (already annotated). +- `AbstractCommand::atime()` — the time format string has no behavioral test that can pin every mutant; annotate if a survivor appears. + +Prefer adding a killing test over ignoring whenever the mutant represents real behavior (e.g. `filters()` version/limit branches must be killed by `fetchAppliedRespectsVersionAndLimit`). + +- [ ] **Step 4: Re-run until green** + +Run: `make tests` +Expected: tests PASS and MSI ≥ 99. + +- [ ] **Step 5: Commit** + +```bash +git add -A +git commit -m "test: reach Infection MSI >= 99 and pass static analysis" +``` + +- [ ] **Step 6 (optional): Request review** + +The repo ships a `migration-safety-reviewer` agent that audits changes to the transaction/command layer against the dialect contract. Consider running it (or `superpowers:requesting-code-review`) before merging. + +--- + +## Self-Review (performed while writing this plan) + +**Spec coverage:** Mode enum (Task 2), native Driver constructor (Task 6), Connection over Client (Task 3), no-op Transaction (Task 3), AbstractCommand + MutationCommand + JournalCommand (Tasks 4-5), two setup.sql (Tasks 4-5), real-ClickHouse Testo tests + workflow (Tasks 1, 3-7), `.docker/ClickHouse` wired into make tests + CI (Task 1), Infection MSI ≥ 99 (Task 10), README/CHANGELOG (Task 9), example (Task 8), database assumed to exist (Driver throws only on empty name; no `CREATE DATABASE`). All spec sections map to a task. + +**Type consistency:** `Mode::setupDir()` returns `'mutation'`/`'appendonly'` and is consumed identically by `Driver::getSetupPath()` and the test helper directory layout. `ClientStatement::fetchRecord()` returns `array` matching `StatementInterface` and `CommandInterface::fetchApplied()`. `AbstractCommand::filters()` returns `[string, string, array]` consumed identically in both commands. `Driver::makeConcreteCommand()` returns `AbstractCommand` (parent of both concrete commands). Constructor parameter order (`host, port, database, mode, username?, password?, options?`) is identical in `Driver`, the test helper, and the example. + +**Placeholder scan:** No TBD/TODO; every code step contains complete code; the AppendOnly workflow test is written out in full (not "similar to Task 7"). diff --git a/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md b/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md index 0d7e0cd..f671097 100644 --- a/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md +++ b/docs/superpowers/specs/2026-05-28-clickhouse-driver-design.md @@ -145,9 +145,10 @@ The driver's own `Command` implementations run statements **directly** through `AbstractCommand` holds `ConnectionInterface` + `Config`, the `dryRun` guard, the shared `exec(Context)` (used for fixtures/repeatable, identical in both modes), binding/param helpers, and an `atime()` helper that returns the current -UTC time at millisecond precision (`(new DateTimeImmutable())->format('Y-m-d H:i:s.v')`) +UTC time at microsecond precision (`(new DateTimeImmutable())->format('Y-m-d H:i:s.u')`) — used as the `:atime` binding for every bookkeeping `INSERT` (both modes, -both `up` and `down`). Subclasses override `fetchApplied`, `up`, `down`. `fetchApplied` builds mode-specific SQL and delegates to +both `up` and `down`). Both version tables use `DateTime64(6)` so the value +parses without truncation and orders reliably. Subclasses override `fetchApplied`, `up`, `down`. `fetchApplied` builds mode-specific SQL and delegates to `$connection->fetchRecord($sql, $params)`. All bookkeeping placeholders use the core's `:name`/`:version`/`:atime` style. @@ -161,8 +162,8 @@ core's `:name`/`:version`/`:atime` style. CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% ( name String, - version UInt64 DEFAULT 0, - atime DateTime DEFAULT now() + version UInt64 DEFAULT 0, + atime DateTime64(6) DEFAULT now64(6) ) ENGINE = ReplacingMergeTree(atime) ORDER BY name; @@ -192,7 +193,7 @@ CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% ( name String, version UInt64 DEFAULT 0, - atime DateTime64(3) DEFAULT now64(3), + atime DateTime64(6) DEFAULT now64(6), active UInt8 DEFAULT 1 -- 1 = applied (up), 0 = rolled back (down) ) ENGINE = MergeTree @@ -219,7 +220,7 @@ ORDER BY (name, atime); return `true`. - `down(Context)`: `dryRun → false`; else `exec($context->query)` then `INSERT INTO %TABLE% (name, version, atime, active) VALUES (:name, :version, :atime, 0)`, - where `:atime` is the current time from `atime()` (millisecond precision) so + where `:atime` is the current time from `atime()` (microsecond precision) so `argMax(atime)` selects the rollback as the latest event; return `true`. ## Known limitations (documented) @@ -228,10 +229,10 @@ ORDER BY (name, atime); version bookkeeping) without a transaction. If the bookkeeping statement fails after a successful migration, it cannot be rolled back. This is inherent to ClickHouse. -- **AppendOnly ordering** relies on millisecond `atime` precision to order - up/down of the same file. If two events collide within the same millisecond - the derived state may be wrong; precision can be raised to `DateTime64(6)` if - this proves a problem in practice. +- **AppendOnly ordering** relies on microsecond `atime` precision + (`DateTime64(6)`) to order up/down of the same file. If two events collide + within the same microsecond the derived state may be wrong; in practice + sequential HTTP round-trips are far enough apart that this does not occur. - **Mutation latency.** `ALTER TABLE ... DELETE` is a mutation; we force synchronous execution with `SETTINGS mutations_sync = 2`. From 5d39329d24336c3b7bd06f741efbc20c20ee0b2d Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:30:57 +0000 Subject: [PATCH 03/21] test: add ClickHouse test harness (docker compose, CI service, smoke test) --- .github/workflows/tests.yml | 23 ++++++++++++- Makefile | 50 ++++++++------------------- compose.yaml | 33 ++++++++++++++++++ tests/SmokeTest.php | 25 ++++++++++++++ tests/Support/ClickHouse.php | 66 ++++++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 37 deletions(-) create mode 100644 compose.yaml create mode 100644 tests/SmokeTest.php create mode 100644 tests/Support/ClickHouse.php diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c4d7c8c..5379463 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,22 @@ jobs: name: unit tests runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server:26.3-alpine + ports: + - 8123:8123 + env: + CLICKHOUSE_DB: main + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + options: >- + --health-cmd "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + strategy: fail-fast: false matrix: @@ -23,7 +39,7 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-version }} - extensions: pdo xdebug + extensions: curl xdebug coverage: xdebug env: fail-fast: true @@ -39,3 +55,8 @@ jobs: run: vendor/bin/testo env: XDEBUG_MODE: coverage + CLICKHOUSE_HOST: 127.0.0.1 + CLICKHOUSE_PORT: "8123" + CLICKHOUSE_DB: main + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" diff --git a/Makefile b/Makefile index 9adf694..cfd3cc2 100644 --- a/Makefile +++ b/Makefile @@ -71,44 +71,22 @@ check: ## run analysis tools make phpstan infection: - docker build \ - --build-arg PHP_VERSION=$(PHP_VERSION) \ - --build-arg USER=$(USER) \ - --build-arg WORKDIR=/app \ - --target tests \ - -t app_cli .docker/php/cli - - docker run --init -it --rm \ - -u $(USER) \ - -v "$$(pwd):/app" \ - -w /app \ - app_cli ./vendor/bin/infection \ - --coverage=/app/runtime/coverage \ - --threads=max \ - --skip-initial-tests - docker image rm -f app_cli + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose up -d --build --wait clickhouse + - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/infection \ + --coverage=/app/runtime/coverage \ + --threads=max \ + --skip-initial-tests + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose down -v tests: - docker build \ - --build-arg PHP_VERSION=$(PHP_VERSION) \ - --build-arg USER=$(USER) \ - --build-arg WORKDIR=/app \ - --target tests \ - -t app_cli .docker/php/cli - - docker run --init -it --rm \ - -u $(USER) \ - -v "$$(pwd):/app" \ - -w /app \ - app_cli ./vendor/bin/testo \ - --coverage --log-junit=/app/runtime/coverage/junit.xml - - docker run --init -it --rm \ - -u $(USER) \ - -v "$$(pwd):/app" \ - -w /app \ - app_cli ./vendor/bin/infection \ - --coverage=/app/runtime/coverage \ - --threads=max \ - --skip-initial-tests - docker image rm -f app_cli + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose up -d --build --wait clickhouse + - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/testo \ + --coverage --log-junit=/app/runtime/coverage/junit.xml + - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/infection \ + --coverage=/app/runtime/coverage \ + --threads=max \ + --skip-initial-tests + UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose down -v ## Application diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..27548f3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,33 @@ +services: + clickhouse: + build: .docker/ClickHouse + environment: + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8123/ping"] + interval: 2s + timeout: 5s + retries: 30 + + cli: + build: + context: .docker/php/cli + target: tests + args: + PHP_VERSION: "${PHP_VERSION:-8.3}" + UID: "${UID:-10001}" + WORKDIR: /app + volumes: + - ".:/app" + working_dir: /app + environment: + CLICKHOUSE_HOST: clickhouse + CLICKHOUSE_PORT: "8123" + CLICKHOUSE_DB: main + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + depends_on: + clickhouse: + condition: service_healthy diff --git a/tests/SmokeTest.php b/tests/SmokeTest.php new file mode 100644 index 0000000..5f253c8 --- /dev/null +++ b/tests/SmokeTest.php @@ -0,0 +1,25 @@ +ping(true)); + } + + public function selectOneReturnsOne(): void + { + $rows = ClickHouse::client()->select('SELECT 1 AS one')->rows(); + + Assert::same((int) $rows[0]['one'], 1); + } +} diff --git a/tests/Support/ClickHouse.php b/tests/Support/ClickHouse.php new file mode 100644 index 0000000..60cefb3 --- /dev/null +++ b/tests/Support/ClickHouse.php @@ -0,0 +1,66 @@ + getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', + 'port' => (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + 'username' => getenv('CLICKHOUSE_USER') ?: 'default', + 'password' => getenv('CLICKHOUSE_PASSWORD') ?: '', + ]); + $client->database(self::database()); + + return $client; + } + + public static function driver(Mode $mode): Driver + { + return new Driver( + host: getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', + port: (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + database: self::database(), + mode: $mode, + username: getenv('CLICKHOUSE_USER') ?: 'default', + password: getenv('CLICKHOUSE_PASSWORD') ?: '', + ); + } + + /** + * Drops the version table, then runs the mode's setup.sql for the given table. + */ + public static function resetTable(Mode $mode, string $table = 'migration'): void + { + $client = self::client(); + $client->write("DROP TABLE IF EXISTS {$table}"); + + $file = __DIR__ . '/../../src/connection/clickhouse/' . $mode->setupDir() . '/migration/setup.sql'; + $sql = (string) file_get_contents($file); + foreach (array_filter(array_map('trim', explode(';', $sql))) as $statement) { + $client->write(str_replace('%SYSTEM_TABLE%', $table, $statement)); + } + } + + public static function dropTable(string $table = 'migration'): void + { + self::client()->write("DROP TABLE IF EXISTS {$table}"); + } + + private static function database(): string + { + return getenv('CLICKHOUSE_DB') ?: 'main'; + } +} From 3413f328a2b412505e4cffce099d516be51b36f6 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:34:50 +0000 Subject: [PATCH 04/21] test: address review (compose CLICKHOUSE_DB, guarded setup read and smoke row) --- compose.yaml | 1 + tests/SmokeTest.php | 1 + tests/Support/ClickHouse.php | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 27548f3..f741c75 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,6 +2,7 @@ services: clickhouse: build: .docker/ClickHouse environment: + CLICKHOUSE_DB: main CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: "" CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" diff --git a/tests/SmokeTest.php b/tests/SmokeTest.php index 5f253c8..1c58cc8 100644 --- a/tests/SmokeTest.php +++ b/tests/SmokeTest.php @@ -20,6 +20,7 @@ public function selectOneReturnsOne(): void { $rows = ClickHouse::client()->select('SELECT 1 AS one')->rows(); + Assert::count($rows, 1); Assert::same((int) $rows[0]['one'], 1); } } diff --git a/tests/Support/ClickHouse.php b/tests/Support/ClickHouse.php index 60cefb3..930b546 100644 --- a/tests/Support/ClickHouse.php +++ b/tests/Support/ClickHouse.php @@ -48,7 +48,10 @@ public static function resetTable(Mode $mode, string $table = 'migration'): void $client->write("DROP TABLE IF EXISTS {$table}"); $file = __DIR__ . '/../../src/connection/clickhouse/' . $mode->setupDir() . '/migration/setup.sql'; - $sql = (string) file_get_contents($file); + $sql = file_get_contents($file); + if ($sql === false) { + throw new \RuntimeException("Cannot read setup file: {$file}"); + } foreach (array_filter(array_map('trim', explode(';', $sql))) as $statement) { $client->write(str_replace('%SYSTEM_TABLE%', $table, $statement)); } From 919ff5e1bfb01db2280b5e0a1f1877bd5f1b54a0 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:37:14 +0000 Subject: [PATCH 05/21] feat: add Mode enum (Mutation, AppendOnly) --- src/Mode.php | 34 ++++++++++++++++++++++++++++++++++ tests/ModeTest.php | 23 +++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 src/Mode.php create mode 100644 tests/ModeTest.php diff --git a/src/Mode.php b/src/Mode.php new file mode 100644 index 0000000..7966024 --- /dev/null +++ b/src/Mode.php @@ -0,0 +1,34 @@ + 'mutation', + self::AppendOnly => 'appendonly', + }; + } +} diff --git a/tests/ModeTest.php b/tests/ModeTest.php new file mode 100644 index 0000000..defa453 --- /dev/null +++ b/tests/ModeTest.php @@ -0,0 +1,23 @@ +setupDir(), 'mutation'); + } + + public function appendOnlySetupDir(): void + { + Assert::same(Mode::AppendOnly->setupDir(), 'appendonly'); + } +} From ac23fc3d8408d0f30ea65576618a7ac7bc6445ff Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:40:43 +0000 Subject: [PATCH 06/21] feat: add Connection and no-op Transaction over ClickHouseDB\Client --- src/internal/ClientStatement.php | 45 ++++++++++++++++++ src/internal/Connection.php | 28 +++++++++++ src/internal/Transaction.php | 46 ++++++++++++++++++ tests/ConnectionTest.php | 81 ++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100644 src/internal/ClientStatement.php create mode 100644 src/internal/Connection.php create mode 100644 src/internal/Transaction.php create mode 100644 tests/ConnectionTest.php diff --git a/src/internal/ClientStatement.php b/src/internal/ClientStatement.php new file mode 100644 index 0000000..654e284 --- /dev/null +++ b/src/internal/ClientStatement.php @@ -0,0 +1,45 @@ + $params + * @return array + */ + #[Override] + public function fetchRecord(string $query, array $params = []): array + { + /** @var list $rows */ + $rows = $this->client->select($query, $params)->rows(); + + $result = []; + foreach ($rows as $row) { + $result[$row['name']] = (int) $row['version']; + } + + /** @var array */ + return $result; + } + + /** + * @param non-empty-string $query + * @param array $params + */ + #[Override] + public function exec(string $query, array $params = []): void + { + $this->client->write($query, $params); + } +} diff --git a/src/internal/Connection.php b/src/internal/Connection.php new file mode 100644 index 0000000..ad017e1 --- /dev/null +++ b/src/internal/Connection.php @@ -0,0 +1,28 @@ +client); + } +} diff --git a/src/internal/Transaction.php b/src/internal/Transaction.php new file mode 100644 index 0000000..51cadbf --- /dev/null +++ b/src/internal/Transaction.php @@ -0,0 +1,46 @@ +write('DROP TABLE IF EXISTS conn_test'); + $client->write('CREATE TABLE conn_test (name String, version UInt64) ENGINE = Memory'); + } + + public function execInsertsAndFetchRecordMapsNameToVersion(): void + { + $connection = new Connection(ClickHouse::client()); + + $connection->exec( + 'INSERT INTO conn_test (name, version) VALUES (:name, :version)', + ['name' => 'a.sql', 'version' => 7], + ); + $connection->exec( + 'INSERT INTO conn_test (name, version) VALUES (:name, :version)', + ['name' => 'b.sql', 'version' => 9], + ); + + $applied = $connection->fetchRecord('SELECT name, version FROM conn_test ORDER BY name'); + + Assert::same($applied, ['a.sql' => 7, 'b.sql' => 9]); + } + + public function fetchRecordReturnsEmptyArrayWhenNoRows(): void + { + $connection = new Connection(ClickHouse::client()); + + Assert::same($connection->fetchRecord('SELECT name, version FROM conn_test'), []); + } + + public function beginTransactionReturnsTransaction(): void + { + $connection = new Connection(ClickHouse::client()); + + Assert::instanceOf($connection->beginTransaction(), TransactionInterface::class); + } + + public function transactionIsNoOp(): void + { + $transaction = new Transaction(ClickHouse::client()); + + Assert::false($transaction->isActive()); + Assert::true($transaction->commit()); + Assert::false($transaction->rollback()); + } + + public function transactionExecRunsImmediately(): void + { + $transaction = new Transaction(ClickHouse::client()); + + $transaction->exec( + 'INSERT INTO conn_test (name, version) VALUES (:name, :version)', + ['name' => 'c.sql', 'version' => 3], + ); + + Assert::same( + $transaction->fetchRecord('SELECT name, version FROM conn_test'), + ['c.sql' => 3], + ); + } +} From 3293955b98141d2d0b9c8a82237f606eb00a8cc5 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:46:54 +0000 Subject: [PATCH 07/21] feat: add Mutation mode (ReplacingMergeTree command + setup.sql) --- .../clickhouse/mutation/migration/setup.sql | 8 ++ src/internal/command/AbstractCommand.php | 68 +++++++++++ src/internal/command/MutationCommand.php | 71 +++++++++++ tests/command/MutationCommandTest.php | 113 ++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 src/connection/clickhouse/mutation/migration/setup.sql create mode 100644 src/internal/command/AbstractCommand.php create mode 100644 src/internal/command/MutationCommand.php create mode 100644 tests/command/MutationCommandTest.php diff --git a/src/connection/clickhouse/mutation/migration/setup.sql b/src/connection/clickhouse/mutation/migration/setup.sql new file mode 100644 index 0000000..30ab8fd --- /dev/null +++ b/src/connection/clickhouse/mutation/migration/setup.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% +( + name String, + version UInt64 DEFAULT 0, + atime DateTime64(6) DEFAULT now64(6) +) +ENGINE = ReplacingMergeTree(atime) +ORDER BY name diff --git a/src/internal/command/AbstractCommand.php b/src/internal/command/AbstractCommand.php new file mode 100644 index 0000000..a8cb019 --- /dev/null +++ b/src/internal/command/AbstractCommand.php @@ -0,0 +1,68 @@ +dryRun) { + return false; + } + + $this->connection->exec($context->query); + + return true; + } + + /** + * @return non-empty-string current UTC time, microsecond precision + */ + final protected function atime(): string + { + return (new DateTimeImmutable())->format('Y-m-d H:i:s.u'); + } + + /** + * Shared version/limit fragments for fetchApplied. + * + * @return array{0: string, 1: string, 2: array} + * [0] = "version = :version" or "", [1] = "LIMIT n" or "", [2] = params + */ + final protected function filters(Options $options): array + { + $where = ''; + $params = []; + if ($options->version > 0) { + $where = 'version = :version'; + $params['version'] = $options->version; + } + + $limit = $options->limit > 0 ? 'LIMIT ' . $options->limit : ''; + + return [$where, $limit, $params]; + } +} diff --git a/src/internal/command/MutationCommand.php b/src/internal/command/MutationCommand.php new file mode 100644 index 0000000..6602582 --- /dev/null +++ b/src/internal/command/MutationCommand.php @@ -0,0 +1,71 @@ +filters($options); + + $query = 'SELECT name, version FROM ' . $this->config->table . ' FINAL' + . ($where !== '' ? ' WHERE ' . $where : '') + . ' ORDER BY atime DESC, name DESC' + . ($limit !== '' ? ' ' . $limit : ''); + + return $this->connection->fetchRecord($query, $params); + } + + #[Override] + public function up(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->connection->exec( + 'INSERT INTO ' . $this->config->table + . ' (name, version, atime) VALUES (:name, :version, :atime)', + [ + 'name' => $context->filename, + 'version' => $context->version, + 'atime' => $this->atime(), + ], + ); + + return true; + } + + #[Override] + public function down(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->connection->exec( + 'ALTER TABLE ' . $this->config->table + . ' DELETE WHERE name = :name SETTINGS mutations_sync = ' . self::MUTATIONS_SYNC, + ['name' => $context->filename], + ); + + return true; + } +} diff --git a/tests/command/MutationCommandTest.php b/tests/command/MutationCommandTest.php new file mode 100644 index 0000000..aa59858 --- /dev/null +++ b/tests/command/MutationCommandTest.php @@ -0,0 +1,113 @@ +command()->fetchApplied(), []); + } + + public function upRecordsVersionAndFetchAppliedReturnsIt(): void + { + $command = $this->command(); + + $applied = $command->up(new Context( + dbName: 'clickhouse/main', + filename: '0001_a.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_a (id UInt64) ENGINE = Memory', + version: 100, + )); + + Assert::true($applied); + Assert::same($command->fetchApplied(), ['0001_a.sql' => 100]); + } + + public function downRemovesTheVersionRow(): void + { + $command = $this->command(); + $context = new Context( + dbName: 'clickhouse/main', + filename: '0001_a.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_a (id UInt64) ENGINE = Memory', + version: 100, + ); + $command->up($context); + + $applied = $command->down(new Context( + dbName: 'clickhouse/main', + filename: '0001_a.sql', + query: 'DROP TABLE IF EXISTS demo_a', + version: 100, + )); + + Assert::true($applied); + Assert::same($command->fetchApplied(), []); + } + + public function fetchAppliedRespectsVersionAndLimit(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + + Assert::same($command->fetchApplied(new Options(version: 20)), ['b.sql' => 20]); + Assert::count($command->fetchApplied(new Options(limit: 1)), 1); + } + + public function dryRunReturnsFalseAndRecordsNothing(): void + { + $command = $this->command(); + + $result = $command->up(new Context( + dbName: 'd', + filename: 'a.sql', + query: 'SELECT 1', + version: 10, + dryRun: true, + )); + + Assert::false($result); + Assert::same($command->fetchApplied(), []); + } + + public function execRunsQueryWithoutBookkeeping(): void + { + $command = $this->command(); + + $result = $command->exec(new Context( + dbName: 'd', + filename: 'fixture.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_fx (id UInt64) ENGINE = Memory', + )); + + Assert::true($result); + Assert::same($command->fetchApplied(), []); + } +} From 8d77a36c22d2495d7440a10e897cd2528aeb6bf4 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 09:53:45 +0000 Subject: [PATCH 08/21] test: cover down() dry-run; declare abstract command methods explicitly --- src/internal/command/AbstractCommand.php | 12 ++++++++++++ tests/command/MutationCommandTest.php | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/internal/command/AbstractCommand.php b/src/internal/command/AbstractCommand.php index a8cb019..d9d1124 100644 --- a/src/internal/command/AbstractCommand.php +++ b/src/internal/command/AbstractCommand.php @@ -38,6 +38,18 @@ public function exec(Context $context): bool return true; } + /** + * @return array + */ + #[Override] + abstract public function fetchApplied(Options $options = new Options()): array; + + #[Override] + abstract public function up(Context $context): bool; + + #[Override] + abstract public function down(Context $context): bool; + /** * @return non-empty-string current UTC time, microsecond precision */ diff --git a/tests/command/MutationCommandTest.php b/tests/command/MutationCommandTest.php index aa59858..474fe16 100644 --- a/tests/command/MutationCommandTest.php +++ b/tests/command/MutationCommandTest.php @@ -97,6 +97,23 @@ public function dryRunReturnsFalseAndRecordsNothing(): void Assert::same($command->fetchApplied(), []); } + public function dryRunOnDownReturnsFalseAndKeepsTheRow(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + + $result = $command->down(new Context( + dbName: 'd', + filename: 'a.sql', + query: 'SELECT 1', + version: 10, + dryRun: true, + )); + + Assert::false($result); + Assert::same($command->fetchApplied(), ['a.sql' => 10]); + } + public function execRunsQueryWithoutBookkeeping(): void { $command = $this->command(); From c9ca2b0fbc22a1f92582eb89d6a2abb330eabd5d Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:01:00 +0000 Subject: [PATCH 09/21] feat: add AppendOnly mode (journal command + setup.sql) --- .../clickhouse/appendonly/migration/setup.sql | 9 ++ src/internal/command/JournalCommand.php | 83 +++++++++++++ tests/command/JournalCommandTest.php | 109 ++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 src/connection/clickhouse/appendonly/migration/setup.sql create mode 100644 src/internal/command/JournalCommand.php create mode 100644 tests/command/JournalCommandTest.php diff --git a/src/connection/clickhouse/appendonly/migration/setup.sql b/src/connection/clickhouse/appendonly/migration/setup.sql new file mode 100644 index 0000000..dd4c6c8 --- /dev/null +++ b/src/connection/clickhouse/appendonly/migration/setup.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS %SYSTEM_TABLE% +( + name String, + version UInt64 DEFAULT 0, + atime DateTime64(6) DEFAULT now64(6), + active UInt8 DEFAULT 1 +) +ENGINE = MergeTree +ORDER BY (name, atime) diff --git a/src/internal/command/JournalCommand.php b/src/internal/command/JournalCommand.php new file mode 100644 index 0000000..3a4294d --- /dev/null +++ b/src/internal/command/JournalCommand.php @@ -0,0 +1,83 @@ +filters($options); + + $query = 'SELECT name, last_version AS version FROM (' + . 'SELECT name,' + . ' argMax(version, atime) AS last_version,' + . ' argMax(active, atime) AS last_active,' + . ' max(atime) AS last_atime' + . ' FROM ' . $this->config->table + . ' GROUP BY name' + . ') WHERE last_active = 1' + . ($where !== '' ? ' AND last_version = :version' : '') + . ' ORDER BY last_atime DESC, name DESC' + . ($limit !== '' ? ' ' . $limit : ''); + + return $this->connection->fetchRecord($query, $params); + } + + #[Override] + public function up(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->insertJournal($context->filename, $context->version, 1); + + return true; + } + + #[Override] + public function down(Context $context): bool + { + if ($context->dryRun) { + return false; + } + + $this->connection->exec($context->query); + $this->insertJournal($context->filename, $context->version, 0); + + return true; + } + + /** + * @param non-empty-string $name + * @param non-negative-int $version + * @throws \Throwable + */ + private function insertJournal(string $name, int $version, int $active): void + { + $this->connection->exec( + 'INSERT INTO ' . $this->config->table + . ' (name, version, atime, active) VALUES (:name, :version, :atime, :active)', + [ + 'name' => $name, + 'version' => $version, + 'atime' => $this->atime(), + 'active' => $active, + ], + ); + } +} diff --git a/tests/command/JournalCommandTest.php b/tests/command/JournalCommandTest.php new file mode 100644 index 0000000..7a0cad2 --- /dev/null +++ b/tests/command/JournalCommandTest.php @@ -0,0 +1,109 @@ +command()->fetchApplied(), []); + } + + public function upAppendsAndFetchAppliedReturnsLatestActive(): void + { + $command = $this->command(); + + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + Assert::same($command->fetchApplied(), ['a.sql' => 100]); + } + + public function downAppendsTombstoneAndFetchAppliedExcludesIt(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + $command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + Assert::same($command->fetchApplied(), []); + } + + public function reUpAfterDownIsActiveAgain(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + $command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 200)); + + Assert::same($command->fetchApplied(), ['a.sql' => 200]); + } + + public function fetchAppliedRespectsVersionAndLimit(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + + Assert::same($command->fetchApplied(new Options(version: 20)), ['b.sql' => 20]); + Assert::count($command->fetchApplied(new Options(limit: 1)), 1); + } + + public function dryRunReturnsFalseAndRecordsNothing(): void + { + $command = $this->command(); + + $result = $command->up(new Context( + dbName: 'd', + filename: 'a.sql', + query: 'SELECT 1', + version: 10, + dryRun: true, + )); + + Assert::false($result); + Assert::same($command->fetchApplied(), []); + } + + public function dryRunOnDownReturnsFalseAndKeepsTheRow(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + $result = $command->down(new Context( + dbName: 'd', + filename: 'a.sql', + query: 'SELECT 1', + version: 100, + dryRun: true, + )); + + Assert::false($result); + Assert::same($command->fetchApplied(), ['a.sql' => 100]); + } +} From e2d13b2d4890f19e9263e2ab10f2582bec1a9394 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:09:28 +0000 Subject: [PATCH 10/21] feat: add ClickHouse Driver wiring Mode to Command and setup path --- src/Driver.php | 142 +++++++++++++++++++++++++++++++++++++++++++ tests/DriverTest.php | 77 +++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 src/Driver.php create mode 100644 tests/DriverTest.php diff --git a/src/Driver.php b/src/Driver.php new file mode 100644 index 0000000..c5af223 --- /dev/null +++ b/src/Driver.php @@ -0,0 +1,142 @@ + $options phpClickHouse passthrough (settings + connectParams extras) + * @throws ConfigurationException + */ + public function __construct( + string $host, + int $port, + string $database, + private readonly Mode $mode, + ?string $username = null, + ?string $password = null, + array $options = [], + ) { + if ($database === '') { + throw new ConfigurationException('ClickHouseDriver: database must not be empty.'); + } + + $this->dbname = strtolower($database); + + /** @var array $settings */ + $settings = $options['settings'] ?? []; + unset($options['settings']); + + $connectParams = [ + 'host' => $host, + 'port' => $port, + 'username' => $username ?? 'default', + 'password' => $password ?? '', + ] + $options; + + $db = $database; + $this->clientFactory = static function () use ($connectParams, $settings, $db): Client { + $client = new Client($connectParams, $settings); + $client->database($db); + + return $client; + }; + } + + #[Override] + public function getName(): string + { + return 'clickhouse'; + } + + #[Override] + public function getSourceName(): string + { + return $this->dbname; + } + + /** + * @throws OutOfBoundsException if the package install path cannot be resolved + */ + #[Override] + public function getSetupPath(): string + { + return path('dbschemix/clickhouse') + . '/src/connection/clickhouse/' . $this->mode->setupDir() . '/migration/'; + } + + #[Override] + public function makeCommand(Config $config): CommandInterface + { + return $this->makeConcreteCommand($this->makeConnection(), $config); + } + + private function makeConcreteCommand(ConnectionInterface $connection, Config $config): AbstractCommand + { + return match ($this->mode) { + Mode::Mutation => new MutationCommand($connection, $config), + Mode::AppendOnly => new JournalCommand($connection, $config), + }; + } + + /** + * @infection-ignore-all + * @throws ConnectionException + */ + private function makeConnection(): ConnectionInterface + { + $timeout = 300; + + if (!$this->connectionInstance instanceof Connection || $this->connectionTimer < time()) { + $this->connectionTimer = time() + $timeout; + try { + return $this->connectionInstance = new Connection(($this->clientFactory)()); + } catch (Throwable $exception) { + throw new ConnectionException($this, $exception); + } + } + + return $this->connectionInstance; + } +} diff --git a/tests/DriverTest.php b/tests/DriverTest.php new file mode 100644 index 0000000..053c03d --- /dev/null +++ b/tests/DriverTest.php @@ -0,0 +1,77 @@ +getName(), 'clickhouse'); + } + + public function sourceNameIsLowercasedDatabase(): void + { + $driver = new \dbschemix\clickhouse\Driver( + host: '127.0.0.1', + port: 8123, + database: 'MainDB', + mode: Mode::Mutation, + ); + + Assert::same($driver->getSourceName(), 'maindb'); + } + + public function emptyDatabaseThrows(): void + { + try { + new \dbschemix\clickhouse\Driver(host: '127.0.0.1', port: 8123, database: '', mode: Mode::Mutation); + Assert::fail('expected ConfigurationException'); + } catch (ConfigurationException) { + Assert::true(true); + } + } + + public function setupPathMatchesMutationMode(): void + { + $path = ClickHouse::driver(Mode::Mutation)->getSetupPath(); + + Assert::true(str_ends_with($path, '/src/connection/clickhouse/mutation/migration/')); + Assert::true(is_file($path . 'setup.sql')); + } + + public function setupPathMatchesAppendOnlyMode(): void + { + $path = ClickHouse::driver(Mode::AppendOnly)->getSetupPath(); + + Assert::true(str_ends_with($path, '/src/connection/clickhouse/appendonly/migration/')); + Assert::true(is_file($path . 'setup.sql')); + } + + public function makeCommandReturnsMutationCommandForMutationMode(): void + { + Assert::instanceOf( + ClickHouse::driver(Mode::Mutation)->makeCommand(new Config()), + MutationCommand::class, + ); + } + + public function makeCommandReturnsJournalCommandForAppendOnlyMode(): void + { + Assert::instanceOf( + ClickHouse::driver(Mode::AppendOnly)->makeCommand(new Config()), + JournalCommand::class, + ); + } +} From 26e3830d31985776267faafcab1423f9390bcdca Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:13:20 +0000 Subject: [PATCH 11/21] refactor: document getSetupPath return and extract CONNECTION_TTL constant --- src/Driver.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Driver.php b/src/Driver.php index c5af223..b7148b4 100644 --- a/src/Driver.php +++ b/src/Driver.php @@ -27,6 +27,8 @@ */ final class Driver implements DriverInterface { + private const int CONNECTION_TTL = 300; + /** * @var Closure():Client */ @@ -97,6 +99,7 @@ public function getSourceName(): string } /** + * @return non-empty-string * @throws OutOfBoundsException if the package install path cannot be resolved */ #[Override] @@ -126,10 +129,8 @@ private function makeConcreteCommand(ConnectionInterface $connection, Config $co */ private function makeConnection(): ConnectionInterface { - $timeout = 300; - if (!$this->connectionInstance instanceof Connection || $this->connectionTimer < time()) { - $this->connectionTimer = time() + $timeout; + $this->connectionTimer = time() + self::CONNECTION_TTL; try { return $this->connectionInstance = new Connection(($this->clientFactory)()); } catch (Throwable $exception) { From ff0535c4935c377cd539bacea792f034f5097120 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:16:45 +0000 Subject: [PATCH 12/21] test: add end-to-end Migrator workflow tests for both modes --- tests/workflow/MigratorAppendOnlyTest.php | 113 +++++++++++++++++++++ tests/workflow/MigratorMutationTest.php | 114 ++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 tests/workflow/MigratorAppendOnlyTest.php create mode 100644 tests/workflow/MigratorMutationTest.php diff --git a/tests/workflow/MigratorAppendOnlyTest.php b/tests/workflow/MigratorAppendOnlyTest.php new file mode 100644 index 0000000..3cd5547 --- /dev/null +++ b/tests/workflow/MigratorAppendOnlyTest.php @@ -0,0 +1,113 @@ +write('DROP TABLE IF EXISTS demo_orders'); + } + + #[AfterClass] + public static function tearDownClass(): void + { + ClickHouse::dropTable(); + ClickHouse::client()->write('DROP TABLE IF EXISTS demo_orders'); + array_map('unlink', glob(self::MIGRATION_DIR . '/*.sql') ?: []); + @rmdir(self::MIGRATION_DIR); + } + + private function migrator(): Migrator + { + $migration = new Migration( + path: self::MIGRATION_DIR, + driver: ClickHouse::driver(Mode::AppendOnly), + config: new Config(), + ); + + return new Migrator([$migration]); + } + + public function upThenFetchAppliedThenDown(): void + { + $migrator = $this->migrator(); + $migrator->init(); + + $migrator->up(); + $applied = $this->command()->fetchApplied(); + Assert::count($applied, 1); + Assert::true($this->tableExists('demo_orders')); + + $migrator->down(new InputOptions(version: $this->onlyVersion($applied))); + Assert::same($this->command()->fetchApplied(), []); + Assert::false($this->tableExists('demo_orders')); + } + + public function redoReappliesMigrations(): void + { + $migrator = $this->migrator(); + $migrator->init(); + $migrator->up(); + + $migrator->redo(new InputOptions(version: $this->onlyVersion($this->command()->fetchApplied()))); + + Assert::count($this->command()->fetchApplied(), 1); + Assert::true($this->tableExists('demo_orders')); + } + + private function command(): \dbschemix\core\command\CommandInterface + { + return ClickHouse::driver(Mode::AppendOnly)->makeCommand(new Config()); + } + + /** + * @param array $applied + * @return non-negative-int + */ + private function onlyVersion(array $applied): int + { + return (int) current($applied); + } + + private function tableExists(string $table): bool + { + $rows = ClickHouse::client() + ->select("EXISTS TABLE {$table}") + ->rows(); + + return (int) $rows[0]['result'] === 1; + } + + private static function writeMigrations(): void + { + if (!is_dir(self::MIGRATION_DIR)) { + mkdir(self::MIGRATION_DIR, 0o775, true); + } + + file_put_contents( + self::MIGRATION_DIR . '/0001_orders.sql', + "-- @up\nCREATE TABLE IF NOT EXISTS demo_orders (id UInt64) ENGINE = MergeTree ORDER BY id\n\n" + . "-- @down\nDROP TABLE IF EXISTS demo_orders\n", + ); + } +} diff --git a/tests/workflow/MigratorMutationTest.php b/tests/workflow/MigratorMutationTest.php new file mode 100644 index 0000000..41ec2f3 --- /dev/null +++ b/tests/workflow/MigratorMutationTest.php @@ -0,0 +1,114 @@ +write('DROP TABLE IF EXISTS demo_orders'); + } + + #[AfterClass] + public static function tearDownClass(): void + { + ClickHouse::dropTable(); + ClickHouse::client()->write('DROP TABLE IF EXISTS demo_orders'); + array_map('unlink', glob(self::MIGRATION_DIR . '/*.sql') ?: []); + @rmdir(self::MIGRATION_DIR); + } + + private function migrator(): Migrator + { + $migration = new Migration( + path: self::MIGRATION_DIR, + driver: ClickHouse::driver(Mode::Mutation), + config: new Config(), + ); + + return new Migrator([$migration]); + } + + public function upThenFetchAppliedThenDown(): void + { + $migrator = $this->migrator(); + $migrator->init(); + + $migrator->up(); + $applied = $this->command()->fetchApplied(); + Assert::count($applied, 1); + Assert::true($this->tableExists('demo_orders')); + + $migrator->down(new InputOptions(version: $this->onlyVersion($applied))); + Assert::same($this->command()->fetchApplied(), []); + Assert::false($this->tableExists('demo_orders')); + } + + public function redoReappliesMigrations(): void + { + $migrator = $this->migrator(); + $migrator->init(); + $migrator->up(); + + $migrator->redo(new InputOptions(version: $this->onlyVersion($this->command()->fetchApplied()))); + + Assert::count($this->command()->fetchApplied(), 1); + Assert::true($this->tableExists('demo_orders')); + } + + private function command(): \dbschemix\core\command\CommandInterface + { + return ClickHouse::driver(Mode::Mutation)->makeCommand(new Config()); + } + + /** + * @param array $applied + * @return non-negative-int + */ + private function onlyVersion(array $applied): int + { + return (int) current($applied); + } + + private function tableExists(string $table): bool + { + $rows = ClickHouse::client() + ->select("EXISTS TABLE {$table}") + ->rows(); + + return (int) $rows[0]['result'] === 1; + } + + private static function writeMigrations(): void + { + if (!is_dir(self::MIGRATION_DIR)) { + mkdir(self::MIGRATION_DIR, 0o775, true); + } + + file_put_contents( + self::MIGRATION_DIR . '/0001_orders.sql', + "-- @up\nCREATE TABLE IF NOT EXISTS demo_orders (id UInt64) ENGINE = MergeTree ORDER BY id\n\n" + . "-- @down\nDROP TABLE IF EXISTS demo_orders\n", + ); + } +} From 03e0beb9ac719c14f02cb4d4ff24b3e9974716a3 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:20:08 +0000 Subject: [PATCH 13/21] test: drop unused Options import in MigratorMutationTest Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/workflow/MigratorMutationTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/workflow/MigratorMutationTest.php b/tests/workflow/MigratorMutationTest.php index 41ec2f3..e560bad 100644 --- a/tests/workflow/MigratorMutationTest.php +++ b/tests/workflow/MigratorMutationTest.php @@ -7,7 +7,6 @@ use dbschemix\clickhouse\Mode; use dbschemix\clickhouse\tests\Support\ClickHouse; use dbschemix\core\Config; -use dbschemix\core\command\Options; use dbschemix\core\InputOptions; use dbschemix\core\Migration; use dbschemix\core\Migrator; From 34ffa16fb7916495d7ac70181d3820b45f1f409b Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:22:32 +0000 Subject: [PATCH 14/21] docs: add runnable ClickHouse migration example Co-Authored-By: Claude Opus 4.7 (1M context) --- example/migration/main/0001_demo.sql | 12 +++++++++ example/run.php | 40 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 example/migration/main/0001_demo.sql create mode 100644 example/run.php diff --git a/example/migration/main/0001_demo.sql b/example/migration/main/0001_demo.sql new file mode 100644 index 0000000..ae9b77d --- /dev/null +++ b/example/migration/main/0001_demo.sql @@ -0,0 +1,12 @@ +-- @up +CREATE TABLE IF NOT EXISTS demo_events +( + id UInt64, + name String, + ts DateTime DEFAULT now() +) +ENGINE = MergeTree +ORDER BY id + +-- @down +DROP TABLE IF EXISTS demo_events diff --git a/example/run.php b/example/run.php new file mode 100644 index 0000000..ef9d10d --- /dev/null +++ b/example/run.php @@ -0,0 +1,40 @@ +init(); +$migrator->up(); + +$command = $driver->makeCommand(new Config()); +echo "Applied migrations (mode={$mode->name}):\n"; +foreach ($command->fetchApplied() as $name => $version) { + echo " {$name} => {$version}\n"; +} From 425b2df3a24046245aa675d1bb937a0686324931 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:26:57 +0000 Subject: [PATCH 15/21] docs: document ClickHouse driver usage and modes Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 11 +++++++++++ README.md | 43 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29..eaf5f78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## [Unreleased] + +### Added +- ClickHouse migration driver with two version-table strategies selectable via + `Mode` (`Mutation`, `AppendOnly`). +- Native `Driver(host, port, database, mode, username?, password?, options?)` + configuration over `smi2/phpclickhouse`. +- End-to-end Migrator workflow tests against a real ClickHouse (local docker + compose + CI service). diff --git a/README.md b/README.md index a434acd..67c45f4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,46 @@ # Database Migrator: ClickHouse +A [dbschemix](https://dbschemix.github.io/) migration driver for ClickHouse, +built on [smi2/phpclickhouse](https://github.com/smi2/phpClickHouse). + +## Usage + +```php +use dbschemix\clickhouse\Driver; +use dbschemix\clickhouse\Mode; +use dbschemix\core\Config; +use dbschemix\core\Migration; +use dbschemix\core\Migrator; + +$driver = new Driver( + host: '127.0.0.1', + port: 8123, + database: 'main', + mode: Mode::Mutation, // or Mode::AppendOnly + username: 'default', + password: '', + // options: ['settings' => [...], 'https' => true, ...] // phpClickHouse passthrough +); + +$migrator = new Migrator([ + new Migration(path: __DIR__ . '/migration/main', driver: $driver, config: new Config()), +]); + +$migrator->init(); // creates the version table +$migrator->up(); // applies pending migrations +``` + +### Modes + +- `Mode::Mutation` — version table is a `ReplacingMergeTree`; reads use `FINAL`, + rollback removes the row via a synchronous `ALTER TABLE ... DELETE`. +- `Mode::AppendOnly` — append-only journal; every `up`/`down` appends a row and + the current state is derived with `argMax`. Nothing is physically deleted. + +> ClickHouse has no transactions: `up`/`down` run the migration and the version +> bookkeeping as separate statements, so a failure between them cannot be rolled +> back. The target database must already exist. + ### Static analysis To run static analysis: @@ -21,7 +62,7 @@ make fix ### Testing The package is tested with -- [PHPUnit](https://phpunit.de/) +- [testo](https://php-testo.github.io/) - [Infection](https://github.com/infection/infection) To run tests: From 70a8b0aa2e953d19bfcb6bfde1a905aff4d29773 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 10:32:30 +0000 Subject: [PATCH 16/21] chore: satisfy phpcs/psalm/phpstan across src, tests, example Co-Authored-By: Claude Opus 4.7 (1M context) --- example/run.php | 11 +++++++++-- src/internal/ClientStatement.php | 2 ++ tests/DriverTest.php | 4 ++-- tests/Support/ClickHouse.php | 16 +++++++++++++--- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/example/run.php b/example/run.php index ef9d10d..9803f1b 100644 --- a/example/run.php +++ b/example/run.php @@ -14,13 +14,20 @@ $mode = ($argv[1] ?? 'mutation') === 'appendonly' ? Mode::AppendOnly : Mode::Mutation; +$port = (int) (getenv('CLICKHOUSE_PORT') ?: '8123'); +if ($port < 1) { + $port = 8123; +} + +$password = getenv('CLICKHOUSE_PASSWORD') ?: null; + $driver = new Driver( host: getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', - port: (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + port: $port, database: getenv('CLICKHOUSE_DB') ?: 'main', mode: $mode, username: getenv('CLICKHOUSE_USER') ?: 'default', - password: getenv('CLICKHOUSE_PASSWORD') ?: '', + password: $password, ); $migration = new Migration( diff --git a/src/internal/ClientStatement.php b/src/internal/ClientStatement.php index 654e284..45129ab 100644 --- a/src/internal/ClientStatement.php +++ b/src/internal/ClientStatement.php @@ -4,6 +4,7 @@ namespace dbschemix\clickhouse\internal; +use ClickHouseDB\Exception\TransportException; use Override; /** @@ -17,6 +18,7 @@ trait ClientStatement * @param non-empty-string $query * @param array $params * @return array + * @throws TransportException */ #[Override] public function fetchRecord(string $query, array $params = []): array diff --git a/tests/DriverTest.php b/tests/DriverTest.php index 053c03d..4b97d43 100644 --- a/tests/DriverTest.php +++ b/tests/DriverTest.php @@ -38,8 +38,8 @@ public function emptyDatabaseThrows(): void try { new \dbschemix\clickhouse\Driver(host: '127.0.0.1', port: 8123, database: '', mode: Mode::Mutation); Assert::fail('expected ConfigurationException'); - } catch (ConfigurationException) { - Assert::true(true); + } catch (ConfigurationException $exception) { + Assert::instanceOf($exception, ConfigurationException::class); } } diff --git a/tests/Support/ClickHouse.php b/tests/Support/ClickHouse.php index 930b546..6d2df6b 100644 --- a/tests/Support/ClickHouse.php +++ b/tests/Support/ClickHouse.php @@ -18,7 +18,7 @@ public static function client(): Client { $client = new Client([ 'host' => getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', - 'port' => (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + 'port' => self::port(), 'username' => getenv('CLICKHOUSE_USER') ?: 'default', 'password' => getenv('CLICKHOUSE_PASSWORD') ?: '', ]); @@ -31,11 +31,11 @@ public static function driver(Mode $mode): Driver { return new Driver( host: getenv('CLICKHOUSE_HOST') ?: '127.0.0.1', - port: (int) (getenv('CLICKHOUSE_PORT') ?: '8123'), + port: self::port(), database: self::database(), mode: $mode, username: getenv('CLICKHOUSE_USER') ?: 'default', - password: getenv('CLICKHOUSE_PASSWORD') ?: '', + password: getenv('CLICKHOUSE_PASSWORD') ?: null, ); } @@ -66,4 +66,14 @@ private static function database(): string { return getenv('CLICKHOUSE_DB') ?: 'main'; } + + /** + * @return positive-int + */ + private static function port(): int + { + $port = (int) (getenv('CLICKHOUSE_PORT') ?: '8123'); + + return $port < 1 ? 8123 : $port; + } } From 63db68d6af481cfaac79084a82048f44913bb811 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 11:07:06 +0000 Subject: [PATCH 17/21] test: kill Infection survivors; ignore equivalent config/cast mutants Add targeted tests for exec side-effect, ORDER BY under LIMIT 1, FINAL dedup, non-dry-run return values, and the inactive tombstone. Justifiably ignore the credential/settings coalesce (only feeds the unexposed Client ctor) and the defensive version int-cast (value already decodes as int from the transport). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Driver.php | 25 +++++++++++++++-- src/internal/ClientStatement.php | 1 + tests/command/JournalCommandTest.php | 34 +++++++++++++++++++++++ tests/command/MutationCommandTest.php | 40 +++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/src/Driver.php b/src/Driver.php index b7148b4..20592db 100644 --- a/src/Driver.php +++ b/src/Driver.php @@ -65,7 +65,27 @@ public function __construct( } $this->dbname = strtolower($database); + $this->clientFactory = self::makeClientFactory($host, $port, $database, $username, $password, $options); + } + /** + * @param non-empty-string $host + * @param positive-int $port + * @param non-empty-string $database + * @param non-empty-string|null $username + * @param non-empty-string|null $password + * @param array $options + * @return Closure():Client + * @infection-ignore-all credential/settings coalesce only feeds the un-exposed Client ctor; not observable + */ + private static function makeClientFactory( + string $host, + int $port, + string $database, + ?string $username, + ?string $password, + array $options, + ): Closure { /** @var array $settings */ $settings = $options['settings'] ?? []; unset($options['settings']); @@ -77,10 +97,9 @@ public function __construct( 'password' => $password ?? '', ] + $options; - $db = $database; - $this->clientFactory = static function () use ($connectParams, $settings, $db): Client { + return static function () use ($connectParams, $settings, $database): Client { $client = new Client($connectParams, $settings); - $client->database($db); + $client->database($database); return $client; }; diff --git a/src/internal/ClientStatement.php b/src/internal/ClientStatement.php index 45129ab..b25a35d 100644 --- a/src/internal/ClientStatement.php +++ b/src/internal/ClientStatement.php @@ -19,6 +19,7 @@ trait ClientStatement * @param array $params * @return array * @throws TransportException + * @infection-ignore-all version already arrives as int from the JSON transport here; the (int) cast is defensive */ #[Override] public function fetchRecord(string $query, array $params = []): array diff --git a/tests/command/JournalCommandTest.php b/tests/command/JournalCommandTest.php index 7a0cad2..295b307 100644 --- a/tests/command/JournalCommandTest.php +++ b/tests/command/JournalCommandTest.php @@ -106,4 +106,38 @@ public function dryRunOnDownReturnsFalseAndKeepsTheRow(): void Assert::false($result); Assert::same($command->fetchApplied(), ['a.sql' => 100]); } + + public function fetchAppliedLimitOnePicksMostRecentlyAppliedByAtime(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + + Assert::same($command->fetchApplied(new Options(limit: 1)), ['a.sql' => 10]); + } + + public function upReturnsTrueOnNonDryRun(): void + { + Assert::true($this->command()->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100))); + } + + public function downReturnsTrueOnNonDryRun(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + Assert::true($command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100))); + } + + public function downWritesInactiveTombstone(): void + { + $command = $this->command(); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + $command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); + + $rows = ClickHouse::client() + ->select("SELECT active FROM migration WHERE name = 'a.sql' ORDER BY atime DESC LIMIT 1") + ->rows(); + Assert::same((int) $rows[0]['active'], 0); + } } diff --git a/tests/command/MutationCommandTest.php b/tests/command/MutationCommandTest.php index 474fe16..dbab6ab 100644 --- a/tests/command/MutationCommandTest.php +++ b/tests/command/MutationCommandTest.php @@ -127,4 +127,44 @@ public function execRunsQueryWithoutBookkeeping(): void Assert::true($result); Assert::same($command->fetchApplied(), []); } + + public function execActuallyRunsTheQueryAgainstClickHouse(): void + { + $command = $this->command(); + + $command->exec(new Context( + dbName: 'd', + filename: 'fixture.sql', + query: 'CREATE TABLE IF NOT EXISTS demo_exec_fx (id UInt64) ENGINE = Memory', + )); + + $rows = ClickHouse::client() + ->select("SELECT count() AS c FROM system.tables WHERE database = currentDatabase() AND name = 'demo_exec_fx'") + ->rows(); + Assert::same((int) $rows[0]['c'], 1); + + ClickHouse::client()->write('DROP TABLE IF EXISTS demo_exec_fx'); + } + + public function fetchAppliedLimitOnePicksMostRecentlyAppliedByAtime(): void + { + $command = $this->command(); + // 'a.sql' is applied last (greatest atime) but sorts first by name — + // so a missing ORDER BY atime DESC cannot coincidentally still return it. + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); + + Assert::same($command->fetchApplied(new Options(limit: 1)), ['a.sql' => 10]); + } + + public function fetchAppliedUsesFinalToKeepLatestRowPerName(): void + { + $command = $this->command(); + $client = ClickHouse::client(); + // Two parts for the same name; FINAL must collapse to the greatest-atime row. + $client->write("INSERT INTO migration (name, version, atime) VALUES ('a.sql', 10, '2020-01-01 00:00:00.000000')"); + $client->write("INSERT INTO migration (name, version, atime) VALUES ('a.sql', 20, '2020-01-02 00:00:00.000000')"); + + Assert::same($command->fetchApplied(), ['a.sql' => 20]); + } } From 7e44a2dc51d033096bff8a0532dbec0ee5d8eb4f Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 11:15:01 +0000 Subject: [PATCH 18/21] test: kill JournalCommand ORDER BY mutant via full ordered fetchApplied assertion Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/command/JournalCommandTest.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/command/JournalCommandTest.php b/tests/command/JournalCommandTest.php index 295b307..fd754a6 100644 --- a/tests/command/JournalCommandTest.php +++ b/tests/command/JournalCommandTest.php @@ -107,6 +107,23 @@ public function dryRunOnDownReturnsFalseAndKeepsTheRow(): void Assert::same($command->fetchApplied(), ['a.sql' => 100]); } + public function fetchAppliedReturnsRowsOrderedByAtimeDescending(): void + { + $command = $this->command(); + // Applied oldest -> newest, so atime(a) < atime(b) < atime(c). + // ORDER BY last_atime DESC must return them newest-first: c, b, a. + // Names ascend a,b,c, so a missing ORDER BY (storage/group order) yields a + // different key order and the strict (order-sensitive) comparison fails. + $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 1)); + $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 2)); + $command->up(new Context(dbName: 'd', filename: 'c.sql', query: 'SELECT 1', version: 3)); + + Assert::same( + $command->fetchApplied(), + ['c.sql' => 3, 'b.sql' => 2, 'a.sql' => 1], + ); + } + public function fetchAppliedLimitOnePicksMostRecentlyAppliedByAtime(): void { $command = $this->command(); From c13325538092044658c422720a9efdc1f6e0f0ac Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 11:28:22 +0000 Subject: [PATCH 19/21] test: harden atime-ordering tests; document makeConnection ignore rationale Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Driver.php | 2 +- tests/command/JournalCommandTest.php | 3 +++ tests/command/MutationCommandTest.php | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Driver.php b/src/Driver.php index 20592db..7e61908 100644 --- a/src/Driver.php +++ b/src/Driver.php @@ -143,7 +143,7 @@ private function makeConcreteCommand(ConnectionInterface $connection, Config $co } /** - * @infection-ignore-all + * @infection-ignore-all connection TTL cache policy is not observable through the public API * @throws ConnectionException */ private function makeConnection(): ConnectionInterface diff --git a/tests/command/JournalCommandTest.php b/tests/command/JournalCommandTest.php index fd754a6..a521309 100644 --- a/tests/command/JournalCommandTest.php +++ b/tests/command/JournalCommandTest.php @@ -115,7 +115,9 @@ public function fetchAppliedReturnsRowsOrderedByAtimeDescending(): void // Names ascend a,b,c, so a missing ORDER BY (storage/group order) yields a // different key order and the strict (order-sensitive) comparison fails. $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 1)); + usleep(1000); $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 2)); + usleep(1000); $command->up(new Context(dbName: 'd', filename: 'c.sql', query: 'SELECT 1', version: 3)); Assert::same( @@ -128,6 +130,7 @@ public function fetchAppliedLimitOnePicksMostRecentlyAppliedByAtime(): void { $command = $this->command(); $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + usleep(1000); $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); Assert::same($command->fetchApplied(new Options(limit: 1)), ['a.sql' => 10]); diff --git a/tests/command/MutationCommandTest.php b/tests/command/MutationCommandTest.php index dbab6ab..2c4cc85 100644 --- a/tests/command/MutationCommandTest.php +++ b/tests/command/MutationCommandTest.php @@ -152,6 +152,7 @@ public function fetchAppliedLimitOnePicksMostRecentlyAppliedByAtime(): void // 'a.sql' is applied last (greatest atime) but sorts first by name — // so a missing ORDER BY atime DESC cannot coincidentally still return it. $command->up(new Context(dbName: 'd', filename: 'b.sql', query: 'SELECT 1', version: 20)); + usleep(1000); $command->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 10)); Assert::same($command->fetchApplied(new Options(limit: 1)), ['a.sql' => 10]); From d7d4d31860084c59d8b058e8572b186161897714 Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 11:30:51 +0000 Subject: [PATCH 20/21] style: wrap long SQL literals in command tests under the 120-char limit Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/command/JournalCommandTest.php | 9 +++++++-- tests/command/MutationCommandTest.php | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/command/JournalCommandTest.php b/tests/command/JournalCommandTest.php index a521309..3a0bdcd 100644 --- a/tests/command/JournalCommandTest.php +++ b/tests/command/JournalCommandTest.php @@ -138,7 +138,9 @@ public function fetchAppliedLimitOnePicksMostRecentlyAppliedByAtime(): void public function upReturnsTrueOnNonDryRun(): void { - Assert::true($this->command()->up(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100))); + Assert::true($this->command()->up( + new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100) + )); } public function downReturnsTrueOnNonDryRun(): void @@ -156,7 +158,10 @@ public function downWritesInactiveTombstone(): void $command->down(new Context(dbName: 'd', filename: 'a.sql', query: 'SELECT 1', version: 100)); $rows = ClickHouse::client() - ->select("SELECT active FROM migration WHERE name = 'a.sql' ORDER BY atime DESC LIMIT 1") + ->select( + "SELECT active FROM migration WHERE name = 'a.sql' " + . "ORDER BY atime DESC LIMIT 1" + ) ->rows(); Assert::same((int) $rows[0]['active'], 0); } diff --git a/tests/command/MutationCommandTest.php b/tests/command/MutationCommandTest.php index 2c4cc85..914571a 100644 --- a/tests/command/MutationCommandTest.php +++ b/tests/command/MutationCommandTest.php @@ -139,7 +139,10 @@ public function execActuallyRunsTheQueryAgainstClickHouse(): void )); $rows = ClickHouse::client() - ->select("SELECT count() AS c FROM system.tables WHERE database = currentDatabase() AND name = 'demo_exec_fx'") + ->select( + "SELECT count() AS c FROM system.tables " + . "WHERE database = currentDatabase() AND name = 'demo_exec_fx'" + ) ->rows(); Assert::same((int) $rows[0]['c'], 1); @@ -163,8 +166,14 @@ public function fetchAppliedUsesFinalToKeepLatestRowPerName(): void $command = $this->command(); $client = ClickHouse::client(); // Two parts for the same name; FINAL must collapse to the greatest-atime row. - $client->write("INSERT INTO migration (name, version, atime) VALUES ('a.sql', 10, '2020-01-01 00:00:00.000000')"); - $client->write("INSERT INTO migration (name, version, atime) VALUES ('a.sql', 20, '2020-01-02 00:00:00.000000')"); + $client->write( + "INSERT INTO migration (name, version, atime) " + . "VALUES ('a.sql', 10, '2020-01-01 00:00:00.000000')" + ); + $client->write( + "INSERT INTO migration (name, version, atime) " + . "VALUES ('a.sql', 20, '2020-01-02 00:00:00.000000')" + ); Assert::same($command->fetchApplied(), ['a.sql' => 20]); } From 7644f05acdf93f202710c896778e88fa6a97687a Mon Sep 17 00:00:00 2001 From: Dmitriy Krivopalov Date: Thu, 28 May 2026 15:22:57 +0300 Subject: [PATCH 21/21] fix style --- .docker/ClickHouse/Dockerfile | 3 +++ .docker/ClickHouse/init.sh | 3 +++ Makefile | 17 ++++++++++------- compose.yaml | 19 +++++++++++++++++++ composer.json | 2 +- example/run.php | 2 +- src/internal/command/AbstractCommand.php | 2 ++ src/internal/command/JournalCommand.php | 6 +++++- src/internal/command/MutationCommand.php | 4 ++++ tests/DriverTest.php | 5 +++-- tests/Support/ClickHouse.php | 3 ++- tests/workflow/MigratorAppendOnlyTest.php | 3 ++- tests/workflow/MigratorMutationTest.php | 3 ++- 13 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 .docker/ClickHouse/Dockerfile create mode 100644 .docker/ClickHouse/init.sh diff --git a/.docker/ClickHouse/Dockerfile b/.docker/ClickHouse/Dockerfile new file mode 100644 index 0000000..e9d1800 --- /dev/null +++ b/.docker/ClickHouse/Dockerfile @@ -0,0 +1,3 @@ +FROM clickhouse/clickhouse-server:26.3-alpine + +COPY init.sh /docker-entrypoint-initdb.d/ diff --git a/.docker/ClickHouse/init.sh b/.docker/ClickHouse/init.sh new file mode 100644 index 0000000..20f570c --- /dev/null +++ b/.docker/ClickHouse/init.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +clickhouse-client --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" --query "CREATE DATABASE IF NOT EXISTS main;" diff --git a/Makefile b/Makefile index cfd3cc2..1b739db 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ VERSION ?= $$(git rev-parse --verify HEAD) USER = $$(id -u) ARGS = $(filter-out $@,$(MAKECMDGOALS)) DOCKER_RUN = docker run --init -it --rm -u ${USER} -v "$$(pwd):/app" -w /app +DOCKER_COMPOSE = UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html .PHONY: help tests fix check @@ -70,23 +71,25 @@ check: ## run analysis tools make psalm make phpstan +## Tests + infection: - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose up -d --build --wait clickhouse - - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/infection \ + $(DOCKER_COMPOSE) up -d --build --wait clickhouse + - $(DOCKER_COMPOSE) run --rm cli ./vendor/bin/infection \ --coverage=/app/runtime/coverage \ --threads=max \ --skip-initial-tests - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose down -v + $(DOCKER_COMPOSE) down -v tests: - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose up -d --build --wait clickhouse - - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/testo \ + $(DOCKER_COMPOSE) up -d --build --wait clickhouse + - $(DOCKER_COMPOSE) run --rm cli ./vendor/bin/testo \ --coverage --log-junit=/app/runtime/coverage/junit.xml - - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose run --rm cli ./vendor/bin/infection \ + - $(DOCKER_COMPOSE) run --rm cli ./vendor/bin/infection \ --coverage=/app/runtime/coverage \ --threads=max \ --skip-initial-tests - UID=$(USER) PHP_VERSION=$(PHP_VERSION) docker compose down -v + $(DOCKER_COMPOSE) down -v ## Application diff --git a/compose.yaml b/compose.yaml index f741c75..43b5ce4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,22 @@ +x-default-logging: &default-logging + driver: local + options: + max-size: "5m" + max-file: "3" + +name: dbschemix services: clickhouse: + container_name: dbschemix_clickhouse build: .docker/ClickHouse + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M + reservations: + cpus: '0.2' + memory: 256M environment: CLICKHOUSE_DB: main CLICKHOUSE_USER: default @@ -11,8 +27,10 @@ services: interval: 2s timeout: 5s retries: 30 + logging: *default-logging cli: + container_name: dbschemix_clickhouse_cli build: context: .docker/php/cli target: tests @@ -32,3 +50,4 @@ services: depends_on: clickhouse: condition: service_healthy + logging: *default-logging diff --git a/composer.json b/composer.json index 44359b0..c24538c 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,6 @@ { "name": "dbschemix/clickhouse", - "description": "dbschemix: database migration clickhouse", + "description": "dbschemix: clickhouse database migration driver", "type": "library", "license": "MIT", "authors": [ diff --git a/example/run.php b/example/run.php index 9803f1b..503e0c6 100644 --- a/example/run.php +++ b/example/run.php @@ -41,7 +41,7 @@ $migrator->up(); $command = $driver->makeCommand(new Config()); -echo "Applied migrations (mode={$mode->name}):\n"; +echo "Applied migrations (mode=$mode->name):\n"; foreach ($command->fetchApplied() as $name => $version) { echo " {$name} => {$version}\n"; } diff --git a/src/internal/command/AbstractCommand.php b/src/internal/command/AbstractCommand.php index d9d1124..9d73e21 100644 --- a/src/internal/command/AbstractCommand.php +++ b/src/internal/command/AbstractCommand.php @@ -6,6 +6,7 @@ use DateTimeImmutable; use Override; +use Throwable; use dbschemix\core\command\CommandInterface; use dbschemix\core\command\Options; use dbschemix\core\connection\ConnectionInterface; @@ -25,6 +26,7 @@ public function __construct( /** * Runs a query with no version bookkeeping (fixtures / repeatable). + * @throws Throwable */ #[Override] public function exec(Context $context): bool diff --git a/src/internal/command/JournalCommand.php b/src/internal/command/JournalCommand.php index 3a4294d..976a393 100644 --- a/src/internal/command/JournalCommand.php +++ b/src/internal/command/JournalCommand.php @@ -5,6 +5,7 @@ namespace dbschemix\clickhouse\internal\command; use Override; +use Throwable; use dbschemix\core\command\Options; use dbschemix\core\Context; @@ -16,6 +17,9 @@ */ final class JournalCommand extends AbstractCommand { + /** + * @throws Throwable + */ #[Override] public function fetchApplied(Options $options = new Options()): array { @@ -65,7 +69,7 @@ public function down(Context $context): bool /** * @param non-empty-string $name * @param non-negative-int $version - * @throws \Throwable + * @throws Throwable */ private function insertJournal(string $name, int $version, int $active): void { diff --git a/src/internal/command/MutationCommand.php b/src/internal/command/MutationCommand.php index 6602582..544e9f5 100644 --- a/src/internal/command/MutationCommand.php +++ b/src/internal/command/MutationCommand.php @@ -7,6 +7,7 @@ use Override; use dbschemix\core\command\Options; use dbschemix\core\Context; +use Throwable; /** * Mode::Mutation — ReplacingMergeTree version table read with FINAL, @@ -18,6 +19,9 @@ final class MutationCommand extends AbstractCommand { private const int MUTATIONS_SYNC = 2; + /** + * @throws Throwable + */ #[Override] public function fetchApplied(Options $options = new Options()): array { diff --git a/tests/DriverTest.php b/tests/DriverTest.php index 4b97d43..b18ec4c 100644 --- a/tests/DriverTest.php +++ b/tests/DriverTest.php @@ -4,6 +4,7 @@ namespace dbschemix\clickhouse\tests; +use dbschemix\clickhouse\Driver; use dbschemix\clickhouse\internal\command\JournalCommand; use dbschemix\clickhouse\internal\command\MutationCommand; use dbschemix\clickhouse\Mode; @@ -23,7 +24,7 @@ public function nameIsClickhouse(): void public function sourceNameIsLowercasedDatabase(): void { - $driver = new \dbschemix\clickhouse\Driver( + $driver = new Driver( host: '127.0.0.1', port: 8123, database: 'MainDB', @@ -36,7 +37,7 @@ public function sourceNameIsLowercasedDatabase(): void public function emptyDatabaseThrows(): void { try { - new \dbschemix\clickhouse\Driver(host: '127.0.0.1', port: 8123, database: '', mode: Mode::Mutation); + new Driver(host: '127.0.0.1', port: 8123, database: '', mode: Mode::Mutation); Assert::fail('expected ConfigurationException'); } catch (ConfigurationException $exception) { Assert::instanceOf($exception, ConfigurationException::class); diff --git a/tests/Support/ClickHouse.php b/tests/Support/ClickHouse.php index 6d2df6b..594dd74 100644 --- a/tests/Support/ClickHouse.php +++ b/tests/Support/ClickHouse.php @@ -7,6 +7,7 @@ use ClickHouseDB\Client; use dbschemix\clickhouse\Driver; use dbschemix\clickhouse\Mode; +use RuntimeException; /** * Test-only helper: builds clients/drivers from connection env and manages the @@ -50,7 +51,7 @@ public static function resetTable(Mode $mode, string $table = 'migration'): void $file = __DIR__ . '/../../src/connection/clickhouse/' . $mode->setupDir() . '/migration/setup.sql'; $sql = file_get_contents($file); if ($sql === false) { - throw new \RuntimeException("Cannot read setup file: {$file}"); + throw new RuntimeException("Cannot read setup file: {$file}"); } foreach (array_filter(array_map('trim', explode(';', $sql))) as $statement) { $client->write(str_replace('%SYSTEM_TABLE%', $table, $statement)); diff --git a/tests/workflow/MigratorAppendOnlyTest.php b/tests/workflow/MigratorAppendOnlyTest.php index 3cd5547..5156cb1 100644 --- a/tests/workflow/MigratorAppendOnlyTest.php +++ b/tests/workflow/MigratorAppendOnlyTest.php @@ -6,6 +6,7 @@ use dbschemix\clickhouse\Mode; use dbschemix\clickhouse\tests\Support\ClickHouse; +use dbschemix\core\command\CommandInterface; use dbschemix\core\Config; use dbschemix\core\InputOptions; use dbschemix\core\Migration; @@ -75,7 +76,7 @@ public function redoReappliesMigrations(): void Assert::true($this->tableExists('demo_orders')); } - private function command(): \dbschemix\core\command\CommandInterface + private function command(): CommandInterface { return ClickHouse::driver(Mode::AppendOnly)->makeCommand(new Config()); } diff --git a/tests/workflow/MigratorMutationTest.php b/tests/workflow/MigratorMutationTest.php index e560bad..1be699e 100644 --- a/tests/workflow/MigratorMutationTest.php +++ b/tests/workflow/MigratorMutationTest.php @@ -6,6 +6,7 @@ use dbschemix\clickhouse\Mode; use dbschemix\clickhouse\tests\Support\ClickHouse; +use dbschemix\core\command\CommandInterface; use dbschemix\core\Config; use dbschemix\core\InputOptions; use dbschemix\core\Migration; @@ -75,7 +76,7 @@ public function redoReappliesMigrations(): void Assert::true($this->tableExists('demo_orders')); } - private function command(): \dbschemix\core\command\CommandInterface + private function command(): CommandInterface { return ClickHouse::driver(Mode::Mutation)->makeCommand(new Config()); }