From deb1e8f551bc4b7acc225ca0d30ccf53533aab87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Fri, 27 Mar 2026 09:30:49 +0100 Subject: [PATCH 1/6] Add basic integration test --- .github/workflows/dm_integration_basic.yaml | 252 ++++++++++++++++++ dm/tests/integration_basic/r/basic.result | 4 + dm/tests/integration_basic/setup.sql | 12 + .../integration_basic/source-mariadb.yaml | 8 + dm/tests/integration_basic/source-mysql.yaml | 8 + dm/tests/integration_basic/t/basic.test | 1 + dm/tests/integration_basic/task-mariadb.yaml | 19 ++ dm/tests/integration_basic/task-mysql.yaml | 19 ++ 8 files changed, 323 insertions(+) create mode 100644 .github/workflows/dm_integration_basic.yaml create mode 100644 dm/tests/integration_basic/r/basic.result create mode 100644 dm/tests/integration_basic/setup.sql create mode 100644 dm/tests/integration_basic/source-mariadb.yaml create mode 100644 dm/tests/integration_basic/source-mysql.yaml create mode 100644 dm/tests/integration_basic/t/basic.test create mode 100644 dm/tests/integration_basic/task-mariadb.yaml create mode 100644 dm/tests/integration_basic/task-mysql.yaml diff --git a/.github/workflows/dm_integration_basic.yaml b/.github/workflows/dm_integration_basic.yaml new file mode 100644 index 0000000000..36d7464b2d --- /dev/null +++ b/.github/workflows/dm_integration_basic.yaml @@ -0,0 +1,252 @@ +name: Basic Integration Tests + +on: + schedule: + - cron: '0 3 * * *' + workflow_dispatch: + pull_request: + +jobs: + mysql-integration: + strategy: + fail-fast: false + matrix: + mysql_version: + # MySQL 5.7 doens't enable binlogs by default and there are no easy options for enabling it. + # - '5.7' + - '8.0' + - '8.4' + - '9.6' + runs-on: ubuntu-latest + services: + mysql: + image: container-registry.oracle.com/mysql/community-server:${{ matrix.mysql_version }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: test + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + tidb: + image: 'pingcap/tidb:v8.5.5' + ports: + - 4000:4000 + env: + DM_MASTER_ADDR: "127.0.0.1:8261" + DM_WORKER_ADDR: "127.0.0.1:8262" + steps: + # Some steps may fail on some versions, but the end result should be the same. + # That's why we set continue-on-error and run this in multiple commands. + - name: Try to enable GTID + continue-on-error: true + run: | + mysql -h 127.0.0.1 -u root -e "SET GLOBAL ENFORCE_GTID_CONSISTENCY=ON" + mysql -h 127.0.0.1 -u root -e "SET GLOBAL gtid_mode=OFF_PERMISSIVE" + mysql -h 127.0.0.1 -u root -e "SET GLOBAL gtid_mode=ON_PERMISSIVE" + mysql -h 127.0.0.1 -u root -e "SET GLOBAL gtid_mode=ON" + - name: Check out code + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Build DM binary + run: make dm + - name: Show versions + shell: bash -xe {0} + run: | + ./bin/dm-master -V + ./bin/dm-worker -V + ./bin/dmctl -V + mysql -h 127.0.0.1 -u root -BNe "SELECT VERSION()" + mysql -h 127.0.0.1 -P 4000 -u root -BNe "SELECT VERSION()" + - name: Run DM + run: | + ./bin/dm-master -master-addr ${DM_MASTER_ADDR} -log-file dm_master.log -log-format json & + ./bin/dm-worker -join ${DM_MASTER_ADDR} -worker-addr ${DM_WORKER_ADDR} -log-file dm_worker.log -log-format json & + - name: Show DM status + shell: bash -xe {0} + run: | + ./bin/dmctl list-member + ./bin/dmctl query-status + - name: Setup source + run: ./bin/dmctl operate-source create dm/tests/integration_basic/source-mysql.yaml | tee dm_setup_source.log + - name: Start task + run: ./bin/dmctl start-task dm/tests/integration_basic/task-mysql.yaml | tee dm_setup_task.log + + # Here we check that dm_setup_task.log contains a line with `"result": true`. + # Unfortunately dm_setup_task.log doesn't contain valid JSON so we can't use jq here. + # https://github.com/pingcap/tiflow/issues/11736 + - name: Check task result + run: | + grep '"result": true' dm_setup_task.log + + - name: Check status + run: | + ./bin/dmctl operate-source show + ./bin/dmctl query-status + ./bin/dmctl query-status mysql-to-tidb + - name: Run input + run: mysql -h 127.0.0.1 -u root -v < dm/tests/integration_basic/setup.sql + - name: Check status + run: | + ./bin/dmctl operate-source show + ./bin/dmctl query-status + ./bin/dmctl query-status mysql-to-tidb + - name: install mysql-tester + run: | + go install github.com/pingcap/mysql-tester/src@latest + mv ~/go/bin/src ~/go/bin/mysql-tester + - name: check logs + continue-on-error: true + run: | + jq 'select(.level == "DPANIC") | .message + ": " + .error' dm_worker.log + jq 'select(.level == "ERROR") | .message + ": " + .error' dm_worker.log + # - name: record validation test + # working-directory: ./dm/tests/integration_basic + # run: ~/go/bin/mysql-tester -record basic + # - name: show results + # working-directory: ./dm/tests/integration_basic + # run: cat r/basic.result + - name: run validation tests + working-directory: ./dm/tests/integration_basic + run: ~/go/bin/mysql-tester + - name: Check status + run: | + ./bin/dmctl operate-source show + ./bin/dmctl query-status + ./bin/dmctl query-status mysql-to-tidb + - name: upload artifacts + uses: actions/upload-artifact@v7 + with: + name: dm-logs-mysql-${{ matrix.mysql_version }} + path: dm*log + + mariadb-integration: + strategy: + fail-fast: false + matrix: + mariadb_version: + - '10.11' + - '11.4' + - '11.8' + - '12.3-rc' + runs-on: ubuntu-latest + services: + tidb: + image: 'pingcap/tidb:v8.5.5' + ports: + - 4000:4000 + env: + DM_MASTER_ADDR: "127.0.0.1:8261" + DM_WORKER_ADDR: "127.0.0.1:8262" + steps: + # Enabling binlogs with a services container is hard, so use docker directly. + # Note that FLUSH PRIVILEGES is done to make sure there is at least one GTID + # as otherwise the GTID format isn't detected correctly and this error happens: + # "message":"failed to update GTID set","GTID":"0-1-6", + # "error":"invalid GTID format, must UUID:interval[:interval]" + # https://github.com/pingcap/tiflow/issues/12423 + - name: Run MariaDB + run: | + docker run \ + -d \ + --name mariadb \ + -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 \ + -p 3306:3306 \ + mariadb:${{ matrix.mariadb_version }} \ + --log-bin=mariadb-bin \ + --binlog-format=ROW + until mysqladmin -h 127.0.0.1 -u root ping; do sleep 1; done + mysql -h 127.0.0.1 -u root -e "reset master" + mysql -h 127.0.0.1 -u root -e "flush privileges" + - name: Check out code + uses: actions/checkout@v6 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + - name: Build DM binary + run: make dm + - name: Show versions + shell: bash -xe {0} + run: | + ./bin/dm-master -V + ./bin/dm-worker -V + ./bin/dmctl -V + mysql -h 127.0.0.1 -u root -BNe "SELECT VERSION()" + mysql -h 127.0.0.1 -P 4000 -u root -BNe "SELECT VERSION()" + - name: Run DM + run: | + ./bin/dm-master -master-addr ${DM_MASTER_ADDR} -log-file dm_master.log -log-format json & + ./bin/dm-worker -join ${DM_MASTER_ADDR} -worker-addr ${DM_WORKER_ADDR} -log-file dm_worker.log -log-format json & + - name: Show DM status + shell: bash -xe {0} + run: | + ./bin/dmctl list-member + ./bin/dmctl query-status + - name: Setup source + run: ./bin/dmctl operate-source create dm/tests/integration_basic/source-mariadb.yaml | tee dm_setup_source.log + - name: Start task + run: ./bin/dmctl start-task dm/tests/integration_basic/task-mariadb.yaml | tee dm_setup_task.log + + # Here we check that dm_setup_task.log contains a line with `"result": true`. + # Unfortunately dm_setup_task.log doesn't contain valid JSON so we can't use jq here. + # https://github.com/pingcap/tiflow/issues/11736 + - name: Check task result + run: | + grep '"result": true' dm_setup_task.log + + - name: Check status + run: | + ./bin/dmctl operate-source show + ./bin/dmctl query-status + ./bin/dmctl query-status mariadb-to-tidb + - name: Run input + run: mysql -h 127.0.0.1 -u root -v < dm/tests/integration_basic/setup.sql + - name: Check status + run: | + ./bin/dmctl operate-source show + ./bin/dmctl query-status + ./bin/dmctl query-status mariadb-to-tidb + - name: install mysql-tester + run: | + go install github.com/pingcap/mysql-tester/src@latest + mv ~/go/bin/src ~/go/bin/mysql-tester + # - name: record validation test + # working-directory: ./dm/tests/integration_basic + # run: ~/go/bin/mysql-tester -record basic + # - name: show results + # working-directory: ./dm/tests/integration_basic + # run: cat r/basic.result + - name: debug + run: | + sleep 30 + mysql -h 127.0.0.1 -u root -e "DESCRIBE test_basic.t1" + mysql -h 127.0.0.1 -u root -e "SELECT * FROM test_basic.t1" + mysql -h 127.0.0.1 -u root -P 4000 -e "DESCRIBE test_basic.t1" + mysql -h 127.0.0.1 -u root -P 4000 -e "SELECT * FROM test_basic.t1" + mysql -h 127.0.0.1 -u root -e "SHOW BINARY LOGS" + mysql -h 127.0.0.1 -u root -e "SHOW BINLOG EVENTS" + - name: check logs + continue-on-error: true + run: | + jq 'select(.level == "DPANIC") | .message + ": " + .error' dm_worker.log + jq 'select(.level == "ERROR") | .message + ": " + .error' dm_worker.log + - name: run validation tests + working-directory: ./dm/tests/integration_basic + run: | + ~/go/bin/mysql-tester + - name: Check status + if: ${{ always() }} + run: | + ./bin/dmctl operate-source show + ./bin/dmctl query-status + ./bin/dmctl query-status mariadb-to-tidb | tee dm_task_status.log + - name: upload artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: dm-logs-mariadb-${{ matrix.mariadb_version }} + path: dm*log diff --git a/dm/tests/integration_basic/r/basic.result b/dm/tests/integration_basic/r/basic.result new file mode 100644 index 0000000000..be2fee2ad9 --- /dev/null +++ b/dm/tests/integration_basic/r/basic.result @@ -0,0 +1,4 @@ +select * from test_basic.t1; +id name +1 test 1 +2 test 2 updated diff --git a/dm/tests/integration_basic/setup.sql b/dm/tests/integration_basic/setup.sql new file mode 100644 index 0000000000..5eb179044b --- /dev/null +++ b/dm/tests/integration_basic/setup.sql @@ -0,0 +1,12 @@ +CREATE SCHEMA test_basic COLLATE utf8mb4_general_ci; +USE test_basic; + +CREATE TABLE t1 ( + id int PRIMARY KEY, + name varchar(255) NOT NULL +); + +INSERT INTO t1 VALUES (1, 'test 1'); +INSERT INTO t1 VALUES (2, 'test 2'), (3, 'test 3'); +UPDATE t1 SET name='test 2 updated' WHERE id=2; +DELETE FROM t1 WHERE id=3; diff --git a/dm/tests/integration_basic/source-mariadb.yaml b/dm/tests/integration_basic/source-mariadb.yaml new file mode 100644 index 0000000000..408428dac9 --- /dev/null +++ b/dm/tests/integration_basic/source-mariadb.yaml @@ -0,0 +1,8 @@ +source-id: "mariadb-source-01" +enable-gtid: false +enable-relay: false + +from: + host: "127.0.0.1" + user: "root" + port: 3306 diff --git a/dm/tests/integration_basic/source-mysql.yaml b/dm/tests/integration_basic/source-mysql.yaml new file mode 100644 index 0000000000..1cf48ba56e --- /dev/null +++ b/dm/tests/integration_basic/source-mysql.yaml @@ -0,0 +1,8 @@ +source-id: "mysql-source-01" +enable-gtid: true +enable-relay: false + +from: + host: "127.0.0.1" + user: "root" + port: 3306 diff --git a/dm/tests/integration_basic/t/basic.test b/dm/tests/integration_basic/t/basic.test new file mode 100644 index 0000000000..54819c841f --- /dev/null +++ b/dm/tests/integration_basic/t/basic.test @@ -0,0 +1 @@ +select * from test_basic.t1; diff --git a/dm/tests/integration_basic/task-mariadb.yaml b/dm/tests/integration_basic/task-mariadb.yaml new file mode 100644 index 0000000000..22c0dcd274 --- /dev/null +++ b/dm/tests/integration_basic/task-mariadb.yaml @@ -0,0 +1,19 @@ +--- +name: "mariadb-to-tidb" +task-mode: all + +target-database: + host: "127.0.0.1" + port: 4000 + user: "root" + +mysql-instances: + - source-id: "mariadb-source-01" + filter-rules: ["test"] + +filters: + test: + schema-pattern: "test*" + table-pattern: "*" + events: ["all"] + action: Do diff --git a/dm/tests/integration_basic/task-mysql.yaml b/dm/tests/integration_basic/task-mysql.yaml new file mode 100644 index 0000000000..f77234bdbc --- /dev/null +++ b/dm/tests/integration_basic/task-mysql.yaml @@ -0,0 +1,19 @@ +--- +name: "mysql-to-tidb" +task-mode: all + +target-database: + host: "127.0.0.1" + port: 4000 + user: "root" + +mysql-instances: + - source-id: "mysql-source-01" + filter-rules: ["test"] + +filters: + test: + schema-pattern: "test*" + table-pattern: "*" + events: ["all"] + action: Do From 3500c8591dfad6845373ff68d1f80d1c027c2fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Sat, 18 Apr 2026 09:12:46 -0700 Subject: [PATCH 2/6] Change settings for MariaDB --- .github/workflows/dm_integration_basic.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dm_integration_basic.yaml b/.github/workflows/dm_integration_basic.yaml index 36d7464b2d..c753ec5cb8 100644 --- a/.github/workflows/dm_integration_basic.yaml +++ b/.github/workflows/dm_integration_basic.yaml @@ -157,7 +157,10 @@ jobs: -p 3306:3306 \ mariadb:${{ matrix.mariadb_version }} \ --log-bin=mariadb-bin \ - --binlog-format=ROW + --log-bin-compress=OFF \ + --binlog-format=ROW \ + --binlog-annotate-row-events=OFF \ + --loose-binlog-legacy-event-pos=ON until mysqladmin -h 127.0.0.1 -u root ping; do sleep 1; done mysql -h 127.0.0.1 -u root -e "reset master" mysql -h 127.0.0.1 -u root -e "flush privileges" From aba7f409a47127e65d0e17c88818b0dad69a43d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 21 Apr 2026 14:45:53 +0200 Subject: [PATCH 3/6] Update based on review --- .github/workflows/dm_integration_basic.yaml | 5 ++++- dm/tests/integration_basic/r/basic.result | 2 +- dm/tests/integration_basic/t/basic.test | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dm_integration_basic.yaml b/.github/workflows/dm_integration_basic.yaml index c753ec5cb8..2a9d3612e6 100644 --- a/.github/workflows/dm_integration_basic.yaml +++ b/.github/workflows/dm_integration_basic.yaml @@ -5,6 +5,9 @@ on: - cron: '0 3 * * *' workflow_dispatch: pull_request: + paths: + - 'dm/**' + - 'cmd/dm*/*' jobs: mysql-integration: @@ -215,7 +218,7 @@ jobs: ./bin/dmctl query-status mariadb-to-tidb - name: install mysql-tester run: | - go install github.com/pingcap/mysql-tester/src@latest + go install github.com/pingcap/mysql-tester/src@f2d90ea9522d30c9a8e8d70cc31c7f016ca2801f mv ~/go/bin/src ~/go/bin/mysql-tester # - name: record validation test # working-directory: ./dm/tests/integration_basic diff --git a/dm/tests/integration_basic/r/basic.result b/dm/tests/integration_basic/r/basic.result index be2fee2ad9..a334f1abd6 100644 --- a/dm/tests/integration_basic/r/basic.result +++ b/dm/tests/integration_basic/r/basic.result @@ -1,4 +1,4 @@ -select * from test_basic.t1; +select * from test_basic.t1 order by id; id name 1 test 1 2 test 2 updated diff --git a/dm/tests/integration_basic/t/basic.test b/dm/tests/integration_basic/t/basic.test index 54819c841f..f0b4b4c468 100644 --- a/dm/tests/integration_basic/t/basic.test +++ b/dm/tests/integration_basic/t/basic.test @@ -1 +1 @@ -select * from test_basic.t1; +select * from test_basic.t1 order by id; From 0e1c542ea5daaf4944d91ef9df1420b60aa48057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 21 Apr 2026 14:55:46 +0200 Subject: [PATCH 4/6] Add readiness check --- .github/workflows/dm_integration_basic.yaml | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/dm_integration_basic.yaml b/.github/workflows/dm_integration_basic.yaml index 2a9d3612e6..612c192635 100644 --- a/.github/workflows/dm_integration_basic.yaml +++ b/.github/workflows/dm_integration_basic.yaml @@ -68,6 +68,26 @@ jobs: run: | ./bin/dm-master -master-addr ${DM_MASTER_ADDR} -log-file dm_master.log -log-format json & ./bin/dm-worker -join ${DM_MASTER_ADDR} -worker-addr ${DM_WORKER_ADDR} -log-file dm_worker.log -log-format json & + - name: Wait for DM readiness + run: | + for i in $(seq 1 30); do + if curl -sf http://${DM_MASTER_ADDR}/status > /dev/null 2>&1; then + echo "dm-master is ready after ${i}s" + break + fi + echo "waiting for dm-master... (${i}/30)" + sleep 1 + done + for i in $(seq 1 30); do + if curl -sf http://${DM_WORKER_ADDR}/status > /dev/null 2>&1; then + echo "dm-worker is ready after ${i}s" + exit 0 + fi + echo "waiting for dm-worker... (${i}/30)" + sleep 1 + done + echo "timed out waiting for DM readiness" + exit 1 - name: Show DM status shell: bash -xe {0} run: | @@ -187,6 +207,26 @@ jobs: run: | ./bin/dm-master -master-addr ${DM_MASTER_ADDR} -log-file dm_master.log -log-format json & ./bin/dm-worker -join ${DM_MASTER_ADDR} -worker-addr ${DM_WORKER_ADDR} -log-file dm_worker.log -log-format json & + - name: Wait for DM readiness + run: | + for i in $(seq 1 30); do + if curl -sf http://${DM_MASTER_ADDR}/status > /dev/null 2>&1; then + echo "dm-master is ready after ${i}s" + break + fi + echo "waiting for dm-master... (${i}/30)" + sleep 1 + done + for i in $(seq 1 30); do + if curl -sf http://${DM_WORKER_ADDR}/status > /dev/null 2>&1; then + echo "dm-worker is ready after ${i}s" + exit 0 + fi + echo "waiting for dm-worker... (${i}/30)" + sleep 1 + done + echo "timed out waiting for DM readiness" + exit 1 - name: Show DM status shell: bash -xe {0} run: | From d154664ed6cc42f3187df32c2bdece92cb17c3c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 28 Apr 2026 09:21:56 +0200 Subject: [PATCH 5/6] Update to MySQL 9.7 --- .github/workflows/dm_integration_basic.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dm_integration_basic.yaml b/.github/workflows/dm_integration_basic.yaml index 612c192635..0aa59b7d76 100644 --- a/.github/workflows/dm_integration_basic.yaml +++ b/.github/workflows/dm_integration_basic.yaml @@ -1,8 +1,6 @@ name: Basic Integration Tests on: - schedule: - - cron: '0 3 * * *' workflow_dispatch: pull_request: paths: @@ -19,7 +17,7 @@ jobs: # - '5.7' - '8.0' - '8.4' - - '9.6' + - '9.7' runs-on: ubuntu-latest services: mysql: From 53d6577ca4a864264457fdd15824f3cb89894a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 28 Apr 2026 10:36:59 +0200 Subject: [PATCH 6/6] Update based on review --- .github/workflows/dm_integration_basic.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dm_integration_basic.yaml b/.github/workflows/dm_integration_basic.yaml index 0aa59b7d76..a54d0531c9 100644 --- a/.github/workflows/dm_integration_basic.yaml +++ b/.github/workflows/dm_integration_basic.yaml @@ -6,6 +6,10 @@ on: paths: - 'dm/**' - 'cmd/dm*/*' + - 'pkg/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/dm_integration_basic.yaml' jobs: mysql-integration: @@ -30,7 +34,7 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 tidb: - image: 'pingcap/tidb:v8.5.5' + image: 'pingcap/tidb:v8.5.6' ports: - 4000:4000 env: @@ -117,7 +121,7 @@ jobs: ./bin/dmctl query-status mysql-to-tidb - name: install mysql-tester run: | - go install github.com/pingcap/mysql-tester/src@latest + go install github.com/pingcap/mysql-tester/src@f2d90ea9522d30c9a8e8d70cc31c7f016ca2801f mv ~/go/bin/src ~/go/bin/mysql-tester - name: check logs continue-on-error: true