CI: Add basic integration test

.github/workflows/dm_integration_basic.yaml

@@ -0,0 +1,300 @@
+name: Basic Integration Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'dm/**'
+      - 'cmd/dm*/*'
+      - 'pkg/**'
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/dm_integration_basic.yaml'
+
+jobs:
+  mysql-integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        mysql_version:
+          # MySQL 5.7 doens't enable binlogs by default and there are no easy options for enabling it.
+          # - '5.7' 
+          - '8.0'
+          - '8.4'
+          - '9.7'
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: container-registry.oracle.com/mysql/community-server:${{ matrix.mysql_version }}
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+          MYSQL_ROOT_HOST: "%"
+          MYSQL_DATABASE: test
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+      tidb:
+        image: 'pingcap/tidb:v8.5.6'
+        ports:
+          - 4000:4000
+    env:
+      DM_MASTER_ADDR: "127.0.0.1:8261"
+      DM_WORKER_ADDR: "127.0.0.1:8262"
+    steps:
+      # Some steps may fail on some versions, but the end result should be the same.
+      # That's why we set continue-on-error and run this in multiple commands.
+      - name: Try to enable GTID
+        continue-on-error: true
+        run: |
+          mysql -h 127.0.0.1 -u root -e "SET GLOBAL ENFORCE_GTID_CONSISTENCY=ON"
+          mysql -h 127.0.0.1 -u root -e "SET GLOBAL gtid_mode=OFF_PERMISSIVE"
+          mysql -h 127.0.0.1 -u root -e "SET GLOBAL gtid_mode=ON_PERMISSIVE"
+          mysql -h 127.0.0.1 -u root -e "SET GLOBAL gtid_mode=ON"
+      - name: Check out code
+        uses: actions/checkout@v6
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+      - name: Build DM binary
+        run: make dm
+      - name: Show versions
+        shell: bash -xe {0}
+        run: |
+          ./bin/dm-master -V
+          ./bin/dm-worker -V
+          ./bin/dmctl -V
+          mysql -h 127.0.0.1 -u root -BNe "SELECT VERSION()"
+          mysql -h 127.0.0.1 -P 4000 -u root -BNe "SELECT VERSION()"
+      - name: Run DM
+        run: |
+          ./bin/dm-master -master-addr ${DM_MASTER_ADDR} -log-file dm_master.log -log-format json &
+          ./bin/dm-worker -join ${DM_MASTER_ADDR} -worker-addr ${DM_WORKER_ADDR} -log-file dm_worker.log -log-format json &
+      - name: Wait for DM readiness
+        run: |
+          for i in $(seq 1 30); do
+            if curl -sf http://${DM_MASTER_ADDR}/status > /dev/null 2>&1; then
+              echo "dm-master is ready after ${i}s"
+              break
+            fi
+            echo "waiting for dm-master... (${i}/30)"
+            sleep 1
+          done
+          for i in $(seq 1 30); do
+            if curl -sf http://${DM_WORKER_ADDR}/status > /dev/null 2>&1; then
+              echo "dm-worker is ready after ${i}s"
+              exit 0
+            fi
+            echo "waiting for dm-worker... (${i}/30)"
+            sleep 1
+          done
+          echo "timed out waiting for DM readiness"
+          exit 1
+      - name: Show DM status
+        shell: bash -xe {0}
+        run: |
+          ./bin/dmctl list-member
+          ./bin/dmctl query-status
+      - name: Setup source
+        run: ./bin/dmctl operate-source create dm/tests/integration_basic/source-mysql.yaml | tee dm_setup_source.log
+      - name: Start task
+        run: ./bin/dmctl start-task dm/tests/integration_basic/task-mysql.yaml | tee dm_setup_task.log
+
+        # Here we check that dm_setup_task.log contains a line with `"result": true`.
+        # Unfortunately dm_setup_task.log doesn't contain valid JSON so we can't use jq here.
+        # https://github.com/pingcap/tiflow/issues/11736
+      - name: Check task result
+        run: |
+          grep '"result": true' dm_setup_task.log
+
+      - name: Check status
+        run: |
+          ./bin/dmctl operate-source show
+          ./bin/dmctl query-status
+          ./bin/dmctl query-status mysql-to-tidb
+      - name: Run input
+        run: mysql -h 127.0.0.1 -u root -v < dm/tests/integration_basic/setup.sql
+      - name: Check status
+        run: |
+          ./bin/dmctl operate-source show
+          ./bin/dmctl query-status
+          ./bin/dmctl query-status mysql-to-tidb
+      - name: install mysql-tester
+        run: |
+          go install github.com/pingcap/mysql-tester/src@f2d90ea9522d30c9a8e8d70cc31c7f016ca2801f
+          mv ~/go/bin/src ~/go/bin/mysql-tester
+      - name: check logs
+        continue-on-error: true
+        run: |
+          jq 'select(.level == "DPANIC") | .message + ": " + .error' dm_worker.log
+          jq 'select(.level == "ERROR") | .message + ": " + .error' dm_worker.log
+      # - name: record validation test
+      #   working-directory: ./dm/tests/integration_basic
+      #   run: ~/go/bin/mysql-tester -record basic
+      # - name: show results
+      #   working-directory: ./dm/tests/integration_basic
+      #   run: cat r/basic.result
+      - name: run validation tests
+        working-directory: ./dm/tests/integration_basic
+        run: ~/go/bin/mysql-tester
+      - name: Check status
+        run: |
+          ./bin/dmctl operate-source show
+          ./bin/dmctl query-status
+          ./bin/dmctl query-status mysql-to-tidb
+      - name: upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: dm-logs-mysql-${{ matrix.mysql_version }}
+          path: dm*log
+
+  mariadb-integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        mariadb_version:
+          - '10.11'
+          - '11.4'
+          - '11.8'
+          - '12.3-rc'
+    runs-on: ubuntu-latest
+    services:
+      tidb:
+        image: 'pingcap/tidb:v8.5.5'
+        ports:
+          - 4000:4000
+    env:
+      DM_MASTER_ADDR: "127.0.0.1:8261"
+      DM_WORKER_ADDR: "127.0.0.1:8262"
+    steps:
+      # Enabling binlogs with a services container is hard, so use docker directly. 
+      # Note that FLUSH PRIVILEGES is done to make sure there is at least one GTID
+      # as otherwise the GTID format isn't detected correctly and this error happens:
+      # "message":"failed to update GTID set","GTID":"0-1-6",
+      # "error":"invalid GTID format, must UUID:interval[:interval]"
+      # https://github.com/pingcap/tiflow/issues/12423
+      - name: Run MariaDB
+        run: |
+          docker run \
+          -d \
+          --name mariadb \
+          -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 \
+          -p 3306:3306 \
+          mariadb:${{ matrix.mariadb_version }} \
+          --log-bin=mariadb-bin \
+          --log-bin-compress=OFF \
+          --binlog-format=ROW \
+          --binlog-annotate-row-events=OFF \
+          --loose-binlog-legacy-event-pos=ON
+          until mysqladmin -h 127.0.0.1 -u root ping; do sleep 1; done
+          mysql -h 127.0.0.1 -u root -e "reset master"
+          mysql -h 127.0.0.1 -u root -e "flush privileges"
+      - name: Check out code
+        uses: actions/checkout@v6
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+      - name: Build DM binary
+        run: make dm
+      - name: Show versions
+        shell: bash -xe {0}
+        run: |
+          ./bin/dm-master -V
+          ./bin/dm-worker -V
+          ./bin/dmctl -V
+          mysql -h 127.0.0.1 -u root -BNe "SELECT VERSION()"
+          mysql -h 127.0.0.1 -P 4000 -u root -BNe "SELECT VERSION()"
+      - name: Run DM
+        run: |
+          ./bin/dm-master -master-addr ${DM_MASTER_ADDR} -log-file dm_master.log -log-format json &
+          ./bin/dm-worker -join ${DM_MASTER_ADDR} -worker-addr ${DM_WORKER_ADDR} -log-file dm_worker.log -log-format json &
+      - name: Wait for DM readiness
+        run: |
+          for i in $(seq 1 30); do
+            if curl -sf http://${DM_MASTER_ADDR}/status > /dev/null 2>&1; then
+              echo "dm-master is ready after ${i}s"
+              break
+            fi
+            echo "waiting for dm-master... (${i}/30)"
+            sleep 1
+          done
+          for i in $(seq 1 30); do
+            if curl -sf http://${DM_WORKER_ADDR}/status > /dev/null 2>&1; then
+              echo "dm-worker is ready after ${i}s"
+              exit 0
+            fi
+            echo "waiting for dm-worker... (${i}/30)"
+            sleep 1
+          done
+          echo "timed out waiting for DM readiness"
+          exit 1
+      - name: Show DM status
+        shell: bash -xe {0}
+        run: |
+          ./bin/dmctl list-member
+          ./bin/dmctl query-status
+      - name: Setup source
+        run: ./bin/dmctl operate-source create dm/tests/integration_basic/source-mariadb.yaml | tee dm_setup_source.log
+      - name: Start task
+        run: ./bin/dmctl start-task dm/tests/integration_basic/task-mariadb.yaml | tee dm_setup_task.log
+
+        # Here we check that dm_setup_task.log contains a line with `"result": true`.
+        # Unfortunately dm_setup_task.log doesn't contain valid JSON so we can't use jq here.
+        # https://github.com/pingcap/tiflow/issues/11736
+      - name: Check task result
+        run: |
+          grep '"result": true' dm_setup_task.log
+
+      - name: Check status
+        run: |
+          ./bin/dmctl operate-source show
+          ./bin/dmctl query-status
+          ./bin/dmctl query-status mariadb-to-tidb
+      - name: Run input
+        run: mysql -h 127.0.0.1 -u root -v < dm/tests/integration_basic/setup.sql
+      - name: Check status
+        run: |
+          ./bin/dmctl operate-source show
+          ./bin/dmctl query-status
+          ./bin/dmctl query-status mariadb-to-tidb
+      - name: install mysql-tester
+        run: |
+          go install github.com/pingcap/mysql-tester/src@f2d90ea9522d30c9a8e8d70cc31c7f016ca2801f
+          mv ~/go/bin/src ~/go/bin/mysql-tester
+      # - name: record validation test
+      #   working-directory: ./dm/tests/integration_basic
+      #   run: ~/go/bin/mysql-tester -record basic
+      # - name: show results
+      #   working-directory: ./dm/tests/integration_basic
+      #   run: cat r/basic.result
+      - name: debug
+        run: |
+          sleep 30
+          mysql -h 127.0.0.1 -u root -e "DESCRIBE test_basic.t1"
+          mysql -h 127.0.0.1 -u root -e "SELECT * FROM test_basic.t1"
+          mysql -h 127.0.0.1 -u root -P 4000 -e "DESCRIBE test_basic.t1"
+          mysql -h 127.0.0.1 -u root -P 4000 -e "SELECT * FROM test_basic.t1"
+          mysql -h 127.0.0.1 -u root -e "SHOW BINARY LOGS"
+          mysql -h 127.0.0.1 -u root -e "SHOW BINLOG EVENTS"
+      - name: check logs
+        continue-on-error: true
+        run: |
+          jq 'select(.level == "DPANIC") | .message + ": " + .error' dm_worker.log
+          jq 'select(.level == "ERROR") | .message + ": " + .error' dm_worker.log
+      - name: run validation tests
+        working-directory: ./dm/tests/integration_basic
+        run: |
+          ~/go/bin/mysql-tester
+      - name: Check status
+        if: ${{ always() }}
+        run: |
+          ./bin/dmctl operate-source show
+          ./bin/dmctl query-status
+          ./bin/dmctl query-status mariadb-to-tidb | tee dm_task_status.log
+      - name: upload artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: dm-logs-mariadb-${{ matrix.mariadb_version }}
+          path: dm*log

dm/tests/integration_basic/r/basic.result

@@ -0,0 +1,4 @@
+select * from test_basic.t1 order by id;
+id	name
+1	test 1
+2	test 2 updated

dm/tests/integration_basic/setup.sql

@@ -0,0 +1,12 @@
+CREATE SCHEMA test_basic COLLATE utf8mb4_general_ci;
+USE test_basic;
+
+CREATE TABLE t1 (
+  id int PRIMARY KEY,
+  name varchar(255) NOT NULL
+);
+
+INSERT INTO t1 VALUES (1, 'test 1');
+INSERT INTO t1 VALUES (2, 'test 2'), (3, 'test 3');
+UPDATE t1 SET name='test 2 updated' WHERE id=2;
+DELETE FROM t1 WHERE id=3;

dm/tests/integration_basic/source-mariadb.yaml

@@ -0,0 +1,8 @@
+source-id: "mariadb-source-01"
+enable-gtid: false
+enable-relay: false
+
+from:
+  host: "127.0.0.1"
+  user: "root"
+  port: 3306

dm/tests/integration_basic/source-mysql.yaml

@@ -0,0 +1,8 @@
+source-id: "mysql-source-01"
+enable-gtid: true
+enable-relay: false
+
+from:
+  host: "127.0.0.1"
+  user: "root"
+  port: 3306

dm/tests/integration_basic/t/basic.test

@@ -0,0 +1 @@
+select * from test_basic.t1 order by id;

dm/tests/integration_basic/task-mariadb.yaml

@@ -0,0 +1,19 @@
+---
+name: "mariadb-to-tidb"
+task-mode: all
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "root"
+
+mysql-instances:
+  - source-id: "mariadb-source-01"
+    filter-rules: ["test"]
+
+filters:
+  test:
+    schema-pattern: "test*"
+    table-pattern: "*"
+    events: ["all"]
+    action: Do

dm/tests/integration_basic/task-mysql.yaml

@@ -0,0 +1,19 @@
+---
+name: "mysql-to-tidb"
+task-mode: all
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "root"
+
+mysql-instances:
+  - source-id: "mysql-source-01"
+    filter-rules: ["test"]
+
+filters:
+  test:
+    schema-pattern: "test*"
+    table-pattern: "*"
+    events: ["all"]
+    action: Do