diff --git a/docs/design/2026-03-18-replace-round-with-cast-decimal.md b/docs/design/2026-03-18-replace-round-with-cast-decimal.md new file mode 100644 index 0000000000..db18605c2d --- /dev/null +++ b/docs/design/2026-03-18-replace-round-with-cast-decimal.md @@ -0,0 +1,172 @@ +# Replace `round()` with `CAST(... AS DECIMAL(65, 30))` in sync-diff-inspector + +**Goal:** Replace `round()` with `CAST(... AS DECIMAL(65, 30))` for float/double columns in sync-diff-inspector to enable TiKV coprocessor pushdown and fix the zero-value NULL bug. + +**Architecture:** Two code sites in `sync_diff_inspector/utils/utils.go` generate SQL that wraps float/double columns with `round()`. Replace both with `CAST(... AS DECIMAL(65, 30))`, update the stale comments, and fix the unit test assertion. + +**Tech Stack:** Go, MySQL/TiDB SQL + +Issue: https://github.com/pingcap/tidb-tools/issues/859 + +## Background + +sync-diff-inspector compares data between an upstream database (MySQL or TiDB) and a +downstream database (TiDB) using a two-phase strategy: + +1. **Checksum phase** (`GetCountAndMD5Checksum`): For each chunk of rows, compute a + `BIT_XOR` of per-row MD5 hashes on both sides. If the checksums and counts match, the + chunk is identical -- no further work needed. +2. **Row-by-row phase** (`GetTableRowsQueryFormat`): If the checksums differ, fetch the + actual rows from both sides, iterate through them in primary-key order, and emit + INSERT/DELETE/REPLACE fix-SQL for each mismatch. + +Both phases must normalize `FLOAT`/`DOUBLE` columns before comparison because the IEEE 754 +binary representation of a floating-point value can produce slightly different decimal +strings across database engines or versions. Without normalization, two databases holding +identical data could yield different checksums. + +## Problem + +The current normalization wraps float/double columns with `round()`: + +```sql +-- For FLOAT (~7 significant digits): +round(`col`, 5 - floor(log10(abs(`col`)))) + +-- For DOUBLE (~15 significant digits): +round(`col`, 14 - floor(log10(abs(`col`)))) +``` + +This has two problems: + +### 1. `round()` cannot be pushed down to TiKV + +TiDB's expression pushdown blocklist +([infer_pushdown.go](https://github.com/pingcap/tidb/blob/07f4eda04057923b0588eb8d62be45962fa7658b/pkg/expression/infer_pushdown.go#L160)) +includes `round`. When a checksum query contains `round`, the entire aggregation +(`BIT_XOR`, `MD5`, `CONCAT_WS`, ...) must execute on the TiDB node rather than being +distributed across TiKV coprocessors. + +Verified by `EXPLAIN` on TiDB v8.5: + +``` +-- With round(): aggregation stays on root (TiDB) +StreamAgg_8 root funcs:bit_xor(...) +└─Projection root cast(round(...)) + └─TableReader root + └─TableFullScan cop[tikv] + +-- With CAST(... AS DECIMAL): aggregation pushed to cop[tikv] +HashAgg_11 root funcs:bit_xor(Column#5) +└─TableReader root data:HashAgg_5 + └─HashAgg_5 cop[tikv] funcs:bit_xor(cast(conv(substring(md5(concat_ws(...)))))) + └─TableFullScan cop[tikv] +``` + +For large tables this is a significant performance bottleneck: every row must be sent from +TiKV to TiDB before the checksum can be computed, instead of computing partial checksums +locally on each TiKV region. + +### 2. `round()` returns NULL when the column value is zero + +`log10(abs(0))` is mathematically undefined (`-inf`), so the expression +`round(0, 5 - floor(-inf))` evaluates to `NULL`: + +```sql +mysql> SELECT round(0.0, 5-floor(log10(abs(0.0)))); ++--------------------------------------+ +| round(0.0, 5-floor(log10(abs(0.0)))) | ++--------------------------------------+ +| NULL | ++--------------------------------------+ +``` + +The code comments acknowledge this: + +```go +// When col value is 0, the result is NULL. +// But we can use ISNULL to distinguish between null and 0. +``` + +The checksum query appends `ISNULL(col)` columns to the `CONCAT_WS` so that a genuine +NULL and a zero produce different hash inputs. This works but is a workaround for a problem +that should not exist. + +## Solution + +Replace `round(col, N - floor(log10(abs(col))))` with `CAST(col AS DECIMAL(65, 30))`. + +### Why `CAST(... AS DECIMAL)` works + +**Pushdown-compatible.** `CAST` to `DECIMAL` is not on TiDB's pushdown blocklist. The +`EXPLAIN` output above confirms the full checksum expression is pushed to `cop[tikv]`. + +**Deterministic.** For any given IEEE 754 float/double bit pattern, `CAST(col AS +DECIMAL(65, 30))` always produces the same decimal string. This guarantees that the same +stored value yields the same checksum on both upstream and downstream, which is the only +correctness requirement for sync-diff-inspector. + +**Handles zero correctly.** `CAST(0 AS DECIMAL(65, 30))` returns +`0.000000000000000000000000000000`, not NULL. The existing `ISNULL` columns in the +checksum query continue to work, and the zero-value edge case is no longer special. + +### Precision analysis + +`DECIMAL(65, 30)` allows up to 35 integer digits and 30 fractional digits. This is the +maximum scale permitted by MySQL/TiDB. + +**FLOAT (single precision, ~7 significant digits):** + +| Value magnitude | Significant digits preserved by DECIMAL(65,30) | Sufficient? | +|---|---|---| +| > 1e35 | All significant digits in integer part (up to 35 digits) | Float only has ~7 sig digits, 35 integer digits is more than enough | +| 1e-24 to 1e35 | All ~7 significant digits | Yes | +| < 1e-24 | Fewer than 7 sig digits (fractional part truncated at 30 places) | Edge case -- values this small are extremely rare in practice | + +**DOUBLE (double precision, ~15 significant digits):** + +| Value magnitude | Significant digits preserved by DECIMAL(65,30) | Sufficient? | +|---|---|---| +| > 1e35 | Up to 35 integer digits; double only has ~15 sig digits | Yes | +| 1e-16 to 1e35 | All ~15 significant digits | Yes | +| < 1e-16 | Fewer than 15 sig digits (e.g., 1.23e-16 keeps 14 sig digits) | Minor edge case | + +Tested on MySQL 8.0: + +```sql +-- Double value at 1e-15: full precision preserved +mysql> SELECT cast(cast(1.234567890123456e-15 as double) as decimal(65,30)); + → 0.000000000000001234567890123456 -- all 15 sig digits preserved + +-- Double value at 1e-16: one sig digit lost +mysql> SELECT cast(cast(1.234567890123456e-16 as double) as decimal(65,30)); + → 0.000000000000000123456789012346 -- 14 sig digits (last digit rounded) +``` + +In practice, float/double columns rarely store values with magnitude below 1e-16. The +current `round()` approach also has precision limitations -- it keeps only 6 or 15 +significant digits and produces NULL for zero. `CAST(... AS DECIMAL(65, 30))` is strictly +better for all practical values. + +### Why use `DECIMAL(65, 30)` specifically? + +- **65** is the maximum precision (total digits) for DECIMAL in MySQL/TiDB. This ensures + we never overflow for any representable float/double value. +- **30** is the maximum scale (fractional digits) for DECIMAL in MySQL/TiDB. This + maximizes the precision preserved for small values. +- Using a single `(65, 30)` for both FLOAT and DOUBLE simplifies the code -- no need for + separate format strings per type. + +### Alternatives considered + +| Approach | Verdict | +|---|---| +| `CAST(col AS DECIMAL(65, 30))` | **Chosen.** Pushdown-compatible, deterministic, fixes zero bug, sufficient precision. | +| `CAST(col AS CHAR)` | Preserves exact representation but may produce different string formats (scientific notation) across MySQL vs TiDB. Pushdown status unverified. | +| Remove wrapper entirely | Not viable. `CONCAT_WS` on raw float produces inconsistent formats (e.g., `1.23457e-6` on one side vs `0.00000123457` on the other). | +| Remove `round` from TiDB pushdown blocklist | Fixes pushdown but not the `log10(0) → NULL` bug. Also introduces upstream dependency — sync-diff-inspector cannot ship the fix independently, and `round` may be on the blocklist for correctness reasons (edge-case behavior differences between TiDB and TiKV). | +| `HEX(col)` (bit-exact comparison) | Zero precision loss and no NULL issue, but **not viable for cross-engine comparison**. The same logical value (e.g., `3.14`) may have slightly different IEEE 754 bit patterns after binlog replication or string-to-float conversion, causing false mismatches. | +| `ROUND(col, fixed_N)` without `log10` | Fixes the zero-value NULL bug by removing `log10(abs(col))`, but `round` is still on TiDB's pushdown blocklist — the core performance problem remains unsolved. | +| `FORMAT(col, 30)` | Forces fixed-point string output, but `FORMAT()` is locale-dependent (inserts thousands separators like `1,234.56`), making results inconsistent across environments. Pushdown status unverified. | +| Hybrid: CAST for checksum, keep `round()` for row-level | Only changes the performance-critical checksum path. However, using different normalization in the two phases risks correctness: if `CAST` and `round` produce different values for the same input, a checksum mismatch would trigger the row-level phase, which could then report spurious diffs or miss real ones. | +| Application-side normalization (Go `strconv.FormatFloat`) | Fully controllable, but **cannot apply to the checksum phase** since checksums are computed server-side via `BIT_XOR(MD5(CONCAT_WS(...)))`. Only useful for the row-level phase, which is not the performance bottleneck. | \ No newline at end of file diff --git a/sync_diff_inspector/config/config.go b/sync_diff_inspector/config/config.go index 842c56c1e6..3098226b00 100644 --- a/sync_diff_inspector/config/config.go +++ b/sync_diff_inspector/config/config.go @@ -58,6 +58,16 @@ const ( UnifiedTimeZone string = "+0:00" ) +// ChecksumAlgorithm specifies the hash function to use for chunk checksumming. +type ChecksumAlgorithm string + +const ( + // MD5 uses MD5 hash function (default for backwards compatibility) + MD5 ChecksumAlgorithm = "md5" + // SHA256 uses SHA256 hash function (for FIPS-compliant environments) + SHA256 ChecksumAlgorithm = "sha256" +) + // TableConfig is the config of table. type TableConfig struct { // table's filter to tell us which table should adapt to this config. @@ -128,6 +138,8 @@ type DataSource struct { Conn *sql.DB SessionConfig SessionConfig `toml:"session" json:"session"` + + ChecksumAlgorithm ChecksumAlgorithm `toml:"checksum-algorithm" json:"checksum-algorithm"` } // IsAutoSnapshot returns true if the tidb_snapshot is expected to automatically @@ -403,6 +415,11 @@ type Config struct { // DMTask string `toml:"dm-task" json:"dm-task"` DMTask string `toml:"dm-task" json:"dm-task"` + // ChecksumAlgorithm specifies the hash function to use for chunk checksumming. + // Options: MD5 or SHA256. Default: MD5 (for backwards compatibility) + // Set to SHA256 for FIPS-compliant environments. + ChecksumAlgorithm ChecksumAlgorithm `toml:"checksum-algorithm" json:"checksum-algorithm"` + DataSources map[string]*DataSource `toml:"data-sources" json:"data-sources"` Routes map[string]*router.TableRule `toml:"routes" json:"routes"` @@ -437,8 +454,10 @@ func NewConfig() *Config { fs.BoolVar(&cfg.CheckStructOnly, "check-struct-only", false, "ignore check table's data") fs.BoolVar(&cfg.SkipNonExistingTable, "skip-non-existing-table", false, "skip validation for tables that don't exist upstream or downstream") fs.BoolVar(&cfg.CheckDataOnly, "check-data-only", false, "ignore check table's struct") + fs.StringVar((*string)(&cfg.ChecksumAlgorithm), "checksum-algorithm", string(MD5), "checksum function: md5, sha256") _ = fs.MarkHidden("check-data-only") + _ = fs.MarkHidden("checksum-algorithm") fs.SortFlags = false return cfg @@ -540,12 +559,13 @@ func (c *Config) adjustConfigByDMSubTasks() (err error) { } dataSources := make(map[string]*DataSource) dataSources["target"] = &DataSource{ - Host: subTaskCfgs[0].To.Host, - Port: subTaskCfgs[0].To.Port, - User: subTaskCfgs[0].To.User, - Password: utils.SecretString(subTaskCfgs[0].To.Password), - SQLMode: sqlMode, - Security: parseTLSFromDMConfig(subTaskCfgs[0].To.Security), + Host: subTaskCfgs[0].To.Host, + Port: subTaskCfgs[0].To.Port, + User: subTaskCfgs[0].To.User, + Password: utils.SecretString(subTaskCfgs[0].To.Password), + SQLMode: sqlMode, + Security: parseTLSFromDMConfig(subTaskCfgs[0].To.Security), + ChecksumAlgorithm: c.ChecksumAlgorithm, } for _, subTaskCfg := range subTaskCfgs { tableRouter, err := router.NewTableRouter(subTaskCfg.CaseSensitive, []*router.TableRule{}) @@ -561,15 +581,15 @@ func (c *Config) adjustConfigByDMSubTasks() (err error) { routeTargetSet[dbutil.TableName(rule.TargetSchema, rule.TargetTable)] = struct{}{} } dataSources[subTaskCfg.SourceID] = &DataSource{ - Host: subTaskCfg.From.Host, - Port: subTaskCfg.From.Port, - User: subTaskCfg.From.User, - Password: utils.SecretString(subTaskCfg.From.Password), - SQLMode: sqlMode, - Security: parseTLSFromDMConfig(subTaskCfg.From.Security), - Router: tableRouter, - - RouteTargetSet: routeTargetSet, + Host: subTaskCfg.From.Host, + Port: subTaskCfg.From.Port, + User: subTaskCfg.From.User, + Password: utils.SecretString(subTaskCfg.From.Password), + SQLMode: sqlMode, + Security: parseTLSFromDMConfig(subTaskCfg.From.Security), + Router: tableRouter, + RouteTargetSet: routeTargetSet, + ChecksumAlgorithm: c.ChecksumAlgorithm, } } c.DataSources = dataSources @@ -585,6 +605,12 @@ func (c *Config) adjustConfigByDMSubTasks() (err error) { // Init initialize the config func (c *Config) Init() (err error) { + checksumAlgo := ChecksumAlgorithm(strings.ToLower(string(c.ChecksumAlgorithm))) + if checksumAlgo != MD5 && checksumAlgo != SHA256 { + return errors.Errorf("checksum-algorithm must be 'md5' or 'sha256', got: %s", c.ChecksumAlgorithm) + } + c.ChecksumAlgorithm = checksumAlgo + if len(c.DMAddr) > 0 { err := c.adjustConfigByDMSubTasks() if err != nil { @@ -597,6 +623,7 @@ func (c *Config) Init() (err error) { return nil } for _, d := range c.DataSources { + d.ChecksumAlgorithm = c.ChecksumAlgorithm routeRuleList := make([]*router.TableRule, 0, len(c.Routes)) d.RouteTargetSet = make(map[string]struct{}) // if we had rules diff --git a/sync_diff_inspector/config/config_test.go b/sync_diff_inspector/config/config_test.go index 8562a5e5f5..753dcc9eb4 100644 --- a/sync_diff_inspector/config/config_test.go +++ b/sync_diff_inspector/config/config_test.go @@ -50,10 +50,10 @@ func TestParseConfig(t *testing.T) { // we might not use the same config to run this test. e.g. MYSQL_PORT can be 4000 require.JSONEq(t, cfg.String(), - "{\"check-thread-count\":4,\"split-thread-count\":5,\"export-fix-sql\":true,\"check-struct-only\":false,\"dm-addr\":\"\",\"dm-task\":\"\",\"data-sources\":{\"mysql1\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null},\"mysql2\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null},\"mysql3\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule3\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null},\"tidb0\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":{\"max_execution_time\":86400,\"tidb_opt_prefer_range_scan\":\"ON\"}}},\"routes\":{\"rule1\":{\"schema-pattern\":\"test_*\",\"table-pattern\":\"t_*\",\"target-schema\":\"test\",\"target-table\":\"t\"},\"rule2\":{\"schema-pattern\":\"test2_*\",\"table-pattern\":\"t2_*\",\"target-schema\":\"test2\",\"target-table\":\"t2\"},\"rule3\":{\"schema-pattern\":\"test2_*\",\"table-pattern\":\"t2_*\",\"target-schema\":\"test\",\"target-table\":\"t\"}},\"table-configs\":{\"config1\":{\"target-tables\":[\"schema*.table*\",\"test2.t2\"],\"Schema\":\"\",\"Table\":\"\",\"ConfigIndex\":0,\"HasMatched\":false,\"IgnoreColumns\":[\"\",\"\"],\"Fields\":[\"\"],\"Range\":\"age \\u003e 10 AND age \\u003c 20\",\"TargetTableInfo\":null,\"Collation\":\"\",\"chunk-size\":0}},\"task\":{\"source-instances\":[\"mysql1\",\"mysql2\",\"mysql3\"],\"source-routes\":null,\"target-instance\":\"tidb0\",\"target-check-tables\":[\"schema*.table*\",\"!c.*\",\"test2.t2\"],\"target-configs\":[\"config1\"],\"output-dir\":\"/tmp/output/config\",\"SourceInstances\":[{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null},{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null},{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule3\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null}],\"TargetInstance\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":{\"max_execution_time\":86400,\"tidb_opt_prefer_range_scan\":\"ON\"}},\"TargetTableConfigs\":[{\"target-tables\":[\"schema*.table*\",\"test2.t2\"],\"Schema\":\"\",\"Table\":\"\",\"ConfigIndex\":0,\"HasMatched\":false,\"IgnoreColumns\":[\"\",\"\"],\"Fields\":[\"\"],\"Range\":\"age \\u003e 10 AND age \\u003c 20\",\"TargetTableInfo\":null,\"Collation\":\"\",\"chunk-size\":0}],\"TargetCheckTables\":[{},{},{}],\"FixDir\":\"/tmp/output/config/fix-on-tidb0\",\"CheckpointDir\":\"/tmp/output/config/checkpoint\",\"HashFile\":\"\"},\"ConfigFile\":\"config_sharding.toml\",\"PrintVersion\":false}") + "{\"check-thread-count\":4,\"split-thread-count\":5,\"export-fix-sql\":true,\"check-struct-only\":false,\"dm-addr\":\"\",\"dm-task\":\"\",\"checksum-algorithm\":\"md5\",\"data-sources\":{\"mysql1\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null,\"checksum-algorithm\":\"md5\"},\"mysql2\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null,\"checksum-algorithm\":\"md5\"},\"mysql3\":{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule3\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null,\"checksum-algorithm\":\"md5\"},\"tidb0\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":{\"max_execution_time\":86400,\"tidb_opt_prefer_range_scan\":\"ON\"},\"checksum-algorithm\":\"md5\"}},\"routes\":{\"rule1\":{\"schema-pattern\":\"test_*\",\"table-pattern\":\"t_*\",\"target-schema\":\"test\",\"target-table\":\"t\"},\"rule2\":{\"schema-pattern\":\"test2_*\",\"table-pattern\":\"t2_*\",\"target-schema\":\"test2\",\"target-table\":\"t2\"},\"rule3\":{\"schema-pattern\":\"test2_*\",\"table-pattern\":\"t2_*\",\"target-schema\":\"test\",\"target-table\":\"t\"}},\"table-configs\":{\"config1\":{\"target-tables\":[\"schema*.table*\",\"test2.t2\"],\"Schema\":\"\",\"Table\":\"\",\"ConfigIndex\":0,\"HasMatched\":false,\"IgnoreColumns\":[\"\",\"\"],\"Fields\":[\"\"],\"Range\":\"age \\u003e 10 AND age \\u003c 20\",\"TargetTableInfo\":null,\"Collation\":\"\",\"chunk-size\":0}},\"task\":{\"source-instances\":[\"mysql1\",\"mysql2\",\"mysql3\"],\"source-routes\":null,\"target-instance\":\"tidb0\",\"target-check-tables\":[\"schema*.table*\",\"!c.*\",\"test2.t2\"],\"target-configs\":[\"config1\"],\"output-dir\":\"/tmp/output/config\",\"SourceInstances\":[{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null,\"checksum-algorithm\":\"md5\"},{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule2\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null,\"checksum-algorithm\":\"md5\"},{\"host\":\"127.0.0.1\",\"port\":3306,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":[\"rule1\",\"rule3\"],\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":null,\"checksum-algorithm\":\"md5\"}],\"TargetInstance\":{\"host\":\"127.0.0.1\",\"port\":4000,\"user\":\"root\",\"password\":\"******\",\"sql-mode\":\"\",\"snapshot\":\"\",\"sql-hint-use-index\":\"\",\"security\":null,\"route-rules\":null,\"Router\":{\"Selector\":{}},\"Conn\":null,\"session\":{\"max_execution_time\":86400,\"tidb_opt_prefer_range_scan\":\"ON\"},\"checksum-algorithm\":\"md5\"},\"TargetTableConfigs\":[{\"target-tables\":[\"schema*.table*\",\"test2.t2\"],\"Schema\":\"\",\"Table\":\"\",\"ConfigIndex\":0,\"HasMatched\":false,\"IgnoreColumns\":[\"\",\"\"],\"Fields\":[\"\"],\"Range\":\"age \\u003e 10 AND age \\u003c 20\",\"TargetTableInfo\":null,\"Collation\":\"\",\"chunk-size\":0}],\"TargetCheckTables\":[{},{},{}],\"FixDir\":\"/tmp/output/config/fix-on-tidb0\",\"CheckpointDir\":\"/tmp/output/config/checkpoint\",\"HashFile\":\"\"},\"ConfigFile\":\"config_sharding.toml\",\"PrintVersion\":false}") hash, err := cfg.Task.ComputeConfigHash() require.NoError(t, err) - require.Equal(t, hash, "5a978bf48039d41b81403d635332493f031bb890a6d4e4d7df77f75e0ccc29f3") + require.Equal(t, hash, "09c59e9563f2f03ec970420b9df37bb06b8b7d31228e65ef9bc8f997ce9a9e20") require.True(t, cfg.TableConfigs["config1"].Valid()) require.NoError(t, os.RemoveAll(cfg.Task.OutputDir)) @@ -75,12 +75,31 @@ func TestError(t *testing.T) { cfg.CheckThreadCount = 1 require.True(t, cfg.CheckConfig()) - // Init + // Checksum algorithm - invalid + cfg.ChecksumAlgorithm = "invalid" + err := cfg.Init() + require.Contains(t, err.Error(), "checksum-algorithm must be 'md5' or 'sha256'") + + // Valid checksum algorithm - sha256 + cfg.ChecksumAlgorithm = "sha256" + cfg.DataSources = nil + err = cfg.Init() + require.NotContains(t, err.Error(), "checksum-algorithm") + require.Equal(t, SHA256, cfg.ChecksumAlgorithm) + + // Valid checksum algorithm - MD5 + cfg.ChecksumAlgorithm = "MD5" + err = cfg.Init() + require.NotContains(t, err.Error(), "checksum-algorithm") + require.Equal(t, MD5, cfg.ChecksumAlgorithm) // normalized to lowercase + + cfg.ChecksumAlgorithm = MD5 cfg.DataSources = make(map[string]*DataSource) + // Init - invalid route cfg.DataSources["123"] = &DataSource{ RouteRules: []string{"111"}, } - err := cfg.Init() + err = cfg.Init() require.Contains(t, err.Error(), "not found source routes for rule 111, please correct the config") } diff --git a/sync_diff_inspector/diff/diff.go b/sync_diff_inspector/diff/diff.go index 6462016512..13c34436bb 100644 --- a/sync_diff_inspector/diff/diff.go +++ b/sync_diff_inspector/diff/diff.go @@ -607,9 +607,9 @@ func (df *Diff) compareChecksumAndGetCount(ctx context.Context, tableRange *spli wg.Add(1) go func() { defer wg.Done() - upstreamInfo = df.upstream.GetCountAndMD5(ctx, tableRange) + upstreamInfo = df.upstream.GetCountAndChecksum(ctx, tableRange) }() - downstreamInfo = df.downstream.GetCountAndMD5(ctx, tableRange) + downstreamInfo = df.downstream.GetCountAndChecksum(ctx, tableRange) wg.Wait() if upstreamInfo.Err != nil { diff --git a/sync_diff_inspector/source/mysql_shard.go b/sync_diff_inspector/source/mysql_shard.go index a4258faf0d..5a633a8476 100644 --- a/sync_diff_inspector/source/mysql_shard.go +++ b/sync_diff_inspector/source/mysql_shard.go @@ -64,7 +64,8 @@ func (a *MySQLTableAnalyzer) AnalyzeSplitter(ctx context.Context, table *common. type MySQLSources struct { tableDiffs []*common.TableDiff - sourceTablesMap map[string][]*common.TableShardSource + sourceTablesMap map[string][]*common.TableShardSource + checksumAlgorithm config.ChecksumAlgorithm } func getMatchedSourcesForTable(sourceTablesMap map[string][]*common.TableShardSource, table *common.TableDiff) []*common.TableShardSource { @@ -99,8 +100,8 @@ func (s *MySQLSources) Close() { } } -// GetCountAndMD5 return count and checksum -func (s *MySQLSources) GetCountAndMD5(ctx context.Context, tableRange *splitter.RangeInfo) *ChecksumInfo { +// GetCountAndChecksum return count and checksum +func (s *MySQLSources) GetCountAndChecksum(ctx context.Context, tableRange *splitter.RangeInfo) *ChecksumInfo { beginTime := time.Now() table := s.tableDiffs[tableRange.GetTableIndex()] chunk := tableRange.GetChunk() @@ -110,7 +111,7 @@ func (s *MySQLSources) GetCountAndMD5(ctx context.Context, tableRange *splitter. for _, ms := range matchSources { go func(ms *common.TableShardSource) { - count, checksum, err := utils.GetCountAndMD5Checksum(ctx, ms.DBConn, ms.OriginSchema, ms.OriginTable, table.Info, chunk.Where, "", chunk.Args) + count, checksum, err := utils.GetCountAndChecksum(ctx, ms.DBConn, ms.OriginSchema, ms.OriginTable, table.Info, chunk.Where, "", chunk.Args, string(s.checksumAlgorithm)) infoCh <- &ChecksumInfo{ Checksum: checksum, Count: count, @@ -404,9 +405,14 @@ func NewMySQLSources(ctx context.Context, tableDiffs []*common.TableDiff, ds []* return nil, errors.Annotatef(err, "please make sure the filter is correct.") } + checksumAlgorithm := config.MD5 + if len(ds) > 0 { + checksumAlgorithm = ds[0].ChecksumAlgorithm + } mss := &MySQLSources{ - tableDiffs: tableDiffs, - sourceTablesMap: sourceTablesMap, + tableDiffs: tableDiffs, + sourceTablesMap: sourceTablesMap, + checksumAlgorithm: checksumAlgorithm, } return mss, nil } diff --git a/sync_diff_inspector/source/source.go b/sync_diff_inspector/source/source.go index 8ca5941e95..d894844a7a 100644 --- a/sync_diff_inspector/source/source.go +++ b/sync_diff_inspector/source/source.go @@ -88,8 +88,8 @@ type Source interface { // there are many workers consume the range from the channel to compare. GetRangeIterator(context.Context, *splitter.RangeInfo, TableAnalyzer, int) (RangeIterator, error) - // GetCountAndMD5 gets the md5 result and the count from given range. - GetCountAndMD5(context.Context, *splitter.RangeInfo) *ChecksumInfo + // GetCountAndChecksum gets the checksum result and the count from given range. + GetCountAndChecksum(context.Context, *splitter.RangeInfo) *ChecksumInfo // GetCountForLackTable gets the count for tables that don't exist upstream or downstream. GetCountForLackTable(context.Context, *splitter.RangeInfo) int64 diff --git a/sync_diff_inspector/source/source_test.go b/sync_diff_inspector/source/source_test.go index 90237b06e3..4f4543c899 100644 --- a/sync_diff_inspector/source/source_test.go +++ b/sync_diff_inspector/source/source_test.go @@ -182,7 +182,7 @@ func TestTiDBSource(t *testing.T) { require.Equal(t, n, tableCase.rangeInfo.GetTableIndex()) countRows := sqlmock.NewRows([]string{"CNT", "CHECKSUM"}).AddRow(123, 456) mock.ExpectQuery("SELECT COUNT.*").WillReturnRows(countRows) - checksum := tidb.GetCountAndMD5(ctx, tableCase.rangeInfo) + checksum := tidb.GetCountAndChecksum(ctx, tableCase.rangeInfo) require.NoError(t, checksum.Err) require.Equal(t, checksum.Count, int64(123)) require.Equal(t, checksum.Checksum, uint64(456)) @@ -403,7 +403,7 @@ func TestMysqlShardSources(t *testing.T) { mock.ExpectQuery("SELECT COUNT.*").WillReturnRows(countRows) } - checksum := shard.GetCountAndMD5(ctx, tableCase.rangeInfo) + checksum := shard.GetCountAndChecksum(ctx, tableCase.rangeInfo) require.NoError(t, checksum.Err) require.Equal(t, checksum.Count, int64(len(dbs))) require.Equal(t, checksum.Checksum, resChecksum) diff --git a/sync_diff_inspector/source/tidb.go b/sync_diff_inspector/source/tidb.go index 6355022387..edeee54efb 100644 --- a/sync_diff_inspector/source/tidb.go +++ b/sync_diff_inspector/source/tidb.go @@ -88,10 +88,11 @@ func (s *TiDBRowsIterator) Next() (map[string]*dbutil.ColumnData, error) { // TiDBSource represents the table in TiDB type TiDBSource struct { - tableDiffs []*common.TableDiff - sourceTableMap map[string]*common.TableSource - snapshot string - sqlHint string + tableDiffs []*common.TableDiff + sourceTableMap map[string]*common.TableSource + snapshot string + sqlHint string + checksumAlgorithm config.ChecksumAlgorithm // bucketSpliterPool is the shared pool to produce chunks using bucket bucketSpliterPool *utils.WorkerPool dbConn *sql.DB @@ -130,8 +131,8 @@ func (s *TiDBSource) Close() { s.dbConn.Close() } -// GetCountAndMD5 returns the checksum info -func (s *TiDBSource) GetCountAndMD5(ctx context.Context, tableRange *splitter.RangeInfo) *ChecksumInfo { +// GetCountAndChecksum returns the checksum info +func (s *TiDBSource) GetCountAndChecksum(ctx context.Context, tableRange *splitter.RangeInfo) *ChecksumInfo { beginTime := time.Now() table := s.tableDiffs[tableRange.GetTableIndex()] chunk := tableRange.GetChunk() @@ -159,9 +160,9 @@ func (s *TiDBSource) GetCountAndMD5(ctx context.Context, tableRange *splitter.Ra } } - count, checksum, err := utils.GetCountAndMD5Checksum( + count, checksum, err := utils.GetCountAndChecksum( ctx, s.dbConn, matchSource.OriginSchema, matchSource.OriginTable, table.Info, - chunk.Where, indexHint, chunk.Args) + chunk.Where, indexHint, chunk.Args, string(s.checksumAlgorithm)) cost := time.Since(beginTime) return &ChecksumInfo{ @@ -328,6 +329,7 @@ func NewTiDBSource( bucketSpliterPool: bucketSpliterPool, version: utils.TryToGetVersion(ctx, ds.Conn), sqlHint: ds.SQLHintUseIndex, + checksumAlgorithm: ds.ChecksumAlgorithm, } return ts, nil } diff --git a/sync_diff_inspector/tests/sync_diff_inspector/shard/run.sh b/sync_diff_inspector/tests/sync_diff_inspector/shard/run.sh index 8795632b1e..f62fdb5c21 100644 --- a/sync_diff_inspector/tests/sync_diff_inspector/shard/run.sh +++ b/sync_diff_inspector/tests/sync_diff_inspector/shard/run.sh @@ -68,67 +68,67 @@ mysql -uroot -h ${MYSQL_HOST} -P ${MYSQL_PORT} -e "insert into Router_test_1.Tbl echo "test router 1: normal rule" sed "s/\"127.0.0.1\"#MYSQL_HOST/\"${MYSQL_HOST}\"/g" ./config_router_1.toml | sed "s/3306#MYSQL_PORT/${MYSQL_PORT}/g" >./config.toml sync_diff_inspector --config=./config.toml -L debug >$OUT_DIR/shard_diff.output || true -check_contains "as CHECKSUM FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log -#check_not_contains "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains_count "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 1 -check_not_contains "as CHECKSUM FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log +#check_not_contains "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains_count "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 1 +check_not_contains "as hash FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log rm -rf $OUT_DIR/* echo "test router 2: only schema rule" sed "s/\"127.0.0.1\"#MYSQL_HOST/\"${MYSQL_HOST}\"/g" ./config_router_2.toml | sed "s/3306#MYSQL_PORT/${MYSQL_PORT}/g" >./config.toml sync_diff_inspector --config=./config.toml -L debug >$OUT_DIR/shard_diff.output || true -check_contains "as CHECKSUM FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log -#check_not_contains "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains_count "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 1 -check_not_contains "as CHECKSUM FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log +#check_not_contains "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains_count "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 1 +check_not_contains "as hash FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log rm -rf $OUT_DIR/* echo "test router 3: other rule" sed "s/\"127.0.0.1\"#MYSQL_HOST/\"${MYSQL_HOST}\"/g" ./config_router_3.toml | sed "s/3306#MYSQL_PORT/${MYSQL_PORT}/g" >./config.toml sync_diff_inspector --config=./config.toml -L debug >$OUT_DIR/shard_diff.output || true -check_not_contains "as CHECKSUM FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log -#check_contains "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains_count "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 2 -check_contains "as CHECKSUM FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log +#check_contains "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains_count "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 2 +check_contains "as hash FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log rm -rf $OUT_DIR/* echo "test router 4: no rule" sed "s/\"127.0.0.1\"#MYSQL_HOST/\"${MYSQL_HOST}\"/g" ./config_router_4.toml | sed "s/3306#MYSQL_PORT/${MYSQL_PORT}/g" >./config.toml sync_diff_inspector --config=./config.toml -L debug >$OUT_DIR/shard_diff.output || true -check_not_contains "as CHECKSUM FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_not_contains "as CHECKSUM FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log -#check_contains "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains_count "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 2 -check_contains "as CHECKSUM FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_not_contains "as hash FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log +#check_contains "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains_count "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 2 +check_contains "as hash FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log rm -rf $OUT_DIR/* echo "test router 5: regex rule" sed "s/\"127.0.0.1\"#MYSQL_HOST/\"${MYSQL_HOST}\"/g" ./config_router_5.toml | sed "s/3306#MYSQL_PORT/${MYSQL_PORT}/g" >./config.toml sync_diff_inspector --config=./config.toml -L debug >$OUT_DIR/shard_diff.output || true -check_contains "as CHECKSUM FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log -#check_contains "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains_count "as CHECKSUM FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 2 -check_contains "as CHECKSUM FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log -check_contains "as CHECKSUM FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`Router_test_0\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_0\`.\`Tbl\`" $OUT_DIR/sync_diff.log +#check_contains "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains_count "as hash FROM \`router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log 2 +check_contains "as hash FROM \`Router_test_1\`.\`tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log +check_contains "as hash FROM \`Router_test_1\`.\`Tbl\`" $OUT_DIR/sync_diff.log rm -rf $OUT_DIR/* echo "shard test passed" diff --git a/sync_diff_inspector/utils/utils.go b/sync_diff_inspector/utils/utils.go index dbf769a427..b64abf5a03 100644 --- a/sync_diff_inspector/utils/utils.go +++ b/sync_diff_inspector/utils/utils.go @@ -164,12 +164,8 @@ func GetTableRowsQueryFormat(schema, table string, tableInfo *model.TableInfo, c } name := dbutil.ColumnName(col.Name.O) - // When col value is 0, the result is NULL. - // But we can use ISNULL to distinguish between null and 0. - if col.FieldType.GetType() == mysql.TypeFloat { - name = fmt.Sprintf("round(%s, 5-floor(log10(abs(%s)))) as %s", name, name, name) - } else if col.FieldType.GetType() == mysql.TypeDouble { - name = fmt.Sprintf("round(%s, 14-floor(log10(abs(%s)))) as %s", name, name, name) + if col.FieldType.GetType() == mysql.TypeFloat || col.FieldType.GetType() == mysql.TypeDouble { + name = fmt.Sprintf("cast(%s as decimal(65, 30)) as %s", name, name) } columnNames = append(columnNames, name) } @@ -857,21 +853,25 @@ func GetTableSize(ctx context.Context, db *sql.DB, schemaName, tableName string) return dataSize.Int64, nil } -// GetCountAndMD5Checksum returns checksum code and count of some data by given condition -func GetCountAndMD5Checksum( +// GetCountAndChecksum returns checksum code and count of some data by given condition +func GetCountAndChecksum( ctx context.Context, db *sql.DB, schemaName, tableName string, tbInfo *model.TableInfo, limitRange string, indexHint string, args []any, + checksumAlgorithm string, ) (int64, uint64, error) { /* - calculate MD5 checksum and count example: - mysql> SELECT COUNT(*) as CNT, BIT_XOR(CAST(CONV(SUBSTRING(MD5(CONCAT_WS(',', `id`, `name`, CONCAT(ISNULL(`id`), ISNULL(`name`)))), 1, 16), 16, 10) AS UNSIGNED) ^ CAST(CONV(SUBSTRING(MD5(CONCAT_WS(',', `id`, `name`, CONCAT(ISNULL(`id`), ISNULL(`name`)))), 17, 16), 16, 10) AS UNSIGNED)) as CHECKSUM FROM `a`.`t`; + calculate checksum and count example (MD5): + mysql> SELECT COUNT(*) as CNT, BIT_XOR(CAST(CONV(SUBSTRING(hash, 1, 16), 16, 10) AS UNSIGNED) ^ CAST(CONV(SUBSTRING(hash, 17, 16), 16, 10) AS UNSIGNED)) as CHECKSUM FROM (SELECT MD5(CONCAT_WS(',', `id`, `name`, CONCAT(ISNULL(`id`), ISNULL(`name`)))) as hash FROM `a`.`t` WHERE TRUE) as t; +--------+---------------------- | CNT | CHECKSUM | +--------+---------------------- | 100000 | 3462532621352132810 | +--------+---------------------- 1 row in set (0.46 sec) + + calculate checksum and count example (SHA256): + mysql> SELECT COUNT(*) as CNT, BIT_XOR(CAST(CONV(SUBSTRING(hash, 1, 16), 16, 10) AS UNSIGNED) ^ CAST(CONV(SUBSTRING(hash, 17, 16), 16, 10) AS UNSIGNED)) as CHECKSUM FROM (SELECT SHA2(CONCAT_WS(',', `id`, `name`, CONCAT(ISNULL(`id`), ISNULL(`name`))), 256) as hash FROM `a`.`t` WHERE TRUE) as t; */ columnNames := make([]string, 0, len(tbInfo.Columns)) columnIsNull := make([]string, 0, len(tbInfo.Columns)) @@ -881,27 +881,33 @@ func GetCountAndMD5Checksum( continue } name := dbutil.ColumnName(col.Name.O) - // When col value is 0, the result is NULL. - // But we can use ISNULL to distinguish between null and 0. - if col.FieldType.GetType() == mysql.TypeFloat { - name = fmt.Sprintf("round(%s, 5-floor(log10(abs(%s))))", name, name) - } else if col.FieldType.GetType() == mysql.TypeDouble { - name = fmt.Sprintf("round(%s, 14-floor(log10(abs(%s))))", name, name) + if col.FieldType.GetType() == mysql.TypeFloat || col.FieldType.GetType() == mysql.TypeDouble { + name = fmt.Sprintf("cast(%s as decimal(65, 30))", name) } columnNames = append(columnNames, name) columnIsNull = append(columnIsNull, fmt.Sprintf("ISNULL(%s)", name)) } - query := fmt.Sprintf("SELECT %s COUNT(*) as CNT, BIT_XOR(CAST(CONV(SUBSTRING(MD5(CONCAT_WS(',', %s, CONCAT(%s))), 1, 16), 16, 10) AS UNSIGNED) ^ CAST(CONV(SUBSTRING(MD5(CONCAT_WS(',', %s, CONCAT(%s))), 17, 16), 16, 10) AS UNSIGNED)) as CHECKSUM FROM %s WHERE %s;", - indexHint, - strings.Join(columnNames, ", "), - strings.Join(columnIsNull, ", "), + var checksumFuncTemplate string + if checksumAlgorithm == "sha256" { + checksumFuncTemplate = "SHA2(%s, 256)" + } else { + checksumFuncTemplate = "MD5(%s)" + } + + concatExpr := fmt.Sprintf("CONCAT_WS(',', %s, CONCAT(%s))", strings.Join(columnNames, ", "), - strings.Join(columnIsNull, ", "), + strings.Join(columnIsNull, ", ")) + + checksumExpr := fmt.Sprintf(checksumFuncTemplate, concatExpr) + + query := fmt.Sprintf("SELECT COUNT(*) as CNT, BIT_XOR(CAST(CONV(SUBSTRING(hash, 1, 16), 16, 10) AS UNSIGNED) ^ CAST(CONV(SUBSTRING(hash, 17, 16), 16, 10) AS UNSIGNED)) as CHECKSUM FROM (SELECT %s %s as hash FROM %s WHERE %s) as t;", + indexHint, + checksumExpr, dbutil.TableName(schemaName, tableName), limitRange, ) - log.Debug("count and checksum", zap.String("sql", query), zap.Reflect("args", args)) + log.Debug("count and checksum", zap.String("sql", query), zap.Reflect("args", args), zap.String("checksum-algorithm", checksumAlgorithm)) var count sql.NullInt64 var checksum uint64 diff --git a/sync_diff_inspector/utils/utils_test.go b/sync_diff_inspector/utils/utils_test.go index eb08551403..01af446954 100644 --- a/sync_diff_inspector/utils/utils_test.go +++ b/sync_diff_inspector/utils/utils_test.go @@ -83,7 +83,7 @@ func TestBasicTableUtilOperation(t *testing.T) { require.NoError(t, err) query, orderKeyCols := GetTableRowsQueryFormat("test", "test", tableInfo, "123") - require.Equal(t, query, "SELECT /*!40001 SQL_NO_CACHE */ `a`, `b`, round(`c`, 5-floor(log10(abs(`c`)))) as `c`, `d` FROM `test`.`test` WHERE %s ORDER BY `a`,`b` COLLATE '123'") + require.Equal(t, query, "SELECT /*!40001 SQL_NO_CACHE */ `a`, `b`, cast(`c` as decimal(65, 30)) as `c`, `d` FROM `test`.`test` WHERE %s ORDER BY `a`,`b` COLLATE '123'") expectName := []string{"a", "b"} for i, col := range orderKeyCols { require.Equal(t, col.Name.O, expectName[i]) @@ -256,7 +256,7 @@ func TestBasicTableUtilOperation(t *testing.T) { require.Equal(t, tableInfo.Indices[0].Columns[1].Offset, 1) } -func TestGetCountAndMD5Checksum(t *testing.T) { +func TestGetCountAndChecksumMD5(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -270,12 +270,33 @@ func TestGetCountAndMD5Checksum(t *testing.T) { mock.ExpectQuery("SELECT COUNT.*FROM `test_schema`\\.`test_table` WHERE \\[23 45\\].*").WithArgs("123", "234").WillReturnRows(sqlmock.NewRows([]string{"CNT", "CHECKSUM"}).AddRow(123, 456)) - count, checksum, err := GetCountAndMD5Checksum(ctx, conn, "test_schema", "test_table", tableInfo, "[23 45]", "", []any{"123", "234"}) + count, checksum, err := GetCountAndChecksum(ctx, conn, "test_schema", "test_table", tableInfo, "[23 45]", "", []any{"123", "234"}, "md5") require.NoError(t, err) require.Equal(t, count, int64(123)) require.Equal(t, checksum, uint64(0x1c8)) } +func TestGetCountAndChecksumSHA256(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + conn, mock, err := sqlmock.New() + require.NoError(t, err) + defer conn.Close() + + createTableSQL := "create table `test`.`test`(`a` int, `c` float, `b` varchar(10), `d` datetime, primary key(`a`, `b`), key(`c`, `d`))" + tableInfo, err := GetTableInfoBySQL(createTableSQL, parser.New()) + require.NoError(t, err) + + // Verify that SHA2 is used in the query + mock.ExpectQuery("SELECT COUNT.*SHA2.*FROM `test_schema`\\.`test_table` WHERE \\[23 45\\].*").WithArgs("123", "234").WillReturnRows(sqlmock.NewRows([]string{"CNT", "CHECKSUM"}).AddRow(456, 789)) + + count, checksum, err := GetCountAndChecksum(ctx, conn, "test_schema", "test_table", tableInfo, "[23 45]", "", []any{"123", "234"}, "sha256") + require.NoError(t, err) + require.Equal(t, count, int64(456)) + require.Equal(t, checksum, uint64(789)) +} + func TestGetApproximateMid(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel()