diff --git a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md index 59c3d6778b9bb..21769fb42abe1 100644 --- a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md @@ -14,7 +14,7 @@ - `pkg/distsql/distsql_test.go` - Tests normal select. - `pkg/distsql/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/distsql/request_builder_test.go` - Tests table handles to KV ranges. -- `pkg/distsql/select_result_test.go` - Tests update coprocessor runtime stats. +- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership/provenance counts, multi-response aggregation, plan-local Cop/CopStream RPC access, independent expected-task coverage, and close-time stats-only handling that cannot replay stale summaries. ## pkg/distsql/context diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index 1a14fcc5a97cd..1ecf4aa555076 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -9,7 +9,8 @@ ## pkg/executor ### Tests -- `pkg/executor/adapter_test.go` - executor: Tests format SQL. +- `pkg/executor/adapter_internal_test.go` - executor: Tests private adapter runtime accounting, including preview RU General Log gates, mutation `cpu_work` plus raw-unit provenance, DML/source/side aggregation, serialization, and deterministic ordering. +- `pkg/executor/adapter_test.go` - executor: Tests statement adapter behavior, including RU v2 finalization and self-describing preview RU General Log completion after lazy SELECT close, DML/early errors, compile-error suppression, cursor-detach gating, repeated finish calls, and EXPLAIN ANALYZE DML commit errors. - `pkg/executor/analyze_test.go` - executor: Tests analyze index extract top n. - `pkg/executor/analyze_utils_test.go` - executor: Tests get analyze panic err. - `pkg/executor/batch_point_get_test.go` - executor: Tests batch point get lock exist key. @@ -29,7 +30,7 @@ - `pkg/executor/executor_failpoint_test.go` - executor: Tests TiDB last txn info commit mode. - `pkg/executor/executor_pkg_test.go` - executor: Tests build KV ranges for index join without CWC. - `pkg/executor/executor_required_rows_test.go` - executor: Tests limit required rows. -- `pkg/executor/explain_test.go` - executor: Tests explain analyze memory. +- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including production-default uncalibrated v4 units and absent totals, distinct read/write request output units, DML PointGet read-request publication, six pushed TiKV operator classes (including an explicitly ordered StreamAgg) with native Unistore's incomplete scan-width fail-closed behavior, bounded scan/transport/point-lookup failures, all raw DML mutation diagnostics with gated mutation `cpu_work`, statement-summary/metrics output versions, restricted/feature-off isolation, and legacy v3 behavior retained as explicitly superseded coverage. - `pkg/executor/explain_unit_test.go` - executor: Tests explain analyze invoke next and close. - `pkg/executor/explainfor_test.go` - executor: Tests explain for. - `pkg/executor/grant_test.go` - executor: Tests grant global. @@ -188,12 +189,12 @@ - `pkg/executor/join/anti_semi_join_probe_test.go` - executor/join: Tests anti-semi join basic. - `pkg/executor/join/bench_test.go` - executor/join: Tests hash table build. - `pkg/executor/join/concurrent_map_test.go` - executor/join: Tests concurrent map. -- `pkg/executor/join/hash_table_v1_test.go` - executor/join: Tests hash row container. +- `pkg/executor/join/hash_table_v1_test.go` - executor/join: Tests hash row container, including V1 NAAJ null-bucket rows in HashJoin state accounting. - `pkg/executor/join/hash_table_v2_test.go` - executor/join: Tests hash table size. - `pkg/executor/join/inner_join_probe_test.go` - executor/join: Tests inner join probe basic. - `pkg/executor/join/inner_join_spill_test.go` - executor/join: Tests inner join spill basic. - `pkg/executor/join/join_row_table_test.go` - executor/join: Tests heap object can move. -- `pkg/executor/join/join_stats_test.go` - executor/join: Tests hash join runtime stats. +- `pkg/executor/join/join_stats_test.go` - executor/join: Tests HashJoin V1/V2 runtime stats, valid-key state-row accounting, overflow rejection, and clone/merge preservation. - `pkg/executor/join/join_table_meta_test.go` - executor/join: Tests join table meta key mode. - `pkg/executor/join/joiner_test.go` - executor/join: Tests required rows. - `pkg/executor/join/left_outer_anti_semi_join_probe_test.go` - executor/join: Tests left-outer anti-semi join probe basic. diff --git a/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md b/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md index 98fb8e2664a31..175902cfe4f0d 100644 --- a/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md @@ -10,7 +10,7 @@ ### Tests - `pkg/metrics/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/metrics/metrics_internal_test.go` - Tests retained labels. +- `pkg/metrics/metrics_internal_test.go` - Tests retained labels and preview-RU bounded metric propagation, including distinct read/write request units, dimension-qualified mutation `cpu_work`, raw mutation samples, and measured zero base-unit samples. - `pkg/metrics/metrics_test.go` - Tests metrics. ## pkg/metrics/common diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index f7e62a9cd68c3..80ea7934e8851 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests LOAD DATA line/field defaults. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, private v4 semantic weights and overflow rejection, independent read/write request units and weights, expression-slot counts for Agg/Join/Window families, Sort CPU work plus TopN input saturation/zero-count/checked bounds, UnionScan direct-child CPU work, plan-local DML cop-reader and PointGet RPC ownership with isolation from statement counters, real distsql ScanDetail ownership/provenance and fail-closed coverage, HashJoin state inputs, mutation normalization into dimension-qualified `cpu_work` with raw diagnostics, and statement-local DML/COMMIT request ownership. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/.agents/skills/tidb-test-guidelines/references/session-case-map.md b/.agents/skills/tidb-test-guidelines/references/session-case-map.md index 944b913714e9e..7dc5396d53549 100644 --- a/.agents/skills/tidb-test-guidelines/references/session-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/session-case-map.md @@ -13,7 +13,7 @@ - `pkg/session/bootstrap_test.go` - session: Tests MySQL DB tables. - `pkg/session/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/session/session_test.go` - session: Tests get start mode. -- `pkg/session/tidb_test.go` - session: Tests do map handle nil. +- `pkg/session/tidb_test.go` - session: Tests domain-map basics, per-statement RUv2 isolation including pessimistic DML/COMMIT write-request snapshots, and all four preview KV mutation APIs including failed attempts. - `pkg/session/upgrade_test.go` - session: Tests upgrade to ver functions check. ## pkg/session/cursor diff --git a/.agents/skills/tidb-test-guidelines/references/util-case-map.md b/.agents/skills/tidb-test-guidelines/references/util-case-map.md index 95410e5b6185f..1097da4ce1c53 100644 --- a/.agents/skills/tidb-test-guidelines/references/util-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/util-case-map.md @@ -215,7 +215,7 @@ ## pkg/util/execdetails ### Tests -- `pkg/util/execdetails/execdetails_test.go` - util/execdetails: Tests string. +- `pkg/util/execdetails/execdetails_test.go` - util/execdetails: Tests runtime-detail formatting, cop summary/task aggregation, and ScanDetail attachment provenance snapshots. - `pkg/util/execdetails/main_test.go` - Configures default goleak settings and registers testdata. ## pkg/util/expensivequery @@ -528,6 +528,7 @@ ### Tests - `pkg/util/stmtsummary/evicted_test.go` - util/stmtsummary: Tests map to evicted count datum. - `pkg/util/stmtsummary/main_test.go` - Configures default goleak settings and registers testdata. +- `pkg/util/stmtsummary/read_billing_test.go` - util/stmtsummary: Tests preview RU dimension aggregation, caps, distinct read/write request units, DML-kind separation for mutation `cpu_work` and raw units, and v4 detail retention with zero legacy convenience totals. - `pkg/util/stmtsummary/statement_summary_test.go` - util/stmtsummary: Tests set up. ## pkg/util/stmtsummary/v2 @@ -535,8 +536,8 @@ ### Tests - `pkg/util/stmtsummary/v2/column_test.go` - util/stmtsummary/v2: Tests column. - `pkg/util/stmtsummary/v2/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/util/stmtsummary/v2/reader_test.go` - util/stmtsummary/v2: Tests time range overlap. -- `pkg/util/stmtsummary/v2/record_test.go` - util/stmtsummary/v2: Tests stmt record. +- `pkg/util/stmtsummary/v2/reader_test.go` - util/stmtsummary/v2: Tests time ranges plus current/history preview RU rows and backward-compatible DML kind reads. +- `pkg/util/stmtsummary/v2/record_test.go` - util/stmtsummary/v2: Tests statement records, including preview RU DML kind aggregation and JSON round trips. - `pkg/util/stmtsummary/v2/stmtsummary_benchmark_test.go` - util/stmtsummary/v2: Tests stmt summary add single workload. - `pkg/util/stmtsummary/v2/stmtsummary_test.go` - util/stmtsummary/v2: Tests stmt window. diff --git a/docs/design/2026-07-01-read-billing-demo-ru-model.md b/docs/design/2026-07-01-read-billing-demo-ru-model.md new file mode 100644 index 0000000000000..a163147645d23 --- /dev/null +++ b/docs/design/2026-07-01-read-billing-demo-ru-model.md @@ -0,0 +1,817 @@ +# Read Billing Demo RU 模型与校准统计面设计 + +> **当前模型说明（2026-07-22）：** 本文描述的是历史 v1-v3 模型，仅保留为迁移与校准背景。当前已实现的规范是 +> `docs/design/2026-07-22-preview-ru-resource-formula-plan.md` 中的 v4：使用语义化资源单位，三个旧 convenience totals +> 对 v4 恒为零，并且权重未校准时不发布 `total_preview_ru`。下文与此冲突的“当前”表述仅适用于 v3。 + +- Author(s): TBD +- Discussion PR: TBD +- Tracking Issue: TBD + +## 目录 + +* [简介](#简介) +* [背景和目标](#背景和目标) +* [当前代码边界](#当前代码边界) +* [模型和支持范围](#模型和支持范围) +* [推荐统计面](#推荐统计面) +* [采集和生命周期](#采集和生命周期) +* [查询和权重校准方法](#查询和权重校准方法) +* [初始权重](#初始权重) +* [性能和容量控制](#性能和容量控制) +* [兼容性和迁移计划](#兼容性和迁移计划) +* [测试设计](#测试设计) +* [备选方案](#备选方案) +* [风险和未决问题](#风险和未决问题) + +## 简介 + +本文描述 read billing demo 的 SELECT 读请求 RU 模型，以及用于 workload 权重校准的 TiDB-native 统计面。 + +模型把读请求执行过程拆成有限数量的 operator class。每个 operator class 都按版本管理权重，权重作用在无系数的 base units 上： + +```text +preview_ru = + fixed_events * fixed_weight + + input_rows * row_weight + + input_bytes * byte_weight + + order_work * order_weight +``` + +`order_work` 只由比较型排序算子产生：full sort 使用 `n * log2(max(n, 2))`，TopN 使用 `n * log2(max(offset + count, 2))`。它作为 double base unit 展示在 `work_rows` 并进入明细统计，不强制转换为 int64 `COUNT`。排序类仍保留真实 `input_rows` base unit，但其 seed `row_weight` 为 0，避免把线性行项与算法 work 重复计费。 + +Join 和 Agg 还会在证据充分时产生 `output_rows`、`output_bytes` shadow units，用于观察结果基数和宽度。它们标记为 `diagnostic_only=true`，不注册 weight，因而不属于上述公式：EXPLAIN 中没有 weight/preview RU，`total_preview_ru` 也不会变化。 + +第一版是校准 demo，不是最终生产 billing 语义。它和现有 RU v2 resource-control 上报保持隔离，并分成两个输出面： + +1. `EXPLAIN ANALYZE FORMAT='RU'`：单语句诊断输出，显示 base units、weight 和 preview RU。 +2. statement-summary-associated statistics tables：workload 级无系数统计面，用于离线重新套任意 weights，不依赖 Prometheus scrape，也不打印 per-SQL log。 + +Prometheus `tidb_read_billing_demo_*` metrics 可以继续作为 bounded observability 辅助，但不能作为 workload testers 的必需接口；校准契约以 SQL 查询面为准。 + +### v3 实现说明 + +当前实现的 `model_version = 'v3'` 延续 v2 的执行期 byte evidence，并为证据充分的 TiKV 一元 cop 链增加保守的输入边估算： + +- TiDB root operator 使用 runtime chunk 的 logical live bytes，`input_source = runtime_chunk_bytes`，`row_width_source = runtime_chunk_avg`。 +- TiKV range scan 使用 scan detail 中的 `processed_key_size / processed_keys * total_keys`，`row_width_source = scan_detail_processed_key_avg`。 +- TiKV Selection、Projection、Limit、TopN、HashAgg 和 StreamAgg 使用直接 child 在 exact plan ID 下聚合到的 runtime `actRows` 作为 `input_rows`，再乘同一 TiKV component 的 `ProcessedKeysSize / ProcessedKeys` 作为估算 `input_bytes`；这类样本标记为 `input_source = runtime_child_act_rows`、`row_width_source = scan_detail_processed_key_avg_estimate`。这里的 exact 只表示 plan-ID attribution，不表示所有 response summary 必然无缺口。 +- Selection、Limit、TopN 保持行形状并向父节点传播平均宽度；Projection、HashAgg 和 StreamAgg 可以消费 child 宽度，但其输出是 width barrier，不能继续向父节点传播。 +- TiDB Join/Agg 的 output shadow units 来自算子自身 runtime chunk；TiKV HashAgg/StreamAgg 的 `output_rows` 来自算子自身 plan ID 下的 cop summary，并且只在该 plan 的 observed summary task 数等于每个已收到 TiKV response 独立登记的 expected task 数时生成。TiKV summary 没有 per-executor output bytes，所以只有这项 coverage 成立、top-cop Agg 直接连接单一普通 reader、且 reader/Agg output rows 相等时，才把 reader logical output bytes 归因成 Agg `output_bytes`。 +- 缺少 exact child summary、唯一 scan/detail 归属或可传播宽度时，以 bounded reason fail closed；SELECT 不产出 partial billable units，DML 只保留其他独立成功算子的 units。 + +这里的 estimated bytes 不是 TiKV executor edge 上真实编码字节数，而是 runtime child rows 与 storage KV 平均宽度的混合 proxy。本文后续保留的 `runtime_act_rows`、`plan_stats`、`schema_type_width`、`schema_fallback`、`operator_helper` 等描述只适用于 v1 历史估算模型或背景说明，不代表 v2/v3 的当前构造规则。v2 的历史契约只覆盖 TiDB runtime bytes 与 TiKV range-scan detail；v3 才增加上述非 scan cop 输入估算。 + +## 背景和目标 + +现有 RU v2 主要是 statement-level billing 逻辑，不适合回答两个问题： + +1. 一条 SELECT 语句里的 TiDB / TiKV 哪些 executor 消耗了 RU； +2. 在 workload 下，如何收集无系数、可重放的原始输入，用真实压测数据回归出更合理的 weights。 + +read billing demo 的目标是： + +1. 让单条语句可以通过 `EXPLAIN ANALYZE FORMAT='RU'` 看到 operator-level 的 base units、weight 和 preview RU； +2. 让 workload 可以通过 SQL 查询收集 `site/op_class/unit` 维度的 `fixed_events`、`input_rows`、`input_bytes` 等基础量； +3. 让 testers 可以在 workload 结束后直接替换 weights 并离线重算 RU，不必重跑 workload； +4. 让 unsupported、unknown input、statement error、aggregation overflow 等失败或降级状态有可查询位置； +5. 在 weights 还没有校准完成前，避免把 demo 结果混入现有 RU v2 billing 或 resource-control consumption 上报。 + +非目标： + +- 不把 per-SQL log 作为校准接口； +- 不要求 downstream testers 搭 Prometheus 或依赖 metrics retention； +- 不在第一版覆盖 TiFlash、MPP、IndexMerge 或复杂 storage path； +- 不把当前 `weight_version = 'v2'` 的 seed weights 解释成生产校准值。 + +## 当前代码边界 + +当前实现已经有以下锚点： + +- `pkg/sessionctx/vardef/tidb_vars.go`、`pkg/sessionctx/variable/session.go`、`pkg/sessionctx/variable/sysvar.go` 定义 `tidb_enable_read_billing_demo`，默认 OFF。 +- `pkg/executor/adapter.go::FinishExecuteStmt` 在 statement 完成时调用 `plannercore.RecordReadBillingDemoForStatement`，随后 `SummaryStmt` 把返回的 read-billing summary 放进 `stmtsummary.StmtExecInfo`。 +- `pkg/session/session.go::recordReadBillingDemoEarlyError` 覆盖 compile / resolve / pre-completion error 等没有正常走到 `FinishExecuteStmt` 的错误出口。 +- `pkg/planner/core/explain_ru.go::buildReadBillingDemoResult` 生成 frozen result，包含 statement status、operator status 和 operator base units。 +- `pkg/planner/core/explain_ru.go::recordReadBillingDemoResult` 当前会写 bounded Prometheus metrics。 +- `pkg/planner/core/explain_ru.go::summarizeReadBillingDemoBaseUnits` 当前只把成功 statement 的 base units 汇总成三个 scalar totals。 +- `pkg/util/stmtsummary/statement_summary.go`、`pkg/util/stmtsummary/reader.go`、`pkg/infoschema/tables.go` 当前在 statement summary v1 路径暴露三列： + - `SUM_READ_BILLING_DEMO_FIXED_EVENTS` + - `SUM_READ_BILLING_DEMO_INPUT_ROWS` + - `SUM_READ_BILLING_DEMO_INPUT_BYTES` +- `pkg/util/stmtsummary/v2/record.go`、`pkg/util/stmtsummary/v2/column.go` 当前在 persistent statement summary v2 路径保持同样三列。 + +三列 totals 可以作为 convenience summary，但它们丢失了 `site/op_class/unit/input_source/input_side` 维度，无法支持 per-opclass weight calibration。因此本设计推荐新增维度化 SQL 查询面，并从同一个 statement summary 生命周期里聚合。 + +## 模型和支持范围 + +### 支持范围和安全 gate + +demo 只接受 side-effect-free SELECT： + +- 普通 `SELECT`； +- set operation，前提是所有叶子都是 side-effect-free SELECT； +- prepared statement / `EXECUTE`，前提是 resolved prepared target 是可支持的 side-effect-free SELECT。 + +以下语句或路径会被拒绝、跳过或 fail closed： + +- 非 SELECT； +- `SELECT ... INTO`； +- locking SELECT； +- 带副作用的函数，例如 lock 函数、sequence mutation、`LAST_INSERT_ID(expr)` 和 `SLEEP`； +- internal / restricted SQL； +- TiFlash、MPP exchange、IndexMerge 和未支持的 operator。 + +当出现 unsupported、unknown input 或 statement error 时，demo 必须记录 status/reason，但不能上报 partial base units 或 partial preview RU。 + +### Base-unit result + +每个 supported 且实际执行的 billable operator 至少产生一个： + +```text +fixed_events = 1 +``` + +大多数 operator 还会产生： + +- `input_rows`：该 operator 消耗的行数或 key 数； +- `input_bytes`：该 operator 消耗的 byte-shaped 输入。 + +Sort 和 TopN 额外产生 `order_work`，用于保存基于 runtime input rows 的算法规模项。 + +TiDB Join/Agg 以及 TiKV Agg 还会按可用证据产生以下 shadow units： + +- `output_rows`：该 operator 实际产生的行数； +- `output_bytes`：可归因到该 operator 输出边界的 logical live bytes。 + +这两个 unit 只进入 EXPLAIN、bounded metrics 和 statement-summary 明细，不进入 weight lookup 或 preview RU 公式。缺少 shadow evidence 只会省略相应 unit，不会让原本完整的 fixed/input/order 公式失败。 + +每个 operator 结果都有以下身份字段： + +- `site`：`tidb`、`tikv`，statement-level status 用 `statement`； +- `op_class`：用于解析权重的有限 class； +- `operator_kind`：真实 plan/operator 名称，只用于观测和调参，不参与 weight lookup； +- `model_version`：当前统计模型版本，当前是 `v3`； +- `weight_version`：当前 weight table 版本，当前是 `v2`；v1 是采用线性 Sort/TopN row 项的历史表。 + +权重按下面的 key 解析： + +```text +site + op_class + weight_version +``` + +因此 TiDB 和 TiKV 即使共享类似的 opclass 名称，例如 `filter_eval`，也可以使用不同权重。 + +### TiKV Cop Executor 模型 + +TiKV 侧不是只按 scan 计费，而是把 cop executor 也纳入模型。第一版包括： + +| TiKV opclass | 覆盖的 operator 形态 | base-unit 输入 | +|---|---|---| +| `kv_range_scan` | table/index range scan | `input_rows = ScanDetail.TotalKeys`；`input_bytes = TotalKeys * (ProcessedKeysSize / ProcessedKeys)` | +| `kv_point_lookup` | point get / batch point get | point lookup work；当前 variable input 基于 runtime rows，fixed event 覆盖 operator setup | +| `filter_eval` | selection | direct child exact actRows × attributable scan-detail average；向父节点传播宽度 | +| `projection_eval` | projection | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | +| `row_limit` | limit | direct child exact actRows × attributable scan-detail average；向父节点传播宽度 | +| `bounded_topn` | TopN | direct child exact actRows × attributable scan-detail average；另有 `order_work = input_rows * log2(max(count, 2))`，其中 pushed plan 必须 `offset=0` 且 `count` 已折叠原 `offset+count`；向父节点传播宽度 | +| `agg_hash` | hash aggregation | direct child exact actRows × attributable scan-detail average；另在独立 expected/observed task coverage 完整时记录 own `output_rows`，可证明时记录 reader-boundary `output_bytes`；输出仍形成 width barrier | +| `agg_stream` | stream aggregation | direct child exact actRows × attributable scan-detail average；另在独立 expected/observed task coverage 完整时记录 own `output_rows`，可证明时记录 reader-boundary `output_bytes`；输出仍形成 width barrier | + +range scan 的 byte 输入使用 processed-key average 外推到 `TotalKeys`，因为 scan 成本不仅和输出行数有关，还和 key/value bytes、MVCC 数据移动、row decode 有关： + +```text +input_bytes = TotalKeys * (ProcessedKeysSize / ProcessedKeys) +``` + +TiKV 非 scan 一元算子的输入估算为： + +```text +input_rows = direct_child_exact_actRows +input_bytes = input_rows * (ProcessedKeysSize / ProcessedKeys) +input_source = runtime_child_act_rows +row_width_source = scan_detail_processed_key_avg_estimate +``` + +TiKV TopN 在上述输入证据成立后额外生成： + +```text +order_work = input_rows * log2(max(count, 2)) +input_source = runtime_ordering_work +``` + +TiKV TopN protobuf 只发送 `Limit=Count`，所以 pushed-down physical TopN 的规范形态是 `Offset=0`、`Count=原始 offset+count`。非零 cop Offset 会以 `invalid_ordering_work` 失败关闭，不按 root 公式猜测。 + +TiKV Agg 的 output shadow evidence 使用独立契约： + +```text +output_rows = current_agg_plan_id_actRows +input_source = runtime_operator_act_rows + +output_bytes = direct_root_reader_runtime_output_bytes +input_source = runtime_reader_output_chunks +row_width_source = runtime_reader_output_chunk_avg +``` + +两个 output shadow 都先要求 Agg observed summary task 数等于 `RuntimeStatsColl` 对每个已收到 TiKV cop response 独立记录的 expected task 数。`selectResult.close` 先关闭底层 iterator 并等待 worker 退出，再且只再快照一次 unconsumed stats，避免漏掉 cancel/wait 期间完成的失败请求。close 收集到的 unconsumed task 没有新 `SelectResponse`，因此只合并 statement-level RPC/scan/time stats 并增加 expected count，绝不重放最后一个已消费 response 的 executor summaries；这样 output shadows 会因 observed/expected 不等而省略。`output_bytes` 还只在 Agg 是普通 TableReader/IndexReader 的唯一直接 cop child、reader 有 byte record、且 reader output rows 等于 Agg output rows 时生成。这里的完整性边界是“所有已收到 response 的 summary 都可见”，不是 TiKV 集群内绝对 task inventory；plan-ID attribution 也不应被解读成协议级 per-executor byte 观测。Agg 会改变 schema 和 value representation，因此绝不使用 `output_rows * (ProcessedKeysSize / ProcessedKeys)`：scan average 描述输入 KV 宽度，不是 aggregate result 宽度。该 reader-boundary 观测也不会解除 Agg 的 width barrier 或被反向用于 input formula。 + +该公式只适用于结构可证明为单 scan、单 detail holder 的 TiKV component。multi-child、多 scan、多 detail holder、child summary 缺失或 task count 可检测为不完整、以及跨 Projection/Agg 的父边都会失败关闭；当前实现不回退 planner/schema width，也不跨 sibling component 借用 scan detail。 + +formula input rows 的数据质量契约是：stats 不存在或 `GetTasks() == 0` 为 missing；`GetTasks() > 0` 且 rows 为 0 是有效零行观测；负 rows 为 invalid；direct child task count 小于同 component 可见最大 task count 时为 incomplete。formula estimator 仍只做 component-relative 检查；如果同一 response 的 summary 在 component 所有 plan ID 上同步缺失，它可能无法发现，因此 v3 input rows 仍是带完整性边界的 estimate，校准时必须结合 coverage 与 failure reason。上述独立 expected-response 计数只收紧 Agg output shadows，不参与、也不改变当前 formula estimator：同步缺失时 formula units 可以继续成功，而 output shadows 会被省略。 + +### TiDB Root Executor 模型 + +TiDB 侧覆盖 root executor CPU、row movement、reader materialization、join 和 local overlay： + +| TiDB opclass | 覆盖的 operator 形态 | base-unit 输入 | +|---|---|---| +| `filter_eval` | root selection | local child runtime rows 和 logical live bytes | +| `projection_eval` | root projection | local child runtime rows 和 logical live bytes | +| `row_limit` | root limit | local child runtime rows 和 logical live bytes | +| `bounded_topn` | root TopN | local child runtime rows、logical live bytes，以及 `order_work = n * log2(max(offset + count, 2))` | +| `full_ordering` | sort | local child runtime rows、logical live bytes，以及 `order_work = n * log2(max(n, 2))` | +| `window_eval` | window executor | local child runtime rows 和 logical live bytes | +| `agg_hash` | hash aggregation | aggregation input rows 和 logical live bytes；自身实际 `output_rows/output_bytes` shadows | +| `agg_stream` | stream aggregation | aggregation input rows 和 logical live bytes；自身实际 `output_rows/output_bytes` shadows | +| `join_hash` | hash join | build/probe 两侧 input rows 和 bytes；自身实际 `output_rows/output_bytes` shadows | +| `join_merge` | merge join | left/right 两侧 input rows 和 bytes；自身实际 `output_rows/output_bytes` shadows | +| `join_lookup` | index lookup join family | left/right 两侧 input rows 和 bytes；自身实际 `output_rows/output_bytes` shadows | +| `reader_receive` | table/index reader receive | runtime output rows 和 logical live bytes | +| `lookup_reader` | index lookup reader orchestration | runtime output rows 和 logical live bytes | +| `overlay_reader` | union scan / local overlay | runtime rows 和 logical live bytes | +| `metadata_reader` | memory / cluster metadata reads | runtime rows 和 logical live bytes | + +Join 的 side 会通过 `input_side` 保留下来： + +- hash join：`build`、`probe`； +- merge join / lookup join：`left`、`right`； +- 其他 operator：`all`。 + +## 推荐统计面 + +推荐新增 statement-summary-associated virtual tables，按 statement summary 的 digest/window/history 生命周期聚合 read billing demo 明细。表名前缀和现有 statement summary 保持一致： + +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS` + +### Base-unit table schema + +每一行表示一个 statement summary window 内、一个 digest/plan_digest 下的一组 base-unit key 聚合。 + +| Column | Type | 含义 | +|---|---|---| +| `SUMMARY_BEGIN_TIME` | timestamp | statement summary window begin | +| `SUMMARY_END_TIME` | timestamp | statement summary window end | +| `STMT_TYPE` | varchar | statement type | +| `SCHEMA_NAME` | varchar | schema | +| `DIGEST` | varchar | SQL digest | +| `DIGEST_TEXT` | blob | normalized SQL，沿用 statement summary 截断策略 | +| `PLAN_DIGEST` | varchar | plan digest；point get 可由 lazy plan digest 补齐 | +| `RESOURCE_GROUP` | varchar | resource group | +| `MODEL_VERSION` | varchar | read billing model version | +| `WEIGHT_VERSION` | varchar | 生成这些 base units 时匹配的 weight version | +| `SITE` | varchar | `tidb` / `tikv` | +| `OP_CLASS` | varchar | bounded opclass | +| `OPERATOR_KIND` | varchar | bounded plan/operator kind | +| `UNIT` | varchar | `fixed_events` / `input_rows` / `input_bytes` / `output_rows` / `output_bytes` / `order_work` / write-side units | +| `INPUT_SOURCE` | varchar | bounded source，例如 `runtime_chunk_bytes` / `scan_detail` / `runtime_child_act_rows` / `runtime_operator_act_rows` / `runtime_reader_output_chunks` / `runtime_ordering_work` / write-side sources | +| `INPUT_SIDE` | varchar | `all` / `build` / `probe` / `left` / `right` | +| `ROW_WIDTH_SOURCE` | varchar | 当前值包括 `runtime_chunk_avg` / `runtime_reader_output_chunk_avg` / `scan_detail_processed_key_avg` / `scan_detail_processed_key_avg_estimate` / `not_applicable`；v1 历史值保留原标签 | +| `VALUE` | double unsigned | 该 key 的 base-unit total | +| `SAMPLE_COUNT` | bigint unsigned | 汇入该 key 的 unit samples 数 | +| `ROW_WIDTH_SUM` | double unsigned | row-width sample sum，用于诊断 modeled bytes | +| `AVG_ROW_WIDTH` | double unsigned | `ROW_WIDTH_SUM / SAMPLE_COUNT` | + +`DIGEST_TEXT` 不是 key，只是查询便利字段。校准时推荐按 `DIGEST`、`PLAN_DIGEST`、`SITE`、`OP_CLASS`、`UNIT` 等机器字段聚合。 + +### Status table schema + +每一行表示一个 statement summary window 内、一个 digest/plan_digest 下的一组 statement/operator status 聚合。 + +| Column | Type | 含义 | +|---|---|---| +| `SUMMARY_BEGIN_TIME` | timestamp | statement summary window begin | +| `SUMMARY_END_TIME` | timestamp | statement summary window end | +| `STMT_TYPE` | varchar | statement type | +| `SCHEMA_NAME` | varchar | schema | +| `DIGEST` | varchar | SQL digest | +| `DIGEST_TEXT` | blob | normalized SQL | +| `PLAN_DIGEST` | varchar | plan digest | +| `RESOURCE_GROUP` | varchar | resource group | +| `MODEL_VERSION` | varchar | read billing model version | +| `WEIGHT_VERSION` | varchar | current weight version at accounting time | +| `SITE` | varchar | `statement` / `tidb` / `tikv` | +| `OP_CLASS` | varchar | statement or operator opclass | +| `OPERATOR_KIND` | varchar | statement or plan/operator kind | +| `STATUS` | varchar | `success` / `unsupported` / `unknown_input` / `error` / `ok` | +| `REASON` | varchar | bounded reason，例如 `unsupported_mpp`、`missing_scan_width_evidence`、`unsupported_cop_width_transform`、`statement_error` | +| `COUNT` | bigint unsigned | 该 status key 的出现次数 | + +失败 statement 只写 status table，不写 base-unit table。成功 statement 写一条 statement-level `success` status，并为每个 billable operator 写 operator-level `ok` status。 + +### 为什么不用 statement summary 单表 JSON 列 + +JSON/encoded column 的写入成本可以较低，但对校准不够直接： + +- testers 需要额外 JSON 解析才能 group by `site/op_class/unit`； +- SQL 里直接套 weights、按 digest/window 过滤和 join 会变复杂； +- 历史持久化中的 schema 演进、旧 JSON 兼容和 partial parse failure 更难测试； +- 失败 status 和 base units 混在一个 encoded blob 里，容易再次变成不可直接观测的状态。 + +窄表把 row explosion 放到查询阶段，而不是把复杂性塞进单列解析；同时它复用 statement summary 的 digest/window/LRU/history 控制。 + +## 采集和生命周期 + +### Statement completion path + +后续实现应把 `RecordReadBillingDemoForStatement` 的返回值从三个 totals 扩成结构化 snapshot，例如： + +```go +type ReadBillingDemoStatementStats struct { + ModelVersion string + WeightVersion string + Statuses []ReadBillingDemoStatusSample + BaseUnits []ReadBillingDemoBaseUnitSample + Totals ReadBillingDemoBaseUnitSummary +} +``` + +其中 `Totals` 从 `BaseUnits` 派生，用于继续填充现有三列；`Statuses` 和 `BaseUnits` 则进入新表所需的维度化聚合。 + +采集顺序： + +1. statement 执行结束或早期错误出口触发 read billing demo hook； +2. `buildReadBillingDemoResult` 生成 frozen result； +3. 如果 result status 是 success： + - 为 statement 写 `success` status； + - 为 billable operator 写 operator `ok` status； + - 把 operator units 展平成 base-unit samples； + - 从 base-unit samples 派生三列 totals； +4. 如果 result status 不是 success： + - 为 statement 或 failed operator 写 status/reason； + - 不写任何 base-unit samples； + - 三列 totals 保持 0； +5. 正常执行结束路径由 `ExecStmt.SummaryStmt` 把结构化 snapshot 挂到 `StmtExecInfo`； +6. statement summary v1/v2 通过同一个 `Add`/`Merge` 路径聚合到当前 window 和 history； +7. 早期错误路径不能假设一定有 `ExecStmt.SummaryStmt`。`session.recordReadBillingDemoEarlyError` 需要在调用 `RecordReadBillingDemoForStatement` 后，把 status-only snapshot 通过一个 read-billing 专用 helper 写入同一套 statement-summary-associated status 聚合，例如 `stmtsummaryv2.AddReadBillingDemoStatusOnly(...)`。该 helper 只记录 status table 所需字段，`PLAN_DIGEST` 可为空，不写 base units，不影响普通 statement summary latency/RU 等通用列。 + +### Statement summary v1 + +v1 in-memory path 位于 `pkg/util/stmtsummary/statement_summary.go`： + +- `stmtSummaryStats` 新增 bounded in-memory maps： + - `ReadBillingDemoBaseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg` + - `ReadBillingDemoStatusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg` +- `newStmtSummaryStats`、`stmtSummaryStats.add`、`ReadBillingDemo...Merge` 负责把 `StmtExecInfo` 的 samples 合并进去； +- status-only early error helper 使用同样的 key 构造和 window 选择逻辑，但只 upsert status map；如果 statement digest 不可用，则使用空 digest/空 plan digest 聚合到当前 window，并保留 bounded reason； +- `stmtSummaryReader` 新增面向 base-unit/status 表的 reader，遍历同一个 current/history/cumulative window 后展开 rows； +- `STATEMENTS_SUMMARY_READ_BILLING_DEMO_*` 表沿用 statement summary 的 auth 行为：无 `PROCESS` privilege 时只能看到自己的 statement 聚合。 + +### Statement summary v2 + +v2 persistent path 位于 `pkg/util/stmtsummary/v2`： + +- `StmtRecord` 新增 JSON-stable entry slices，而不是 `map[struct]...` 这类不能稳定 marshal/unmarshal 的字段： + - `ReadBillingDemoBaseUnitAggs []ReadBillingDemoBaseUnitAggEntry` + - `ReadBillingDemoStatusAggs []ReadBillingDemoStatusAggEntry` +- 每个 entry 直接携带 key 字段和 aggregate value；`StmtRecord.Add` / `StmtRecord.Merge` 可以用临时 canonical string key 或小规模线性 upsert 合并后再保持 entry slice 排序； +- `MemReader` 从当前 `StmtRecord` 展开新表 rows； +- `HistoryReader` 读取历史 JSON log 时，如果旧记录没有这些字段，则 entry slices 为 nil，新表返回 0 行，不影响旧 log 解析； +- evicted aggregate record 也合并 read billing entries。对校准而言，evicted row 只作为覆盖率/容量诊断，不应用来拟合精细 weights，因为 digest/plan 维度已经被合并。 + +### Current 和 history 表 + +当前表读取未持久化窗口，history 表读取内存当前窗口加已持久化历史，与现有 `STATEMENTS_SUMMARY` / `STATEMENTS_SUMMARY_HISTORY` 语义保持一致。 + +如果 user 需要完整 workload run，推荐： + +1. 开启 statement summary； +2. 确认 `tidb_enable_read_billing_demo = ON`； +3. 记录 workload 起止时间； +4. 从 `CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_*` 用 `SUMMARY_BEGIN_TIME` / `SUMMARY_END_TIME` 过滤； +5. 同时查 status 表确认失败和 overflow 比例。 + +## 查询和权重校准方法 + +### 开启采集 + +```sql +SET GLOBAL tidb_enable_read_billing_demo = ON; +``` + +建议只在校准 workload 的 session 或测试集群中开启。生产默认仍然是 OFF。 + +### 第一步：检查覆盖率和失败状态 + +先看 statement/operator status： + +```sql +SELECT + status, + reason, + SUM(count) AS count +FROM information_schema.cluster_statements_summary_history_read_billing_demo_status +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v3' + AND weight_version = 'v2' +GROUP BY status, reason +ORDER BY count DESC; +``` + +如果 `unsupported`、`unknown_input`、`error` 或 `aggregation_overflow` 占比高，说明 base-unit 数据不能代表整个 workload，需要先收敛覆盖范围或降低维度压力。 + +进一步按 operator 看原因： + +```sql +SELECT + site, + op_class, + operator_kind, + status, + reason, + SUM(count) AS count +FROM information_schema.cluster_statements_summary_history_read_billing_demo_status +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v3' + AND weight_version = 'v2' +GROUP BY site, op_class, operator_kind, status, reason +ORDER BY count DESC; +``` + +### 第二步：收集 base units + +按 `site/op_class/unit/source/side` 聚合 base units： + +```sql +SELECT + site, + op_class, + unit, + input_source, + input_side, + SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v3' + AND weight_version = 'v2' +GROUP BY site, op_class, unit, input_source, input_side +ORDER BY site, op_class, unit, input_source, input_side; +``` + +这个查询是校准模型的主要输入，相当于得到无系数特征矩阵： + +```text +X[site, op_class, unit] = 当前时间窗口内观测到的 base-unit total +``` + +校准 v3 TiKV 非 scan estimator 时必须按 unit 使用不同的 source/width-source predicate，避免把 `fixed_events` 或 TopN 的 `order_work` 过滤掉： + +```sql +SELECT op_class, unit, input_source, row_width_source, SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE model_version = 'v3' + AND weight_version = 'v2' + AND site = 'tikv' + AND op_class IN ('filter_eval', 'projection_eval', 'row_limit', 'bounded_topn', 'agg_hash', 'agg_stream') + AND ( + (input_source = 'runtime_child_act_rows' AND unit = 'fixed_events' AND row_width_source = 'not_applicable') + OR (input_source = 'runtime_child_act_rows' AND unit IN ('input_rows', 'input_bytes') AND row_width_source = 'scan_detail_processed_key_avg_estimate') + OR (op_class = 'bounded_topn' AND input_source = 'runtime_ordering_work' AND unit = 'order_work' AND row_width_source = 'not_applicable') + ) +GROUP BY op_class, unit, input_source, row_width_source +ORDER BY op_class, unit, input_source, row_width_source; +``` + +`output_rows/output_bytes` 不应混入上面的 formula feature query。需要分析 Agg output shadows 时单独查询，并保留 source 以区分 coverage-guarded cop rows 与 guarded reader bytes： + +```sql +SELECT op_class, unit, input_source, row_width_source, + SUM(sample_count) AS samples, SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE model_version = 'v3' + AND weight_version = 'v2' + AND site = 'tikv' + AND op_class IN ('agg_hash', 'agg_stream') + AND ( + (unit = 'output_rows' AND input_source = 'runtime_operator_act_rows' AND row_width_source = 'not_applicable') + OR (unit = 'output_bytes' AND input_source = 'runtime_reader_output_chunks' AND row_width_source = 'runtime_reader_output_chunk_avg') + ) +GROUP BY op_class, unit, input_source, row_width_source +ORDER BY op_class, unit, input_source, row_width_source; +``` + +TiDB Join/Agg 使用 `input_source = 'runtime_chunk_bytes'`；同样应作为 shadow diagnostics 单独分析，而不是给当前 weight table 增加 output coefficient。 + +`model_version` 仍是 `v3`，因为 output shadows 不改变 formula 或 weights。这个版本号也意味着历史窗口本身不能标识功能部署边界：部署前的 `v3` 记录没有 output-unit 行，缺行不能解释为零。校准查询必须把 `SUMMARY_BEGIN_TIME` 限定在所有目标 TiDB 节点均已部署该能力之后，并另外检查节点/窗口覆盖率；只有实际存在、且 `VALUE = 0` 的 unit sample 才表示观测到零输出。 + +如果需要区分 plan shape，可以保留 `DIGEST` 和 `PLAN_DIGEST`： + +```sql +SELECT + digest, + plan_digest, + site, + op_class, + unit, + SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v3' + AND weight_version = 'v2' +GROUP BY digest, plan_digest, site, op_class, unit; +``` + +### 第三步：检查 row-width evidence + +v3 row width 既可能来自 TiDB runtime chunk average，也可能是 TiKV scan-detail KV average proxy；后者不是 cop executor edge 的真实 byte sample。拟合前需要按 `row_width_source` 检查它是否稳定： + +```sql +SELECT + site, + op_class, + row_width_source, + SUM(sample_count) AS samples, + SUM(row_width_sum) / NULLIF(SUM(sample_count), 0) AS avg_row_width +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND unit = 'input_bytes' + AND model_version = 'v3' + AND weight_version = 'v2' +GROUP BY site, op_class, row_width_source +ORDER BY site, op_class, row_width_source; +``` + +如果 `scan_detail_processed_key_avg_estimate` 的分布异常，或 residual 随 projection shape、group NDV、task count 系统漂移，拟合出的 byte weight 可能是在补偿 proxy 误差，而不是真实 byte cost。v3 不使用 `schema_fallback`。 + +### 第四步：用任意 weights 重算 RU + +testers 可以把新 weights 放到 CTE、临时表或外部 notebook。SQL 侧的最小形态如下： + +```sql +WITH observed AS ( + SELECT site, op_class, unit, SUM(value) AS value + FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units + WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v3' + AND weight_version = 'v2' + GROUP BY site, op_class, unit +), +weights(site, op_class, fixed_weight, row_weight, byte_weight, order_weight) AS ( + SELECT 'tikv', 'kv_range_scan', 0.070, 0.000045, 0.000020, 0.000000 UNION ALL + SELECT 'tidb', 'projection_eval', 0.020, 0.000020, 0.000004, 0.000000 UNION ALL + SELECT 'tidb', 'full_ordering', 0.080, 0.000000, 0.000012, 0.000070 +) +SELECT + SUM(CASE observed.unit + WHEN 'fixed_events' THEN observed.value * weights.fixed_weight + WHEN 'input_rows' THEN observed.value * weights.row_weight + WHEN 'input_bytes' THEN observed.value * weights.byte_weight + WHEN 'order_work' THEN observed.value * weights.order_weight + ELSE 0 + END) AS recalculated_ru +FROM observed +JOIN weights + ON observed.site = weights.site + AND observed.op_class = weights.op_class; +``` + +因为统计面保留的是 base units，重算时只需要替换 `weights` 数据，不需要重新跑 workload。 + +这里的 `ELSE 0` 是有意的：`output_rows/output_bytes` 没有 weight，不参与当前公式或 weight-version 语义。 + +### 第五步：拟合新 weights + +对每个校准窗口构造： + +```text +y_window = 观测目标成本 +X_window = 按 site/opclass/unit 聚合的 base-unit totals +``` + +`y_window` 可以选择： + +- workload 窗口内 TiDB/TiKV CPU time； +- 现有 RU consumption，用于兼容性对比； +- benchmark 手工标注成本； +- 混合目标，例如 CPU + IO-normalized scan cost。 + +推荐使用非负线性模型： + +```text +minimize || X * w - y ||^2 +subject to w >= 0 +``` + +拟合规则建议： + +1. 按 `site/op_class` 分别拟合 `fixed`、`row`、`byte`、`order` 系数；只有 Sort/TopN 产生 `order_work`。 +2. 保持权重非负。 +3. 使用 hold-out workload window 验证预测误差。 +4. 默认保留符合源码成本形态的单调关系，除非数据强烈反证，例如： + - `agg_hash.row >= agg_stream.row`； + - `bounded_topn.order > 0`，同时保持 `bounded_topn.row = 0` 以避免和 order work 双算； + - `kv_range_scan.byte >= filter_eval.byte`； + - `join_hash.row >= join_merge.row`。 +5. 拟合后的新权重应该发布为新的 `weight_version`，不要静默改变当前 `v2` 或历史 `v1` 的含义。 + +## 初始权重 + +当前 `v2` weights 是 demo calibration seed，不是 production-calibrated billing constants。v2 将 Sort/TopN 的历史线性 row coefficient 移到独立 `order_weight`，因此没有复用 v1 的权重语义。它们来自源码成本形态和相对排序： + +- scan 同时受 row/key 数和 bytes 影响，比纯 expression eval 更重； +- hash aggregation 和 hash join 需要维护 hash map / hash table / aggregate state，row weight 更高； +- stream aggregation 和 merge join 依赖有序输入，通常低于 hash 版本； +- limit 是最轻的 row-shaped operator； +- TopN 需要 order expression 和 heap maintenance，算法 work 是 `n * log2(k)`；full sort 是 `n * log2(n)`； +- TiDB reader class 对 bytes 更敏感，因为它负责接收、materialize chunk 和协调 lookup； +- TiDB 与 TiKV 即使 opclass 名称相同，也运行不同代码路径，因此权重分开设置。 + +当前 `v2` 权重如下： + +| site | opclass | fixed_weight | row_weight | byte_weight | order_weight | 初始设置原因 | +|---|---|---:|---:|---:|---:|---| +| `tikv` | `kv_range_scan` | 0.070 | 0.000045 | 0.000020 | 0 | range iterator、MVCC/key-value movement、row decode 和 scan limiter 使 scan 同时对 row/key 和 bytes 敏感。 | +| `tikv` | `kv_point_lookup` | 0.045 | 0.000030 | 0.000012 | 0 | point get 有 per-key dispatch 和 decode，但没有 range iterator loop。 | +| `tikv` | `filter_eval` | 0.020 | 0.000040 | 0.000006 | 0 | RPN predicate eval 主要按 logical rows 消耗 CPU，byte movement 权重较低。 | +| `tikv` | `projection_eval` | 0.020 | 0.000030 | 0.000006 | 0 | projection 会执行表达式或 materialize column reference，row 成本低于 filter。 | +| `tikv` | `row_limit` | 0.010 | 0.000008 | 0.000002 | 0 | 主要是 pass-through counter / gate，是最轻的 executor。 | +| `tikv` | `bounded_topn` | 0.060 | 0 | 0.000012 | 0.000075 | order expression eval 加 `n * log2(k)` bounded heap maintenance。 | +| `tikv` | `agg_hash` | 0.080 | 0.000100 | 0.000014 | 0 | group expression eval、hash group lookup/insert、aggregate state update。 | +| `tikv` | `agg_stream` | 0.060 | 0.000065 | 0.000010 | 0 | ordered group compare 加 state update，不构建 hash table。 | +| `tidb` | `filter_eval` | 0.020 | 0.000030 | 0.000005 | 0 | TiDB chunk 上的 expression eval，低于 scan 和 hash 类 operator。 | +| `tidb` | `projection_eval` | 0.020 | 0.000020 | 0.000004 | 0 | projection / materialization，per-row 成本较低。 | +| `tidb` | `row_limit` | 0.010 | 0.000006 | 0.000001 | 0 | 轻量 pass-through control。 | +| `tidb` | `bounded_topn` | 0.060 | 0 | 0.000010 | 0.000060 | heap / order expression 的 `n * log2(k)` work。 | +| `tidb` | `full_ordering` | 0.080 | 0 | 0.000012 | 0.000070 | full sort 的 `n * log2(n)` work。 | +| `tidb` | `window_eval` | 0.070 | 0.000070 | 0.000010 | 0 | partition / order / frame state maintenance。 | +| `tidb` | `agg_hash` | 0.070 | 0.000085 | 0.000012 | 0 | hash grouping 和 aggregate state update。 | +| `tidb` | `agg_stream` | 0.050 | 0.000055 | 0.000008 | 0 | ordered streaming aggregate，低于 hash agg。 | +| `tidb` | `join_hash` | 0.110 | 0.000115 | 0.000020 | 0 | build/probe hash table work，是 TiDB 侧最高 row cost 之一。 | +| `tidb` | `join_merge` | 0.090 | 0.000075 | 0.000012 | 0 | ordered comparison / merge，低于 hash join。 | +| `tidb` | `join_lookup` | 0.120 | 0.000120 | 0.000020 | 0 | lookup task orchestration 加 join-side work，固定成本和 row 成本都较高。 | +| `tidb` | `reader_receive` | 0.040 | 0.000025 | 0.000014 | 0 | network/result receive 和 chunk materialization，对 bytes 敏感。 | +| `tidb` | `lookup_reader` | 0.070 | 0.000045 | 0.000016 | 0 | index/table two-phase lookup reader coordination。 | +| `tidb` | `overlay_reader` | 0.050 | 0.000035 | 0.000012 | 0 | UnionScan/local overlay merge。 | +| `tidb` | `metadata_reader` | 0.020 | 0.000008 | 0.000002 | 0 | metadata / memory-table read，故意设得较轻。 | + +## 性能和容量控制 + +### 低开销原则 + +新统计面必须满足： + +- `tidb_enable_read_billing_demo` 默认 OFF，关闭时不构造 read billing result，也不写 statement summary maps； +- 开启后仍只在 statement end 聚合一次，不打印 per-SQL log； +- base-unit/status key 由 bounded enum-like 字段组成，不包含 SQL text、table name、index name、region、store address 或 plan id； +- `DIGEST_TEXT` 和 `PLAN_DIGEST` 复用 statement summary 已有字段，不进入 read-billing map key； +- 读表时才把 map 展开成多行，写入时保持 map 聚合，不为每次执行追加 unbounded slice； +- cluster 表沿用现有 cluster memtable fan-out，不引入新的后台 scrape。 + +### 单 digest/window map 上限 + +statement summary 已有 `max-stmt-count`、`refresh-interval`、`history-size` 和 LRU eviction，但这些只限制 digest/window 数量。read billing 还需要限制单 digest/window 内的维度 key 数。 + +建议第一版使用保守上限： + +```text +max_base_unit_keys_per_record = 256 +max_status_keys_per_record = 128 +``` + +当超过上限： + +- status table 增加 `status='unknown_input'`、`reason='aggregation_overflow'` 的 reserved row； +- 超出上限的新 base-unit key 不再写入 base-unit table，避免把不完整特征矩阵伪装成完整数据； +- 现有已聚合 key 继续累加； +- 三列 totals 仍只从已接收 base-unit samples 派生，并且用户必须通过 status 表判断该窗口是否可用于拟合。 + +overflow row 必须永远可见，不能被 status map 自身的上限挤掉。实现时预留两个不计入 `max_status_keys_per_record` 的 fixed buckets： + +```text +statement/statement/statement unknown_input aggregation_overflow +statement/statement/statement unknown_input status_aggregation_overflow +``` + +如果 base-unit map 超限，写 `aggregation_overflow` bucket。如果 status map 对普通 status key 超限，写 `status_aggregation_overflow` bucket。reserved buckets 只累加 count，不触发新的 overflow，也不写 base units。 + +这个策略牺牲超高维异常 workload 的完整性，但保证整体性能和内存不会被单个 digest 放大。 + +### 估算内存上界 + +粗略上界： + +```text +max_records = stmt-summary.max-stmt-count * (history-size 或 current window) +per_record_read_billing_keys <= 256 + 128 +``` + +实际 key 数通常远低于上限，因为第一版 opclass、unit、input_source 和 input_side 都是有限集合。典型 SELECT digest/window 只会产生数十个 key。 + +## 兼容性和迁移计划 + +1. 保留 `SUM_READ_BILLING_DEMO_FIXED_EVENTS`、`SUM_READ_BILLING_DEMO_INPUT_ROWS`、`SUM_READ_BILLING_DEMO_INPUT_BYTES` 三列作为 convenience totals。 +2. 文档和后续测试明确：三列不能用于 per-opclass calibration，因为它们丢失维度。 +3. 新 base-unit/status 表成为 workload calibration 的主接口。 +4. Prometheus `tidb_read_billing_demo_*` metrics 可继续保留为 optional observability，但不作为 testers 必需依赖。 +5. v2 persisted statement summary 的 JSON 结构新增 read-billing maps；旧日志没有这些字段时读出来为空，不需要 migration。 +6. 如果后续模型稳定并准备生产化，可以再决定是否移除 demo 三列或把 demo 表改名为非 demo 表；第一版不做破坏性删除。 + +## 测试设计 + +后续实现 loop 至少需要补以下测试。 + +### Planner / calculator tests + +- `RecordReadBillingDemoForStatement` 返回结构化 status/base-unit snapshot； +- success path 只为 billable operators 产生 base units； +- unsupported、unknown-input、error path 不产生 base units，但产生 status/reason； +- normal finish path 通过 `StmtExecInfo` 写 status/base units，early error path 通过 status-only helper 写 SQL-visible status； +- `aggregation_overflow` 和 `status_aggregation_overflow` reserved buckets 可见且不会写 partial unknown key； +- current three-column totals 从 base-unit samples 派生，与现有测试保持兼容。 + +### Statement summary v1 tests + +- `stmtSummaryStats.add` 能按 `site/op_class/unit/input_source/input_side/row_width_source` 聚合 value； +- status-only early error helper 能写入 current window 的 status table，且不写 base-unit rows； +- `Merge` / evicted aggregate 能合并 read billing maps； +- `STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS` current/history rows 与 digest/window 对齐； +- `STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS` 能看到 failed billing； +- base-unit/status map 超限时 reserved overflow buckets 不受 cap 影响； +- auth 行为与 statement summary 保持一致。 + +### Statement summary v2 tests + +- `StmtRecord.Add` / `Merge` 聚合 JSON-stable read billing entry slices； +- current reader 展开 base-unit/status rows； +- history reader 能读取新 JSON 字段； +- old history JSON 没有 read-billing 字段时返回 0 行且不报错； +- entry slice marshal/unmarshal 后 key 字段和值不丢失，排序保持 deterministic； +- evicted aggregate 包含 read billing maps，但测试中明确它不适合精细拟合。 + +### Information schema / cluster table tests + +- 新表注册、列顺序、cluster `INSTANCE` 列； +- digest/time predicate extractor 复用 statement summary extractor； +- `CLUSTER_*` 表能查询多个 TiDB instance 的 rows； +- 默认 OFF 时新表无 read-billing rows； +- 开启 demo 后可以用 SQL 直接按 `site/op_class/unit` 重算 RU。 + +### 性能验证 + +- targeted benchmark 比较 demo OFF、demo ON + scalar totals、demo ON + dimension maps； +- 验证正常 workload 下 statement end 额外开销不显著； +- 验证高维异常 workload 触发 overflow 后内存和写入开销保持 bounded。 + +## 备选方案 + +### 继续只用 Prometheus metrics + +不采用。metrics label shape 可以保存 `site/op_class/unit`，但 downstream testers 明确不希望依赖 Prometheus；metrics retention、scrape interval 和 workload 窗口对齐也会增加校准流程复杂度。 + +### 每条 SQL 打日志 + +不采用。per-SQL log 会显著影响 QPS，也会把校准流程变成日志采集和解析问题。 + +### 只扩展现有 statement summary 三列 + +不采用。三列 totals 丢失 `site/op_class/unit`，只能重算一个全局粗粒度模型，不能直接优化不同 opclass 的 weights。 + +### 在 statement summary 单表里放 JSON/encoded column + +暂不采用。它减少表数量，但把校准所需的 group-by 和 join 复杂度推给用户，也让 failed billing status 不够直接。可以作为后续低表面面积方案重新评估。 + +### 每个 operator kind 一个 weight + +不采用。这样会显著提高模型和统计面复杂度，也会让校准更脆弱。当前保留 `operator_kind` 用于观测，但 weight 绑定在有限 opclass 上。 + +### workload 统计面直接上报 preview RU + +不采用。preview RU 是加权结果，会遮住校准需要的 base-unit 原始证据。workload 表上报 base units；单语句调试通过 `EXPLAIN ANALYZE FORMAT='RU'` 看 preview RU。 + +## 风险和未决问题 + +### 风险 + +- 新表会增加 statement summary 内存占用；必须用单 digest/window key 上限和 overflow status 控制。 +- v2 persistent history 会扩大 statement summary JSON record；需要 benchmark 和 history-reader 兼容测试。 +- TiKV 非 scan row width 是 storage KV average proxy，不是 executor edge 真值；byte weight 可能补偿 projection/aggregation 行形状、编码和 task 分布误差。 +- evicted aggregate 会丢 digest/plan 细节，不适合作为精细校准输入；用户需要查询 status 和 evicted 表判断窗口质量。 +- 当前 point lookup miss 的 variable row cost 证据仍有限；没有 requested-key 证据时主要由 fixed event 覆盖。 + +### 未决问题 + +- 第一轮生产化拟合应该选什么目标信号：CPU、现有 RU、latency-normalized cost，还是混合目标？ +- point lookup 是否需要接入 requested key count，使 key miss 也能产生 variable row cost？ +- scan bytes 后续是否应该拆成 total key size、processed key size、value size 等多个 bounded units？ +- `aggregation_overflow` 的默认上限是否需要暴露为配置，还是作为 demo 常量即可？ +- TiFlash、MPP、IndexMerge 何时从 fail-closed 进入 supported opclasses？ diff --git a/docs/design/2026-07-08-write-billing-units-weights-design.md b/docs/design/2026-07-08-write-billing-units-weights-design.md new file mode 100644 index 0000000000000..9b58f00218205 --- /dev/null +++ b/docs/design/2026-07-08-write-billing-units-weights-design.md @@ -0,0 +1,677 @@ +# TiDB 写入计费 Unit 和初始权重设计 + +> **已被取代。** 本文保留为早期多 opclass / rows-cells 方案的历史记录， +> 不再描述当前 preview RU 写侧实现。当前收敛模型见 +> [Preview RU TiDB KV mutation operator](2026-07-10-preview-ru-tidb-kv-mutation.md)。 +> 不得把本文的 L5、rows/cells 或 seed 权重用于 `tidb/kv_mutation`。 + +- Author(s): Codex +- Discussion PR: N/A +- Tracking Issue: N/A + +## 目录 + +* [简介](#简介) +* [目标和非目标](#目标和非目标) +* [当前代码边界](#当前代码边界) +* [模型边界](#模型边界) +* [Taxonomy](#taxonomy) +* [Base Units](#base-units) +* [覆盖语句](#覆盖语句) +* [初始权重](#初始权重) +* [采集和暴露面](#采集和暴露面) +* [失败和降级状态](#失败和降级状态) +* [避免重复计费](#避免重复计费) +* [实现期 instrumentation 需求](#实现期-instrumentation-需求) +* [测试设计](#测试设计) +* [风险和未决问题](#风险和未决问题) + +## 简介 + +本文描述 TiDB 侧写入工作的 billing demo 设计，重点回答写入 executor 应该如何拆分 `unit` 和 `weight`。这是设计文档，不包含产品代码实现。 + +设计沿用 read billing demo 的基本方向：先暴露无系数 base units，再通过 `site/op_class/unit/weight_version` 离线套权重。不要只输出最终加权 RU，因为 workload 校准需要保留足够维度来重算不同权重。 + +写入侧 preview RU 推荐使用同一种可扩展公式形态： + +```text +preview_ru = sum(unit_value * weight(site, op_class, unit, weight_version)) +``` + +其中 `unit` 是有界枚举，不把表名、索引名、SQL 文本、region、store 或 plan id 放进权重 key。`operator_kind` 可以保留用于诊断，但不参与权重解析。 + +## 目标和非目标 + +目标： + +1. 覆盖 TiDB 侧主要 DML 写入 executor：`INSERT`、`INSERT ... SELECT`、`INSERT ... ON DUPLICATE KEY UPDATE`、`REPLACE`、`UPDATE`、`DELETE`、batch DML，以及 `LOAD DATA` 的 TiDB 解析和提交 worker 路径。 +2. 明确 `site`、`op_class`、`unit`、`operator_kind` 的 bounded taxonomy。 +3. 为每个 opclass 定义 base units、初始权重或相对权重顺序。 +4. 区分哪些 unit 可以从现有 executor / statement context / runtime stats 得到，哪些需要新增 instrumentation。 +5. 明确 `INSERT ... SELECT`、`UPDATE`、`DELETE` 等混合读写语句如何避免重复计算读路径和写路径。 +6. 明确 TiDB 写准备、表达式求值、table/index mutation、transaction/2PC orchestration、TiKV storage write cost 的边界。 +7. 让 failed / unsupported / unknown accounting 有 SQL 可查询 status。 + +非目标： + +- 不改变当前 RU v2 resource-control 生产上报。 +- 不把 storage-side TiKV/TiFlash write charging 纳入本设计的第一版权重；TiKV write keys / write bytes 只作为边界和未来校准参考。 +- 不把 commit / 2PC latency 当作 TiDB executor CPU 的替代输入。 +- 不用 `affectedRows` 直接代表物理写入行数。 +- 不要求第一版支持所有 DDL backfill、temporary index merge、FK cascade 的完整成本；这些路径必须能给出 status。 + +## 当前代码边界 + +写入入口和关键路径如下： + +- `pkg/executor/insert_common.go` + - `InsertValues` 保存 INSERT / REPLACE / LOAD DATA 共享状态。 + - `insertRows` 处理 `INSERT/REPLACE ... VALUES` 和 `SET`。 + - `insertRowsFromSelect` 处理 `INSERT/REPLACE ... SELECT`。 + - `evalRow`、`fastEvalRow`、`getRow`、`fillRow` 完成表达式求值、cast、default、generated column、auto id 和 rowid 填充。 + - `batchCheckAndInsert` 处理 `IGNORE` / `LOAD DATA IGNORE` / `LOAD DATA REPLACE` 的 duplicate 检查。 + - `addRecordWithAutoIDHint` 调用 table 层 `AddRecord`，并更新 SQL 语义 row counters。 +- `pkg/executor/insert.go` + - `InsertExec.exec` 分派普通 insert、insert ignore 和 ODKU。 + - `batchUpdateDupRows` 构造 duplicate check keys、prefetch，并对冲突行走 update path。 + - `doDupRowUpdate` 复用 `updateRecord`。 +- `pkg/executor/replace.go` + - `ReplaceExec.exec` 先构造 duplicate check keys，再删除冲突旧行并插入新行。 + - `replaceRow`、`removeIndexRow` 负责 handle / unique-key 冲突处理。 +- `pkg/executor/update.go` 和 `pkg/executor/write.go` + - `UpdateExec.updateRows` 从 child executor 读取待更新行。 + - `composeNewRow` / `fastComposeNewRow` 执行 assignment 表达式和 cast。 + - `updateRecord` 比较 old/new row，处理 no-op、handle changed、generated/on-update-now、FK、`RemoveRecord + AddRecord` 或 `UpdateRecord`。 +- `pkg/executor/delete.go` + - `deleteSingleTableByChunk` / `deleteMultiTablesByChunk` 从 child executor 读取待删行。 + - `removeRow` 调用 table 层 `RemoveRecord` 并处理 FK。 +- `pkg/executor/load_data.go` + - `LoadDataExec.Next` 进入 local / remote load。 + - `LoadDataWorker.load` 创建 reader、encode worker 和 commit worker pipeline。 + - `encodeWorker.readOneBatchRows` / `parserData2TableData` 做 parser 输出读取、列映射、user variable、`SET` 表达式、cast/default/generated column。 + - `commitWorker.checkAndInsertOneBatch` 按 worker task 复用 insert / duplicate-check 写入路径；不要把每个 task 等同为单独的 statement commit。 +- `pkg/table/tables/tables.go` 和 `pkg/table/tables/index.go` + - `TableCommon.AddRecord` 构造 record key/value、写 memBuffer、创建索引。 + - `TableCommon.UpdateRecord` 重建受影响索引、编码新 row、写 memBuffer。 + - `TableCommon.RemoveRecord` delete record key 和 index keys。 + - `index.create` / `index.Delete` 生成 index key/value，做 unique check、temp index double-write、assertion 和 memBuffer mutation。 +- `pkg/sessionctx/stmtctx/stmtctx.go` + - `RecordRows`、`CopiedRows`、`UpdatedRows`、`DeletedRows`、`TouchedRows`、`AffectedRows` 是 MySQL 兼容语义计数，不是物理写入成本。 +- `pkg/executor/adapter.go` 和 `pkg/util/execdetails/ruv2_metrics.go` + - 当前 RU v2 从 `CommitDetails.WriteKeys` / `WriteSize` 汇总写入相关指标，但显式事务、batch DML 和 pipelined DML 的归因边界不同于 executor-local base units。 + +## 模型边界 + +写 billing demo 第一版按 `site` 分成三层： + +| Site | 含义 | 第一版动作 | +|---|---|---| +| `statement` | statement-level status | 只写 status | +| `tidb` | TiDB executor、row prepare、table/index mutation 构造、memBuffer mutation、LOAD DATA decode | 计入 base units | +| `tikv` | storage read/write、lock、2PC、raft/write bytes | 第一版不直接计入 TiDB 写 base units；可作为独立 storage billing 或 status/reference | + +TiDB 写计费只覆盖 TiDB 本地工作。下面这些工作不应在 TiDB write opclass 中计入： + +- `INSERT ... SELECT`、`UPDATE`、`DELETE` child plan 的 scan/filter/join/read executor 成本。它们应由 read billing demo 继续计入。 +- duplicate / FK check 过程中实际落到 TiKV 的 `txn.Get`、`txn.BatchGet`、snapshot iterator 的 storage read 成本。 +- pessimistic lock acquisition、`txn.LockKeys`、prewrite、commit、resolve lock、region grouping、local latch、raftstore write。 +- TiKV / TiFlash storage engine write amplification。 + +TiDB 写侧可以记录 duplicate probe keys、mutation keys、mutation bytes 等无系数输入，但这些 unit 的权重必须解释为 TiDB 本地 key construction / encoding / memBuffer work，而不是 storage RPC 或 2PC 成本。 + +## Taxonomy + +### Site + +第一版使用： + +- `statement` +- `tidb` + +保留未来扩展： + +- `tikv` +- `tiflash` + +### Op Class + +第一版推荐的 TiDB 写 opclasses： + +| Op class | 主要覆盖 | 代码锚点 | +|---|---|---| +| `write_row_prepare` | VALUES/SELECT/LOAD DATA 输入行到 table row 的表达式求值、cast、default、generated column、auto id/rowid/on-update-now | `InsertValues.evalRow`、`fastEvalRow`、`getRow`、`fillRow`、`UpdateExec.composeNewRow`、`parserData2TableData` | +| `write_duplicate_probe` | 构造 handle / unique index probe keys，批量 duplicate 检查的 TiDB 本地循环和 key handling | `getKeysNeedCheck`、`prefetchUniqueIndices`、`batchCheckAndInsert`、`batchUpdateDupRows`、`ReplaceExec.replaceRow` | +| `write_row_compare` | UPDATE / ODKU old/new row 比较、changed/no-op 判断、handle changed 判断 | `updateRecord` | +| `write_row_mutation` | record key/value encode，record key set/delete，row-level memBuffer mutation | `TableCommon.AddRecord`、`UpdateRecord`、`RemoveRecord` | +| `write_index_mutation` | index key/value encode，index set/delete，unique-index memBuffer checks，MV index expansion，temp-index double write | `TableCommon.addIndices`、`rebuildUpdateRecordIndices`、`removeRowIndices`、`index.create`、`index.Delete` | +| `write_foreign_key` | FK check/cascade 的 TiDB 本地 row/key preparation | `FKCheckExec`、`FKCascadeExec`、`checkRows`、`onRemoveRowForFK` | +| `write_txn_orchestrate` | batch DML statement commit boundary、`NewTxnInStmt`、`txn.MayFlush` 调用等 TiDB 事务编排事件 | `doBatchInsert`、`doBatchDelete`、`txn.MayFlush` callers | +| `load_data_decode` | LOAD DATA 文件行解析、列映射、user vars、SET 表达式进入 row prepare 前的 TiDB 本地处理 | `LoadDataWorker.load`、`encodeWorker.readOneBatchRows`、`parserData2TableData` | + +`operator_kind` 是 bounded diagnostic 维度，建议使用具体执行路径而不是表名或索引名： + +- `insert_values` +- `insert_select` +- `insert_on_duplicate` +- `replace` +- `update` +- `delete` +- `load_data` +- `batch_insert` +- `batch_delete` +- `table_add_record` +- `table_update_record` +- `table_remove_record` +- `index_create` +- `index_delete` +- `foreign_key_check` +- `foreign_key_cascade` +- `txn_may_flush` +- `batch_dml_stmt_commit` + +### Unit + +第一版推荐 unit 枚举： + +| Unit | 含义 | +|---|---| +| `fixed_events` | opclass 发生次数；用于保留固定开销 | +| `input_rows` | opclass 消耗的逻辑输入行数 | +| `input_cells` | 被表达式、cast、default、compare 处理的列值数量；可近似 row * column | +| `input_bytes` | TiDB 本地处理的 row/file/value byte-shaped 输入 | +| `probe_keys` | duplicate / FK / unchanged-lock 等准备或检查的 key 数 | +| `probe_key_bytes` | probe keys 的 key bytes | +| `mutation_keys` | TiDB 写入 memBuffer 的 record/index set/delete key 数 | +| `mutation_bytes` | TiDB 构造并写入 memBuffer 的 key/value bytes；delete 只计 key bytes | +| `changed_rows` | 发生真实 row mutation 的逻辑行数 | +| `unchanged_rows` | UPDATE/ODKU/REPLACE 中比较后没有真实 row mutation 的行数 | +| `conflict_rows` | duplicate probe 找到冲突并进入 ignore/update/replace 分支的行数 | +| `batch_events` | batch DML / LOAD DATA commit worker / statement transaction boundary 事件 | + +这些 units 都是 coefficient-free。下游校准可以选择只使用其中一部分；TiDB 仍应保留所有可验证维度。 + +## Base Units + +### `write_row_prepare` + +覆盖： + +- INSERT VALUES / SET 的表达式求值和 cast。 +- INSERT SELECT 从 child row 到 table row 的 cast/default/generated/auto id。 +- UPDATE assignment 表达式、cast、generated/on-update-now。 +- LOAD DATA 行解析后的列映射、SET 表达式、cast/default/generated。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每次 executor 或 worker batch 发生 | 是 | +| `input_rows` | `InsertValues.rowCount` delta、`StmtCtx.RecordRows`、UPDATE child rows、LOAD DATA batch count | 部分需要 | +| `input_cells` | insert/update 当前已有 `rowsColMultiply` 口径，LOAD DATA 可用 parsed columns | 需要保存为维度化样本 | +| `input_bytes` | `types.EstimatedMemUsage`、LOAD DATA raw bytes、row datum estimated size | 需要 | + +`StmtCtx.AffectedRows` 不可用于这里，因为 ODKU、REPLACE、CLIENT_FOUND_ROWS 和 FK trigger 会改变 MySQL 语义。 + +### `write_duplicate_probe` + +覆盖： + +- `getKeysNeedCheck` 为每个 row 生成 handle key 和 unique index keys。 +- `prefetchUniqueIndices` / `prefetchConflictedOldRows` 的 TiDB key collection。 +- `batchCheckAndInsert`、ODKU、REPLACE 中逐 key duplicate handling。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 duplicate-check batch | 是 | +| `input_rows` | 被检查的 row 数 | `toBeCheckedRows` 长度可得 | +| `probe_keys` | handle key + unique keys + temp-index fallback keys | 是 | +| `probe_key_bytes` | probe key len sum | 是 | +| `conflict_rows` | duplicate path 找到冲突的 row 数 | 是 | + +注意：实际 `txn.Get` / `BatchGet` 的 storage read cost 不在本 opclass 权重里。若未来 TiKV read/write billing 覆盖这些请求，应通过 storage site 或独立 opclass 处理。 + +### `write_row_compare` + +覆盖： + +- UPDATE 和 ODKU 中 old/new row 比较。 +- no-op update / unchanged replace 判断。 +- handle changed 判断。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 update/ODKU compare path | 是 | +| `input_rows` | 进入 `updateRecord` 的 row 数 | 是 | +| `input_cells` | `len(t.Cols())` 或 writable columns | 是 | +| `changed_rows` | `updateRecord` changed=true | 是 | +| `unchanged_rows` | `updateRecord` changed=false 或 identical replace | 是 | + +`unchanged_rows` 保留为 base unit，是因为 no-op UPDATE 仍有 TiDB 比较和 lock-key preparation 成本，但不能被当成 row mutation。 + +### `write_row_mutation` + +覆盖： + +- `AddRecord` record key/value encode 和 memBuffer set。 +- `UpdateRecord` record value re-encode 和 memBuffer set。 +- `RemoveRecord` record key delete。 +- handle changed UPDATE 中的 remove + add 两段 mutation。 +- REPLACE 中删除旧 row 后插入新 row。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | `AddRecord` / `UpdateRecord` / `RemoveRecord` 调用 | 是 | +| `input_rows` | row mutation 数，remove + add 分开计 | 是 | +| `input_cells` | encoded / checked writable columns | 是 | +| `mutation_keys` | record key set/delete 数 | 是 | +| `mutation_bytes` | record key bytes + encoded row value bytes；delete 只计 key bytes | 是 | + +`CommitDetails.WriteKeys` / `WriteSize` 不适合作为第一版来源：它们发生在 commit/2PC 边界，显式事务会偏向 `COMMIT` 语句，pipelined DML 还存在 flush 归并口径风险。 + +### `write_index_mutation` + +覆盖： + +- AddRecord 创建 writable secondary indexes。 +- UpdateRecord 删除旧 index、创建新 index，跳过 untouched index 的优化。 +- RemoveRecord 删除 deletable indexes。 +- unique index 的 local/memBuffer 检查和 assertion 设置。 +- MV index expansion。 +- DDL 状态下的 temp index / double-write。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每次进入 index mutation path | 是 | +| `input_rows` | 参与 index mutation 的 row 数 | 是 | +| `input_cells` | index columns + affected columns + partial-index condition columns | 是 | +| `mutation_keys` | 实际 memBuffer set/delete 的 index key 数；temp-index double-write 分开计 | 是 | +| `mutation_bytes` | index key bytes + index value bytes；delete 只计 key bytes | 是 | +| `probe_keys` | unique index existence check keys | 是 | +| `probe_key_bytes` | unique index check key bytes | 是 | + +不要把 `index_name` 放入维度。索引数量和索引 key bytes 已经反映写放大；`operator_kind=index_create/index_delete` 足够诊断路径。 + +### `write_foreign_key` + +覆盖： + +- FK check rows/key preparation。 +- FK cascade 调用前后的 TiDB 本地 row preparation。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 FK checker/cascade executor | 是 | +| `input_rows` | 被检查或 cascade 的 rows | 部分可从 FK runtime stats 推导 | +| `probe_keys` | FK check keys | 是 | +| `probe_key_bytes` | FK key bytes | 是 | + +第一版可先对 simple FK check 计 base units；复杂 cascade 若无法完整归因，应写 status `unsupported` / `unsupported_fk_cascade`，不能发 partial units。 + +### `write_txn_orchestrate` + +覆盖： + +- batch insert/delete 每批 `StmtCommit` + `NewTxnInStmt`。 +- `txn.MayFlush` 调用事件。 +- LOAD DATA commit worker batch 事件。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 事务编排事件 | 是 | +| `batch_events` | batch DML commit boundary / LOAD DATA worker task 数 | 是 | +| `input_rows` | batch 中 rows | 可从 batch count 得到 | + +本 opclass 只表示 TiDB 事务编排固定成本，不包含 prewrite/commit/lock 的 storage/2PC 成本。 + +### `load_data_decode` + +覆盖： + +- 文件行读取和 parser 输出。 +- column list、user variables、`SET` 表达式。 +- restrictive / non-restrictive load 的 warning/error handling。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 load stream / batch | 是 | +| `input_rows` | parsed rows | 可从 controller/encoder batch count 得到 | +| `input_cells` | parsed fields + SET expressions | 是 | +| `input_bytes` | raw file bytes 或 parser consumed bytes | 是 | + +后续插入阶段继续复用 `write_row_prepare`、`write_duplicate_probe`、`write_row_mutation`、`write_index_mutation`。 + +## 覆盖语句 + +### INSERT VALUES / SET + +推荐发出： + +- `write_row_prepare` +- `write_row_mutation` +- `write_index_mutation` +- `write_txn_orchestrate`，仅 batch DML、pipelined flush 或显式需要 statement transaction boundary 时 + +普通 INSERT 的 duplicate check 如果走 lazy check，只在 `write_index_mutation` 中记录 unique index key construction / memBuffer flags；不要把 future prewrite constraint check 算进 TiDB executor。 + +### INSERT ... SELECT + +读路径由 read billing 计费： + +- child SELECT 的 scan/filter/join/aggregation 继续使用 read billing units。 +- write billing 从 `insertRowsFromSelect` 从 child chunk 取出的 rows 开始，只计 row prepare 和 table/index mutation。 +- `input_source` 建议记录为 `child_output_rows`，避免把 child executor 的 input rows 再计一次。 + +### INSERT IGNORE + +推荐发出： + +- `write_row_prepare` +- `write_duplicate_probe` +- 对未冲突 rows 发出 `write_row_mutation` / `write_index_mutation` +- 对冲突 rows 发出 `conflict_rows`，但不发 mutation units + +### INSERT ... ON DUPLICATE KEY UPDATE + +推荐发出： + +- `write_row_prepare`，对 candidate insert rows +- `write_duplicate_probe` +- 对非冲突 rows 发出 insert mutation units +- 对冲突 rows 发出 `write_row_compare` +- 对 changed duplicate rows 发出 update mutation units +- 对 unchanged duplicate rows 发出 `unchanged_rows`，不发 row/index mutation units + +ODKU 的 `AffectedRows += 2` 是 MySQL 语义，不能作为 mutation row 数。 + +### REPLACE + +REPLACE 的 physical effect 应拆成 duplicate probe、delete mutation、insert mutation： + +- candidate row 发出 `write_row_prepare` +- duplicate path 发出 `write_duplicate_probe` +- 每个真正被删除的旧 row 发出 delete-flavored `write_row_mutation` / `write_index_mutation` +- 每个真正插入的新 row 发出 insert-flavored `write_row_mutation` / `write_index_mutation` +- identical row 不发 mutation units，只发 `unchanged_rows` + +这和 `AffectedRows` 的 1/2 语义分离。 + +### UPDATE + +读路径由 read billing 计费： + +- child executor 找 matched rows 的 scan/join/filter 成本属于 read side。 +- write billing 从 `UpdateExec.updateRows` 读取到 matched row 后开始。 + +推荐发出： + +- `write_row_prepare`，assignment/cast/generated/on-update-now +- `write_row_compare` +- handle unchanged 且 changed：`UpdateRecord` 作为 update-flavored row/index mutation +- handle changed：`RemoveRecord + AddRecord` 拆成 delete + insert mutation +- no-op update：只发 compare units 和 `unchanged_rows` + +### DELETE + +读路径由 read billing 计费： + +- child executor 找待删 rows 的成本属于 read side。 +- write billing 从 `DeleteExec.deleteOneRow` / `removeRow` 开始。 + +推荐发出： + +- `write_row_mutation`，record key delete +- `write_index_mutation`，deletable index delete +- batch delete 额外发 `write_txn_orchestrate` + +Multi-table DELETE 应按每个实际删除的 table row 发 mutation units，但不把 table id 纳入维度。 + +### Batch DML + +`tidb_batch_insert` / `tidb_batch_delete` 会在单条 SQL 内多次 `StmtCommit` 并 `NewTxnInStmt`。base units 应仍归属于原始 digest/window： + +- row/index units 累加到原 statement。 +- `write_txn_orchestrate.batch_events` 记录 batch boundary 次数。 +- 不新增 batch number 维度，避免高基数。 + +### LOAD DATA + +LOAD DATA 首版建议覆盖两个阶段： + +1. `load_data_decode`：文件 decode、列映射、user variables、SET expression。 +2. insert-like write path：按 `ERROR` / `IGNORE` / `REPLACE` 复用 INSERT / duplicate / REPLACE 的 units。 + +远端对象存储读取、网络下载和 decompression worker 是否属于 TiDB billing 需要单独决定。若没有可靠 byte evidence，先发 status `unknown_input` / `missing_load_data_bytes`，不要估算。 + +## 初始权重 + +以下权重只用于 demo preview 和离线试算，不代表生产校准结果。数值选择原则： + +- `fixed_events` 只覆盖小额固定成本； +- `input_cells` 捕捉表达式、cast、比较的 per-cell CPU； +- `probe_keys` / `probe_key_bytes` 只覆盖 TiDB key construction 和 local handling，不覆盖 TiKV read RPC； +- `mutation_keys` / `mutation_bytes` 覆盖 TiDB row/index encode 和 memBuffer set/delete，不覆盖 storage write； +- index mutation 通常比 row mutation 更贵，因为它包含 index value/key generation、partial/MV/unique 处理； +- duplicate probe、row compare、LOAD DATA decode 根据源码中的额外逻辑给予更高 per-row/per-key 权重。 + +推荐 seed weights，单位是 preview RU per unit： + +| Site | Op class | `fixed_events` | `input_rows` | `input_cells` | `input_bytes` | `probe_keys` | `probe_key_bytes` | `mutation_keys` | `mutation_bytes` | `changed_rows` | `unchanged_rows` | `conflict_rows` | `batch_events` | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `tidb` | `write_row_prepare` | 0.030 | 0.000020 | 0.000006 | 0.000001 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| `tidb` | `write_duplicate_probe` | 0.040 | 0.000015 | 0 | 0 | 0.000025 | 0.000001 | 0 | 0 | 0 | 0 | 0.000040 | 0 | +| `tidb` | `write_row_compare` | 0.030 | 0.000015 | 0.000005 | 0.000001 | 0 | 0 | 0 | 0 | 0.000010 | 0.000006 | 0 | 0 | +| `tidb` | `write_row_mutation` | 0.050 | 0.000020 | 0.000004 | 0.000001 | 0 | 0 | 0.000035 | 0.000004 | 0.000010 | 0 | 0 | 0 | +| `tidb` | `write_index_mutation` | 0.060 | 0.000015 | 0.000006 | 0.000001 | 0.000020 | 0.000001 | 0.000050 | 0.000005 | 0 | 0 | 0 | 0 | +| `tidb` | `write_foreign_key` | 0.050 | 0.000020 | 0.000004 | 0.000001 | 0.000030 | 0.000001 | 0 | 0 | 0 | 0 | 0 | 0 | +| `tidb` | `write_txn_orchestrate` | 0.020 | 0.000003 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.010 | +| `tidb` | `load_data_decode` | 0.060 | 0.000025 | 0.000007 | 0.000002 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + +相对顺序： + +```text +write_index_mutation + > write_row_mutation + > write_duplicate_probe ~= load_data_decode + > write_row_prepare ~= write_row_compare + > write_txn_orchestrate +``` + +这个顺序来自当前源码工作量： + +- `index.create` / `index.Delete` 需要 index value/key generation、MV expansion、unique/temp-index 处理和 memBuffer flags。 +- `AddRecord` / `UpdateRecord` / `RemoveRecord` 需要 row encoding、constraint check、assertion 和 record key mutation。 +- duplicate probe 在 `getKeysNeedCheck`、`prefetchUniqueIndices`、`batchCheckAndInsert` 中构造多组 keys 并循环处理冲突。 +- row prepare 和 row compare 主要是 expression/cast/default/generated/compare 的 per-row/per-cell CPU。 +- transaction orchestration 的本地固定开销存在，但 storage/2PC 成本不在本 opclass。 + +## 采集和暴露面 + +推荐新增与 read billing demo 平行的写侧开关和 SQL 面： + +- `tidb_enable_write_billing_demo`，默认 OFF。 +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_STATUS` + +Base-unit table columns 应沿用 read billing demo 的主要维度，并把 read-only 的 `ROW_WIDTH_SOURCE` 泛化为 `WIDTH_SOURCE` 或保留兼容命名： + +| Column | 含义 | +|---|---| +| statement summary window / digest / plan digest / resource group columns | 沿用 statement summary | +| `MODEL_VERSION` | write billing model version | +| `WEIGHT_VERSION` | 生成样本时使用的 weight version | +| `SITE` | `tidb` | +| `OP_CLASS` | bounded opclass | +| `OPERATOR_KIND` | bounded diagnostic kind | +| `UNIT` | bounded unit | +| `INPUT_SOURCE` | `executor_counter` / `stmtctx_counter` / `runtime_stats` / `mem_buffer_delta` / `encoded_bytes` / `load_data_parser` | +| `INPUT_SIDE` | `all`；必要时可用 `old` / `new` / `delete` / `insert`，但不要放 table/index name | +| `VALUE` | unit total | +| `SAMPLE_COUNT` | sample count | + +Status table 应和 read billing demo 一致，支持 statement-level 和 operator-level status。 + +Prometheus 可以新增 bounded observability metrics： + +- `tidb_write_billing_demo_statements_total` +- `tidb_write_billing_demo_operator_status_total` +- `tidb_write_billing_demo_base_units_total` + +但 workload 校准契约应以 SQL 查询面为准，不能要求 testers 依赖 Prometheus retention。 + +## 失败和降级状态 + +推荐 status： + +| Status | 含义 | +|---|---| +| `success` | statement-level accounting 完整 | +| `ok` | operator-level accounting 完整 | +| `unsupported` | 语句或路径明确不支持 | +| `unknown_input` | 必要输入缺失，不能安全估算 | +| `error` | statement 执行失败 | + +推荐 reason： + +| Reason | 触发场景 | +|---|---| +| `statement_error` | executor 返回错误 | +| `missing_plan` | 没有可归因 plan | +| `missing_stmtctx` | statement context 不可用 | +| `missing_runtime_stats` | 必要 runtime stats 不可用 | +| `missing_mutation_stats` | table/index mutation 没有 instrumentation | +| `missing_encoded_bytes` | bytes unit 需要的 encoded length 不可用 | +| `missing_load_data_bytes` | LOAD DATA 无可靠 input bytes | +| `unsupported_statement` | 非目标语句 | +| `unsupported_fk_cascade` | FK cascade 无法完整归因 | +| `unsupported_temporary_table` | temporary table 口径不同且未支持 | +| `unsupported_temp_index_merge` | DDL temp-index merge / double-write 无法完整归因 | +| `unsupported_internal_sql` | restricted/internal SQL | +| `aggregation_overflow` | base-unit key cap overflow | +| `status_aggregation_overflow` | status key cap overflow | + +失败或 unsupported statement 不写 base units。若某个必需 opclass 缺失输入，应 fail closed：写 status，跳过该 statement 的所有 base units，避免 partial units 被误用于校准。 + +## 避免重复计费 + +1. 读写混合语句： + - child SELECT / scan / join / filter 由 read billing 负责； + - write billing 只从 DML executor 消费 child rows 后开始； + - `input_source=child_output_rows` 可标明边界。 +2. duplicate / FK storage probes： + - TiDB write side 只计 key construction 和本地 loop； + - `txn.Get` / `BatchGet` / snapshot scan 成本属于 storage/read path； + - 不把 probe RPC latency 或 processed keys 乘入 TiDB write weights。 +3. row/index mutation 与 commit details： + - `mutation_keys` / `mutation_bytes` 是 TiDB 构造并放入 memBuffer 的 key/value； + - `CommitDetails.WriteKeys` / `WriteSize` 是 2PC mutation 口径，显式事务可能归到 COMMIT； + - 两者不能同时作为同一层权重输入。 +4. transaction orchestration： + - `write_txn_orchestrate` 只计本地 fixed/batch event； + - prewrite/commit/lock/resolve-lock/local-latch 不属于本 opclass。 +5. affected rows： + - `AffectedRows` 只用于 SQL 兼容展示； + - physical mutation rows 必须从 `AddRecord` / `UpdateRecord` / `RemoveRecord` 或新增 counters 得到。 + +## 实现期 Instrumentation 需求 + +可直接复用或改造的现有输入： + +| 输入 | 当前来源 | 限制 | +|---|---|---| +| statement type / digest / plan digest / resource group | statement summary | 可复用 | +| `RecordRows` / `CopiedRows` / `UpdatedRows` / `DeletedRows` / `TouchedRows` | `StatementContext` | SQL 语义，不够物理 | +| `rowsColMultiply` | `InsertValues.rowsColMultiply`、`UpdateExec.rowsColMultiplyForPreparedRow`、delete row/column count | 当前只进 RU v2 scalar counter，需要结构化保存 | +| insert/update runtime stats | `InsertRuntimeStat`、`updateRuntimeStats` | time/RPC 诊断，不是 base-unit source | +| commit write keys/size | `CommitDetails` / RU v2 metrics | storage/2PC 边界，不适合作为 TiDB executor-local source | + +需要新增： + +1. Statement-scoped `WriteBillingDemoStats`，与 read billing stats 类似，包含 status samples 和 base-unit samples。 +2. Table/index mutation instrumentation： + - record set/delete key count； + - record key bytes； + - encoded row value bytes； + - index set/delete key count； + - index key/value bytes； + - temp-index double-write count。 +3. Duplicate probe instrumentation： + - per row handle key count； + - unique key count； + - probe key bytes； + - conflict rows； + - temp-index fallback probes。 +4. Row prepare / compare instrumentation： + - input rows； + - input cells； + - estimated input bytes； + - changed / unchanged row counts。 +5. LOAD DATA instrumentation： + - parser consumed bytes； + - parsed rows / fields； + - SET expression cells； + - restrictive/non-restrictive skipped rows。 +6. Statement summary v1/v2 aggregation maps and JSON-stable v2 entry slices，保留 overflow status。 + +## 测试设计 + +后续实现应至少覆盖： + +1. INSERT VALUES： + - 无索引、普通二级索引、unique index、common handle、auto id。 + - 验证 row prepare、row mutation、index mutation units。 +2. INSERT SELECT： + - 验证 read side 和 write side 不重复计算 child input rows。 +3. INSERT IGNORE： + - 无冲突、handle 冲突、unique 冲突。 + - 冲突 rows 只产生 duplicate probe / conflict units，不产生 mutation units。 +4. ODKU： + - 无冲突插入、冲突更新、冲突 no-op。 + - `AffectedRows` 与 mutation rows 分离。 +5. REPLACE： + - identical row、replace one row、多个 unique key 触发多次 remove。 +6. UPDATE： + - no-op、普通 update、handle changed update、multi-table update。 +7. DELETE： + - single-table、multi-table、batch delete。 +8. LOAD DATA： + - ERROR / IGNORE / REPLACE 三种 duplicate mode； + - local 和 remote reader 的 input bytes 缺失时 status 行。 +9. FK： + - simple FK check； + - unsupported cascade status-only。 +10. Statement summary： + - v1 current/history、v2 persistent current/history、cluster table； + - base-unit aggregation overflow 和 status aggregation overflow； + - error / unsupported / unknown-input 不写 base units。 +11. RU v2 隔离： + - 开启 write billing demo 不改变 resource-control RU 上报； + - `CommitDetails.WriteKeys/WriteSize` 不被误当成 TiDB write base units。 + +## 风险和未决问题 + +1. `mutation_bytes` 的最佳口径需要实现时固定：建议用 TiDB 实际构造的 key bytes + value bytes，而不是 memBuffer memory footprint。 +2. 显式事务中一条 DML 产生 mutation，另一条 COMMIT 提交。若未来要 storage write billing，需要单独解决语句归因；本文只解决 TiDB executor-local write preparation。 +3. Pipelined DML 当前 commit details 可能无法完整反映 flushed keys/size。不要用该口径验证 TiDB write units。 +4. Temporary table、temp index merge、DDL backfill 期间的 double-write 口径复杂，第一版可以 status-only 或以 bounded units 记录 double-write count。 +5. FK cascade 可能递归触发多条 row mutation。第一版应先保证不漏报 status，再决定是否把 cascade 作为独立 nested write opclass。 +6. `LOAD DATA` 的文件读取、解压、对象存储网络是否属于 TiDB write billing 需要产品层确认；本设计只覆盖 parser/decode 和 insert-like write path。 diff --git a/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md b/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md new file mode 100644 index 0000000000000..fdc60790dd8e1 --- /dev/null +++ b/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md @@ -0,0 +1,143 @@ +# Preview RU TiDB KV mutation operator + +> **Current-model notice (2026-07-22):** the mutation lifecycle evidence in +> this document remains valid, but its v3 two-weight and commit-detail formulas +> are historical. The implemented v4 formula and statement-local DML/COMMIT +> request ownership are normative in +> `docs/design/2026-07-22-preview-ru-resource-formula-plan.md`. +> In that v4 contract, normalized mutation work is emitted as the shared +> `cpu_work` unit and identified by `site=tidb`, `op_class=kv_mutation`, +> `operator_kind=memdb_mutation`, `input_source=stmt_memdb_mutation_calls`, and +> `input_side=all`; it is not a separate weight-bearing unit. + +## Status and scope + +This document defines the write-side model implemented by the preview RU demo. +It does not change production RUv2 charging, reporting, or configuration. + +The model intentionally covers only the narrow TiDB work that constructs and +encodes record/index KVs and prepares foreground transaction MemDB mutations. +Scan, reader, filter, projection, join, sort, row preparation, comparisons, +constraints, foreign keys, and auto-ID work remain separate plan or future +synthetic operators. Missing coverage is reported as partial; it is not folded +into a rows, cells, or DML-specific mutation formula. + +## Operators and formulas + +TiDB statement-local mutation work is represented by one shared operator: + +```text +site=tidb +op_class=kv_mutation +operator_kind=memdb_mutation +scope=statement_attempted +source=stmt_memdb_mutation_calls + +RU_tidb_mutation = encoded_mutation_count * tidb_mutation_count_weight + + encoded_mutation_bytes * tidb_mutation_byte_weight +``` + +All INSERT, UPDATE, DELETE, INSERT IGNORE, and UPSERT paths use this same weight +key. `dml_kind` is a diagnostic statement-summary dimension and does not +participate in `(site, op_class, weight_version)` lookup. The corresponding +`DML_KIND` information-schema column is appended after the existing base-unit +columns so older column ordinals remain stable. + +The mutation weights are independent calibration slots. They are currently +zero and output is explicitly marked `uncalibrated`; they do not alias RUv2 L5, +RUv2 write-key, scan-byte, or TiKV coefficients. Shadow units `set_count`, +`delete_count`, `key_bytes`, and `value_bytes` have zero weight. + +TiKV transaction payload remains separate: + +```text +site=tikv +op_class=kv_write +operator_kind=txn_prewrite +scope=txn_prewrite_payload +source=commit_detail + +RU_tikv_write = write_keys * tikv_write_key_weight + + write_byte * tikv_write_byte_weight +``` + +`write_byte` is retained for preview-output compatibility and is annotated with +the semantic name `write_bytes`. Region and write-RPC counts remain zero-weight +diagnostics. TiKV coefficients are not used by the TiDB mutation operator. + +## Exact mutation semantics + +- Each attempted `Set`, `SetWithFlags`, `Delete`, or `DeleteWithFlags` counts + once after key/value encoding and before the underlying MemDB call. +- Set bytes are `len(key)+len(value)`; delete bytes are `len(key)`. +- A failed MemDB call still counts because encoding and call preparation have + already occurred. +- Same-key overwrites, pessimistic statement retries, and mutations later + removed by staging cleanup or rollback remain counted. +- `UpdateFlags`, staging release/cleanup, lock-only operations, commit-time net + mutations, and local-temporary-table apply copies are not counted as new + foreground encoded mutations. +- The recorder lives on `StatementContext`. It is allocated only when the demo + flag is on or `EXPLAIN ANALYZE FORMAT='RU'` explicitly requests preview data. +- Both restricted-SQL entry points execute through the internal-statement path, + including their current-session form, so internal/meta DML cannot become a + foreground mutation sample merely because the preview flag is enabled. + +The transaction wrapper dynamically resolves the current statement context. +This preserves per-statement attribution during optimistic history replay. +Because a statement summary already emitted before a later optimistic COMMIT +cannot be rewritten, retryable explicit transactions are marked partial for +optimistic replay attribution. + +## Availability and lifecycle + +- Autocommit DML normally exposes TiDB mutation and TiKV prewrite operators. +- Explicit-transaction DML always retains statement-local TiDB units and its + read tree. Missing commit detail creates a partial TiKV operator only. The + unavailable operator remains visible in `FORMAT='RU'` as an unweighted + status row with its site, class, scope, and reason; it is not reduced to a + statement-level summary note. + The final COMMIT emits the transaction-scoped TiKV prewrite payload with an + empty `dml_kind`; it is never guessed or backfilled onto earlier DML. +- No-op/zero-match DML emits both TiDB primary units with value zero. A missing + TiKV commit detail is partial rather than a successful zero payload. +- Pipelined mutation units remain valid, while missing TiKV logical flush + payload is reported as `pipelined_tikv_payload_unsupported`. Pipelined + transactions allocate a `CommitDetails` object without populating its + `WriteKeys/WriteSize`; those empty fields are not published as zero units. +- Deprecated batch DML keeps one statement recorder across its internal + transaction switches. The TiDB mutation count therefore includes every + batch exactly once; available commit details continue to accumulate on the + statement context. +- Ancillary DML CPU is currently partial for every supported DML kind. The gate + cannot yet distinguish a simple DELETE from one that performs unmodeled FK + checks or cascade orchestration, so DELETE is conservatively partial too. + REPLACE remains rejected by the existing gate. IMPORT INTO and other + non-foreground mutation paths are not represented by this operator. +- Local temporary table encoding is counted once on the foreground transaction + MemDB. Its later copy to session temporary data does not pass through the + recorder, and TiKV commit detail is unavailable/partial. + +## Plan-tree composition + +DML is flattened before result construction. The DML root is a non-billable +wrapper, while its SELECT/CTE/scalar-subquery and scan/filter/reader descendants +continue through the existing preview operator classifier. The mutation and +available TiKV write operators are appended once. Summary RU is the sum of all +weighted unit rows. An unsupported node or a node with missing runtime evidence +is emitted as an unweighted partial status row, and traversal continues so it +cannot hide independently usable descendants. These diagnostic rows do not +enter the summary total. + +## Primary code locations + +- Recorder and snapshot: `pkg/sessionctx/stmtctx/stmtctx.go` +- MemDB/transaction interception and statement lifecycle: + `pkg/session/txn.go`, `pkg/session/session.go` +- Preview composition, weights, status, and output: + `pkg/planner/core/explain_ru.go` +- Structured `dml_kind` diagnostic: + `pkg/util/stmtsummary/read_billing.go`, `pkg/infoschema/tables.go` +- Regression coverage: + `pkg/session/tidb_test.go`, `pkg/planner/core/common_plans_test.go`, + `pkg/executor/explain_test.go` diff --git a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md new file mode 100644 index 0000000000000..6642c2c16be96 --- /dev/null +++ b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md @@ -0,0 +1,605 @@ +# Stabilize preview RU around resource-shaped operator formulas + +This ExecPlan is a living document. Keep `Progress`, `Surprises & Discoveries`, `Decision Log`, and `Outcomes & Retrospective` up to date as work proceeds. + +Reference: `PLANS.md` at repository root. This plan must be maintained according to it. + +## Purpose / Big Picture + +The current preview RU demo uses an operator-specific matrix of fixed, row, byte, and ordering weights. That matrix is useful for collecting evidence, but it does not express the intended model directly: expression evaluation should consume one shared CPU weight, scans should consume one scan-byte weight, transport should consume network-byte and request weights, and stateful operators should add small hash-table or join-output terms. + +After this plan is implemented, `EXPLAIN ANALYZE FORMAT='RU'`, the bounded metrics, the general-log detail, and the statement-summary detail tables will expose the same coefficient-free resource work units. A reader can reproduce every operator's preview RU with seven named weights and can see exactly which current runtime detail supplied each input. The preview remains isolated from production RUv2 charging and resource-control reporting. + +This plan first froze the model, field mappings, failure behavior, migration, and validation, and now records the implementation and verification evidence from the implementation worktree. + +## Progress + +- [x] (2026-07-22 09:00Z) Inspected the current preview RU implementation and its closest tests in `pkg/planner/core/explain_ru.go`, `pkg/planner/core/common_plans_test.go`, and `pkg/executor/explain_test.go`. +- [x] (2026-07-22 09:20Z) Verified current row, byte, scan-detail, cop-task, RUv2 response-byte/RPC, aggregation-output, inline Projection, and reader-child evidence in code. +- [x] (2026-07-22 09:35Z) Resolved the formula, expression-count, reader-attribution, write-unit, failure, and version-migration decisions in this document. +- [x] (2026-07-22 10:05Z) Incorporated the explicit de-duplication constraints for Sort/TopN inline Projection CPU and IndexJoin inner-reader requests. +- [x] (2026-07-22 11:15Z) Incorporated the final ordering clarification: Sort owns `n*log(n)`, TopN owns `n*log(k)` with `k` derived from offset plus count, and expression evaluation remains Projection-owned. +- [x] (2026-07-22 11:40Z) A fresh-context release reviewer re-read the updated user handoff, plan, and code and concluded: “当前版本无需必要修改”. +- [x] (2026-07-22 11:45Z) Applied the Ready profile to this design-only diff: no Go/Bazel trigger was present and the staged Markdown patch passed `git diff --cached --check`. +- [x] (2026-07-22 13:05Z) A later fresh-context completion audit found that explicit-transaction DML write RPCs are statement-local and cannot all be deferred to COMMIT; revised ownership to charge each DML and COMMIT from its own frozen `RUV2Metrics` snapshot. +- [x] (2026-07-22 13:35Z) A new no-inherited-context reviewer inspected the corrected plan and current code and concluded exactly: “当前版本无需必要修改”. Because this iteration changed a substantive ownership rule, the convergence policy still requires another fresh-context iteration before design-loop completion. +- [x] (2026-07-22 14:20Z) A further fresh-context reviewer independently audited the original formula contract, current exec-detail sources, operator ownership, degradation rules, migration, and corrected statement-local write-request lifecycle and concluded exactly: “当前版本无需必要修改”. No substantive design change was made, so the required convergence review is complete. +- [x] (2026-07-22 19:10Z) Implemented Milestone 1: introduced the private six-weight v4 container, semantic work units, centralized expression-slot counts, checked ordering arithmetic, and exact formula tests without changing production RUv2. +- [x] (2026-07-22 19:35Z) Implemented Milestone 2: constructed root/cop/read-transport/write units from existing frozen details and added only the narrow HashJoin `HashTableRows()` runtime interface with V1/V2 state accounting. +- [x] (2026-07-22 20:05Z) Implemented Milestone 3: migrated EXPLAIN, metrics, statement summary, General Log, version labels, legacy-total behavior, documentation, and focused regression tests atomically. +- [x] (2026-07-22 20:55Z) Completed the implementation Ready profile: `make bazel_prepare`, all six prescribed failpoint-wrapped package test sets, the tagged metrics test, `make lint`, and `git diff --check` passed; the HashJoin set additionally ran `TestHashRowContainer` for V1 NAAJ state coverage. +- [x] (2026-07-22 22:10Z) A completion audit found and fixed a v4 migration regression: the write constructor had stopped consulting the existing statement mutation snapshot's `Pipelined` flag. Pipelined DML and COMMIT now retain mutation evidence but fail the write-request component closed with `pipelined_tikv_payload_unsupported`; focused DML and COMMIT regressions cover the restored gate. +- [x] (2026-07-22 04:25+08:00) The final completion audit found that the ordering gate rejected residual scalar functions but did not prove that column `ByItems` belonged to the direct child schema. Sort/TopN now validate the executed flat child, including Projection expression/schema alignment, and a regression was observed failing before the fix and passing afterward. +- [x] (2026-07-22 04:30+08:00) Recompleted the Ready gate after the ordering fix: the full planner/core and executor target sets passed with failpoint cleanup, the earlier same-state join/session/statement-summary/metrics runs remained applicable, `make lint` passed after the code edit, and `git diff --check` plus API/range review found no remaining mismatch. +- [x] (2026-07-22 05:35+08:00) A requirement-by-requirement completion audit found that the private weight container omitted its specified immutable version, so a test-only calibrated set could publish a total under the production uncalibrated label. Added version ownership and validation to the weight container, propagated its active version to every output, and observed the focused regression fail before the fix and pass afterward. +- [x] (2026-07-22 05:45+08:00) Recompleted the full Ready gate from the current worktree: `make bazel_prepare` produced no generated diff; all six failpoint-wrapped target groups, the HashJoin NAAJ coverage, and tagged metrics tests passed; `make lint`, `git diff --check`, API-boundary searches, and the baseline-to-HEAD scope review passed. Failpoint refcount returned to zero after every run. +- [x] (2026-07-22 07:15+08:00) A subsequent output-version audit found that background Prometheus statement, operator-status, base-unit, and row-width series carried only `model_version`. Added the bounded active `weight_version` label to all four families and froze one active version per result recording; the new metric-contract regression failed before the change and passed afterward. +- [x] (2026-07-22 07:20+08:00) Recompleted the Ready gate after the Prometheus provenance fix: `make bazel_prepare` produced no generated diff; all six failpoint-wrapped target groups, HashJoin NAAJ coverage, and tagged metrics tests passed with failpoint refcount returning to zero; `make lint`, `git diff --check`, call-site search, version-label review, and the baseline scope review passed. +- [x] (2026-07-22 15:42+08:00) Applied the final write-unit contract: normalized mutation work now reuses `cpu_work`, is emitted only under a valid calibrated weight snapshot, and remains distinguishable through `site=tidb`, `op_class=kv_mutation`, `operator_kind=memdb_mutation`, `input_source=stmt_memdb_mutation_calls`, and `input_side=all`. The six raw mutation diagnostics remain visible when uncalibrated. +- [x] (2026-07-22 15:53+08:00) Completed the revised write-unit Ready gate: the new regression failed against the old independent-unit behavior and passed after restoration; all six failpoint-wrapped target groups, tagged metrics tests, `make lint`, residual-name search, and `git diff --check` passed with failpoint refcount returning to zero. +- [x] (2026-07-22 16:09+08:00) Split the shared request contract into `read_request_count`/`read_request_weight` and `write_request_count`/`write_request_weight`, preserving the existing statement-scope ownership and fail-closed rules. Updated EXPLAIN, General Log, Prometheus, statement-summary, formula, DML, and COMMIT regressions plus the affected test case maps. +- [x] (2026-07-22 16:14+08:00) Completed the request-split Ready gate: both reader and writer regressions failed when their constructors were temporarily restored to the old shared `request_count` output, then passed with the split implementation. All six failpoint-wrapped target groups, tagged metrics tests, `make lint`, residual-name search, and `git diff --check` passed with failpoint refcount returning to zero. The Bazel gate found no trigger, so `make bazel_prepare` was not required. +- [x] (2026-07-22 18:45+08:00) Corrected TopN ordering work to saturate its legal `offset+count` heap bound at actual input rows and to emit zero work when `count=0`. The focused regression failed against the unsaturated formula and passed after the fix; overflow remains fail-closed for every positive count. +- [x] (2026-07-22 18:47+08:00) Completed the TopN correction Ready gate: the full preview-RU planner target, EXPLAIN/metrics/general-log executor targets, `make lint`, and `git diff --check` passed with failpoint refcount returning to zero. The Bazel gate found no trigger, so `make bazel_prepare` was not required. +- [x] (2026-07-23 23:35+08:00) Reproduced the real-TiKV non-Scan publication failure in code: distsql attaches response ScanDetail to the component's last plan ID while the scan leaf receives only an execution summary, and the v4 estimator counted both the real holder and the leaf's default-zero detail as holders. Three rounds of ownership, fail-closed, and validation review converged on explicit TiDB-side ScanDetail attachment provenance without changing the distsql ownership or TiKV protocol. +- [x] (2026-07-24 00:05+08:00) Added a regression that exactly uses `RecordOneCopTask` for the scan leaf and `RecordCopStats` for the parent holder. It failed before the fix with `ambiguous_cop_scan_width` and passed after `RuntimeStatsColl` began recording ScanDetail attachments and the estimator validated holder/task coverage from one consistent snapshot. +- [x] (2026-07-24 00:20+08:00) Completed the provenance regression matrix: explicit empty detail, repeated same-holder attachments, attachment-without-holder-summary, independent scan-summary and attachment coverage failures, no holder, partial tuple, multi-holder ambiguity, ExecutorId remap, and statement-atomic failure are all covered. Independent implementation and test reviewers both concluded LGTM with no P0/P1/P2. +- [x] (2026-07-24 00:33+08:00) Completed cleanup-safe real-TiKV validation against PD/TiKV `v9.0.0-beta.2.pre-363-g8cabcf1` and the current-worktree TiDB binary. Selection, Projection, Limit, TopN, HashAgg, and ordered StreamAgg all succeeded in `FORMAT='RU'`; the Selection probe emitted `tikv/filter_eval cpu_work=4` and `tikv/kv_range_scan scan_bytes=212.5` identically through General Log, Prometheus, and statement summary. The exact TiUP tag was cleaned and the PD endpoint was unreachable afterward. +- [x] (2026-07-24 00:36+08:00) Completed the final Ready gate after the stable ordered-StreamAgg fixture revision: targeted execdetails, distsql, planner/core, and executor tests passed; `make lint` and `git diff --check` passed; failpoint refcount was zero. The Bazel gate found no added/moved Go file, import change, new top-level test, Bazel edit, or module edit, so `make bazel_prepare` was not required. +- [x] (2026-07-24 01:20+08:00) Added statement-scoped RPC-only preview units for closed-set read-only PointGet and BatchPointGet plans. The implementation emits exactly one synthetic point-lookup operator, retains physical lookup rows as non-billable diagnostics, and keeps locking, DML, mixed cop-reader, and unknown producer shapes fail-closed. +- [x] (2026-07-24 01:35+08:00) Observed the focused EXPLAIN regression fail against the prior `ambiguous_reader_transport_producers` behavior and pass after the implementation. Planner, EXPLAIN, plan-digest, General Log, Prometheus, and statement-summary regressions passed; `make bazel_prepare`, `make lint`, and `git diff --check` passed. A cleanup-safe real-TiKV run confirmed PointGet and BatchPointGet each publish only one statement-scoped `read_request_count=1` unit and no uncalibrated total. +- [x] (2026-07-24 02:05+08:00) Bound UnionScan's intentionally simple CPU formula to its direct child's actual runtime rows: `cpu_work=input_rows`. The focused regression failed while `overlay_reader` was non-billable and passed after the formula was enabled, with no expression-count multiplier or new runtime datum. +- [x] (2026-07-24 02:25+08:00) Completed UnionScan output and Ready verification. A cleanup-safe real-TiKV transaction produced UnionScan output rows 4 over direct-child rows 3 and published exactly `tidb/overlay_reader/unionscan cpu_work=3` from `runtime_child_act_rows`; the focused preview-RU planner suite, `make lint`, and `git diff --check` passed with failpoint refcount zero. The Bazel gate found no import, file, top-level-test, Bazel, or module trigger, so `make bazel_prepare` was not required. +- [x] (2026-07-24 12:20+08:00) Enabled DML PointGet and BatchPointGet request publication from their plan-local `SnapshotRuntimeStats.GetCmdRPCCount`. The focused regression failed against the blanket DML rejection and passed after the change; the DML statement producer set remains open, so pessimistic-lock and ancillary requests cannot be copied from the statement RUv2 counter into PointGet. +- [x] (2026-07-24 12:26+08:00) A cleanup-safe real-TiKV run with the current-worktree TiDB binary confirmed `source=snapshot_runtime_stats`: the autocommit UPDATE published PointGet `read_request_count=1`, while the explicit pessimistic UPDATE published PointGet `read_request_count=0` and independently retained `write_request_count=1`. The unique TiUP tag was removed, the PD endpoint was unreachable afterward, and the pre-existing playground tags were preserved. +- [x] (2026-07-24 12:27+08:00) Completed the Ready gate: `make bazel_prepare` generated only the required planner/core TiKV RPC dependencies; the focused planner and EXPLAIN failpoint-wrapped regressions passed with refcount returning to zero; `make server`, `make lint`, and `git diff --check` passed. + +## Surprises & Discoveries + +- Observation: root executor byte accounting is logical live chunk bytes, not encoded network bytes. + Evidence: `pkg/util/execdetails/runtime_stats.go` defines `BasicRuntimeStats.inputBytes/outputBytes`, and `pkg/executor/internal/exec/executor.go` records child/output chunk bytes around `Next`. + +- Observation: exact TiKV response bytes and chargeable read/write RPC counters exist at statement scope, while PointGet and BatchPointGet additionally retain plan-local RPC counts in their snapshot runtime stats. + Evidence: `pkg/util/execdetails/ruv2_metrics.go` exposes `TiKVCoprocessorResponseBytes`, `ResourceManagerReadCnt`, and `ResourceManagerWriteCnt`; `pkg/executor/point_get.go` and `pkg/executor/batch_point_get.go` register `SnapshotRuntimeStats` under their plan IDs and query `GetCmdRPCCount` for `CmdGet` and `CmdBatchGet`. + +- Observation: `selectResultRuntimeStats` already counts cop responses internally, but proportional or per-reader network-byte attribution is not available from current exec details. + Evidence: `pkg/distsql/select_result.go` keeps `copRespTime` and `reqStat` inside the unexported `selectResultRuntimeStats`, while response bytes are drained into statement-level `RUV2Metrics`. + +- Observation: HashAgg runtime details do not expose a separate group-count counter, but the physical Agg node's own actual output rows are the number of materialized group states for this simple model. + Evidence: `RuntimeStatsColl` provides own-plan `GetActRows`; the current preview implementation already publishes Agg output-row shadows from this value and verifies TiKV expected/observed task coverage. + +- Observation: HashJoin state is row-backed rather than one-entry-per-distinct-key, and the exact admitted row count exists below the exec-details boundary. + Evidence: `pkg/executor/join/hash_table_v1.go::Len` counts ordinary inserted row pointers, while NAAJ null-key rows are separately retained in `hashNANullBucket.entries`; `pkg/executor/join/join_row_table.go::validKeyCount` counts v2 rows eligible for the hash lookup structure. Current HashJoin runtime stats expose timing/collision data but not the total admitted lookup-state row count. + +- Observation: IndexJoin's inner lookup work is executed by dynamically built reader children, whose physical network requests already enter statement-level read RPC details. + Evidence: `pkg/executor/join/index_lookup_join.go::fetchInnerResults` drains `task.innerExec`, while the reader paths use DistSQL and feed `RUV2Metrics.ResourceManagerReadCnt`; adding the private inner task count as another request term would charge the same lookup twice. + +- Observation: inline Projection materializes scalar Sort/TopN `ByItems`, while ordinary column keys need no scalar evaluation; injection is not universal for pushed cop plans. + Evidence: `pkg/planner/core/rule_inject_extra_projection.go::InjectProjBelowSort` injects only when a `ByItems` expression is a `ScalarFunction`, and root post-optimization does not rewrite every pushed cop plan. V4 therefore assigns expression evaluation exclusively to an actual Projection, assigns one aggregate sorting-complexity term to Sort/TopN regardless of key count, and fails closed if a scalar ordering expression remains unmaterialized. + +- Observation: completed IndexHashJoin plans do not retain the ordinary IndexJoin equality representation. + Evidence: `pkg/planner/core/exhaust_physical_plans.go::completePhysicalIndexJoin` clears `EqualConditions` after deriving `OuterHashKeys` and `InnerHashKeys`; IndexMergeJoin instead exposes executable `CompareFuncs` and optional `OuterCompareFuncs`. Counting only the embedded IndexJoin fields would undercount these subtypes. + +- Observation: zero-valued RUv2 getters do not prove that their payload was present, and the read-RPC counter is broader than cop response bytes. + Evidence: absent/bypassed `RUV2Metrics` reads as zero; `ResourceManagerReadCnt` covers TiKV read RPC producers including unsupported point/ancillary paths, while `TiKVCoprocessorResponseBytes` covers cop responses. Runtime task coverage and a closed producer set are therefore required before zero or statement-wide totals are attributable. + +- Observation: write RPC counters are statement-local even inside an explicit transaction. + Evidence: `session.executeStmtImpl` installs the current statement's `RUV2Metrics`, `ExecStmt.finalizeStatementRUV2Metrics` drains raw/commit details into it before preview construction, and `TestRUV2MetricsIsolatedPerStatementInExplicitTxn` proves successive statements use distinct instances. A pessimistic DML's write requests therefore cannot be deferred to the later COMMIT snapshot. + +- Observation: MockStore does not synthesize resource-manager write RPC detail for pessimistic KV traffic. + Evidence: the new session lifecycle test observes zero from a real MockStore DML before injection. It therefore injects finalized values into the actual per-statement `RUV2Metrics` objects to prove DML/COMMIT snapshot separation, while private core construction tests independently prove that values 7 and 2 become distinct `write_request_count` units. + +- Observation: pipelined completeness is already preserved in the statement-local mutation snapshot and on the COMMIT statement before transaction invalidation. + Evidence: `stmtctx.PreviewKVMutationSnapshot.Pipelined`, `LazyTxn.previewKVMutationRecorder`, and `session.markPreviewKVMutationTxnPipelined` provide the lifecycle evidence. The v4 write constructor must consume that flag before publishing `ResourceManagerWriteCnt`; a present counter alone does not prove complete pipelined logical-flush attribution. + +- Observation: a non-scalar Sort/TopN `ByItems` expression alone does not prove that inline expression evaluation was materialized by the executed child. + Evidence: `InjectProjBelowSort` rewrites scalar ordering expressions to newly allocated Projection columns, but the preview constructor previously accepted any column without checking the direct child's schema. Column membership and Projection expression/schema alignment are therefore required presence evidence before publishing ordering work. + +- Observation: a constant output-side weight version does not prove that the coefficient set used to calculate a total has that version. + Evidence: the initial v4 implementation kept `readBillingDemoWeightVersion` outside `readBillingDemoWeights`; private calibrated formula fixtures could therefore make `total_preview_ru` available while EXPLAIN, statement summary, and metrics still labeled the result `v3-resource-formula-uncalibrated`. + +- Observation: `model_version` alone is insufficient provenance for background Prometheus units whose semantics can change with the weight container. + Evidence: mutation-derived `cpu_work` depends on the versioned `MutationBytesPerCPUUnit`, while the four `tidb_read_billing_demo_*` families previously omitted `weight_version`; a rolling transition could therefore merge distinct v4 contracts into the same series. + +- Observation: distsql response ScanDetail ownership differs from executor-summary ownership. + Evidence: `pkg/distsql/select_result.go::updateCopRuntimeStats` records each valid execution summary under its exact `copPlanIDs` entry but attaches the response-level ScanDetail only through `RecordCopStats` on the last plan ID. `pkg/distsql/select_result_test.go::TestUpdateCopRuntimeStats` proves that the scan leaf has tasks and zero detail while its parent holds the merged detail. + +- Observation: a zero-valued `CopRuntimeStats.GetScanDetail()` does not prove an observed empty scan. + Evidence: both a summary-only scan leaf and an actually attached all-zero ScanDetail read back as the zero struct. The previous `tasks>0 && detail==zero` sentinel therefore double-counted the leaf beside a real parent holder and could also turn a missing or partial detail into a false observed zero. + +- Observation: native EmbedUnistore is not a complete scan-width oracle for this regression. + Evidence: its cop response supplies processed versions but not the complete total/processed-size tuple required by `readBillingDemoRangeScanInput`. A test-only response enrichment may exercise output wiring, but the corrected Ready evidence must include real TiKV. + +- Observation: a bare `stream_agg()` hint did not produce StreamAgg consistently across native EmbedUnistore and real TiKV. + Evidence: the stable validation shape also orders the result and uses `order_index(table, idx_b)`, making the ordered index property explicit; ordinary EXPLAIN and `FORMAT='RU'` then both show `StreamAgg cop[tikv]`. + +- Observation: TiKV client-go already classifies every successfully completed TiKV `CmdGet` and `CmdBatchGet` RPC as one raw read RPC, and TiDB drains that counter into the statement's frozen `ResourceManagerReadCnt`. + Evidence: `client-go/internal/client/completedTiKVRUV2RPCCount` returns `(1, 0)` for non-write TiKV requests and its regression names both Get and BatchGet; `config.UpdateTiKVRUV2FromExecDetailsV2` adds that count to `kvrpcpb.RUV2.ReadRpcCount`; `ExecStmt.finalizeStatementRUV2Metrics` drains it through `SyncRUV2MetricsFromRUDetails`. + +- Observation: PointGet and BatchPointGet have no attributable response-byte counter in the frozen preview details, while their statement read-RPC counter may cover more than one plan node. + Evidence: `RUV2Metrics.TiKVCoprocessorResponseBytes` covers coprocessor responses rather than Get/BatchGet payloads, and `ResourceManagerReadCnt` is statement-scoped rather than keyed by physical plan ID. Per-node publication would therefore invent byte work and duplicate RPC work for multi-lookup statements. + +- Observation: UnionScan merges snapshot rows from its direct child with transaction mem-buffer rows, but the frozen preview details expose only the direct child's exact actual-row count. + Evidence: `UnionScanExec.getOneRow` merges `getSnapshotRow` and `getAddedRow`; `RuntimeStatsColl` records the direct child and UnionScan output rows but no separate count of mem-buffer rows considered by the merge. Output rows cannot reconstruct input work because deletes, replacements, and predicates can suppress rows. + +## Decision Log + +- Decision: replace, rather than layer on top of, the v3 fixed/row/byte opclass matrix for new samples. + Rationale: keeping both billable models would double count and make `cpu_weight` cease to mean one expression-slot evaluation. Historical v3 samples remain queryable under their model version. + Date/Author: 2026-07-22 / Codex. + +- Decision: use seven semantic RU weights: `cpu_weight`, `scan_weight`, `net_weight`, `read_request_weight`, `write_request_weight`, `hash_table_weight`, and `join_weight`. + Rationale: read and write requests have independent calibration coefficients and externally distinct units. Operator identity remains a diagnostic dimension, not a weight key. + Date/Author: 2026-07-22 / Codex. + +- Decision: Sort CPU work is `rows * log2(max(rows, 2))`; for positive `count`, TopN CPU work is `rows * log2(max(min(rows, k), 2))`, where `k = offset + count`; `count=0` produces zero TopN work. + Rationale: this is the final requested ownership split. Inline Projection alone charges scalar expression evaluation; the ordering node charges aggregate algorithmic work, with no expression/key-count multiplier. TopN retains at most the rows that actually exist, so a configured heap bound larger than input cardinality cannot increase work. Offset remains part of the legal heap bound. The implementation rejects `offset + count` overflow for positive counts instead of wrapping; zero/one rows use base two so positive-count work is finite and deterministic, while `count=0` matches the executor's no-child-read fast path. + Date/Author: 2026-07-22 / Codex. + +- Decision: count top-level executable expression slots, not recursive AST nodes. + Rationale: plan fields provide a stable, cheap, deterministic count. Recursive counts would make rewrites of an equivalent scalar expression change billing without runtime evidence and would blur the meaning of one calibrated slot. + Date/Author: 2026-07-22 / Codex. + +- Decision: aggregate reader transport once per statement under `reader_transport`, while retaining TableReader, IndexReader, IndexLookup, and IndexMerge kinds as bounded diagnostics. + Rationale: the existing byte and RPC details are statement-scoped and all listed reader kinds share the same weights. Algebraically, charging the totals once equals summing per-reader formulas. Proportional allocation by logical output bytes is rejected because it invents attribution and is badly biased for IndexLookup/IndexMerge internal legs. + Date/Author: 2026-07-22 / Codex. + +- Decision: publish PointGet and BatchPointGet read RPCs once per statement under `id=point_lookup@statement`, `site=tikv`, and `op_class=kv_point_lookup`, with only `read_request_count`. + Rationale: pure PointGet and pure BatchPointGet use their respective operator kind; a statement containing both uses `mixed_point_lookup`. Read-only lookups use the closed producer-set gate and the chargeable statement counter. DML keeps the statement producer set open and instead sums only plan-local `CmdGet`/`CmdBatchGet` counts, which excludes pessimistic-lock writes and unrelated ancillary reads by construction. No CPU, scan, or network work is inferred; missing plan-local runtime evidence remains fail-closed. + Date/Author: 2026-07-24 / Codex. + +- Decision: make root UnionScan weight-bearing with exactly `cpu_work = direct_child_actual_rows`. + Rationale: this is the requested simple formula and reuses the same stable `runtime_child_act_rows` source as other root unary operators. UnionScan emits no `expression_count` and does not infer mem-buffer input rows from output rows. A future model that charges the second input would require an explicit executor datum and a separate design revision. + Date/Author: 2026-07-24 / Codex. + +- Decision: interpret the requested HashJoin `distinct_rows` term as runtime `hash_state_rows`, the cumulative rows actually admitted into hash lookup state; interpret the HashAgg term as `group_rows = own output rows`. + Rationale: duplicate join keys still allocate/probe row-backed entries, while null/filter-rejected build rows do not enter that structure. Exact admitted rows are therefore more faithful than all build-child rows, and counting unique keys would require a second hash set solely for billing. This is the one permitted Join-only exec-details addition. + Date/Author: 2026-07-22 / Codex. + +- Decision: IndexJoin has no additional request term at the Join node and requires no request-related runtime datum. + Rationale: its extra lookups execute through inner reader children and are already included in the statement reader-transport request counter. Charging inner lookup tasks again would duplicate request cost. The optional Join exec-detail permission is therefore not exercised for IndexJoin/request accounting; v4 uses it only for HashJoin `hash_state_rows`. + Date/Author: 2026-07-22 / Codex. + +- Decision: keep calibrated-weight injection private to `pkg/planner/core` formula tests; package-external end-to-end tests exercise the production-default uncalibrated contract. + Rationale: `pkg/executor/explain_test.go` is `package executor_test`, while the preview weight container intentionally remains private and has no session/global knob. A public or failpoint configuration surface solely for tests would weaken the initial contract. + Date/Author: 2026-07-22 / Codex. + +- Decision: freeze the three legacy statement-summary convenience totals as v3-only and leave them zero for v4 samples. + Rationale: `fixed_events`, `input_rows`, and `input_bytes` cannot safely represent the new semantic units. V4 consumers must use the versioned detail table; adding a parallel convenience schema and its v1/v2 persistence migration is outside this minimal model change. + Date/Author: 2026-07-22 / Codex. + +- Decision: ship `model_version='v4'` with `weight_version='v3-resource-formula-uncalibrated'` and delete the executable v3 weight map. + Rationale: historical rows already contain values and versions, while private per-statement results do not survive process upgrade. Keeping a second calculation path would create an unnecessary mixed-model state; an eventual calibrated set must receive another immutable version. + Date/Author: 2026-07-22 / Codex. + +- Decision: combine write mutation count and bytes into `cpu_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit`, then apply `cpu_weight` once. Distinguish mutation CPU from expression CPU through the operator dimensions rather than a separate unit name. + Rationale: normalization produces expression-equivalent CPU work, so reusing the shared semantic unit preserves one CPU coefficient. `mutation_bytes_per_cpu_unit` is a versioned normalization constant, not an RU coefficient; the complete weight snapshot must be calibrated and valid before mutation-derived `cpu_work` is emitted. Raw mutation count/byte diagnostics remain available without calibration. + Date/Author: 2026-07-22 / Codex. + +- Decision: charge write requests to the statement whose frozen `RUV2Metrics` snapshot contains them; an explicit-transaction DML and the eventual COMMIT each own only their respective snapshots. + Rationale: TiDB installs a fresh `RUV2Metrics` object for every statement and finalizes raw RU details into that statement before preview construction. Pessimistic DML can issue write RPCs before COMMIT, so deferring all request work to COMMIT would permanently omit those DML-local requests. A cross-statement accumulator is unnecessary and would weaken the existing statement-local contract. + Date/Author: 2026-07-22 / Codex. + +- Decision: expose read and write requests as `read_request_count` and `write_request_count`, with independent `read_request_weight` and `write_request_weight` coefficients. + Rationale: the counters already have distinct authoritative sources and workload behavior, and independent unit names prevent downstream consumers from collapsing them before calibration. This changes only preview-v4 semantics; statement-scope ownership, fail-closed gates, and production RUv2 remain unchanged. + Date/Author: 2026-07-22 / Codex. + +- Decision: do not guess numerical v4 weights or silently map the heterogeneous v3 opclass weights. + Rationale: there is no evidence-backed one-to-one mapping. The implementation first publishes v4 base units with `uncalibrated_weights` and no total until an explicit v4 weight set and positive mutation normalization are installed. The preview flag is off by default, so this is safer than presenting arbitrary numbers as RU. + Date/Author: 2026-07-22 / Codex. + +- Decision: validate Sort/TopN ordering columns against the direct executed flat child schema, and additionally require a Projection child's schema width to match its expression list. + Rationale: this proves the ownership handoff to the child Projection without charging expression CPU at the ordering node, while preserving ordinary column and constant ordering. Relying only on optimizer invariants would violate the explicit fail-closed contract for missing or misaligned Projection schema. + Date/Author: 2026-07-22 / Codex. + +- Decision: make the private weight container own its immutable version and reject calibration under the shipped uncalibrated label. + Rationale: coefficients and their version are one atomic contract. Deriving all output labels from the active container prevents a future calibrated set from publishing a total under stale metadata, while the production default remains explicitly `v3-resource-formula-uncalibrated` and cannot publish totals. + Date/Author: 2026-07-22 / Codex. + +- Decision: add the active weight version to every background preview Prometheus family, not only to base-unit samples. + Rationale: keeping statement status, operator status, units, and row-width observations on the same bounded version key makes the Prometheus output an atomic contract and prevents partial cross-version aggregation. The version is immutable per binary configuration, so the extra label remains bounded. + Date/Author: 2026-07-22 / Codex. + +- Decision: record TiDB-internal ScanDetail attachment counts and use them, rather than detail values, to identify the unique component holder. + Rationale: `RecordCopStats(scan != nil)` already knows whether a response attached ScanDetail, including a legitimate all-zero detail, while `RecordOneCopTask` is summary-only. Counting attachments under the original plan ID preserves the existing distsql/EXPLAIN ownership contract, distinguishes empty from absent evidence, and makes multiple holders fail closed without relying on flat-tree traversal order. This is internal provenance metadata, not a new TiKV protocol field or formula input. + Date/Author: 2026-07-23 / Codex. + +- Decision: validate scan executor summaries and ScanDetail attachments as independent coverage channels. + Rationale: the scan leaf must have `observed summaries == expected responses > 0`, while the unique holder must have `attachment records == holder expected == scan expected`. The holder's own execution summary is not required for scan bytes because `RecordCopStats` can receive complete ScanDetail with a nil summary. Only after both channels are complete may an all-zero tuple mean an observed empty scan. + Date/Author: 2026-07-23 / Codex. + +- Decision: keep the broader non-Scan direct-child expected-task hardening outside this focused ownership correction. + Rationale: the existing relative `maxSummaryTasks` gate can miss the case where every executor omits the same response summary, but changing every unary/Agg/DML path is not required to fix the confirmed double-holder failure and needs its own calibrated-release regression matrix. + Date/Author: 2026-07-23 / Codex. + +## Outcomes & Retrospective + +The design is stable after the required convergence review. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(min(n, offset+count))` work without repeating Projection expression evaluation, treats zero-count TopN as zero work, validates ordering-column ownership against the executed child schema, removes IndexJoin request double charging, and defines a no-guess migration. The implementation now publishes exactly seven weight-bearing v4 semantic units, reuses dimension-qualified `cpu_work` for normalized mutation preparation, retains the raw mutation diagnostics, and keeps production weights intentionally uncalibrated. It atomically binds any future calibrated coefficients to their immutable output version across EXPLAIN, statement summary, general log, and all background Prometheus families, removes the executable v3 formula from production, gives read and write requests independent unit names and weights while preserving statement-scoped ownership, and preserves the pipelined write-request fail-closed gate. Runtime additions remain narrow: HashJoin exposes actual admitted state rows as a formula datum, while cop stats record only TiDB-side ScanDetail attachment provenance needed to distinguish an observed empty scan from absent detail. MockStore cannot produce real resource-manager write RPC details, so the session lifecycle test injects finalized observations into the actual fresh per-statement snapshots; private construction tests separately verify their exact 7/2 unit values. The ownership correction now passes the complete focused regression matrix and real-TiKV output verification: six non-Scan cop operator classes publish their units, and one Selection sample agrees across EXPLAIN, General Log, Prometheus, and statement summary without publishing an uncalibrated total. The subsequent point-lookup extension adds no runtime datum: pure read-only PointGet and BatchPointGet statements publish one exact RPC-only synthetic operator from the chargeable statement counter, while DML lookups publish only their existing plan-local snapshot RPC counts and leave the broader statement producer set open. UnionScan likewise reuses existing evidence and the existing CPU weight: it now publishes direct-child actual rows as `cpu_work` without charging its conditions or inventing a mem-buffer input estimate. + +## Context and Orientation + +Preview RU is an observational model. It emits coefficient-free work units and optionally multiplies them by preview-only weights. It must not modify the RU charged by TiKV, the RUv2 total reported to resource control, or scheduling decisions. + +The current constructor and renderer live in `pkg/planner/core/explain_ru.go`. `buildReadBillingDemoResult` freezes one statement result, `readBillingDemoRootUnits` derives TiDB root units, `readBillingDemoCopUnits` derives TiKV cop units, and `readBillingDemoUnitPreviewRU` applies weights. The same result feeds `EXPLAIN ANALYZE FORMAT='RU'`, metrics, statement summary, and logging. The later implementation must keep one frozen result as the sole source for all outputs. + +`RuntimeStatsColl` in `pkg/util/execdetails/runtime_stats.go` maps physical plan IDs to root and cop runtime details. Root `BasicRuntimeStats` gives actual output rows and logical chunk bytes. Cop `CopRuntimeStats` gives executor-produced rows, task count, and `ScanDetail`. `RUV2Metrics` in `pkg/util/execdetails/ruv2_metrics.go` gives statement-level TiKV response bytes and read/write RPC counts. Physical plan structs in `pkg/planner/core/operator/physicalop` give expression lists; these are static plan metadata, not new runtime observations. + +In this document, a work unit is a coefficient-free non-negative finite number. A weight converts one work unit into preview RU. A physical operator is one node in the executed physical plan. An expression slot is one top-level expression/function/key comparison that the physical operator evaluates per input row. `cpu_weight` prices one expression-equivalent CPU-work unit; expression evaluation, ordering comparisons, Limit row handling, and normalized mutation preparation can all produce such units. A missing value differs from an observed zero; missing required evidence fails closed, while observed zero remains billable as zero. + +## Stable formula contract + +For one statement, weighted preview RU is the sum of the following unit families: + + preview_ru = + cpu_work * cpu_weight + + scan_bytes * scan_weight + + net_bytes * net_weight + + read_request_count * read_request_weight + + write_request_count * write_request_weight + + hash_state_rows * hash_table_weight + + join_output_rows * join_weight + +All arithmetic is float64 with explicit negative, NaN, infinity, and overflow rejection. Integer counters are converted only after validating that they are non-negative. Each pre-aggregation operator result retains `site`, `op_class`, `operator_kind`, `operator_id`, `source`, and, for joins, `input_side`. Physical results use the executed plan's Explain ID. Statement-scoped synthetic results use reserved, non-plan IDs: `reader_transport@statement`, `point_lookup@statement`, `mutation@statement`, and `txn_write@statement`. Statement-summary detail intentionally aggregates away `operator_id`; its remaining bounded dimensions and version still preserve formula provenance. + +There is no billable fixed-event term in v4. Setup costs that correlate with remote fanout use the direction-specific request unit; other constant setup costs stay outside this intentionally simple model. + +### Operator formulas + +| Operator | v4 formula | Required inputs | +|---|---|---| +| Selection | `rows * n_expr * cpu_weight` | direct child actual rows; selection expression slots | +| Projection | `rows * n_expr * cpu_weight` | direct child actual rows; projected expression slots | +| Sort | `rows * log2(max(rows,2)) * cpu_weight` | direct child actual rows; scalar expression evaluation belongs to inline Projection | +| TopN | `count == 0 ? 0 : rows * log2(max(min(rows,k),2)) * cpu_weight` | direct child actual rows; checked `k=offset+count` for positive count; scalar expression evaluation belongs to inline Projection | +| TableScan / IndexScan | `scan_bytes * scan_weight` | attributable TiKV `ScanDetail` | +| TableReader / IndexReader / IndexLookup / IndexMerge transport | `net_bytes * net_weight + read_request_count * read_request_weight` | statement `RUV2Metrics`, emitted once | +| PointGet / BatchPointGet | `read_request_count * read_request_weight` | statement `RUV2Metrics`, emitted once for a closed point-lookup producer set | +| UnionScan | `rows * cpu_weight` | direct child actual rows; no expression-count multiplier | +| StreamAgg | `rows * n_expr * cpu_weight` | direct child actual rows; group and aggregate slots | +| HashAgg | `rows * n_expr * cpu_weight + group_rows * hash_table_weight` | StreamAgg inputs plus own actual output rows | +| MergeJoin | `(left_rows + right_rows) * n_expr * cpu_weight + output_rows * join_weight` | both child rows, join slots, own output rows | +| HashJoin | `(left_rows + right_rows) * n_expr * cpu_weight + hash_state_rows * hash_table_weight + output_rows * join_weight` | both child rows, join slots, one Join runtime state count, own output rows | +| IndexJoin family | `(left_rows + right_rows) * n_expr * cpu_weight + output_rows * join_weight` | both child rows, join slots, own output rows; inner reader already owns requests | +| Limit | `rows * cpu_weight` | direct child actual rows | +| Window | `rows * n_expr * cpu_weight` | direct child rows and the refined Window slot count below | +| Write mutation and commit | `cpu_work * cpu_weight + write_request_count * write_request_weight` | existing mutation recorder and statement/commit RUv2 details; mutation provenance is carried by operator dimensions | + +The table is normative. Diagnostic `input_rows`, logical chunk bytes, output bytes, mutation component counters, and operator status may still be emitted, but they have no weight and never enter `preview_ru`. + +### Expression-slot count + +`n_expr` is a non-negative integer derived from the executed physical plan. It is stored as a diagnostic unit so offline recomputation does not need the original plan object. + +Selection uses `len(PhysicalSelection.Conditions)`. Projection uses `len(PhysicalProjection.Exprs)`, including inline ordering expressions materialized for Sort/TopN and column pass-through expressions because the executor still materializes an output column. + +Sort and TopN do not emit an expression-count unit and do not multiply algorithmic work by `len(ByItems)` or `len(PartitionBy)`. Ordinary column, constant, and multi-key ordering are covered by the single aggregate sorting term. Before publishing that term, inspect `ByItems`: a remaining `ScalarFunction` proves that expression evaluation was not materialized into a child Projection, so the operator fails closed with `missing_ordering_projection` rather than charging expression CPU at Sort/TopN. If `ByItems` references columns produced by an aligned child `PhysicalProjection`, that Projection's normal formula owns all of its `Exprs` once. A missing/misaligned Projection schema also fails closed. For positive-count TopN, compute `k=offset+count` with checked unsigned addition, saturate it to `effective_k=min(uint64(rows), k)`, and only then convert it to float; do not use `count` alone. A zero-count TopN emits zero ordering work regardless of offset because the executor returns without reading its child. + +StreamAgg and HashAgg use `len(GroupByItems) + len(AggFuncs)`. One aggregate function descriptor is one slot regardless of partial/final/complete mode and regardless of argument count. This intentionally avoids a two-phase special model. `COUNT(*)` therefore has one slot rather than zero. Ordered aggregate arguments remain inside their aggregate function slot for v4. + +For joins, one join-key pair or executable comparison function is one slot, not two column slots. Add the lengths of the remaining `LeftConditions`, `RightConditions`, and `OtherConditions`. HashJoin key pairs come from `EqualConditions + NAEqualConditions`; MergeJoin key comparisons come from `CompareFuncs`. + +The IndexJoin family is counted by concrete subtype, because its completed physical plans do not share one key representation: + +- `PhysicalIndexJoin`: count aligned `OuterJoinKeys`/`InnerJoinKeys`, the remaining left/right/other conditions, and `len(CompareFilters.OpType)`. +- `PhysicalIndexHashJoin`: count aligned `OuterHashKeys`/`InnerHashKeys` rather than cleared `EqualConditions`, then the remaining left/right/other conditions and `len(CompareFilters.OpType)`. +- `PhysicalIndexMergeJoin`: count `CompareFuncs`, plus `OuterCompareFuncs` only when `NeedOuterSort` is true, then the remaining left/right/other conditions and `len(CompareFilters.OpType)`. + +The implementation must centralize this in one helper. It must reject mismatched aligned key slices, a `NeedOuterSort`/`OuterCompareFuncs` structural inconsistency, or another impossible subtype layout instead of selecting an arbitrary side or falling back to stale embedded fields. + +Window refines the original direction without adding a new weight: + + n_expr = + len(WindowFuncDescs) + + len(PartitionBy) + + len(OrderBy) + + len(Frame.Start.CalcFuncs, if present) + + len(Frame.End.CalcFuncs, if present) + +This covers per-row function evaluation, partition/order comparisons, and dynamic frame-bound calculation. It does not add a partition-size, frame-width, or buffering term because current exec details do not expose those values and the requested model should remain simple. + +For every expression-based operator, `n_expr == 0` is valid only when the physical operator genuinely contains no expression slot. The resulting CPU work is zero. The implementation must not silently clamp `n_expr` to one; intrinsic work is represented only where the formula explicitly supplies it, such as Limit. + +### Row and state semantics + +For a TiDB root operator, `rows` is the sum of direct root children's own actual output rows from `BasicRuntimeStats.GetActRows`. Unary operators normally have one child. A missing child statistic is not zero. Reader-like leaf nodes do not reuse their own output rows as an input for the CPU formulas because reader transport has a separate formula. + +For a pushed TiKV unary operator, `rows` is the exact-plan-ID actual output rows of its direct cop child. Existing v3 expected/observed task coverage rules remain: no tasks means missing; negative rows are invalid; fewer observed tasks than the component's known coverage is incomplete. The model must not fall back to optimizer estimates. + +HashAgg `group_rows` is the operator's own actual output rows. Each physical partial or final Agg node is charged independently from its own input and output; there is no phase multiplier. TiKV Agg keeps the existing independent expected-response versus observed-summary coverage gate before its own rows are accepted. + +HashJoin `hash_state_rows` is the cumulative number of build rows actually admitted into hash lookup structures. Duplicate keys count once per admitted row; rows rejected by build filters and ordinary null-key rows that are not retained for lookup do not count. V1 null-aware anti join is the exception: its null-key rows are stored and probed through `hashNANullBucket.entries`, so they do count. If spilling rebuilds hash state in another round, admissions in that round count again because the state construction work repeats. This value comes from the HashJoin executor, not from build-child output rows. + +Expose it through one narrow read-only interface in `pkg/util/execdetails`: + + type HashTableRuntimeStats interface { + RuntimeStats + HashTableRows() int64 + } + +Both private v1 and v2 HashJoin runtime stats implement the getter. V1 records successful `hashTable.Len() + len(hashNANullBucket.entries)` for NAAJ, with a nil bucket contributing zero; v2 records the sum of `validKeyCount` admitted for each completed build round. Clone/Merge preserve the cumulative counter. Tests cover ordinary null rejection and V1 NAAJ null-bucket inclusion. Missing or negative state rows fail with `missing_hash_state_rows` or `invalid_hash_state_rows`. No unique-key set is introduced. + +MergeJoin, HashJoin, and IndexJoin `output_rows` is the join node's own `BasicRuntimeStats.GetActRows`. It is the only permitted output term; output bytes remain diagnostic only. Both join inputs and the output are required even when observed as zero. + +### Scan bytes + +TableScan and IndexScan use the current v3 scan-byte proxy: + + if TotalKeys == 0 and ProcessedKeys == 0 and ProcessedKeysSize == 0: + scan_bytes = 0 + otherwise: + scan_bytes = TotalKeys * ProcessedKeysSize / ProcessedKeys + +The all-zero case is an observed zero only when the scan leaf has `observed execution summaries == expected responses > 0` and the unique ScanDetail holder has `attachment records == holder expected == scan expected`. An attachment record is one `RecordCopStats` call with a non-nil ScanDetail under the original plan ID; a summary-only `RecordOneCopTask` is not an attachment. The holder's own execution summary is independent and is not required for scan bytes. Otherwise the value is missing or incomplete. In the nonzero case, all three values must be positive and the result must be finite. This proxy preserves scan work for MVCC/skipped keys while using only current `ScanDetail`; it is labeled `scan_detail_processed_key_avg_estimate`, not presented as encoded response bytes. + +Each scan detail must be attributable to exactly one scan component. Multi-scan IndexMerge is supported by evaluating each partial scan component separately; it must not share one detail across siblings. Ambiguous or absent attribution fails the affected statement according to the atomicity rules below. + +### Reader transport + +The four named reader families share one statement-level transport formula. The constructor identifies all executed TableReader, IndexReader, IndexLookup, and IndexMerge nodes, then emits exactly one `id=reader_transport@statement`, `site=tidb`, `op_class=reader_transport` operator with a bounded `operator_kind` set: `table_reader`, `index_reader`, `index_lookup`, `index_merge`, or `mixed_reader`. + +`net_bytes` is `RUV2Metrics.TiKVCoprocessorResponseBytes()`. `read_request_count` is `RUV2Metrics.ResourceManagerReadCnt()`. Both are snapshots from the same frozen statement details used by existing preview outputs. The unit source is `ruv2_metrics`; logical `BasicRuntimeStats.GetOutputBytes` remains a diagnostic and is never substituted for transport bytes. + +The statement-wide counters are publishable only when the executed flat plan proves a closed producer set: every possible TiKV read-RPC producer for the statement belongs to those four supported cop-reader families. The initial implementation uses this exact algorithm: + +1. Only a read-only `SELECT` can pass the closed-set gate. If a DML flat plan contains any supported reader, its reader-transport component is always `unknown_input/ambiguous_reader_transport_producers`; v4 has no DML allowlist because current details cannot exclude uniqueness, locking, FK, transaction, or other ancillary reads. +2. Walk the complete executed `FlatPhysicalPlan`, including IndexJoin inner plans. Classify `*physicalop.PhysicalTableReader` with `StoreType == kv.TiKV`, `*physicalop.PhysicalIndexReader`, `*physicalop.PhysicalIndexLookUpReader`, and `*physicalop.PhysicalIndexMergeReader` as supported producers. +3. Reject any `*physicalop.PointGetPlan` or `*physicalop.BatchPointGetPlan` from the cop-reader transport component; a separate point-lookup component below may publish it only when the complete statement contains point lookup producers and no cop reader or other open read producer. Also reject any `PhysicalTableReader` whose `StoreType != kv.TiKV`, `*physicalop.PhysicalExchangeReceiver`, and `*physicalop.PhysicalExchangeSender`. Any node that the existing preview classifier marks as a reader/store-access class but that is not one of the four supported cop-reader types also opens the set. This catches TiFlash/MPP and future external reader types without treating a new producer as free. +4. Other already-supported CPU, join, aggregation, wrapper, scan-descendant, UnionScan, MemTable, and TableDual nodes are not independent TiKV read-RPC producers and do not open the set. Any structurally unknown plan node continues to fail through the existing unsupported-operator gate, so transport is never published alongside an unknown tree. + +A supported reader mixed with any rejected producer is not partially charged from the total counter: SELECT fails atomically; DML marks only its reader-transport component unknown. This is conservative because the current statement counter cannot subtract unsupported RPCs. + +Presence and zero handling are normative: + +- A nil, bypassed, or otherwise unavailable RUv2 snapshot is missing, even though public getters return zero. +- Inspect `GetTasks()` and `GetExpectedCopTasks()` for every supported reader/cop descendant. If any observed or expected cop task exists, `net_bytes == 0 && read_request_count == 0` is missing rather than free. +- `net_bytes > 0 && read_request_count == 0` is invalid. `read_request_count > 0 && net_bytes == 0` is valid for an empty cop response. +- `net_bytes == 0 && read_request_count == 0` is an observed zero only when a present, non-bypassed frozen RUv2 snapshot exists, no supported descendant has an observed or expected cop task, every supported reader root has observed zero output rows, and the producer set is closed. This represents an empty range/no-request execution. + +If no supported reader executed, no reader-transport operator is emitted. Unsupported producers retain explicit bounded status rows until an attributable mapping is designed. The bounded transport reasons are `missing_reader_transport_details` for presence/coverage failure and `ambiguous_reader_transport_producers` for an open producer set. + +### Point lookup RPC transport + +PointGet and BatchPointGet plans emit request work once as `id=point_lookup@statement`, `site=tikv`, `op_class=kv_point_lookup`, `input_side=all`, and unit `read_request_count`. Its bounded `operator_kind` is `point_get`, `batch_point_get`, or `mixed_point_lookup` when both physical kinds occur. The physical PointGet/BatchPointGet plan rows remain non-billable diagnostics. + +For read-only statements, a present, non-bypassed frozen RUv2 snapshot is required and `input_source=ruv2_metrics`. For DML, every PointGet/BatchPointGet plan ID must have snapshot runtime stats and `input_source=snapshot_runtime_stats`; the component sums only `CmdGet` or `CmdBatchGet` as appropriate. Zero is valid because a point lookup can be satisfied without a Get/BatchGet RPC, for example through a pessimistic lock request that returns the value. A negative count or overflow is invalid. The component emits no `cpu_work`, `scan_bytes`, or `net_bytes`, because current details expose neither point-lookup executor CPU nor attributable Get/BatchGet response bytes. + +For a read-only statement, the point-lookup producer set is closed only when its complete flat plan contains one or more non-locking PointGet/BatchPointGet nodes and no cop-reader producer, TiFlash/MPP producer, or unknown store-access producer. The DML statement producer set is always open: uniqueness checks, foreign-key reads, pessimistic locks, or other transaction work may add requests, so its statement RUv2 read count is never assigned to PointGet. Plan-local snapshot stats still allow PointGet to publish its own exact Get/BatchGet RPC count even when other DML producers exist. + +### IndexJoin request de-duplication + +IndexJoin, IndexHashJoin, and IndexMergeJoin do not emit a Join-local `read_request_count`. Their dynamic inner executors use TableReader, IndexReader, or IndexLookup paths, so the resulting physical read RPCs are already present in the statement-level reader-transport `ResourceManagerReadCnt`. PhysicalIndexMergeReader remains a supported standalone transport producer, but `dataReaderBuilder.BuildExecutorForIndexJoin` does not construct it as an IndexJoin inner path. + +This is the explicit de-duplication refinement to the initial simple IndexJoin formula: + + initial: IndexJoin CPU + lookup requests + output rows + v4: IndexJoin CPU + output rows + reader_transport already charges all inner physical requests once + +The existing private inner task counter remains an EXPLAIN timing diagnostic and is not converted into request RU. IndexJoin adds no detail; the only v4 runtime extension is HashJoin's state-row getter above. + +### Write work + +The existing statement-local mutation recorder remains authoritative. Its complete semantics for this plan are: + +- Count each attempted foreground `Set`, `SetWithFlags`, `Delete`, or `DeleteWithFlags` once after encoding and before calling MemDB. A failed MemDB call still counts. Set bytes are `len(key)+len(value)`; delete bytes are `len(key)`. +- Same-key overwrites, pessimistic statement retries, and mutations later removed by staging cleanup or ROLLBACK remain counted because their encoding/preparation CPU already occurred. `UpdateFlags`, staging release/cleanup, lock-only operations, commit-time net mutations, and local-temporary-table apply copies do not create another foreground mutation. +- The recorder is statement-local and dynamically follows the current `StatementContext`, including optimistic history replay. Restricted/internal SQL never becomes a foreground sample. Retryable explicit transactions whose already-emitted statements cannot be rewritten are marked `optimistic_replay_attribution_unsupported`/partial rather than pretending exact attribution. +- No-op or zero-match DML has a present recorder with zero count and bytes. Deprecated batch DML keeps one recorder across its internal transaction switches, so every batch attempt is counted once. Local-temporary-table encoding is counted once at its foreground MemDB write. + +`docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md` remains background evidence, but the bullets above are the normative v4 contract. + +The v4 mutation normalization derives the shared CPU unit: + + cpu_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit + +`mutation_bytes_per_cpu_unit` is a positive, finite, versioned normalization constant with units bytes per expression-equivalent CPU-work unit. It is stored alongside the v4 weights and included in output metadata. It is not independently multiplied by RU. If it is unset, zero, negative, NaN, or infinite, mutation base components remain visible but weighted v4 total is unavailable with `uncalibrated_weights`. + +The externally weight-bearing semantic units are fixed to `cpu_work`, `scan_bytes`, `net_bytes`, `read_request_count`, `write_request_count`, `hash_state_rows`, and `join_output_rows`. Mutation-derived `cpu_work` uses `site=tidb`, `op_class=kv_mutation`, `operator_kind=memdb_mutation`, `input_source=stmt_memdb_mutation_calls`, and `input_side=all`; consumers must use these dimensions to distinguish it from expression CPU work. + +`write_request_count` is `RUV2Metrics.ResourceManagerWriteCnt()` and uses its independent `write_request_weight`; read transport uses `read_request_weight`. The frozen snapshot must exist and be non-bypassed. Every DML statement, whether autocommit or inside an explicit transaction, emits the write-request count present in its own finalized snapshot alongside its statement-local mutation work. This is required for pessimistic transactions, whose DML statements can issue nonzero write RPCs before COMMIT. The eventual COMMIT emits only the write requests in its own fresh snapshot, with empty `dml_kind`; it neither absorbs nor back-attributes earlier DML requests. Thus each physical request is owned once by the statement whose RU details recorded it. A nonzero mutation with a missing request payload is partial, while an observed zero DML request count is valid only when the non-bypassed statement snapshot is present and finalized. A known empty transaction COMMIT likewise emits observed zero request work; an absent lifecycle snapshot is missing, not zero. SQL `ROLLBACK` remains unsupported in v4 and emits neither a zero unit nor a total: current routing has no complete rollback-RPC attribution, and declaring it free would be unsafe. + +Pipelined transactions retain valid mutation units but mark the write-request component `pipelined_tikv_payload_unsupported` until current details prove a complete logical flush request count. Deprecated batch DML accumulates available write-request snapshots across internal transaction switches; any missing switch makes the request component partial. Optimistic retry/replay keeps the mutation behavior above and marks unavailable request attribution partial. Thus no retry, pipeline, or batch path publishes a known-incomplete zero. + +Mutation count and bytes continue to be emitted as zero-weight diagnostics so calibration can change `mutation_bytes_per_cpu_unit` offline. `CommitDetails.WriteKeys/WriteSize` are not substituted for the TiDB mutation unit. + +## Availability, atomicity, and degraded behavior + +The existing preview gates remain: the feature is default off; `EXPLAIN ANALYZE FORMAT='RU'` explicitly enables collection; unsupported side-effecting/locking/internal paths are rejected; production resource control is untouched. `*ast.RollbackStmt` is explicitly routed to `unsupported/unsupported_statement` before the ordinary SELECT gate, so it cannot be mistaken for a missing-plan SELECT or an observed-zero write. + +For side-effect-free SELECT, billing is statement-atomic. If any executed supported operator lacks a required input, the statement records status and reason but emits no billable v4 units and no `total_preview_ru`. This prevents a partial plan from looking cheap. Diagnostic status rows may still identify every missing operator. + +For DML, read-tree, mutation, and statement-local write-request components keep independent status. COMMIT has its own write-request component because explicit transactions separate statement lifetimes. Complete components may retain coefficient-free units for calibration, but a statement-level weighted total is absent unless every component expected at that lifecycle point is complete and the weight set is calibrated. + +Observed zero is accepted only with presence evidence: an existing root stat, a cop stat with complete task coverage, a HashJoin state-row runtime stat, a mutation recorder snapshot, or a frozen RUv2 snapshot plus the reader consistency checks above. Missing, negative, overflowed, NaN, and infinite inputs have bounded reasons. New reasons are `missing_expression_count`, `missing_ordering_projection`, `invalid_topn_bound`, `missing_reader_transport_details`, `ambiguous_reader_transport_producers`, `missing_hash_state_rows`, `invalid_hash_state_rows`, and `uncalibrated_weights`. + +No fallback may use optimizer estimated rows, schema-estimated widths, plan `netDataSize`, string parsing of `EXPLAIN` runtime text, or proportional allocation of statement counters. + +## Weight units and migration + +The v4 weight container is preview-only: + + type previewRUWeights struct { + Version string + CPUPerWorkUnit float64 + ScanPerByte float64 + NetworkPerByte float64 + ReadRequest float64 + WriteRequest float64 + HashTablePerRow float64 + JoinPerOutputRow float64 + MutationBytesPerCPUUnit float64 + Calibrated bool + } + +The seven RU fields have units stated by their names. `MutationBytesPerCPUUnit` is a normalization, not RU. Validation requires a nonempty new version, finite non-negative RU weights, a positive finite mutation normalization, and `Calibrated=true` before any weighted total is published. Formula tests in `pkg/planner/core` use the private container directly with small deterministic values. No exported setter, session/global variable, or failpoint is added. Package-external executor tests see the production default and assert `uncalibrated_weights`, coefficient-free units, and no `total_preview_ru` until a later calibration change supplies production values. + +Set the exact constants `model_version='v4'` and `weight_version='v3-resource-formula-uncalibrated'`. Do not reuse the old model `v3` or weight `v2` labels. The weight-version string intentionally describes the shipped state; a later calibrated weight set must use another immutable version rather than changing values behind this label. Existing statement-summary detail already carries model and weight versions, so old rows remain self-describing and are not rewritten. Queries that compare workload windows must group by both versions. + +The existing `ReadBillingDemoBaseUnitSummary` and its infoschema convenience columns (`fixed_events`, `input_rows`, and `input_bytes`) are frozen legacy-v3 views. V4 samples contribute zero to all three, and v4 consumers use the versioned base-unit detail rows instead. Do not reinterpret an old column as `cpu_work`, and do not merge v3 and v4 in those totals. This avoids a cross-version semantic lie and avoids expanding the v1/v2 statement-summary persistence schema in this milestone. Tests must cover v3 legacy aggregation unchanged, v4 legacy totals zero, and v4 detail surviving memory/history readers. + +There is no migration of `config.RUV2`, TiKV client RU coefficients, or resource-group settings. Those configure production RUv2 and have different semantics. Delete the current internal `readBillingDemoWeights` map, `readBillingDemoResolveWeights`, and v3 formula application in the same atomic implementation change. `readBillingDemoResult` is constructed and rendered within one statement in one process; no frozen result crosses a process-upgrade boundary, while historical statement-summary rows already store their unit values and versions and are never recomputed from this map. Therefore no v3 calculation compatibility branch is needed or permitted. + +All outputs must switch together: EXPLAIN unit rows, `total_preview_ru`, Prometheus base units, statement-summary detail, and general log. A mixed state where logs use v4 while EXPLAIN uses v3 is not accepted. General-log aggregation must extend its key and serialized object to retain `DMLKind`, `InputSource`, and `InputSide`; otherwise distinct v4 units with the same operator/unit label would collapse and lose their provenance. `operator_id` remains an internal/EXPLAIN identity and is intentionally absent from the bounded statement-summary and general-log aggregation keys. + +## Plan of Work + +### Milestone 1: represent v4 work without changing collection + +In `pkg/planner/core/explain_ru.go`, add bounded v4 unit names (`cpu_work`, `expression_count`, `scan_bytes`, `net_bytes`, `read_request_count`, `write_request_count`, `hash_state_rows`, and `join_output_rows`), the validated weight container, and one formula application function. Replace opclass-specific billable lookup for new results with semantic-unit lookup. Keep diagnostic legacy units zero-weight. + +Add physical-plan helpers that return expression-slot count for every supported concrete type. Unit-test exact counts for simple and compound Selection, Projection, Agg, Join, and Window plans, including the distinct ordinary IndexJoin, IndexHashJoin, and IndexMergeJoin key representations. For Sort/TopN, test that a root scalar expression backed by inline Projection is evaluated only at Projection, ordinary-column/multi-key plans still receive exactly one aggregate sorting term, and a TiKV pushed TopN with an unmaterialized scalar `ByItems` fails `missing_ordering_projection`. Test checked `offset+count`, a nonzero offset, saturation when `k` exceeds actual input rows, a legal bound near `MaxUint64`, positive-count overflow rejection, zero-count fast-path work, and zero/one-row boundaries. At this milestone, custom unit fixtures in internal `package core` tests use the private calibrated weight container to prove exact algebra and invalid-number rejection before runtime constructors change. + +Acceptance: formula tests with injected weights reproduce hand-calculated totals; no production RUv2 API or configuration changes. + +### Milestone 2: construct units from authoritative details + +Refactor root and cop constructors in `pkg/planner/core/explain_ru.go` around the field mappings in this document. Preserve current exact child-plan attribution and task coverage code where it satisfies the new rows/scan rules. Add statement-scope reader transport from the frozen `RUV2Metrics` snapshot only after proving the closed producer set and the presence/task gates above; support multiple IndexMerge scan components without allocating transport twice. + +Use the existing flat-plan build/probe/left/right labels for join rows and the join node's own rows for output. Do not expose or add an IndexJoin lookup-task counter: its dynamic inner readers are already included by the statement-scope reader transport unit. + +Add `HashTableRuntimeStats` in `pkg/util/execdetails/runtime_stats.go`. Implement it for both HashJoin runtime-stat versions in `pkg/executor/join/hash_join_stats.go`, recording successful v1/v2 state admissions at the existing build completion points in `hash_join_v1.go` and `hash_join_v2.go`. Do not add a unique-key collector. The later ScanDetail ownership correction may add only an attachment-presence counter and consistent coverage snapshot to `RuntimeStatsColl`; that metadata does not enter a formula and must not alter distsql ownership. + +Change write construction in `pkg/planner/core/explain_ru.go` to derive mutation `cpu_work` only from a valid calibrated normalization/weight snapshot and use the current statement's finalized write RPC count for both DML and COMMIT. Do not carry request counts across statements or defer explicit-transaction DML requests to COMMIT. Retain raw mutation units as diagnostics even when the derived unit is unavailable. + +Acceptance: every required formula input can be traced to the source table below; source searches show exactly one new Join-only state-row counter and no other runtime field. + +### Milestone 3: output migration and behavioral coverage + +Bump the constants and renderer in `pkg/planner/core/explain_ru.go`, including `buildReadBillingDemoStatementStats`, `summarizeReadBillingDemoBaseUnits`, the EXPLAIN row builders, and `recordReadBillingDemoMetrics`. Use `pkg/metrics/explain_ru.go::{RecordReadBillingDemoStatement, RecordReadBillingDemoOperatorStatus, AddReadBillingDemoBaseUnits, ObserveExplainRURow}` for the bounded v4 labels; their public signatures need change only if a required existing provenance dimension is absent. + +Update aggregation keys and entry conversion in `pkg/util/stmtsummary/read_billing.go`, statement accumulation plus the legacy-v3-only `ReadBillingDemoBaseUnitSummary` behavior in `pkg/util/stmtsummary/statement_summary.go`, and verify `pkg/util/stmtsummary/v2/record.go` persistence/merge plus `pkg/util/stmtsummary/v2/reader.go::{readBillingDemoRowsFromRecord, readBillingDemoBaseUnitColumnValue}` preserve all v4 detail dimensions. Do not add new convenience columns. In `pkg/infoschema/tables.go`, keep the three legacy columns but change their comments to state v3-only/zero-for-v4 semantics; retain the versioned detail table schema unless the existing columns cannot carry one of the frozen dimensions. + +In `pkg/executor/adapter.go`, extend `readBillingDemoGeneralLogUnit`, `buildReadBillingDemoGeneralLogUnits`, and `readBillingDemoGeneralLogUnit.MarshalLogObject` so `DMLKind`, `InputSource`, and `InputSide` participate in aggregation, sorting, and serialization. Apply the exact model/weight versions and output semantics atomically across EXPLAIN, metrics, statement summary, and general log. Update `docs/design/2026-07-01-read-billing-demo-ru-model.md` and `docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md` to point to this v4 contract rather than retaining contradictory current-model claims. + +Extend the existing internal suites instead of creating a new planner casetest category. Keep formula and constructor tests near `pkg/planner/core/common_plans_test.go`; extend `pkg/executor/explain_test.go` for end-to-end RU output and de-duplication; keep transaction lifecycle tests in `pkg/session/tidb_test.go`. Update corresponding `.agents/skills/tidb-test-guidelines/references/*-case-map.md` files when test files change. + +Acceptance: internal formula tests observe exact calibrated totals. EXPLAIN, metrics hooks, statement summary, and log tests observe identical v4 coefficient-free units and the production-default `uncalibrated_weights`/absent-total state; unsupported and missing-evidence cases also have no weighted total, for their specific bounded reasons. + +## Authoritative field map + +| Formula input | Source today | Attribution and validation | New runtime data? | +|---|---|---|---| +| root `rows` | `RuntimeStatsColl` direct child `BasicRuntimeStats.GetActRows` | exact child plan ID must exist | no | +| cop `rows` | direct child `CopRuntimeStats.GetActRows/GetTasks` | exact plan ID and coverage checks | no | +| `n_expr` | concrete `physicalop` plan fields | centralized type switch, structural validation | no; immutable plan metadata | +| `scan_bytes` | `CopRuntimeStats.GetScanDetail` plus TiDB attachment provenance | unique holder by non-nil `RecordCopStats` attachment; scan summaries and holder attachments each cover all expected responses | **yes, TiDB-only attachment count; no protocol field or formula term** | +| `net_bytes` | `RUV2Metrics.TiKVCoprocessorResponseBytes` | once per statement; non-bypassed presence, descendant task gate, closed read producer set | no | +| reader `read_request_count` | `RUV2Metrics.ResourceManagerReadCnt` | once per statement only when every read-RPC producer is attributable to supported cop readers | no | +| point-lookup `read_request_count` | read-only: `RUV2Metrics.ResourceManagerReadCnt`; DML: plan-local `SnapshotRuntimeStats.GetCmdRPCCount` | once per statement; read-only requires a closed producer set, DML sums exact `CmdGet`/`CmdBatchGet` counts across PointGet/BatchPointGet plan IDs while the statement producer set stays open | no | +| HashAgg `group_rows` | Agg node own runtime rows | TiKV additionally needs expected/observed coverage | no | +| HashJoin `hash_state_rows` | v1 hash-table `Len` plus NAAJ null-bucket entries; v2 row-table `validKeyCount` | completed build round, cumulative across rebuilds | **yes, Join only** | +| Join `output_rows` | Join node own `BasicRuntimeStats.GetActRows` | executed root stat required | no | +| `mutation_count/bytes` | `StatementContext` preview mutation recorder | current attempted-call semantics | no | +| `write_request_count` | current statement's `RUV2Metrics.ResourceManagerWriteCnt` | finalized, present, non-bypassed snapshot; each DML and COMMIT owns only its own count | no | + +## Concrete Steps + +The design loop must first commit this file and hand the implementation loop the exact commit hash as ``. From the original repository root, create the required independent branch/worktree with these commands; the committed plan arrives through Git, so no untracked file copy is permitted: + + preview_ru_design_commit= + preview_ru_impl_worktree=/DATA/disk4/yiding/gocode/tidb.worktrees/preview-ru-v4-impl + git cat-file -e "${preview_ru_design_commit}^{commit}" + git worktree add -b preview-ru-v4-impl "$preview_ru_impl_worktree" "$preview_ru_design_commit" + cd "$preview_ru_impl_worktree" + test "$(git rev-parse HEAD)" = "$preview_ru_design_commit" + test -f docs/design/2026-07-22-preview-ru-resource-formula-plan.md + pwd + git branch --show-current + git status --short + +The final `git status --short` must be empty before implementation begins, `pwd` must be `/DATA/disk4/yiding/gocode/tidb.worktrees/preview-ru-v4-impl`, and `git branch --show-current` must print `preview-ru-v4-impl`. If an external orchestration loop creates the worktree, these three facts plus `` are mandatory handoff evidence; the implementation loop must stop rather than reuse the design-loop worktree or guess another base revision. + +Then inspect local changes and apply the Bazel preparation gate: + + git status --short + git diff --name-status + git diff -U0 -- '*.go' + +Run `make bazel_prepare` if the actual diff changes a Go import section, adds/moves/removes a Go file, adds a top-level `func TestXxx(t *testing.T)`, changes Bazel targets, or hits another trigger in `AGENTS.md`. The implementation should normally extend existing tests, but it must use the actual diff rather than assume the gate result. If run, review generated `BUILD.bazel`/`.bzl` changes and include only those caused by the implementation. + +Implement milestones in order. During WIP, run the smallest targeted tests. The affected packages use failpoints, so use the cleanup-safe wrapper rather than raw `go test` where the package scan finds failpoint use: + + ./tools/check/failpoint-go-test.sh pkg/planner/core -run 'TestExplainRU(PlanFormulaAndOperatorClasses|ComponentSnapshotStatusAndWeights)|TestReadBillingDemo' + ./tools/check/failpoint-go-test.sh pkg/executor -run 'TestExplainAnalyzeFormatRU|TestReadBillingDemoMetricsHook|TestReadBillingDemoGeneralLogUnits|TestWriteSlowLog' + ./tools/check/failpoint-go-test.sh pkg/executor/join -run 'Test.*HashJoin.*RuntimeStats' + ./tools/check/failpoint-go-test.sh pkg/session -run 'TestPreviewKVMutationRecorder|TestRUV2Metrics(IsolatedPerStatementInExplicitTxn|WriteRequestsInPessimisticTxn)' + ./tools/check/failpoint-go-test.sh pkg/util/stmtsummary -run 'TestReadBillingDemo(BaseUnitsToDatum|StructuredRowsToDatum|AggregationCaps|DMLKindAggregation|ReservedStatusMergeBypassesStatusCap)' + ./tools/check/failpoint-go-test.sh pkg/util/stmtsummary/v2 -run 'TestStmtRecordReadBillingDemoStructuredStats|TestReadBillingDemo(MemReader|HistoryReader)' + go test -tags=intest,deadlock ./pkg/metrics -run 'TestExplainRUMetrics|TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues' + +The metrics package currently has no failpoint use, hence the raw targeted command above. If a package scan changes at implementation time, follow `docs/agents/testing-flow.md`, switch to the matching wrapper/raw form, and record that evidence. + +At Ready, run the minimum targeted set again after any required `make bazel_prepare`, then run: + + make lint + +Do not run `make bazel_lint_changed`. Formula unit tests do not require a live TiKV cluster, but the ScanDetail ownership correction must complete one scoped real-TiKV Ready verification because the defect was observed only with the complete real response tuple and native EmbedUnistore cannot supply that evidence. + +## Validation and Acceptance + +The implementation is accepted only when all of the following are observable. + +With private test weights set to simple values inside `pkg/planner/core`, table-driven formula tests show exact results for every operator row in the formula table, including zero rows, one row, multiple expressions, multi-key joins, all three IndexJoin-family key representations, V1 NAAJ null-bucket state, and Window frame expressions. Sort uses `log2(max(rows,2))`; positive-count TopN uses `log2(max(min(rows,offset+count),2))` with checked addition, while zero-count TopN emits zero work. Cases cover nonzero offset, `k>rows`, a legal bound near `MaxUint64`, and overflow rejection. Neither ordering operator has an expression/key-count multiplier, and unmaterialized scalar ordering expressions fail closed rather than being charged there. + +End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, RPC-only PointGet/BatchPointGet, UnionScan, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. An explicit pessimistic transaction case must prove that a DML-local nonzero `ResourceManagerWriteCnt` is emitted by that DML, the later COMMIT emits only its own fresh-snapshot count, and neither count is lost or duplicated. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. + +A multi-reader or IndexMerge case proves that statement `net_bytes` and `read_request_count` appear once, while every scan retains its own `scan_bytes`. PointGet and BatchPointGet cases prove that a pure read-only plan publishes the statement `read_request_count` exactly once under the bounded lookup kind, publishes no inferred CPU/scan/network unit, accepts a present zero counter, and rejects locking reads, bypassed/missing detail, and mixed point/cop-reader producer sets. A DML PointGet case proves that plan-local `CmdGet`/`CmdBatchGet` counts publish independently of a missing or larger statement counter, while missing exec details stays unknown. A UnionScan case proves that `tidb/overlay_reader/unionscan` publishes `cpu_work` equal to its direct child's exact actual rows, carries `runtime_child_act_rows`, and has no expression-count multiplier. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. DML and COMMIT cases prove that `write_request_count` remains statement-local and uses a coefficient independent from read requests. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. + +Missing root stats, incomplete cop summaries, ambiguous scan details, missing reader transport, invalid expression structure, invalid mutation normalization, negative inputs, overflow, NaN, and infinity all fail closed with bounded reasons. SELECT produces no partial billable total. DML preserves complete independent units but does not claim a complete statement total. + +Search and API review prove that runtime additions are limited to HashJoin state rows and the TiDB-only ScanDetail attachment count. The latter is provenance only and never enters a formula. `config.RUV2`, TiKV request charging, `ReportRUV2Consumption`, and resource-control behavior are unchanged. + +Statement-summary detail, Prometheus metrics hooks, EXPLAIN rows, and general-log details are built from the same frozen result and agree on unit values. General-log records retain DML kind, input source, and input side. Historical v3 rows remain distinguishable by version; legacy three-column convenience totals keep v3 behavior and remain zero for v4, whose memory/history-reader details remain queryable. + +## Idempotence and Recovery + +All formula construction is read-only over frozen plan/runtime snapshots and must be safe to call repeatedly. Unit construction must not drain `RUDetails`; use the already synchronized/frozen `RUV2Metrics` snapshot exactly as the current preview path does. + +`make bazel_prepare`, formatting, and targeted tests are safe to rerun. The failpoint test wrapper always disables failpoints during cleanup. If a milestone leaves mixed output versions, revert only that milestone's focused changes or finish all output consumers before running behavioral tests; never commit a mixed v3/v4 renderer state. + +If the runtime source cannot prove a required input, add a bounded status reason and keep the formula unavailable. Do not recover by parsing runtime strings or by introducing an estimate not recorded in this plan. Any newly discovered formula datum still requires revisiting this design. The ScanDetail attachment count is a provenance exception recorded here after real-TiKV evidence showed that a zero value alone cannot distinguish missing detail from an observed empty scan. + +## Artifacts and Notes + +Current evidence commands used while drafting this plan included: + + rg -n 'type (BasicRuntimeStats|CopRuntimeStats|RuntimeStatsColl)' pkg/util/execdetails/runtime_stats.go + rg -n 'TiKVCoprocessorResponseBytes|ResourceManagerReadCnt|ResourceManagerWriteCnt|Bypass' pkg/util/execdetails/ruv2_metrics.go + rg -n 'innerWorker.task|type indexLookUpJoinRuntimeStats' pkg/executor/join + rg -n 'type Physical(Selection|Projection|TopN|Sort|HashAgg|StreamAgg|HashJoin|MergeJoin|IndexJoin|Window|UnionScan)' pkg/planner/core/operator/physicalop + +The evidence establishes availability and location, not completion of the later implementation. + +The final ScanDetail-ownership evidence additionally includes a cleanup-safe real-TiKV run with one current-worktree TiDB, one PD, and one TiKV. It observed `cpu_work=4` for `site=tikv, op_class=filter_eval, operator_kind=selection` and `scan_bytes=212.5` for the corresponding table scan in General Log, Prometheus, and statement summary. The shipped `v3-resource-formula-uncalibrated` version left total preview RU absent as required. + +The point-lookup evidence additionally includes a cleanup-safe real-TiKV run using the current-worktree TiDB binary and TiKV isolation. `SELECT ... WHERE primary_key = 1` emitted `id=point_lookup@statement, site=tikv, op_class=kv_point_lookup, operator_kind=point_get, read_request_count=1`; the two-key `IN` form emitted the same dimensions with `operator_kind=batch_point_get` and `read_request_count=1`. Neither row contained CPU, scan, or network work, and the uncalibrated summary contained no total preview RU. The exact TiUP tag and data directory were removed and all dedicated ports were closed afterward. + +The UnionScan evidence includes a cleanup-safe real-TiKV transaction over three committed rows plus one uncommitted inserted row. Ordinary `EXPLAIN ANALYZE` reported `UnionScan actRows=4` and its direct `TableReader actRows=3`; `FORMAT='RU'` emitted `site=tidb, op_class=overlay_reader, operator_kind=unionscan, input_rows=3` and `cpu_work=3`, both sourced from `runtime_child_act_rows`, with no expression-count unit and no uncalibrated total. The exact `preview-ru-unionscan-v4` TiUP tag and data directory were removed and ports 25000, 23379, 41160, and 24930 were closed afterward. + +## Interfaces and Dependencies + +Keep all v4 model types private to `pkg/planner/core` except the narrow `execdetails.HashTableRuntimeStats` read interface and the `RuntimeStatsColl` coverage-snapshot method needed by planner/core. Do not export the preview weight container, concrete executor-private Join stats, or add public session/global variables. + +The implementation depends only on existing TiDB packages: `physicalop` for immutable plan expressions, `execdetails` for runtime rows/scan/RUv2 details, the statement mutation recorder for write inputs, and existing statement-summary/metrics/log renderers. It adds no third-party dependency and no protocol field. + +At milestone completion, the key internal interfaces should have these conceptual signatures: + + func previewRUExpressionCount(plan base.Plan) (int64, bool) + func previewRUFormulaUnits(plan base.Plan, details previewRUDetails) ([]previewRUUnit, previewRUStatus) + func previewRUForUnit(unit previewRUUnit, weights previewRUWeights) (weight, ru float64, ok bool) + type HashTableRuntimeStats interface { + RuntimeStats + HashTableRows() int64 + } + +The exact private names may follow nearby conventions, but the semantics, data sources, one-Join-runtime-datum boundary, and de-duplication rules in this plan are mandatory. + +Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, positive-count TopN owns `n*log(min(n,k))` with checked `k=offset+count`, zero-count TopN owns zero work, and inner readers own IndexJoin request cost. HashJoin exposes actual admitted hash-state rows instead of approximating them with all build rows. A later fresh-context audit corrected write-request ownership: explicit-transaction DML and COMMIT each charge only the write RPCs in their own finalized statement snapshot, preventing pessimistic DML requests from being lost. + +Revision note (2026-07-23): real TiKV proved that non-Scan execution summaries were present but all units were suppressed by false ScanDetail ambiguity. The plan now records non-nil ScanDetail attachment counts under the original plan ID, validates scan-summary and attachment coverage independently, preserves true empty scans, and requires a scoped real-TiKV Ready verification. This is a minimal evidence/provenance revision; formulas, weights, protocol, distsql ownership, output dimensions, and production RUv2 remain unchanged. + +Revision note (2026-07-24): PointGet and BatchPointGet now consume only the existing statement-scoped TiKV read-RPC counter. The plan adds one synthetic point-lookup publisher with a closed producer set and retains fail-closed handling wherever the counter may include locking, DML, cop-reader, MPP, or unknown work. No CPU or byte estimate, runtime field, protocol change, or new weight-bearing unit was introduced. + +Revision note (2026-07-24): UnionScan now reuses the existing `cpu_work` semantic unit with `input_source=runtime_child_act_rows` and value equal to the direct child's actual rows. This intentionally simple first formula does not multiply by UnionScan conditions and does not estimate its transaction mem-buffer input. + +Revision note (2026-07-24): DML PointGet and BatchPointGet no longer fail solely because they execute under DML. Their existing plan-local snapshot runtime stats provide structural `CmdGet`/`CmdBatchGet` counts, while the DML statement producer set remains open and its RUv2 read counter is not assigned to PointGet. This lets the lookup report its own requests without absorbing pessimistic-lock writes or unrelated ancillary reads. diff --git a/pkg/distsql/BUILD.bazel b/pkg/distsql/BUILD.bazel index 05416ed307778..2252000f73550 100644 --- a/pkg/distsql/BUILD.bazel +++ b/pkg/distsql/BUILD.bazel @@ -47,6 +47,7 @@ go_library( "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//metrics", "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//tikvrpc/interceptor", "@com_github_tikv_client_go_v2//util", "@org_golang_google_grpc//metadata", @@ -101,6 +102,7 @@ go_test( "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//util", "@org_uber_go_goleak//:goleak", ], ) diff --git a/pkg/distsql/select_result.go b/pkg/distsql/select_result.go index fe53353333744..15dc58dff13b0 100644 --- a/pkg/distsql/select_result.go +++ b/pkg/distsql/select_result.go @@ -47,6 +47,7 @@ import ( "github.com/pingcap/tipb/go-tipb" tikvmetrics "github.com/tikv/client-go/v2/metrics" "github.com/tikv/client-go/v2/tikv" + "github.com/tikv/client-go/v2/tikvrpc" clientutil "github.com/tikv/client-go/v2/util" "go.uber.org/zap" ) @@ -658,6 +659,21 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr } r.stats.mergeCopRuntimeStats(copStats, respTime) + if copStats.TimeDetail.ProcessTime > 0 { + r.ctx.CPUUsage.MergeTikvCPUTime(copStats.TimeDetail.ProcessTime) + } + if r.storeType == kv.TiKV { + // Record the independent expectation before validating individual + // summaries so a response with all summaries missing is still visible. + r.ctx.RuntimeStatsColl.RecordExpectedCopTasks(r.copPlanIDs) + } + if forUnconsumedStats { + // Close-time runtime stats do not have a newly unmarshaled SelectResponse. + // Keep the generic RPC/scan/time details above, but never replay summaries + // from the last consumed response. + return + } + // If hasExecutor is true, it means the summary is returned from TiFlash. hasExecutor := false for _, detail := range r.selectResp.GetExecutionSummaries() { @@ -680,9 +696,6 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr } } } - if copStats.TimeDetail.ProcessTime > 0 { - r.ctx.CPUUsage.MergeTikvCPUTime(copStats.TimeDetail.ProcessTime) - } if hasExecutor { if len(r.copPlanIDs) > 0 { r.ctx.RuntimeStatsColl.RecordCopStats(r.copPlanIDs[len(r.copPlanIDs)-1], r.storeType, copStats.ScanDetail, copStats.TimeDetail, nil) @@ -704,7 +717,7 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr // for TiFlash streaming call(BatchCop and MPP), it is by design that only the last response will // carry the execution summaries, so it is ok if some responses have no execution summaries, should // not trigger an error log in this case. - if !forUnconsumedStats && !(r.storeType == kv.TiFlash && len(r.selectResp.GetExecutionSummaries()) == 0) { + if !(r.storeType == kv.TiFlash && len(r.selectResp.GetExecutionSummaries()) == 0) { logutil.Logger(ctx).Warn("invalid cop task execution summaries length", zap.Int("expected", len(r.copPlanIDs)), zap.Int("received", len(r.selectResp.GetExecutionSummaries()))) @@ -764,16 +777,10 @@ func (r *selectResult) close() error { r.memConsume(-respSize) } if r.ctx != nil { - if unconsumed, ok := r.resp.(copr.HasUnconsumedCopRuntimeStats); ok && unconsumed != nil { - unconsumedCopStats := unconsumed.CollectUnconsumedCopRuntimeStats() - for _, copStats := range unconsumedCopStats { - _ = r.updateCopRuntimeStats(context.Background(), copStats, time.Duration(0), true) - r.ctx.ExecDetails.MergeCopExecDetails(&copStats.CopExecDetails, 0) - } - } - } - if r.stats != nil && r.ctx != nil { defer func() { + if r.stats == nil { + return + } if ci, ok := r.resp.(copr.CopInfo); ok { r.stats.buildTaskDuration = ci.GetBuildTaskElapsed() batched, fallback := ci.GetStoreBatchInfo() @@ -788,7 +795,17 @@ func (r *selectResult) close() error { r.ctx.RuntimeStatsColl.RegisterStats(r.rootPlanID, r.stats) }() } - return r.resp.Close() + closeErr := r.resp.Close() + if r.ctx != nil { + if unconsumed, ok := r.resp.(copr.HasUnconsumedCopRuntimeStats); ok && unconsumed != nil { + unconsumedCopStats := unconsumed.CollectUnconsumedCopRuntimeStats() + for _, copStats := range unconsumedCopStats { + _ = r.updateCopRuntimeStats(context.Background(), copStats, time.Duration(0), true) + r.ctx.ExecDetails.MergeCopExecDetails(&copStats.CopExecDetails, 0) + } + } + } + return closeErr } type selRespChannelIter struct { @@ -1035,6 +1052,14 @@ func (s *selectResultRuntimeStats) mergeCopRuntimeStats(copStats *copr.CopRuntim } } +// GetCmdRPCCount returns the number of RPCs issued by this SelectResult for cmd. +func (s *selectResultRuntimeStats) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 { + if s == nil || s.reqStat == nil { + return 0 + } + return int64(s.reqStat.GetCmdRPCCount(cmd)) +} + func (s *selectResultRuntimeStats) Clone() execdetails.RuntimeStats { newRs := selectResultRuntimeStats{ copRespTime: execdetails.Percentile[execdetails.Duration]{}, diff --git a/pkg/distsql/select_result_test.go b/pkg/distsql/select_result_test.go index e08a4ecea5061..a1184501304ae 100644 --- a/pkg/distsql/select_result_test.go +++ b/pkg/distsql/select_result_test.go @@ -32,8 +32,37 @@ import ( "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikv" + "github.com/tikv/client-go/v2/tikvrpc" + tikvutil "github.com/tikv/client-go/v2/util" ) +type closeAppendingRuntimeStatsResponse struct { + stats []*copr.CopRuntimeStats + statsOnClose *copr.CopRuntimeStats + closeErr error + closed bool + collectCalls int +} + +func (*closeAppendingRuntimeStatsResponse) Next(context.Context) (kv.ResultSubset, error) { + return nil, nil +} + +func (r *closeAppendingRuntimeStatsResponse) Close() error { + r.closed = true + r.stats = append(r.stats, r.statsOnClose) + return r.closeErr +} + +func (r *closeAppendingRuntimeStatsResponse) CollectUnconsumedCopRuntimeStats() []*copr.CopRuntimeStats { + r.collectCalls++ + if !r.closed { + return nil + } + return r.stats +} + func TestUpdateCopRuntimeStats(t *testing.T) { ctx := mock.NewContext() ctx.GetSessionVars().StmtCtx = stmtctx.NewStmtCtx() @@ -72,6 +101,155 @@ func TestUpdateCopRuntimeStats(t *testing.T) { sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", BackoffSleep: backOffSleep}}, 0, false) require.Equal(t, "tikv_task:{time:1ns, loops:1}", ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(1234).String()) require.Equal(t, sr.stats.backoffSleep["RegionMiss"], time.Duration(500)) + + newSummary := func(rows uint64) *tipb.ExecutorExecutionSummary { + one := uint64(1) + return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &one, NumProducedRows: &rows, NumIterations: &one} + } + update := func(scanRows, parentRows uint64, detail *tikvutil.ScanDetail) { + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{ + newSummary(scanRows), + newSummary(parentRows), + }} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", ScanDetail: detail}, + }, 0, false)) + } + + const scanPlanID = 2001 + const parentPlanID = 2002 + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + sr.copPlanIDs = []int{scanPlanID, parentPlanID} + update(4, 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + scanStats := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + require.Equal(t, int64(4), scanStats.GetActRows()) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Zero(t, scanStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int64(2), parentStats.GetActRows()) + require.Equal(t, int32(1), parentStats.GetTasks()) + require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int64(100), parentStats.GetScanDetail().ProcessedKeysSize) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, scanDetailRecords, scanObservedTasks, scanExpectedTasks := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(scanPlanID) + _, parentDetailRecords, parentObservedTasks, parentExpectedTasks := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Zero(t, scanDetailRecords) + require.Equal(t, int32(1), scanObservedTasks) + require.Equal(t, int32(1), scanExpectedTasks) + require.Equal(t, int32(1), parentDetailRecords) + require.Equal(t, int32(1), parentObservedTasks) + require.Equal(t, int32(1), parentExpectedTasks) + + update(3, 1, &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}) + require.Equal(t, int64(7), scanStats.GetActRows()) + require.Equal(t, int32(2), scanStats.GetTasks()) + require.Equal(t, int64(3), parentStats.GetActRows()) + require.Equal(t, int32(2), parentStats.GetTasks()) + require.Equal(t, int64(7), parentStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int64(140), parentStats.GetScanDetail().ProcessedKeysSize) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, scanDetailRecords, scanObservedTasks, scanExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(scanPlanID) + _, parentDetailRecords, parentObservedTasks, parentExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Zero(t, scanDetailRecords) + require.Equal(t, int32(2), scanObservedTasks) + require.Equal(t, int32(2), scanExpectedTasks) + require.Equal(t, int32(2), parentDetailRecords) + require.Equal(t, int32(2), parentObservedTasks) + require.Equal(t, int32(2), parentExpectedTasks) + + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{newSummary(4), nil}} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", ScanDetail: &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}}, + }, 0, false)) + scanStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int32(0), parentStats.GetTasks()) + require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, parentDetailRecords, parentObservedTasks, parentExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Equal(t, int32(1), parentDetailRecords) + require.Zero(t, parentObservedTasks) + require.Equal(t, int32(1), parentExpectedTasks) + + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + update(4, 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{nil, newSummary(1)}} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", ScanDetail: &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}}, + }, 0, false)) + scanStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int32(2), parentStats.GetTasks()) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{nil, nil}} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee"}, + }, 0, false)) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int32(2), parentStats.GetTasks()) + require.Equal(t, int32(3), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(3), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, parentDetailRecords, _, parentExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Equal(t, int32(2), parentDetailRecords) + require.Equal(t, int32(3), parentExpectedTasks) + + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + sr.stats = &selectResultRuntimeStats{} + update(4, 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + scanStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + reqStats := tikv.NewRegionRequestRuntimeStats() + reqStats.RecordRPCRuntimeStats(tikvrpc.CmdCop, time.Millisecond) + reqStats.RecordRPCRuntimeStats(tikvrpc.CmdCopStream, time.Millisecond) + closeErr := fmt.Errorf("close failed") + resp := &closeAppendingRuntimeStatsResponse{ + statsOnClose: &copr.CopRuntimeStats{ReqStats: reqStats}, + closeErr: closeErr, + } + sr.resp = resp + require.Equal(t, closeErr, sr.close()) + require.True(t, resp.closed) + require.Equal(t, 1, resp.collectCalls) + require.Equal(t, uint32(1), sr.stats.reqStat.GetCmdRPCCount(tikvrpc.CmdCop)) + require.Equal(t, int64(1), sr.stats.GetCmdRPCCount(tikvrpc.CmdCop)) + require.Equal(t, int64(1), sr.stats.GetCmdRPCCount(tikvrpc.CmdCopStream)) + require.Equal(t, int64(4), scanStats.GetActRows()) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int64(2), parentStats.GetActRows()) + require.Equal(t, int32(1), parentStats.GetTasks()) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + + onlyReqStats := tikv.NewRegionRequestRuntimeStats() + onlyReqStats.RecordRPCRuntimeStats(tikvrpc.CmdCop, time.Millisecond) + onlyResp := &closeAppendingRuntimeStatsResponse{ + statsOnClose: &copr.CopRuntimeStats{ReqStats: onlyReqStats}, + } + const onlyUnconsumedRootID = 3001 + const onlyUnconsumedCopID = 3002 + onlyUnconsumed := selectResult{ + ctx: ctx.GetDistSQLCtx(), + rootPlanID: onlyUnconsumedRootID, + copPlanIDs: []int{onlyUnconsumedCopID}, + storeType: kv.TiKV, + resp: onlyResp, + } + require.NoError(t, onlyUnconsumed.close()) + require.NotNil(t, onlyUnconsumed.stats) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(onlyUnconsumedCopID)) + require.True(t, ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.ExistsRootStats(onlyUnconsumedRootID)) } func TestNewSelRespChannelIter(t *testing.T) { diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel index 9b26968e75bc8..ac1c2856843c5 100644 --- a/pkg/executor/BUILD.bazel +++ b/pkg/executor/BUILD.bazel @@ -528,6 +528,7 @@ go_test( "//pkg/util/set", "//pkg/util/sqlexec", "//pkg/util/sqlkiller", + "//pkg/util/stmtsummary", "//pkg/util/stmtsummary/v2:stmtsummary", "//pkg/util/syncutil", "//pkg/util/tableutil", diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index b3a826f033e9a..245a49297735f 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -19,6 +19,7 @@ import ( "fmt" "math" "runtime/trace" + "slices" "strconv" "strings" "sync" @@ -283,14 +284,33 @@ func (a *recordSet) Finish() error { } func (a *recordSet) Close() error { + return a.close(nil) +} + +func (a *recordSet) close(lastErr error) error { err := a.Finish() if err != nil { logutil.BgLogger().Error("close recordSet error", zap.Error(err)) } - a.stmt.CloseRecordSet(a.txnStartTS, errors.Join(a.lastErrs...)) + a.stmt.CloseRecordSet(a.txnStartTS, errors.Join(errors.Join(a.lastErrs...), lastErr)) return err } +// CloseRecordSetWithError closes an executor-owned record set and forwards an +// execution error to its single statement-completion path. +func CloseRecordSetWithError(rs sqlexec.RecordSet, lastErr error) error { + switch recordSet := rs.(type) { + case *recordSet: + return recordSet.close(lastErr) + case *chunkRowRecordSet: + recordSet.execStmt.CloseRecordSet(recordSet.execStmt.Ctx.GetSessionVars().TxnCtx.StartTS, lastErr) + return nil + default: + closeErr := rs.Close() + return errors.Join(closeErr, errors.New("record set does not support closing with an execution error")) + } +} + // OnFetchReturned implements commandLifeCycle#OnFetchReturned func (a *recordSet) OnFetchReturned() { a.stmt.LogSlowQuery(a.txnStartTS, len(a.lastErrs) == 0, true) @@ -298,6 +318,11 @@ func (a *recordSet) OnFetchReturned() { // TryDetach creates a new `RecordSet` which doesn't depend on the current session context. func (a *recordSet) TryDetach() (sqlexec.RecordSet, bool, error) { + // A detached static record set no longer reaches ExecStmt completion. Keep + // the live record set only when the preview RU completion event is enabled. + if a.stmt != nil && readBillingDemoGeneralLogEnabled(a.stmt.Ctx) { + return nil, false, nil + } e, ok := Detach(a.executor) if !ok { return nil, false, nil @@ -366,11 +391,13 @@ type ExecStmt struct { Ctx sessionctx.Context // LowerPriority represents whether to lower the execution priority of a query. - LowerPriority bool - isPreparedStmt bool - isSelectForUpdate bool - retryCount uint - retryStartTime time.Time + LowerPriority bool + isPreparedStmt bool + isSelectForUpdate bool + resolvedPreparedStmt ast.StmtNode + retryCount uint + retryStartTime time.Time + readBillingDemoGeneralLogEmitted atomic.Bool // Phase durations are splited into two parts: 1. trying to lock keys (but // failed); 2. the final iteration of the retry loop. Here we use @@ -770,11 +797,19 @@ func (a *ExecStmt) inheritContextFromExecuteStmt() { a.Ctx.SetValue(sessionctx.QueryString, executePlan.Stmt.Text()) a.OutputNames = executePlan.OutputNames() a.isPreparedStmt = true + a.resolvedPreparedStmt = executePlan.Stmt a.Plan = executePlan.Plan a.Ctx.GetSessionVars().StmtCtx.SetPlan(executePlan.Plan) } } +func (a *ExecStmt) readBillingDemoStmtNode() ast.StmtNode { + if a.resolvedPreparedStmt != nil { + return a.resolvedPreparedStmt + } + return a.StmtNode +} + func (a *ExecStmt) getSQLForProcessInfo() string { sql := a.Text() if simple, ok := a.Plan.(*plannercore.Simple); ok && simple.Statement != nil { @@ -1689,10 +1724,15 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo } a.finalizeStatementRUV2Metrics() + // Lazy SELECT metrics are emitted when the record set is closed. If a + // client abandons a result without closing/draining it, this first demo can + // miss that statement instead of guessing an incomplete status. + readBillingDemoStats := plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) + a.logReadBillingDemoGeneralLog(readBillingDemoStats) a.updateNetworkTrafficStatsAndMetrics() // `LowSlowQuery` and `SummaryStmt` must be called before recording `PrevStmt`. a.LogSlowQuery(txnTS, succ, hasMoreResults) - a.SummaryStmt(succ) + a.SummaryStmt(succ, readBillingDemoStats) a.observeStmtFinishedForTopProfiling() a.UpdatePlanCacheRuntimeInfo() if sessVars.StmtCtx.IsTiFlash.Load() { @@ -1751,6 +1791,136 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo a.Ctx.ReportUsageStats() } +const readBillingDemoGeneralLogMessage = "GENERAL_LOG_RU_UNITS" + +type readBillingDemoGeneralLogUnit struct { + site string + opClass string + operatorKind string + dmlKind string + unit string + inputSource string + inputSide string + value float64 +} + +func (u readBillingDemoGeneralLogUnit) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("site", u.site) + enc.AddString("op_class", u.opClass) + enc.AddString("operator_kind", u.operatorKind) + enc.AddString("dml_kind", u.dmlKind) + enc.AddString("unit", u.unit) + enc.AddString("input_source", u.inputSource) + enc.AddString("input_side", u.inputSide) + enc.AddFloat64("value", u.value) + return nil +} + +type readBillingDemoGeneralLogUnits []readBillingDemoGeneralLogUnit + +func (units readBillingDemoGeneralLogUnits) MarshalLogArray(enc zapcore.ArrayEncoder) error { + for _, unit := range units { + if err := enc.AppendObject(unit); err != nil { + return err + } + } + return nil +} + +func buildReadBillingDemoGeneralLogUnits(stats stmtsummary.ReadBillingDemoStatementStats) readBillingDemoGeneralLogUnits { + totals := make(map[readBillingDemoGeneralLogUnit]float64, len(stats.BaseUnits)) + for _, sample := range stats.BaseUnits { + key := readBillingDemoGeneralLogUnit{ + site: sample.Site, + opClass: sample.OpClass, + operatorKind: sample.OperatorKind, + dmlKind: sample.DMLKind, + unit: sample.Unit, + inputSource: sample.InputSource, + inputSide: sample.InputSide, + } + totals[key] += sample.Value + } + + units := make(readBillingDemoGeneralLogUnits, 0, len(totals)) + for unit, value := range totals { + unit.value = value + units = append(units, unit) + } + slices.SortFunc(units, func(left, right readBillingDemoGeneralLogUnit) int { + if cmp := strings.Compare(left.site, right.site); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.opClass, right.opClass); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.operatorKind, right.operatorKind); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.dmlKind, right.dmlKind); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.unit, right.unit); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.inputSource, right.inputSource); cmp != 0 { + return cmp + } + return strings.Compare(left.inputSide, right.inputSide) + }) + return units +} + +func readBillingDemoGeneralLogEnabled(sctx sessionctx.Context) bool { + if sctx == nil || !vardef.ProcessGeneralLog.Load() { + return false + } + sessVars := sctx.GetSessionVars() + if sessVars == nil || !sessVars.EnableReadBillingDemo || sessVars.InRestrictedSQL || + sessVars.StmtCtx == nil || sessVars.StmtCtx.InRestrictedSQL { + return false + } + return true +} + +func canLogReadBillingDemoGeneralLog(sctx sessionctx.Context, stats stmtsummary.ReadBillingDemoStatementStats) bool { + return readBillingDemoGeneralLogEnabled(sctx) && stats.ModelVersion != "" && stats.WeightVersion != "" +} + +// LogReadBillingDemoGeneralLog emits a preview RU completion event for paths +// that finish before an ExecStmt reaches FinishExecuteStmt. +func LogReadBillingDemoGeneralLog(sctx sessionctx.Context, stats stmtsummary.ReadBillingDemoStatementStats) { + if !canLogReadBillingDemoGeneralLog(sctx, stats) { + return + } + emitReadBillingDemoGeneralLog(sctx, stats) +} + +func emitReadBillingDemoGeneralLog(sctx sessionctx.Context, stats stmtsummary.ReadBillingDemoStatementStats) { + sessVars := sctx.GetSessionVars() + normalizedSQL, digest := sessVars.StmtCtx.SQLDigest() + digestString := "" + if digest != nil { + digestString = digest.String() + } + logutil.GeneralLogger.Info(readBillingDemoGeneralLogMessage, + zap.Uint64("conn", sessVars.ConnectionID), + zap.String("model_version", stats.ModelVersion), + zap.String("weight_version", stats.WeightVersion), + zap.String("normalized_sql", normalizedSQL), + zap.String("sql_digest", digestString), + zap.Array("units", buildReadBillingDemoGeneralLogUnits(stats)), + ) +} + +func (a *ExecStmt) logReadBillingDemoGeneralLog(stats stmtsummary.ReadBillingDemoStatementStats) { + if !canLogReadBillingDemoGeneralLog(a.Ctx, stats) || + !a.readBillingDemoGeneralLogEmitted.CompareAndSwap(false, true) { + return + } + emitReadBillingDemoGeneralLog(a.Ctx, stats) +} + func (a *ExecStmt) recordAffectedRows2Metrics() { sessVars := a.Ctx.GetSessionVars() if affectedRows := sessVars.StmtCtx.AffectedRows(); affectedRows > 0 { @@ -2179,7 +2349,7 @@ func (digest planDigestAlias) planDigestDumpTriggerCheck(config *traceevent.Dump } // SummaryStmt collects statements for information_schema.statements_summary -func (a *ExecStmt) SummaryStmt(succ bool) { +func (a *ExecStmt) SummaryStmt(succ bool, readBillingDemoStats stmtsummary.ReadBillingDemoStatementStats) { sessVars := a.Ctx.GetSessionVars() var userString string if sessVars.User != nil { @@ -2283,6 +2453,8 @@ func (a *ExecStmt) SummaryStmt(succ bool) { stmtExecInfo.KeyspaceID = keyspaceID stmtExecInfo.RUDetail = ruDetail stmtExecInfo.TotalRUV2 = calculateStatementTotalRUV2(sessVars.RUV2Metrics, sessVars.RUV2Weights(), ruDetail) + stmtExecInfo.ReadBillingDemoStats = readBillingDemoStats + stmtExecInfo.ReadBillingDemoBaseUnits = readBillingDemoStats.Totals stmtExecInfo.ResourceGroupName = sessVars.StmtCtx.ResourceGroupName stmtExecInfo.CPUUsages = sessVars.SQLCPUUsages.GetCPUUsages() stmtExecInfo.PlanCacheUnqualified = sessVars.StmtCtx.PlanCacheUnqualified() diff --git a/pkg/executor/adapter_internal_test.go b/pkg/executor/adapter_internal_test.go index 7aa2237ece5e4..07e2d68e39d0e 100644 --- a/pkg/executor/adapter_internal_test.go +++ b/pkg/executor/adapter_internal_test.go @@ -29,11 +29,14 @@ import ( "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/auth" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/pingcap/tidb/pkg/util/topsql" topsqlmock "github.com/pingcap/tidb/pkg/util/topsql/collector/mock" topsqlstate "github.com/pingcap/tidb/pkg/util/topsql/state" @@ -44,6 +47,8 @@ import ( metastorage "github.com/tikv/pd/client/clients/metastorage" "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) type stmtStatsTestContext struct { @@ -97,6 +102,159 @@ func resetTopProfilingStateForTest(t *testing.T) { }) } +func TestReadBillingDemoGeneralLogUnits(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + sctx := mock.NewContext() + sessVars := sctx.GetSessionVars() + sessVars.ConnectionID = 42 + sessVars.StmtCtx.OriginalSQL = "select 12345, 'secret_literal'" + expectedNormalizedSQL, expectedDigest := parser.NormalizeDigest(sessVars.StmtCtx.OriginalSQL) + stats := stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v4", + WeightVersion: "test-v4-calibrated", + BaseUnits: []stmtsummary.ReadBillingDemoBaseUnitSample{ + { + Site: "tikv", OpClass: "join_hash", OperatorKind: "hashjoin", Unit: "input_rows", Value: 2, + DMLKind: "insert", InputSource: "child", InputSide: "left", RowWidthSource: "scan", RowWidth: 4, + }, + { + Site: "tikv", OpClass: "join_hash", OperatorKind: "hashjoin", Unit: "input_rows", Value: 3, + DMLKind: "update", InputSource: "runtime", InputSide: "right", RowWidthSource: "schema", RowWidth: 8, + }, + {Site: "tidb", OpClass: "agg_hash", OperatorKind: "hashagg", Unit: "output_rows", Value: 1}, + {Site: "tidb", OpClass: "agg_hash", OperatorKind: "hashagg", Unit: "input_bytes", Value: 8}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "cpu_work", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 3.5}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "encoded_mutation_count", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 2}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "encoded_mutation_bytes", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 15}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "set_count", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 2}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "delete_count", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 0}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "key_bytes", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 5}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "value_bytes", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 10}, + {Site: "tidb", OpClass: "reader_transport", OperatorKind: "mixed_reader", Unit: "read_request_count", InputSource: "ruv2_metrics", InputSide: "all", Value: 4}, + {Site: "tikv", OpClass: "kv_write", OperatorKind: "txn_write", DMLKind: "update", Unit: "write_request_count", InputSource: "ruv2_metrics", InputSide: "all", Value: 2}, + }, + } + + // Neither individual gate is sufficient. + sessVars.EnableReadBillingDemo = true + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + vardef.ProcessGeneralLog.Store(true) + sessVars.EnableReadBillingDemo = false + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + + // Internal SQL is excluded whether marked at session or statement scope. + sessVars.EnableReadBillingDemo = true + sessVars.InRestrictedSQL = true + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + sessVars.InRestrictedSQL = false + sessVars.StmtCtx.InRestrictedSQL = true + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + sessVars.StmtCtx.InRestrictedSQL = false + + LogReadBillingDemoGeneralLog(sctx, stats) + entries := recorded.TakeAll() + require.Len(t, entries, 1) + require.Equal(t, readBillingDemoGeneralLogMessage, entries[0].Message) + fields := entries[0].ContextMap() + require.Len(t, fields, 6) + require.Equal(t, uint64(42), fields["conn"]) + require.Equal(t, "v4", fields["model_version"]) + require.Equal(t, "test-v4-calibrated", fields["weight_version"]) + require.Equal(t, expectedNormalizedSQL, fields["normalized_sql"]) + require.Equal(t, expectedDigest.String(), fields["sql_digest"]) + require.NotContains(t, fields["normalized_sql"], "12345") + require.NotContains(t, fields["normalized_sql"], "secret_literal") + require.NotContains(t, fields, "sql") + rawUnits, ok := fields["units"].([]any) + require.True(t, ok) + require.Len(t, rawUnits, 13) + + units := make([]map[string]any, 0, len(rawUnits)) + for _, rawUnit := range rawUnits { + unit, ok := rawUnit.(map[string]any) + require.True(t, ok) + require.Len(t, unit, 8) + for _, field := range []string{"site", "op_class", "operator_kind", "dml_kind", "unit", "input_source", "input_side", "value"} { + require.Contains(t, unit, field) + } + for _, internalField := range []string{"row_width_source", "row_width", "operator_id"} { + require.NotContains(t, unit, internalField) + } + units = append(units, unit) + } + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "dml_kind": "", "unit": "input_bytes", "input_source": "", "input_side": "", "value": float64(8), + }, units[0]) + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "dml_kind": "", "unit": "output_rows", "input_source": "", "input_side": "", "value": float64(1), + }, units[1]) + expectedMutationValues := map[string]float64{ + "cpu_work": 3.5, "delete_count": 0, "encoded_mutation_bytes": 15, "encoded_mutation_count": 2, + "key_bytes": 5, "set_count": 2, "value_bytes": 10, + } + for i, unitName := range []string{"cpu_work", "delete_count", "encoded_mutation_bytes", "encoded_mutation_count", "key_bytes", "set_count", "value_bytes"} { + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "kv_mutation", "operator_kind": "memdb_mutation", "dml_kind": "insert", + "unit": unitName, "input_source": "stmt_memdb_mutation_calls", "input_side": "all", "value": expectedMutationValues[unitName], + }, units[i+2]) + } + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "reader_transport", "operator_kind": "mixed_reader", "dml_kind": "", "unit": "read_request_count", "input_source": "ruv2_metrics", "input_side": "all", "value": float64(4), + }, units[9]) + require.Equal(t, map[string]any{ + "site": "tikv", "op_class": "join_hash", "operator_kind": "hashjoin", "dml_kind": "insert", "unit": "input_rows", "input_source": "child", "input_side": "left", "value": float64(2), + }, units[10]) + require.Equal(t, map[string]any{ + "site": "tikv", "op_class": "join_hash", "operator_kind": "hashjoin", "dml_kind": "update", "unit": "input_rows", "input_source": "runtime", "input_side": "right", "value": float64(3), + }, units[11]) + require.Equal(t, map[string]any{ + "site": "tikv", "op_class": "kv_write", "operator_kind": "txn_write", "dml_kind": "update", "unit": "write_request_count", "input_source": "ruv2_metrics", "input_side": "all", "value": float64(2), + }, units[12]) + + // A valid completed snapshot with no unit samples stays structured and does + // not invent one statement status from the snapshot's multi-status model. + LogReadBillingDemoGeneralLog(sctx, stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v2", + }) + entries = recorded.TakeAll() + require.Len(t, entries, 1) + fields = entries[0].ContextMap() + require.Len(t, fields, 6) + require.NotContains(t, fields, "status") + require.NotContains(t, fields, "reason") + rawUnits, ok = fields["units"].([]any) + require.True(t, ok) + require.Empty(t, rawUnits) + + // Explicitly initialized empty identity is safe and does not require a + // digest object to be present. + emptyIdentityCtx := mock.NewContext() + emptyIdentityCtx.GetSessionVars().EnableReadBillingDemo = true + emptyIdentityCtx.GetSessionVars().StmtCtx.InitSQLDigest("", nil) + LogReadBillingDemoGeneralLog(emptyIdentityCtx, stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v2", + }) + entries = recorded.TakeAll() + require.Len(t, entries, 1) + fields = entries[0].ContextMap() + require.Equal(t, "", fields["normalized_sql"]) + require.Equal(t, "", fields["sql_digest"]) +} + func newExecStmtWithStmtStatsForTest(goCtx context.Context, t *testing.T) (*ExecStmt, *stmtstats.StatementStats) { t.Helper() diff --git a/pkg/executor/adapter_test.go b/pkg/executor/adapter_test.go index 75bf763df0100..b3b25647dc0bc 100644 --- a/pkg/executor/adapter_test.go +++ b/pkg/executor/adapter_test.go @@ -30,10 +30,12 @@ import ( "github.com/pingcap/tidb/pkg/config" distsqlctx "github.com/pingcap/tidb/pkg/distsql/context" "github.com/pingcap/tidb/pkg/executor" + "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/auth" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/resourcegroup" "github.com/pingcap/tidb/pkg/session" "github.com/pingcap/tidb/pkg/sessionctx/slowlogrule" @@ -43,6 +45,7 @@ import ( "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tidb/pkg/util/sqlexec" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -58,6 +61,15 @@ type mockRUV2ConsumptionReporter struct { tiflashRU float64 } +func requireReadBillingDemoGeneralLogIdentity(t *testing.T, entry observer.LoggedEntry, sql string) { + t.Helper() + normalizedSQL, digest := parser.NormalizeDigest(sql) + fields := entry.ContextMap() + require.Equal(t, normalizedSQL, fields["normalized_sql"]) + require.Equal(t, digest.String(), fields["sql_digest"]) + require.NotContains(t, fields, "sql") +} + func (*mockRUV2ConsumptionReporter) ReportConsumption(_ string, _ *rmpb.Consumption) {} func (m *mockRUV2ConsumptionReporter) ReportRUV2Consumption(resourceGroupName string, tikvRUV2, tidbRUV2, tiflashRUV2 float64) { @@ -644,6 +656,398 @@ func TestFinishExecuteStmtSyncsTiDBRUV2FromRUDetails(t *testing.T) { close(done) wg.Wait() }) + + t.Run("preview RU general log waits for SELECT close", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + selectSQL := "select 12345, 'secret_literal'" + rs, err := tk.Exec(selectSQL) + require.NoError(t, err) + require.NotNil(t, rs) + beforeClose := recorded.TakeAll() + require.Len(t, beforeClose, 1) + require.Equal(t, "GENERAL_LOG", beforeClose[0].Message) + startFields := beforeClose[0].ContextMap() + // Keep the existing start record's field contract unchanged; preview + // units belong only to the separate completion record. + require.Len(t, startFields, 11) + require.Equal(t, selectSQL, startFields["sql"]) + require.NotContains(t, startFields, "model_version") + require.NotContains(t, startFields, "weight_version") + require.NotContains(t, startFields, "units") + + require.NoError(t, rs.Close()) + afterClose := recorded.TakeAll() + require.Len(t, afterClose, 1) + require.Equal(t, "GENERAL_LOG_RU_UNITS", afterClose[0].Message) + completionFields := afterClose[0].ContextMap() + require.Len(t, completionFields, 6) + require.Contains(t, completionFields, "conn") + require.Contains(t, completionFields, "model_version") + require.Contains(t, completionFields, "weight_version") + require.Contains(t, completionFields, "normalized_sql") + require.Contains(t, completionFields, "sql_digest") + require.Contains(t, completionFields, "units") + require.NotContains(t, completionFields, "sql") + requireReadBillingDemoGeneralLogIdentity(t, afterClose[0], selectSQL) + require.NotContains(t, completionFields["normalized_sql"], "12345") + require.NotContains(t, completionFields["normalized_sql"], "secret_literal") + }) + + t.Run("preview RU general log publishes point lookup rpc only", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table point_get_rpc_ru_units (id int primary key, v int)") + tk.MustExec("insert into point_get_rpc_ru_units values (1, 10), (2, 20)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + + testCases := []struct { + name string + querySQL string + operatorKind string + requests int64 + rows []string + }{ + {"point get", "select v from point_get_rpc_ru_units where id = 1", "point_get", 3, []string{"10"}}, + {"batch point get", "select v from point_get_rpc_ru_units where id in (1, 2) order by v", "batch_point_get", 4, []string{"10", "20"}}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recorded.TakeAll() + ctx := execdetails.ContextWithInitializedExecDetails(context.Background()) + execdetails.RUV2MetricsFromContext(ctx).AddResourceManagerReadCnt(tc.requests) + tk.MustQueryWithContext(ctx, tc.querySQL).Check(testkit.Rows(tc.rows...)) + + entries := recorded.TakeAll() + var completionEntries []observer.LoggedEntry + for _, entry := range entries { + if entry.Message == "GENERAL_LOG_RU_UNITS" { + completionEntries = append(completionEntries, entry) + } + } + require.Len(t, completionEntries, 1) + requireReadBillingDemoGeneralLogIdentity(t, completionEntries[0], tc.querySQL) + rawUnits, ok := completionEntries[0].ContextMap()["units"].([]any) + require.True(t, ok) + pointUnits := 0 + for _, rawUnit := range rawUnits { + unit, ok := rawUnit.(map[string]any) + require.True(t, ok) + if unit["op_class"] != "kv_point_lookup" { + continue + } + pointUnits++ + require.Equal(t, "tikv", unit["site"]) + require.Equal(t, tc.operatorKind, unit["operator_kind"]) + require.Equal(t, "read_request_count", unit["unit"]) + require.Equal(t, "ruv2_metrics", unit["input_source"]) + require.Equal(t, "all", unit["input_side"]) + require.Equal(t, float64(tc.requests), unit["value"]) + } + require.Equal(t, 1, pointUnits) + }) + } + }) + + t.Run("preview RU general log DML completion is self describing", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table dml_ru_units_identity (id int primary key, v int)") + tk.MustExec("insert into dml_ru_units_identity values (12345, 1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + updateSQL := "update dml_ru_units_identity set v = 98765 where id = 12345" + tk.MustExec(updateSQL) + entries := recorded.TakeAll() + require.Len(t, entries, 2) + // DML currently completes inside runStmt before its deferred legacy + // GENERAL_LOG. Identity makes the completion independent of this order. + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[0].Message) + require.Equal(t, "GENERAL_LOG", entries[1].Message) + requireReadBillingDemoGeneralLogIdentity(t, entries[0], updateSQL) + normalizedSQL := entries[0].ContextMap()["normalized_sql"] + require.NotContains(t, normalizedSQL, "98765") + require.NotContains(t, normalizedSQL, "12345") + }) + + t.Run("preview RU general log skips pre-defer compile error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + _, err := tk.Exec("select * from missing_ru_units_identity") + require.Error(t, err) + // Compile fails before legacy GENERAL_LOG is registered. Do not emit an + // orphan completion that has no corresponding executed statement. + require.Empty(t, recorded.TakeAll()) + }) + + t.Run("preview RU general log covers prepared point get setup error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table point_get_setup_error_ru_units (id int primary key, v int)") + tk.MustExec("insert into point_get_setup_error_ru_units values (12345, 1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + querySQL := "select v from point_get_setup_error_ru_units where id = ?" + failpointName := "github.com/pingcap/tidb/pkg/session/mockGetTSFail" + require.NoError(t, failpoint.Enable(failpointName, "return")) + failpointEnabled := true + t.Cleanup(func() { + if failpointEnabled { + require.NoError(t, failpoint.Disable(failpointName)) + } + }) + ctx := failpoint.WithHook(context.Background(), func(_ context.Context, name string) bool { + return name == failpointName + }) + _, err := tk.ExecWithContext(ctx, querySQL, 12345) + require.NoError(t, failpoint.Disable(failpointName)) + failpointEnabled = false + require.Error(t, err) + + entries := recorded.TakeAll() + require.Len(t, entries, 2) + require.Equal(t, "GENERAL_LOG", entries[0].Message) + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[1].Message) + requireReadBillingDemoGeneralLogIdentity(t, entries[1], querySQL) + rawUnits, ok := entries[1].ContextMap()["units"].([]any) + require.True(t, ok) + require.Empty(t, rawUnits) + }) + + t.Run("preview RU general log disables cursor detach only when needed", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table detach_ru_units (a int)") + tk.MustExec("insert into detach_ru_units values (1), (2)") + tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true) + + // General Log alone preserves the existing detach behavior. + vardef.ProcessGeneralLog.Store(true) + rs, err := tk.Exec("select * from detach_ru_units where a > ?", 0) + require.NoError(t, err) + detachable := rs.(sqlexec.DetachableRecordSet) + detached, ok, err := detachable.TryDetach() + require.NoError(t, err) + require.True(t, ok) + require.NotNil(t, detached) + require.NoError(t, detached.Close()) + + // With both gates enabled, keep the live record set so Close can freeze + // runtime evidence and emit exactly one completion event. + vardef.ProcessGeneralLog.Store(false) + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + rs, err = tk.Exec("select * from detach_ru_units where a > ?", 0) + require.NoError(t, err) + detachable = rs.(sqlexec.DetachableRecordSet) + detached, ok, err = detachable.TryDetach() + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, detached) + beforeClose := recorded.TakeAll() + require.Len(t, beforeClose, 1) + require.Equal(t, "GENERAL_LOG", beforeClose[0].Message) + require.NoError(t, rs.Close()) + afterClose := recorded.TakeAll() + require.Len(t, afterClose, 1) + require.Equal(t, "GENERAL_LOG_RU_UNITS", afterClose[0].Message) + }) + + t.Run("preview RU general log covers early statement error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table early_error_ru_units (a int primary key)") + tk.MustExec("insert into early_error_ru_units values (1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + tk.MustExec("begin pessimistic") + tk.MustQuery("select * from early_error_ru_units where a = 1 for update").Check(testkit.Rows("1")) + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + atomic.StoreUint32(&tk.Session().GetSessionVars().TxnCtx.LockExpire, 1) + t.Cleanup(func() { + atomic.StoreUint32(&tk.Session().GetSessionVars().TxnCtx.LockExpire, 0) + }) + + _, err := tk.Exec("select 1") + atomic.StoreUint32(&tk.Session().GetSessionVars().TxnCtx.LockExpire, 0) + require.Error(t, err) + entries := recorded.TakeAll() + require.Len(t, entries, 2) + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[0].Message) + require.Equal(t, "GENERAL_LOG", entries[1].Message) + require.Equal(t, "select 1", entries[1].ContextMap()["sql"]) + requireReadBillingDemoGeneralLogIdentity(t, entries[0], "select 1") + rawUnits, ok := entries[0].ContextMap()["units"].([]any) + require.True(t, ok) + require.Empty(t, rawUnits) + }) + + t.Run("preview RU general log covers explain analyze DML commit error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + tk.MustExec("set tidb_retry_limit = 0") + tk.MustExec("create table explain_commit_error_ru_units (id int primary key, v int)") + tk.MustExec("insert into explain_commit_error_ru_units values (1, 1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + explainSQL := "explain analyze update explain_commit_error_ru_units set v = v + 1 where id = 1" + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/session/mockCommitError8942", `return(true)`)) + failpointEnabled := true + t.Cleanup(func() { + if failpointEnabled { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/session/mockCommitError8942")) + } + }) + rs, err := tk.Exec(explainSQL) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/session/mockCommitError8942")) + failpointEnabled = false + require.Nil(t, rs) + require.Error(t, err) + require.True(t, kv.ErrTxnRetryable.Equal(err), err) + entries := recorded.TakeAll() + require.Len(t, entries, 2) + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[0].Message) + require.Equal(t, "GENERAL_LOG", entries[1].Message) + requireReadBillingDemoGeneralLogIdentity(t, entries[0], explainSQL) + digest := entries[0].ContextMap()["sql_digest"] + + vardef.ProcessGeneralLog.Store(false) + tk.MustQuery("select v from explain_commit_error_ru_units where id = 1").Check(testkit.Rows("1")) + tk.MustQuery("select status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest = ? and site = 'statement'", digest). + Check(testkit.Rows("error statement_error 1")) + }) + + t.Run("preview RU general log emits once across repeated finish", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(true) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + ctx := mock.NewContext() + sessVars := ctx.GetSessionVars() + sessVars.EnableReadBillingDemo = true + sessVars.StartTime = time.Now() + sessVars.StmtCtx.StmtType = "Select" + sessVars.StmtCtx.OriginalSQL = "select 1" + sessVars.StmtCtx.ResetSQLDigest(sessVars.StmtCtx.OriginalSQL) + goCtx := execdetails.ContextWithInitializedExecDetails(context.Background()) + sessVars.RUV2Metrics = execdetails.RUV2MetricsFromContext(goCtx) + execStmt := &executor.ExecStmt{ + Ctx: ctx, + GoCtx: goCtx, + StmtNode: &ast.SelectStmt{}, + } + + execStmt.FinishExecuteStmt(0, nil, false) + execStmt.FinishExecuteStmt(0, nil, false) + entries := recorded.TakeAll() + completionCount := 0 + for _, entry := range entries { + if entry.Message == "GENERAL_LOG_RU_UNITS" { + completionCount++ + } + } + require.Equal(t, 1, completionCount) + }) } func TestSlowLogMaxPerSec(t *testing.T) { diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go index 7644929184311..8ffa1e7c5f473 100644 --- a/pkg/executor/builder.go +++ b/pkg/executor/builder.go @@ -2616,10 +2616,18 @@ func (b *executorBuilder) buildMemTable(v *physicalop.PhysicalMemTable) exec.Exe case strings.ToLower(infoschema.TableStatementsSummary), strings.ToLower(infoschema.TableStatementsSummaryHistory), strings.ToLower(infoschema.TableStatementsSummaryEvicted), + strings.ToLower(infoschema.TableStatementsSummaryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.TableStatementsSummaryReadBillingDemoStatus), + strings.ToLower(infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus), strings.ToLower(infoschema.TableTiDBStatementsStats), strings.ToLower(infoschema.ClusterTableStatementsSummary), strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory), strings.ToLower(infoschema.ClusterTableStatementsSummaryEvicted), + strings.ToLower(infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus), + strings.ToLower(infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus), strings.ToLower(infoschema.ClusterTableTiDBStatementsStats): var extractor *plannercore.StatementsSummaryExtractor if v.Extractor != nil { diff --git a/pkg/executor/compiler.go b/pkg/executor/compiler.go index b686423358f67..c478dacd5331a 100644 --- a/pkg/executor/compiler.go +++ b/pkg/executor/compiler.go @@ -153,6 +153,9 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecS OutputNames: names, Ti: &TelemetryInfo{}, } + if preparedObj != nil && preparedObj.PreparedAst != nil { + stmt.resolvedPreparedStmt = preparedObj.PreparedAst.Stmt + } // Use cached plan if possible. if preparedObj != nil && plannercore.IsSafeToReusePointGetExecutor(c.Ctx, is, preparedObj) { if exec, isExec := finalPlan.(*plannercore.Execute); isExec { diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 8d7e04e436d6b..037d4197e0a0b 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -16,18 +16,30 @@ package executor_test import ( "bytes" + "context" "encoding/json" "fmt" + "os" + "path/filepath" "regexp" "strconv" "strings" "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/metrics" + "github.com/pingcap/tidb/pkg/parser/auth" plannercore "github.com/pingcap/tidb/pkg/planner/core" + "github.com/pingcap/tidb/pkg/session" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/sqlexec" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -496,6 +508,7 @@ func TestExplainFormatInCtx(t *testing.T) { types.ExplainFormatTiDBJSON, types.ExplainFormatCostTrace, types.ExplainFormatPlanCache, + types.ExplainFormatRU, } tk.MustExec("select * from t") @@ -515,6 +528,978 @@ func TestExplainFormatInCtx(t *testing.T) { } } +func TestExplainAnalyzeFormatRUOutput(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + rows := tk.MustQuery("explain analyze format='ru' select 1").Rows() + require.NotEmpty(t, rows) + require.Equal(t, "summary", rows[0][0]) + require.Equal(t, "total_preview_ru", rows[0][2]) + require.Equal(t, "summary_total", rows[0][15]) + requireExplainRUPlanRow(t, rows) + + tk.MustExec("drop table if exists explain_ru_t") + tk.MustExec("create table explain_ru_t(a int primary key, b varchar(20))") + tk.MustExec("insert into explain_ru_t values (1, 'x'), (2, 'yy')") + _, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' select * from explain_ru_t where a > 0") + require.Error(t, err) + require.Contains(t, err.Error(), "status=unknown_input") + require.True(t, + strings.Contains(err.Error(), "reason=missing_scan_width_evidence operator=tikv/kv_range_scan/tablerangescan") || + strings.Contains(err.Error(), "reason=missing_reader_transport_details operator=tidb/reader_transport/table_reader"), + err.Error(), + ) + + for _, tc := range []struct { + sql string + operatorKind string + }{ + {"explain analyze format='ru' select * from explain_ru_t where a = 1", "point_get"}, + {"explain analyze format='ru' select * from explain_ru_t where a in (1, 2)", "batch_point_get"}, + } { + rows, err = queryExplainRURowsOrErrWithContext(execdetails.ContextWithInitializedExecDetails(context.Background()), t, tk, tc.sql) + require.NoError(t, err, tc.sql) + require.NotEmpty(t, rows, tc.sql) + requireExplainRUOperatorClass(t, rows, "tikv/kv_point_lookup") + require.GreaterOrEqual(t, explainRUCountUnitValue(t, rows, "tikv/kv_point_lookup", "read_request_count"), 0.0) + for _, row := range rows { + if len(row) != 17 || row[0] != "plan" || row[3] != "tikv/kv_point_lookup" { + continue + } + require.Equal(t, tc.operatorKind, row[2], tc.sql) + require.NotContains(t, []any{"cpu_work", "scan_bytes", "net_bytes"}, row[11], tc.sql) + } + } +} + +func TestExplainAnalyzeFormatRUTiKVCopOperatorClasses(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_cop") + tk.MustExec("create table explain_ru_cop(a int primary key, b int, c varchar(20), key idx_b(b))") + tk.MustExec("insert into explain_ru_cop values (1, 10, 'a'), (2, 20, 'bb'), (3, 20, 'ccc'), (4, 30, 'dddd')") + + cases := []struct { + sql string + nonScanOpClass string + planOperator string + }{ + { + sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) where b > 10", + nonScanOpClass: "tikv/filter_eval", + planOperator: "Selection", + }, + { + sql: "explain analyze format='ru' select a from explain_ru_cop where b > 10", + nonScanOpClass: "tikv/projection_eval", + planOperator: "Projection", + }, + { + sql: "explain analyze format='ru' select * from explain_ru_cop limit 2", + nonScanOpClass: "tikv/row_limit", + planOperator: "Limit", + }, + { + sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) order by c limit 2", + nonScanOpClass: "tikv/bounded_topn", + planOperator: "TopN", + }, + { + sql: "explain analyze format='ru' select /*+ agg_to_cop(), hash_agg() */ b, count(*) from explain_ru_cop group by b", + nonScanOpClass: "tikv/agg_hash", + planOperator: "HashAgg", + }, + { + sql: "explain analyze format='ru' select /*+ stream_agg(), agg_to_cop(), order_index(explain_ru_cop, idx_b) */ b, count(*) from explain_ru_cop group by b order by b", + nonScanOpClass: "tikv/agg_stream", + planOperator: "StreamAgg", + }, + } + for _, tc := range cases { + ordinarySQL := strings.Replace(tc.sql, "explain analyze format='ru'", "explain analyze", 1) + seenPushedOperator := false + for _, row := range tk.MustQuery(ordinarySQL).Rows() { + rowText := fmt.Sprint(row) + if strings.Contains(rowText, tc.planOperator) && strings.Contains(rowText, "cop[tikv]") { + seenPushedOperator = true + break + } + } + require.True(t, seenPushedOperator, "%s did not produce %s in cop[tikv]", tc.sql, tc.planOperator) + + _, err := queryExplainRURowsOrErr(t, tk, tc.sql) + require.Error(t, err, tc.nonScanOpClass) + require.Contains(t, err.Error(), "status=unknown_input", tc.sql) + // Native EmbedUnistore returns execution summaries for these pushed + // operators but does not populate the complete scan-width tuple. + require.Contains(t, err.Error(), "reason=missing_scan_width_evidence", tc.sql) + require.Contains(t, err.Error(), "operator=tikv/kv_range_scan/", tc.sql) + } +} + +func queryExplainRURowsOrErr(t *testing.T, tk *testkit.TestKit, sql string) ([][]any, error) { + return queryExplainRURowsOrErrWithContext(context.Background(), t, tk, sql) +} + +func queryExplainRURowsOrErrWithContext(ctx context.Context, t *testing.T, tk *testkit.TestKit, sql string) ([][]any, error) { + t.Helper() + rs, err := tk.ExecWithContext(ctx, sql) + if err != nil { + if rs != nil { + require.NoError(t, rs.Close()) + } + return nil, err + } + fields := rs.Fields() + chunkRows, err := session.GetRows4Test(context.Background(), tk.Session(), rs) + closeErr := rs.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + anyRows := make([][]any, len(chunkRows)) + for i, row := range chunkRows { + anyRows[i] = make([]any, row.Len()) + for j := range row.Len() { + if row.IsNull(j) { + anyRows[i][j] = "" + continue + } + datum := row.GetDatum(j, &fields[j].Column.FieldType) + value, err := datum.ToString() + if err != nil { + return nil, err + } + anyRows[i][j] = value + } + } + return anyRows, nil +} + +func requireExplainRUPlanRow(t *testing.T, rows [][]any) { + t.Helper() + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" { + continue + } + require.NotEmpty(t, row[1]) + require.NotEmpty(t, row[2]) + require.Contains(t, fmt.Sprint(row[3]), "/") + require.NotEmpty(t, row[4]) + require.NotEmpty(t, row[8]) + require.NotEmpty(t, row[11]) + require.NotEmpty(t, row[12]) + require.Empty(t, row[13]) + require.Empty(t, row[14]) + require.NotEmpty(t, row[15]) + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v3-resource-formula-uncalibrated") + return + } + require.Fail(t, "missing FORMAT='RU' plan row") +} + +func requireNoExplainRUOperatorClass(t *testing.T, rows [][]any, operatorClass string) { + t.Helper() + for _, row := range rows { + require.Len(t, row, 17) + if row[0] == "plan" && row[3] == operatorClass { + require.Failf(t, "unexpected FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) + } + } +} + +func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorClass string) { + t.Helper() + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" || row[3] != operatorClass { + continue + } + require.NotEmpty(t, row[12], "missing semantic unit value for %s row %v", operatorClass, row) + require.Empty(t, row[13], "uncalibrated v4 must not publish a weight for %s row %v", operatorClass, row) + require.Empty(t, row[14], "uncalibrated v4 must not publish preview RU for %s row %v", operatorClass, row) + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v3-resource-formula-uncalibrated") + return + } + require.Failf(t, "missing weighted FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) +} + +func requireExplainRUDMLRows(t *testing.T, rows [][]any, dmlKind string, requireReadTree, requireTiKV, requireMutations bool) { + t.Helper() + require.NotEmpty(t, rows) + require.Len(t, rows[0], 17) + require.Equal(t, "summary", rows[0][0]) + require.Equal(t, "total_preview_ru", rows[0][2]) + require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v2") + require.Contains(t, fmt.Sprint(rows[0][16]), "mutation_weights_uncalibrated=true") + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_dml_ancillary_work_partial") + summaryPreviewRU, err := strconv.ParseFloat(fmt.Sprint(rows[0][14]), 64) + require.NoError(t, err) + + var mutationCountRows, mutationByteRows, writeKeysRows, writeByteRows, partialWriteRows, readRows int + var mutationCount float64 + var planPreviewRU float64 + seenPlanUnits := make(map[string]struct{}) + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" { + continue + } + if unit := fmt.Sprint(row[11]); unit != "" { + key := fmt.Sprintf("%s/%s/%s/%s/%s", row[1], row[3], unit, row[15], row[16]) + require.NotContains(t, seenPlanUnits, key, "DML plan/unit row must not be emitted twice: %v", row) + seenPlanUnits[key] = struct{}{} + } + if fmt.Sprint(row[14]) != "" { + previewRU, err := strconv.ParseFloat(fmt.Sprint(row[14]), 64) + require.NoError(t, err) + planPreviewRU += previewRU + } + if row[3] != "tidb/kv_mutation" && row[3] != "tikv/kv_write" { + readRows++ + continue + } + require.Contains(t, fmt.Sprint(row[16]), "dml_kind="+dmlKind) + if row[3] == "tikv/kv_write" && row[11] == "" { + partialWriteRows++ + require.Contains(t, fmt.Sprint(row[16]), "status=partial") + require.Contains(t, fmt.Sprint(row[16]), "reason=missing_commit_detail") + require.Contains(t, fmt.Sprint(row[16]), "scope=txn_prewrite_payload") + continue + } + switch row[11] { + case "encoded_mutation_count": + mutationCountRows++ + require.Equal(t, "memdb_mutation", row[2]) + require.Equal(t, "tidb/kv_mutation", row[3]) + require.Equal(t, "stmt_memdb_mutation_calls", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "scope=statement_attempted") + require.Contains(t, fmt.Sprint(row[16]), "uncalibrated=true") + mutationCount, err = strconv.ParseFloat(fmt.Sprint(row[12]), 64) + require.NoError(t, err) + case "encoded_mutation_bytes": + mutationByteRows++ + require.Equal(t, "memdb_mutation", row[2]) + require.Equal(t, "tidb/kv_mutation", row[3]) + require.Equal(t, "stmt_memdb_mutation_calls", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "scope=statement_attempted") + case "write_keys": + writeKeysRows++ + require.Equal(t, "txn_prewrite", row[2]) + require.NotEmpty(t, row[12]) + require.NotEmpty(t, row[13]) + require.NotEmpty(t, row[14]) + require.Equal(t, "commit_detail", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "scope=txn_prewrite_payload") + case "write_byte": + writeByteRows++ + require.Equal(t, "txn_prewrite", row[2]) + require.NotEmpty(t, row[10]) + require.NotEmpty(t, row[13]) + require.NotEmpty(t, row[14]) + require.Equal(t, "commit_detail", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "semantic_name=write_bytes") + case "prewrite_region_num", "tikv_write_rpc_count": + require.NotEmpty(t, row[12]) + require.Equal(t, "0.000000", row[13]) + require.Equal(t, "0.000000", row[14]) + require.Contains(t, fmt.Sprint(row[16]), "diagnostic_only=true") + } + } + require.Equal(t, 1, mutationCountRows, rows) + require.Equal(t, 1, mutationByteRows, rows) + if requireMutations { + require.Positive(t, mutationCount, rows) + } else { + require.Zero(t, mutationCount, rows) + } + if requireReadTree { + require.Positive(t, readRows, rows) + } + if requireTiKV { + require.Equal(t, 1, writeKeysRows, rows) + require.Equal(t, 1, writeByteRows, rows) + } else { + require.Zero(t, writeKeysRows, rows) + require.Zero(t, writeByteRows, rows) + require.Equal(t, 1, partialWriteRows, rows) + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_missing_commit_detail") + } + require.InDelta(t, planPreviewRU, summaryPreviewRU, 0.000001) +} + +func requireExplainRUOperatorClass(t *testing.T, rows [][]any, operatorClass string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" && row[3] == operatorClass { + return + } + } + require.Failf(t, "missing RU operator class", "operatorClass=%s rows=%v", operatorClass, rows) +} + +func explainRUCountUnitValue(t *testing.T, rows [][]any, operatorClass, unit string) float64 { + t.Helper() + for _, row := range rows { + if len(row) != 17 || row[0] != "plan" || row[3] != operatorClass || row[11] != unit { + continue + } + valueColumn := 12 + switch unit { + case "cpu_work": + valueColumn = 9 + case "scan_bytes", "net_bytes", "encoded_mutation_bytes", "key_bytes", "value_bytes": + valueColumn = 10 + } + value, err := strconv.ParseFloat(fmt.Sprint(row[valueColumn]), 64) + require.NoError(t, err) + return value + } + require.Failf(t, "missing RU count unit", "operatorClass=%s unit=%s rows=%v", operatorClass, unit, rows) + return 0 +} + +func TestExplainAnalyzeFormatRUPlanDigest(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + + originEnableStmtSummary := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_stmt_summary").Rows()[0][0]) + originStmtSummaryRefreshInterval := fmt.Sprint(tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Rows()[0][0]) + originStmtSummaryHistorySize := fmt.Sprint(tk.MustQuery("select @@global.tidb_stmt_summary_history_size").Rows()[0][0]) + defer tk.MustExec(fmt.Sprintf("set global tidb_enable_stmt_summary = %s", originEnableStmtSummary)) + defer tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_refresh_interval = %s", originStmtSummaryRefreshInterval)) + defer tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_history_size = %s", originStmtSummaryHistorySize)) + tk.MustExec("set global tidb_stmt_summary_history_size = 24") + tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + + tk.MustExec("drop table if exists explain_ru_digest") + tk.MustExec("create table explain_ru_digest(a int primary key, b int)") + tk.MustExec("insert into explain_ru_digest values (1, 10)") + tk.MustQuery("select b from explain_ru_digest where a = 1").Check(testkit.Rows("10")) + + digestRows := tk.MustQuery("select plan_digest from information_schema.statements_summary_history where digest_text like 'select `b` from `explain_ru_digest`%' and plan_digest != ''").Rows() + require.NotEmpty(t, digestRows) + planDigest := fmt.Sprint(digestRows[0][0]) + rows, err := queryExplainRURowsOrErr(t, tk, fmt.Sprintf("explain analyze format='ru' '%s'", planDigest)) + require.NoError(t, err) + requireExplainRUOperatorClass(t, rows, "tikv/kv_point_lookup") + require.GreaterOrEqual(t, explainRUCountUnitValue(t, rows, "tikv/kv_point_lookup", "read_request_count"), 0.0) +} + +func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_write_v4") + tk.MustExec("create table explain_ru_write_v4(a int primary key, b varchar(20))") + + rows, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_v4 values (1, 'one')") + require.NoError(t, err) + require.NotEmpty(t, rows) + require.Equal(t, "summary", rows[0][0]) + require.Empty(t, rows[0][14], rows) + require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v3-resource-formula-uncalibrated") + require.Contains(t, fmt.Sprint(rows[0][16]), "uncalibrated_weights") + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_bytes"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Zero(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "key_bytes"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "value_bytes"), rows) + requireExplainRUOperatorUnitAbsent(t, rows, "tidb/kv_mutation", "cpu_work") + require.Zero(t, explainRUCountUnitValue(t, rows, "tikv/kv_write", "write_request_count"), rows) + tk.MustQuery("select * from explain_ru_write_v4").Check(testkit.Rows("1 one")) + + tk.MustExec("begin pessimistic") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write_v4 set b = 'two' where a = 1") + require.NoError(t, err) + requireExplainRUOperatorClass(t, rows, "tikv/kv_point_lookup") + require.GreaterOrEqual(t, explainRUCountUnitValue(t, rows, "tikv/kv_point_lookup", "read_request_count"), 0.0) + requireExplainRUUnitSource(t, rows, "tikv/kv_point_lookup", "read_request_count", "snapshot_runtime_stats") + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + requireExplainRUOperatorUnitAbsent(t, rows, "tidb/kv_mutation", "cpu_work") + require.Zero(t, explainRUCountUnitValue(t, rows, "tikv/kv_write", "write_request_count"), rows) + tk.MustExec("commit") + tk.MustQuery("select * from explain_ru_write_v4").Check(testkit.Rows("1 two")) +} + +func requireExplainRUUnitAbsent(t *testing.T, rows [][]any, unit string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" { + require.NotEqual(t, unit, row[11], rows) + } + } +} + +func requireExplainRUUnitSource(t *testing.T, rows [][]any, operatorClass, unit, source string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" && row[3] == operatorClass && row[11] == unit { + require.Equal(t, source, row[15], rows) + return + } + } + require.Failf(t, "missing RU unit", "operator_class=%s unit=%s rows=%v", operatorClass, unit, rows) +} + +func requireExplainRUOperatorUnitAbsent(t *testing.T, rows [][]any, operatorClass, unit string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" && row[3] == operatorClass { + require.NotEqual(t, unit, row[11], rows) + } + } +} + +func TestExplainAnalyzeFormatRUWriteDMLV3Legacy(t *testing.T) { + t.Skip("v3 per-unit weighting and commit-detail ownership are superseded by the v4 resource formula") + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_write") + tk.MustExec("create table explain_ru_write(a int primary key, b int, c varchar(20), key idx_b(b))") + tk.MustExec("drop table if exists explain_ru_write_src") + tk.MustExec("create table explain_ru_write_src(a int primary key, b int, c varchar(20))") + tk.MustExec("insert into explain_ru_write_src values (2, 20, 'src2'), (3, 30, 'src3')") + + rows, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write values (1, 10, 'aa')") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", false, true, true) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Zero(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 10 aa")) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write select * from explain_ru_write_src where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", true, true, true) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write values (2, 20, 'upsert') on duplicate key update c = values(c)") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "upsert", false, true, true) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert ignore into explain_ru_write values (2, 20, 'ignored')") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert_ignore", false, false, false) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write with cte as (select * from explain_ru_write_src where a = 3) select * from cte") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", true, true, true) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = (select 'scalar') where a = 3") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + + // Changing an indexed column emits one record Set plus an index Delete/Set. + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set b = 21 where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + require.Equal(t, 3.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Equal(t, 1.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'bbb' where b >= 10") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + tk.MustQuery("select a, b, c from explain_ru_write order by a").Check(testkit.Rows("1 10 bbb", "2 21 bbb", "3 30 bbb")) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'never' where a = 999") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, false) + + // A matched but unchanged row only contributes read/compare work. The + // unchanged-key lock bookkeeping must not become a MemDB mutation. + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'bbb' where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, false) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' delete from explain_ru_write where b = 10") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "delete", true, true, true) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Zero(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + tk.MustQuery("select a, b, c from explain_ru_write order by a").Check(testkit.Rows("2 21 bbb", "3 30 bbb")) + + // Keep the join, projection, sort, reader, filter, and scan descendants of + // INSERT SELECT visible alongside the shared mutation/write operators. + tk.MustExec("create table explain_ru_write_join_dst(a int primary key, b int, c varchar(20))") + tk.MustExec("create table explain_ru_write_join_dim(a int primary key, d int)") + tk.MustExec("insert into explain_ru_write_join_dim values (2, 200), (3, 300)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_join_dst select /*+ hash_join(s, d) */ s.a, s.b, s.c from explain_ru_write_src s join explain_ru_write_join_dim d on s.a = d.a where d.d >= 200 order by d.d + s.b desc") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", true, true, true) + for _, operatorClass := range []string{ + "tidb/join_hash", + "tidb/full_ordering", + "tidb/projection_eval", + "tidb/reader_receive", + "tikv/filter_eval", + "tikv/kv_range_scan", + } { + requireExplainRUOperatorClass(t, rows, operatorClass) + } + tk.MustQuery("select * from explain_ru_write_join_dst order by a").Check(testkit.Rows("2 20 src2", "3 30 src3")) + + // An unsupported reader must make only that operator partial. The scans + // below it still belong to the DML plan tree and must remain in the output. + tk.MustExec("set tidb_enable_index_merge=on") + tk.MustExec("create table explain_ru_write_index_merge(id int primary key, a int, b int, c int, key idx_a(a), key idx_b(b))") + tk.MustExec("insert into explain_ru_write_index_merge values (1, 10, 1, 1), (2, 2, 20, 2)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update /*+ use_index_merge(explain_ru_write_index_merge, idx_a, idx_b) */ explain_ru_write_index_merge set c = c + 1 where a = 10 or b = 20") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_unsupported_index_merge") + var hasPartialIndexMerge, hasPartialScan bool + for _, row := range rows { + if len(row) != 17 || row[0] != "plan" || !strings.Contains(fmt.Sprint(row[16]), "status=partial") { + continue + } + note := fmt.Sprint(row[16]) + hasPartialIndexMerge = hasPartialIndexMerge || strings.Contains(note, "reason=unsupported_index_merge") + hasPartialScan = hasPartialScan || + (strings.Contains(strings.ToLower(fmt.Sprint(row[2])), "scan") && strings.Contains(note, "reason=missing_scan_width_evidence")) + } + require.True(t, hasPartialIndexMerge, rows) + require.True(t, hasPartialScan, rows) + + tk.MustExec("begin") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'txn' where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, true) + + // The next statement gets a fresh recorder even though it shares the same + // explicit transaction and the previous statement attempted mutations. + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'never' where a = 999") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, false) + + tk.MustExec("savepoint explain_ru_write_savepoint") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'savepoint' where a = 3") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, true) + tk.MustExec("rollback to savepoint explain_ru_write_savepoint") + tk.MustQuery("select c from explain_ru_write where a = 3").Check(testkit.Rows("bbb")) + tk.MustExec("rollback") + tk.MustQuery("select a, b, c from explain_ru_write order by a").Check(testkit.Rows("2 21 bbb", "3 30 bbb")) + + tk.MustExec("create temporary table explain_ru_write_tmp(a int primary key)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_tmp values (1)") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", false, false, true) + require.Equal(t, 1.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + + originalEnableBatchDML := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_batch_dml").Rows()[0][0]) + defer tk.MustExec("set global tidb_enable_batch_dml=" + originalEnableBatchDML) + tk.MustExec("set global tidb_enable_batch_dml=on") + tk.MustExec("set tidb_batch_insert=on") + tk.MustExec("set tidb_dml_batch_size=1") + tk.MustExec("create table explain_ru_write_batch(a int primary key)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_batch values (1), (2)") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", false, true, true) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) +} + +func TestExplainAnalyzeFormatRUUnsupportedTargetsBeforeExecution(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_dml") + tk.MustExec("create table explain_ru_dml(a int primary key)") + tk.MustExec("insert into explain_ru_dml values (1)") + + err := tk.ExecToErr("explain format='ru' select 1") + require.Error(t, err) + require.Contains(t, err.Error(), "cannot work without 'analyze'") + require.Contains(t, err.Error(), "format=ru") + + err = tk.ExecToErr("explain analyze format=ru select 1") + require.Error(t, err) + + for _, sql := range []string{ + "explain analyze format='ru' replace into explain_ru_dml values (2)", + "explain analyze format='ru' alter table explain_ru_dml add column b int", + "explain analyze format='ru' import into explain_ru_dml from select * from explain_ru_dml", + } { + err = tk.ExecToErr(sql) + require.Error(t, err, sql) + require.Contains(t, err.Error(), "unsupported_non_select", sql) + tk.MustQuery("select * from explain_ru_dml order by a").Check(testkit.Rows("1")) + } + tk.MustQuery("show columns from explain_ru_dml").Check(testkit.Rows("a int(11) NO PRI ")) + + err = tk.ExecToErr("explain analyze format='ru' values (1)") + require.Error(t, err) + + err = tk.ExecToErr("explain analyze format='ru' table explain_ru_dml") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_non_select") + + tk.MustExec("set @explain_ru_var := 7") + err = tk.ExecToErr("explain analyze format='ru' select @explain_ru_var := 9") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + tk.MustQuery("select @explain_ru_var").Check(testkit.Rows("7")) + + tk.MustQuery("select last_insert_id(11)").Check(testkit.Rows("11")) + err = tk.ExecToErr("explain analyze format='ru' select last_insert_id(123)") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + tk.MustQuery("select last_insert_id()").Check(testkit.Rows("11")) + + outFile := filepath.Join(t.TempDir(), "explain_ru.csv") + err = tk.ExecToErr(fmt.Sprintf("explain analyze format='ru' select 1 into outfile %q", outFile)) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + _, statErr := os.Stat(outFile) + require.True(t, os.IsNotExist(statErr)) + + err = tk.ExecToErr("explain analyze format='ru' select * from explain_ru_dml for update skip locked") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_locking_select") + + err = tk.ExecToErr("explain analyze format='ru' select get_lock('explain_ru', 0)") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + tk.MustQuery("select is_free_lock('explain_ru')").Check(testkit.Rows("1")) +} + +func TestReadBillingDemoMetricsHook(t *testing.T) { + metrics.InitExplainRUMetrics() + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + + originEnableStmtSummary := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_stmt_summary").Rows()[0][0]) + defer tk.MustExec(fmt.Sprintf("set global tidb_enable_stmt_summary = %s", originEnableStmtSummary)) + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v4", "v3-resource-formula-uncalibrated") + cpuWork := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "cpu_work", "runtime_child_act_rows", "all", "v4", "v3-resource-formula-uncalibrated") + pointGetRequests := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_point_lookup", "point_get", "read_request_count", "ruv2_metrics", "all", "v4", "v3-resource-formula-uncalibrated") + batchPointGetRequests := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_point_lookup", "batch_point_get", "read_request_count", "ruv2_metrics", "all", "v4", "v3-resource-formula-uncalibrated") + mutationCPUWork := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "cpu_work", "stmt_memdb_mutation_calls", "all", "v4", "v3-resource-formula-uncalibrated") + mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v4", "v3-resource-formula-uncalibrated") + + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Zero(t, readExecutorCounterValue(t, success)) + + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Equal(t, 1.0, readExecutorCounterValue(t, success)) + require.Greater(t, readExecutorCounterValue(t, cpuWork), 0.0) + tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events, sum_read_billing_demo_input_rows, sum_read_billing_demo_input_bytes from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 0 0 0")) + tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and unit = 'cpu_work'`).Check(testkit.Rows("tidb projection_eval projection cpu_work runtime_child_act_rows all v4 v3-resource-formula-uncalibrated 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) + + tk.MustExec("drop table if exists read_billing_demo_v4") + tk.MustExec("create table read_billing_demo_v4(a int primary key)") + beforeMutationCPUWork := readExecutorCounterValue(t, mutationCPUWork) + beforeMutation := readExecutorCounterValue(t, mutationCount) + tk.MustExec("insert into read_billing_demo_v4 values (1), (2)") + require.Equal(t, beforeMutationCPUWork, readExecutorCounterValue(t, mutationCPUWork)) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeMutation) + tk.MustQuery("select site, op_class, operator_kind, dml_kind, unit, input_source, input_side, model_version, weight_version, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_demo_v4`%' and unit in ('encoded_mutation_bytes', 'encoded_mutation_count', 'set_count', 'delete_count', 'key_bytes', 'value_bytes') order by unit").Check(testkit.Rows( + "tidb kv_mutation memdb_mutation insert delete_count stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 0", + "tidb kv_mutation memdb_mutation insert encoded_mutation_bytes stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert encoded_mutation_count stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert key_bytes stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert set_count stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert value_bytes stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + )) + + pointCtx := execdetails.ContextWithInitializedExecDetails(context.Background()) + execdetails.RUV2MetricsFromContext(pointCtx).AddResourceManagerReadCnt(3) + beforePointGetRequests := readExecutorCounterValue(t, pointGetRequests) + tk.MustQueryWithContext(pointCtx, "select * from read_billing_demo_v4 where a = 1").Check(testkit.Rows("1")) + require.Equal(t, beforePointGetRequests+3, readExecutorCounterValue(t, pointGetRequests)) + + batchCtx := execdetails.ContextWithInitializedExecDetails(context.Background()) + execdetails.RUV2MetricsFromContext(batchCtx).AddResourceManagerReadCnt(4) + beforeBatchPointGetRequests := readExecutorCounterValue(t, batchPointGetRequests) + tk.MustQueryWithContext(batchCtx, "select * from read_billing_demo_v4 where a in (1, 2)").Sort().Check(testkit.Rows("1", "2")) + require.Equal(t, beforeBatchPointGetRequests+4, readExecutorCounterValue(t, batchPointGetRequests)) + + tk.MustQuery("select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'select * from `read_billing_demo_v4`%' and op_class = 'kv_point_lookup' order by operator_kind").Check(testkit.Rows( + "tikv kv_point_lookup batch_point_get read_request_count ruv2_metrics all v4 v3-resource-formula-uncalibrated 1 4", + "tikv kv_point_lookup point_get read_request_count ruv2_metrics all v4 v3-resource-formula-uncalibrated 1 3", + )) +} + +func TestReadBillingDemoV3MetricsHookLegacy(t *testing.T) { + t.Skip("v3 formula expectations are superseded by TestReadBillingDemoMetricsHook") + metrics.InitExplainRUMetrics() + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + + originEnableStmtSummary := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_stmt_summary").Rows()[0][0]) + defer tk.MustExec(fmt.Sprintf("set global tidb_enable_stmt_summary = %s", originEnableStmtSummary)) + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + + tk.MustQuery("select @@tidb_enable_read_billing_demo").Check(testkit.Rows("0")) + tk.MustExec("drop table if exists read_billing_demo") + tk.MustExec("create table read_billing_demo(a int primary key)") + tk.MustExec("insert into read_billing_demo values (1), (2)") + + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v3", "v2") + unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v3", "v2") + unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v3", "v2") + errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v3", "v2") + projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v3", "v2") + mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v3", "v2") + mutationBytes := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v3", "v2") + writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_keys", "commit_detail", "all", "v3", "v2") + writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_byte", "commit_detail", "all", "v3", "v2") + + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Equal(t, 0.0, readExecutorCounterValue(t, success)) + + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Equal(t, 1.0, readExecutorCounterValue(t, success)) + require.Equal(t, 1.0, readExecutorCounterValue(t, projectionFixedEvents)) + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events > 0, sum_read_billing_demo_input_rows > 0, sum_read_billing_demo_input_bytes = 0 from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 1 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width = 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_chunk_bytes all v3 v2 1 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) + tk.MustQuery(`select column_name from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and ordinal_position = 1`).Check(testkit.Rows("INSTANCE")) + // Append the diagnostic dimension so consumers that depend on the existing + // base-unit column ordinals keep seeing UNIT and every later legacy column + // at the same position. + tk.MustQuery(`select column_name, ordinal_position from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and column_name in ('UNIT', 'DML_KIND') order by ordinal_position`).Check(testkit.Rows( + "UNIT 14", + "DML_KIND 22", + )) + tk.MustExec("set tidb_enable_read_billing_demo=on") + + beforeRestrictedSuccess := readExecutorCounterValue(t, success) + beforeRestrictedBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) + internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnOthers) + _, _, restrictedErr := tk.Session().GetRestrictedSQLExecutor().ExecRestrictedSQL(internalCtx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, "select 1 + 1") + require.NoError(t, restrictedErr) + require.Equal(t, beforeRestrictedSuccess, readExecutorCounterValue(t, success)) + require.Equal(t, beforeRestrictedBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + + // The statement-taking restricted API must also mark a current-session DML + // as internal. It may modify its target table, but must not become a + // foreground mutation sample merely because the preview flag is enabled. + tk.MustExec("create table read_billing_internal(a int primary key)") + beforeRestrictedSuccess = readExecutorCounterValue(t, success) + beforeRestrictedMutation := readExecutorCounterValue(t, mutationCount) + internalInsert, err := session.ParseWithParams4Test(context.Background(), tk.Session(), "insert into read_billing_internal values (1)") + require.NoError(t, err) + _, _, restrictedErr = tk.Session().GetRestrictedSQLExecutor().ExecRestrictedStmt(internalCtx, internalInsert, sqlexec.ExecOptionUseCurSession) + require.NoError(t, restrictedErr) + tk.MustQuery("select * from read_billing_internal").Check(testkit.Rows("1")) + require.Equal(t, beforeRestrictedSuccess, readExecutorCounterValue(t, success)) + require.Equal(t, beforeRestrictedMutation, readExecutorCounterValue(t, mutationCount)) + + // Preview collection is isolated from production RUv2. Equivalent INSERTs + // with the flag off/on retain identical RUv2 inputs, while only the enabled + // statement publishes preview mutation units. + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("create table read_billing_ruv2_off(a int primary key)") + tk.MustExec("create table read_billing_ruv2_on(a int primary key)") + tk.MustExec("create table read_billing_plain_explain(a int primary key)") + beforeFlagOffMutation := readExecutorCounterValue(t, mutationCount) + tk.MustExec("insert into read_billing_ruv2_off values (1)") + offRUV2 := tk.Session().GetSessionVars().RUV2Metrics.Clone() + require.Nil(t, tk.Session().GetSessionVars().StmtCtx.PreviewKVMutationRecorder) + require.Equal(t, beforeFlagOffMutation, readExecutorCounterValue(t, mutationCount)) + plainExplainRows := tk.MustQuery("explain analyze insert into read_billing_plain_explain values (1)").Rows() + require.NotEmpty(t, plainExplainRows) + require.Nil(t, tk.Session().GetSessionVars().StmtCtx.PreviewKVMutationRecorder) + require.Equal(t, beforeFlagOffMutation, readExecutorCounterValue(t, mutationCount)) + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustExec("insert into read_billing_ruv2_on values (1)") + onRUV2 := tk.Session().GetSessionVars().RUV2Metrics.Clone() + require.Equal(t, offRUV2.ExecutorL5InsertRows(), onRUV2.ExecutorL5InsertRows()) + require.Equal(t, offRUV2.PlanCnt(), onRUV2.PlanCnt()) + require.Equal(t, offRUV2.PlanDeriveStatsPaths(), onRUV2.PlanDeriveStatsPaths()) + require.Equal(t, offRUV2.SessionParserTotal(), onRUV2.SessionParserTotal()) + require.Equal(t, offRUV2.TxnCnt(), onRUV2.TxnCnt()) + require.Equal(t, offRUV2.ResourceManagerReadCnt(), onRUV2.ResourceManagerReadCnt()) + require.Equal(t, offRUV2.ResourceManagerWriteCnt(), onRUV2.ResourceManagerWriteCnt()) + require.Equal(t, offRUV2.WriteKeys(), onRUV2.WriteKeys()) + require.Equal(t, offRUV2.WriteSize(), onRUV2.WriteSize()) + require.Equal(t, offRUV2.PrewriteRegionNum(), onRUV2.PrewriteRegionNum()) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeFlagOffMutation) + + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("prepare read_billing_demo_stmt from 'select ? + 1'") + tk.MustExec("set @read_billing_demo_param := 2") + tk.MustExec("set tidb_enable_read_billing_demo=on") + beforePreparedSuccess := readExecutorCounterValue(t, success) + tk.MustQuery("execute read_billing_demo_stmt using @read_billing_demo_param").Check(testkit.Rows("3")) + require.Equal(t, beforePreparedSuccess+1, readExecutorCounterValue(t, success)) + + beforeBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) + beforeBaseUnitsTotal := readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + beforeMutationCount := readExecutorCounterValue(t, mutationCount) + beforeMutationBytes := readExecutorCounterValue(t, mutationBytes) + beforeWriteKeys := readExecutorCounterValue(t, writeKeys) + beforeWriteByte := readExecutorCounterValue(t, writeByte) + beforeWriteSuccess := readExecutorCounterValue(t, success) + beforeUnsupported := readExecutorCounterValue(t, unsupported) + tk.MustExec("insert into read_billing_demo values (3)") + require.Equal(t, beforeUnsupported, readExecutorCounterValue(t, unsupported)) + require.Equal(t, beforeWriteSuccess+1, readExecutorCounterValue(t, success)) + require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeMutationCount) + require.Greater(t, readExecutorCounterValue(t, mutationBytes), beforeMutationBytes) + require.Greater(t, readExecutorCounterValue(t, writeKeys), beforeWriteKeys) + require.Greater(t, readExecutorCounterValue(t, writeByte), beforeWriteByte) + require.Greater(t, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter), beforeBaseUnitsTotal) + tk.MustQuery("select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_demo`%' and site = 'statement' group by status, reason").Check(testkit.Rows("success none 1")) + tk.MustQuery("select site, op_class, operator_kind, dml_kind, unit, input_source, input_side, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_demo`%' and unit in ('encoded_mutation_bytes', 'encoded_mutation_count', 'write_byte', 'write_keys') order by site, unit").Check(testkit.Rows( + "tidb kv_mutation memdb_mutation insert encoded_mutation_bytes stmt_memdb_mutation_calls all 1", + "tidb kv_mutation memdb_mutation insert encoded_mutation_count stmt_memdb_mutation_calls all 1", + "tikv kv_write txn_prewrite insert write_byte commit_detail all 1", + "tikv kv_write txn_prewrite insert write_keys commit_detail all 1", + )) + + // Explicit-transaction DML keeps statement-local TiDB units and reports a + // missing TiKV payload. The final COMMIT owns the transaction-scoped + // prewrite payload without guessing a DML kind or backfilling prior SQL. + tk.MustExec("create table read_billing_explicit_txn(a int primary key)") + tk.MustExec("begin") + tk.MustExec("insert into read_billing_explicit_txn values (1)") + tk.MustQuery("select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_explicit_txn`%' and site = 'tikv'").Check(testkit.Rows("0")) + tk.MustQuery("select status, reason from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_explicit_txn`%' and site = 'tikv' and op_class = 'kv_write'").Check(testkit.Rows("partial missing_commit_detail")) + tk.MustExec("commit") + tk.MustQuery("select unit, input_source, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'commit' and site = 'tikv' and op_class = 'kv_write' and unit in ('write_byte', 'write_keys') order by unit").Check(testkit.Rows( + "write_byte commit_detail 1", + "write_keys commit_detail 1", + )) + tk.MustQuery("select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'commit' and site = 'tikv' and dml_kind <> ''").Check(testkit.Rows("0")) + + beforeUnknownInput := readExecutorCounterValue(t, unknownInput) + beforeBaseUnits = readExecutorCounterValue(t, projectionFixedEvents) + beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + plannercore.RecordReadBillingDemoForStatement(tk.Session(), nil, nil, nil) + require.Equal(t, beforeUnknownInput+1, readExecutorCounterValue(t, unknownInput)) + require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + beforeUnknownInput = readExecutorCounterValue(t, unknownInput) + beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + tk.MustQuery("select * from read_billing_demo where a > 0").Sort().Check(testkit.Rows("1", "2", "3")) + require.Equal(t, beforeUnknownInput+1, readExecutorCounterValue(t, unknownInput)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + tk.MustQuery(`select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?' and site = 'statement' group by status, reason`).Check(testkit.Rows("unknown_input missing_scan_width_evidence 1")) + tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?'`).Check(testkit.Rows("0")) + + // Inject an error after AddRecord has emitted its MemDB mutation. Statement + // staging cleanup removes the table change but must not erase the attempted + // mutation counters. + tk.MustExec("create table read_billing_fail(a int primary key)") + beforeMutationCount = readExecutorCounterValue(t, mutationCount) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/table/tables/corruptMutations", `return("unknown-command")`)) + err = tk.ExecToErr("insert into read_billing_fail values (1)") + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/table/tables/corruptMutations")) + require.Error(t, err) + tk.MustQuery("select count(*) from read_billing_fail").Check(testkit.Rows("0")) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeMutationCount) + tk.MustQuery("select status, reason from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_fail`%' and site = 'statement'").Check(testkit.Rows("error statement_error")) + tk.MustQuery("select dml_kind, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_fail`%' and site = 'tidb' and op_class = 'kv_mutation' and unit = 'encoded_mutation_count'").Check(testkit.Rows("insert 1")) + + // A retryable autocommit failure replays the whole statement. Both the + // original record Set and the replayed Set are attempted local mutation + // work, even though the injected commit error ultimately rolls them back. + tk.MustExec("set tidb_txn_mode='optimistic'") + tk.MustExec("set tidb_retry_limit=1") + tk.MustExec("create table read_billing_retry(a int primary key, b int)") + tk.MustExec("insert into read_billing_retry values (1, 1)") + beforeMutationCount = readExecutorCounterValue(t, mutationCount) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/session/mockCommitError8942", `return(true)`)) + err = tk.ExecToErr("update read_billing_retry set b = b + 1 where a = 1") + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/session/mockCommitError8942")) + require.Error(t, err) + require.True(t, kv.ErrTxnRetryable.Equal(err), err) + require.Equal(t, beforeMutationCount+2, readExecutorCounterValue(t, mutationCount)) + tk.MustQuery("select b from read_billing_retry where a = 1").Check(testkit.Rows("1")) + tk.MustQuery("select dml_kind, value from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'update `read_billing_retry`%' and site = 'tidb' and op_class = 'kv_mutation' and unit = 'encoded_mutation_count'").Check(testkit.Rows("update 2")) + + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("create table read_billing_compile_error(a int)") + tk.MustExec("prepare read_billing_compile_error_stmt from 'select * from read_billing_compile_error'") + tk.MustExec("drop table read_billing_compile_error") + tk.MustExec("set tidb_enable_read_billing_demo=on") + beforeError := readExecutorCounterValue(t, errorStatus) + beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + err = tk.ExecToErr("execute read_billing_compile_error_stmt") + require.Error(t, err) + require.Equal(t, beforeError+1, readExecutorCounterValue(t, errorStatus)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + beforeEarlyError := readExecutorCounterValue(t, errorStatus) + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = tk.ExecWithContext(canceledCtx, "select 1") + require.Error(t, err) + require.Equal(t, beforeEarlyError+1, readExecutorCounterValue(t, errorStatus)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + tk.MustExec("drop table if exists read_billing_outer") + tk.MustExec("create table read_billing_outer(a int)") + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("insert into read_billing_outer values (0)") + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustExec("set @@tidb_init_chunk_size=1") + rs, err := tk.Exec("select (select t.a from read_billing_demo t where t.a > o.a) from read_billing_outer o") + require.NoError(t, err) + err = rs.Next(context.TODO(), rs.NewChunk(nil)) + require.Error(t, err) + require.NoError(t, rs.Close()) + require.Equal(t, beforeEarlyError+2, readExecutorCounterValue(t, errorStatus)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + tk.MustQuery(`select sum(count) from information_schema.statements_summary_read_billing_demo_status where site = 'statement' and status = 'error' and reason = 'statement_error'`).Check(testkit.Rows("5")) + tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units b join information_schema.statements_summary_read_billing_demo_status s on b.digest = s.digest where s.status in ('unsupported', 'error') and b.digest_text not like 'insert into ` + "`read_billing_fail`" + `%' and b.digest_text not like 'update ` + "`read_billing_retry`" + `%'`).Check(testkit.Rows("0")) +} + +func readExecutorCounterValue(t *testing.T, counter prometheus.Counter) float64 { + t.Helper() + m := &dto.Metric{} + require.NoError(t, counter.Write(m)) + return m.GetCounter().GetValue() +} + +func readExecutorCounterVecTotal(t *testing.T, collector prometheus.Collector) float64 { + t.Helper() + ch := make(chan prometheus.Metric) + go func() { + collector.Collect(ch) + close(ch) + }() + var total float64 + for metric := range ch { + m := &dto.Metric{} + require.NoError(t, metric.Write(m)) + total += m.GetCounter().GetValue() + } + return total +} + func TestExplainImportFromSelect(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/pkg/executor/explainfor_test.go b/pkg/executor/explainfor_test.go index d2238ae879c60..1d25cdd26c045 100644 --- a/pkg/executor/explainfor_test.go +++ b/pkg/executor/explainfor_test.go @@ -336,6 +336,28 @@ func TestExplainDotForExplainPlan(t *testing.T) { require.Contains(t, err.Error(), "explain format 'dot' for connection is not supported now") } +func TestExplainRUForConnectionUnsupported(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + + connIDRows := tk.MustQuery("select connection_id()").Rows() + require.Len(t, connIDRows, 1) + connID := connIDRows[0][0].(string) + + tkProcess := tk.Session().ShowProcess() + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: []*sessmgr.ProcessInfo{tkProcess}}) + err := tk.ExecToErr(fmt.Sprintf("explain format='ru' for connection %s", connID)) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_for_connection") + + tk.MustQuery("select 1") + tkProcess = tk.Session().ShowProcess() + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: []*sessmgr.ProcessInfo{tkProcess}}) + err = tk.ExecToErr(fmt.Sprintf("explain format='ru' for connection %s", connID)) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_for_connection") +} + func TestExplainDotForQuery(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/pkg/executor/internal/exec/BUILD.bazel b/pkg/executor/internal/exec/BUILD.bazel index c258a46e1eb71..f3976f9b54f89 100644 --- a/pkg/executor/internal/exec/BUILD.bazel +++ b/pkg/executor/internal/exec/BUILD.bazel @@ -42,16 +42,22 @@ go_test( ], embed = [":exec"], flaky = True, - shard_count = 8, + shard_count = 9, deps = [ "//pkg/domain", + "//pkg/expression", "//pkg/kv", "//pkg/parser/ast", + "//pkg/parser/mysql", "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", "//pkg/statistics", "//pkg/statistics/handle/usage/indexusage", "//pkg/testkit", + "//pkg/types", "//pkg/types/parser_driver", + "//pkg/util/chunk", + "//pkg/util/execdetails", "//pkg/util/logutil", "//pkg/util/mock", "@com_github_pingcap_tipb//go-tipb", diff --git a/pkg/executor/internal/exec/executor.go b/pkg/executor/internal/exec/executor.go index d0a1efe910bb2..637f8e643b907 100644 --- a/pkg/executor/internal/exec/executor.go +++ b/pkg/executor/internal/exec/executor.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" @@ -41,8 +40,10 @@ import ( // nextIOAcc accumulates input volume for a single Executor.Next() invocation. type nextIOAcc struct { - inRows int64 - inCells int64 + inRows int64 + inCells int64 + inBytes int64 + trackBytes bool } func (a *nextIOAcc) reset() { @@ -51,6 +52,8 @@ func (a *nextIOAcc) reset() { } stdatomic.StoreInt64(&a.inRows, 0) stdatomic.StoreInt64(&a.inCells, 0) + stdatomic.StoreInt64(&a.inBytes, 0) + a.trackBytes = false } func calcCellCount(rows, cols int) int64 { @@ -60,13 +63,16 @@ func calcCellCount(rows, cols int) int64 { return int64(rows) * int64(cols) } -// addInput adds rows and cells (rows*cols) into the accumulator. -func (a *nextIOAcc) addInput(rows, cols int) { +// addInput adds rows, cells (rows*cols), and optional bytes into the accumulator. +func (a *nextIOAcc) addInput(rows, cols int, bytes int64) { if a == nil || rows <= 0 { return } stdatomic.AddInt64(&a.inRows, int64(rows)) stdatomic.AddInt64(&a.inCells, calcCellCount(rows, cols)) + if bytes > 0 { + stdatomic.AddInt64(&a.inBytes, bytes) + } } type nextIOAccKeyType struct{} @@ -392,10 +398,12 @@ type executorStats struct { normalizedSQL string normalizedPlan string inRestrictedSQL bool + collectRuntimeBytes bool } // newExecutorStats creates a new `executorStats` -func newExecutorStats(stmtCtx *stmtctx.StatementContext, id int) executorStats { +func newExecutorStats(vars *variable.SessionVars, id int) executorStats { + stmtCtx := vars.StmtCtx normalizedSQL, sqlDigest := stmtCtx.SQLDigest() normalizedPlan, planDigest := stmtCtx.GetPlanDigest() e := executorStats{ @@ -405,6 +413,7 @@ func newExecutorStats(stmtCtx *stmtctx.StatementContext, id int) executorStats { normalizedPlan: normalizedPlan, planDigest: planDigest, inRestrictedSQL: stmtCtx.InRestrictedSQL, + collectRuntimeBytes: needReadBillingRuntimeBytes(vars), } if stmtCtx.RuntimeStatsColl != nil { @@ -416,6 +425,17 @@ func newExecutorStats(stmtCtx *stmtctx.StatementContext, id int) executorStats { return e } +func needReadBillingRuntimeBytes(vars *variable.SessionVars) bool { + if vars == nil || vars.StmtCtx == nil { + return false + } + if vars.EnableReadBillingDemo { + return true + } + stmtCtx := vars.StmtCtx + return stmtCtx.InExplainStmt && stmtCtx.InExplainAnalyzeStmt && stmtCtx.ExplainFormat == types.ExplainFormatRU +} + // RuntimeStats returns the runtime stats of an executor. func (e *executorStats) RuntimeStats() *execdetails.BasicRuntimeStats { return e.runtimeStats @@ -464,7 +484,7 @@ func NewBaseExecutorV2(vars *variable.SessionVars, schema *expression.Schema, id executorMeta := newExecutorMeta(schema, id, children...) e := BaseExecutorV2{ executorMeta: executorMeta, - executorStats: newExecutorStats(vars.StmtCtx, id), + executorStats: newExecutorStats(vars, id), executorChunkAllocator: newExecutorChunkAllocator(vars, executorMeta.RetFieldTypes()), executorKillerHandler: newExecutorKillerHandler(&vars.SQLKiller), } @@ -512,6 +532,10 @@ func (e *BaseExecutorV2) ruv2NextCache() *ruv2NextCacheState { return &e.ruv2CacheState } +func (e *BaseExecutorV2) needReadBillingRuntimeBytes() bool { + return e.collectRuntimeBytes && e.runtimeStats != nil +} + // BuildNewBaseExecutorV2 builds a new `BaseExecutorV2` based on the configuration of the current base executor. // It's used to build a new sub-executor from an existing executor. For example, the `IndexLookUpExecutor` will use // this function to build `TableReaderExecutor` @@ -630,9 +654,10 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { err = util.GetRecoverError(r) } }() - if e.RuntimeStats() != nil { + runtimeStats := e.RuntimeStats() + if runtimeStats != nil { start := time.Now() - defer func() { e.RuntimeStats().Record(time.Since(start), req.NumRows()) }() + defer func() { runtimeStats.Record(time.Since(start), req.NumRows()) }() } if err := e.HandleSQLKillerSignal(); err != nil { @@ -640,11 +665,12 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { } var ( - regionName string - info ruv2ExecutorMetric - trackRUV2 bool - ruv2Metrics *execdetails.RUV2Metrics - recorder execdetails.ExecutorMetricRecorder + regionName string + info ruv2ExecutorMetric + trackRUV2 bool + trackRuntimeBytes bool + ruv2Metrics *execdetails.RUV2Metrics + recorder execdetails.ExecutorMetricRecorder ) if provider, ok := e.(ruv2CacheProvider); ok { cache := provider.ruv2NextCache() @@ -670,16 +696,19 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { // A tracked-type executor whose statement is bypassed leaves ruv2Metrics nil // and must skip the per-child IO accumulator setup as well as the late update. trackRUV2 = ruv2Metrics != nil + if provider, ok := e.(interface{ needReadBillingRuntimeBytes() bool }); ok { + trackRuntimeBytes = provider.needReadBillingRuntimeBytes() + } r, ctx := tracing.StartRegionEx(ctx, regionName) defer r.End() parentAcc, _ := ctx.Value(nextIOAccKey).(*nextIOAcc) childCount := 0 - if trackRUV2 || parentAcc != nil { + if trackRUV2 || trackRuntimeBytes || parentAcc != nil { childCount = len(e.AllChildren()) } - needLocalAcc := needNextIOAcc(trackRUV2, parentAcc, childCount) + needLocalAcc := needNextIOAcc(trackRUV2 || trackRuntimeBytes, parentAcc, childCount) var myAcc *nextIOAcc if needLocalAcc { // Keep descendant IO local to this executor before optionally bubbling the @@ -687,6 +716,7 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { // local accumulator, and BaseExecutorV2-backed executors can reuse one // across Next() calls. myAcc = getReusableNextIOAcc(e) + myAcc.trackBytes = trackRuntimeBytes ctx = context.WithValue(ctx, nextIOAccKey, myAcc) } @@ -699,8 +729,22 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { outRows := req.NumRows() outCols := req.NumCols() + var outBytes int64 + if trackRuntimeBytes || (parentAcc != nil && parentAcc.trackBytes) { + outBytes = req.LogicalLiveBytes() + } if parentAcc != nil { - parentAcc.addInput(outRows, outCols) + parentAcc.addInput(outRows, outCols, outBytes) + } + + var inRows, inCells, inBytes int64 + if myAcc != nil { + inRows = stdatomic.LoadInt64(&myAcc.inRows) + inCells = stdatomic.LoadInt64(&myAcc.inCells) + inBytes = stdatomic.LoadInt64(&myAcc.inBytes) + } + if trackRuntimeBytes && runtimeStats != nil { + runtimeStats.RecordBytes(inBytes, outBytes) } if !trackRUV2 { @@ -708,11 +752,6 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { return e.HandleSQLKillerSignal() } - var inRows, inCells int64 - if myAcc != nil { - inRows = stdatomic.LoadInt64(&myAcc.inRows) - inCells = stdatomic.LoadInt64(&myAcc.inCells) - } outCells := calcCellCount(outRows, outCols) addRUV2ExecutorMetricCached(ruv2Metrics, info, recorder, inRows, int64(outRows), inCells, outCells) // recheck whether the session/query is killed during the Next() diff --git a/pkg/executor/internal/exec/executor_test.go b/pkg/executor/internal/exec/executor_test.go index 9d67c38a468f8..4944ddb0c14b9 100644 --- a/pkg/executor/internal/exec/executor_test.go +++ b/pkg/executor/internal/exec/executor_test.go @@ -15,8 +15,15 @@ package exec import ( + "context" "testing" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx/variable" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/mock" "github.com/stretchr/testify/require" ) @@ -36,26 +43,28 @@ func TestNextIOAccAddInputCountsRowsWithZeroCols(t *testing.T) { t.Run("add input counts rows with zero cols", func(t *testing.T) { acc := &nextIOAcc{} - acc.addInput(3, 0) + acc.addInput(3, 0, 0) require.Equal(t, int64(3), acc.inRows) require.Equal(t, int64(0), acc.inCells) + require.Equal(t, int64(0), acc.inBytes) }) t.Run("base executor reuses local accumulator state", func(t *testing.T) { exec := newMockNextIOAccExecutor() first := getReusableNextIOAcc(exec) - first.addInput(4, 2) + first.addInput(4, 2, 16) second := getReusableNextIOAcc(exec) require.Same(t, first, second) require.Equal(t, int64(0), second.inRows) require.Equal(t, int64(0), second.inCells) + require.Equal(t, int64(0), second.inBytes) allocs := testing.AllocsPerRun(1000, func() { acc := getReusableNextIOAcc(exec) - acc.addInput(1, 1) + acc.addInput(1, 1, 1) }) require.Less(t, allocs, 1.) }) @@ -70,6 +79,70 @@ func TestNextIOAccAddInputCountsRowsWithZeroCols(t *testing.T) { }) } +type mockRuntimeBytesExecutor struct { + BaseExecutorV2 + produce func(*chunk.Chunk) + returned bool +} + +func newMockRuntimeBytesExecutor(vars *variable.SessionVars, schema *expression.Schema, id int, produce func(*chunk.Chunk), children ...Executor) *mockRuntimeBytesExecutor { + return &mockRuntimeBytesExecutor{ + BaseExecutorV2: NewBaseExecutorV2(vars, schema, id, children...), + produce: produce, + } +} + +func (e *mockRuntimeBytesExecutor) Next(ctx context.Context, req *chunk.Chunk) error { + req.Reset() + if e.returned { + return nil + } + e.returned = true + for _, child := range e.AllChildren() { + childReq := NewFirstChunk(child) + if err := Next(ctx, child, childReq); err != nil { + return err + } + } + if e.produce != nil { + e.produce(req) + } + return nil +} + +func TestNextRecordsRuntimeBytesForReadBilling(t *testing.T) { + ctx := mock.NewContext() + vars := ctx.GetSessionVars() + vars.EnableReadBillingDemo = true + vars.StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + schema := expression.NewSchema( + &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)}, + &expression.Column{RetType: types.NewFieldType(mysql.TypeVarchar)}, + ) + + child := newMockRuntimeBytesExecutor(vars, schema, 1, func(req *chunk.Chunk) { + req.AppendInt64(0, 1) + req.AppendString(1, "abc") + req.AppendInt64(0, 2) + req.AppendString(1, "d") + }) + parent := newMockRuntimeBytesExecutor(vars, schema, 2, func(req *chunk.Chunk) { + req.AppendInt64(0, 3) + req.AppendString(1, "ef") + }, child) + + req := NewFirstChunk(parent) + require.NoError(t, Next(context.Background(), parent, req)) + + childStats := vars.StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(1, false) + parentStats := vars.StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(2, false) + require.True(t, childStats.HasBytes()) + require.True(t, parentStats.HasBytes()) + require.Equal(t, int64(46), childStats.GetOutputBytes()) + require.Equal(t, childStats.GetOutputBytes(), parentStats.GetInputBytes()) + require.Equal(t, int64(28), parentStats.GetOutputBytes()) +} + func TestRUV2ExecutorMetricByTypeIncludesConcreteExecutorTypes(t *testing.T) { cases := map[string]ruv2ExecutorMetric{ "*aggregate.HashAggExec": {level: 2, label: "HashAggExec", useCells: false}, diff --git a/pkg/executor/join/hash_join_stats.go b/pkg/executor/join/hash_join_stats.go index 10d82b323c246..d538b828f252f 100644 --- a/pkg/executor/join/hash_join_stats.go +++ b/pkg/executor/join/hash_join_stats.go @@ -17,6 +17,7 @@ package join import ( "bytes" "fmt" + "math" "strconv" "sync/atomic" "time" @@ -25,6 +26,37 @@ import ( "github.com/pingcap/tidb/pkg/util/execdetails" ) +func addHashTableRows(addr *int64, rows uint64) { + if rows > math.MaxInt64 { + atomic.StoreInt64(addr, -1) + return + } + delta := int64(rows) + for { + current := atomic.LoadInt64(addr) + if current < 0 { + return + } + if delta > math.MaxInt64-current { + if atomic.CompareAndSwapInt64(addr, current, -1) { + return + } + continue + } + if atomic.CompareAndSwapInt64(addr, current, current+delta) { + return + } + } +} + +func mergeHashTableRows(addr *int64, rows int64) { + if rows < 0 { + atomic.StoreInt64(addr, -1) + return + } + addHashTableRows(addr, uint64(rows)) +} + func writeSpilledPartitionNumStatsToString(buf *bytes.Buffer, partitionNum int, spilledPartitionNumPerRound []int) { buf.WriteString("[") fmt.Fprintf(buf, "%d/%d", spilledPartitionNumPerRound[0], partitionNum) @@ -53,8 +85,11 @@ type hashJoinRuntimeStats struct { probe int64 concurrent int maxFetchAndProbe int64 + hashTableRows int64 } +func (e *hashJoinRuntimeStats) HashTableRows() int64 { return atomic.LoadInt64(&e.hashTableRows) } + // Tp implements the RuntimeStats interface. func (*hashJoinRuntimeStats) Tp() int { return execdetails.TpHashJoinRuntimeStats @@ -101,6 +136,7 @@ func (e *hashJoinRuntimeStats) Clone() execdetails.RuntimeStats { probe: e.probe, concurrent: e.concurrent, maxFetchAndProbe: e.maxFetchAndProbe, + hashTableRows: e.HashTableRows(), } } @@ -114,6 +150,7 @@ func (e *hashJoinRuntimeStats) Merge(rs execdetails.RuntimeStats) { e.hashStat.probeCollision += tmp.hashStat.probeCollision e.fetchAndProbe += tmp.fetchAndProbe e.probe += tmp.probe + mergeHashTableRows(&e.hashTableRows, tmp.HashTableRows()) if e.maxFetchAndProbe < tmp.maxFetchAndProbe { e.maxFetchAndProbe = tmp.maxFetchAndProbe } @@ -163,9 +200,12 @@ type hashJoinRuntimeStatsV2 struct { spill spillStats - isHashJoinGA bool + isHashJoinGA bool + hashTableRows int64 } +func (e *hashJoinRuntimeStatsV2) HashTableRows() int64 { return atomic.LoadInt64(&e.hashTableRows) } + func setMaxValue(addr *int64, currentValue int64) { for { value := atomic.LoadInt64(addr) @@ -179,6 +219,7 @@ func setMaxValue(addr *int64, currentValue int64) { } func (e *hashJoinRuntimeStatsV2) reset() { + atomic.StoreInt64(&e.hashTableRows, 0) e.probeCollision = 0 e.fetchAndBuildHashTable = 0 e.partitionData = 0 @@ -308,6 +349,7 @@ func (e *hashJoinRuntimeStatsV2) Clone() execdetails.RuntimeStats { maxBuildHashTableForCurrentRound: e.maxBuildHashTableForCurrentRound, maxProbeForCurrentRound: e.maxProbeForCurrentRound, maxWorkerFetchAndProbeForCurrentRound: e.maxWorkerFetchAndProbeForCurrentRound, + hashTableRows: e.HashTableRows(), } } @@ -326,6 +368,7 @@ func (e *hashJoinRuntimeStatsV2) Merge(rs execdetails.RuntimeStats) { e.maxPartitionData = tmp.maxPartitionData } e.probeCollision += tmp.probeCollision + mergeHashTableRows(&e.hashTableRows, tmp.HashTableRows()) e.fetchAndProbe += tmp.fetchAndProbe e.probe += tmp.probe if e.maxWorkerFetchAndProbe < tmp.maxWorkerFetchAndProbe { diff --git a/pkg/executor/join/hash_join_v1.go b/pkg/executor/join/hash_join_v1.go index 6976ea6a834eb..6e90320e5345a 100644 --- a/pkg/executor/join/hash_join_v1.go +++ b/pkg/executor/join/hash_join_v1.go @@ -1082,6 +1082,8 @@ func (e *HashJoinV1Exec) fetchAndBuildHashTable(ctx context.Context) { if err == nil { if err = <-fetchBuildSideRowsOk; err != nil { e.buildFinished <- err + } else if e.stats != nil && e.RowContainer != nil { + addHashTableRows(&e.stats.hashTableRows, e.RowContainer.hashStateRows()) } } } diff --git a/pkg/executor/join/hash_join_v2.go b/pkg/executor/join/hash_join_v2.go index 51a278bbcce73..7ab2946bf1307 100644 --- a/pkg/executor/join/hash_join_v2.go +++ b/pkg/executor/join/hash_join_v2.go @@ -93,6 +93,24 @@ func (htc *hashTableContext) getAllMemoryUsageInHashTable() int64 { return totalMemoryUsage } +// hashStateRows returns the total number of valid rows in the hash table. +func (htc *hashTableContext) hashStateRows() uint64 { + var rows uint64 + if htc == nil || htc.hashTable == nil { + return 0 + } + for _, table := range htc.hashTable.tables { + if table != nil && table.rowData != nil { + validRows := table.rowData.validKeyCount() + if math.MaxUint64-rows < validRows { + return math.MaxUint64 + } + rows += validRows + } + } + return rows +} + func (htc *hashTableContext) clearHashTable() { partNum := len(htc.hashTable.tables) for i := range partNum { @@ -1324,7 +1342,12 @@ func (e *HashJoinV2Exec) fetchAndBuildHashTableImpl(ctx context.Context) { buildTaskCh := e.createBuildTasks(totalSegmentCnt, wg, errCh, doneCh) e.buildHashTable(buildTaskCh, wg, errCh, doneCh) - waitJobDone(wg, errCh) + if !waitJobDone(wg, errCh) { + return + } + if e.stats != nil { + addHashTableRows(&e.stats.hashTableRows, e.hashTableContext.hashStateRows()) + } } func (e *HashJoinV2Exec) fetchBuildSideRows(ctx context.Context, fetcherAndWorkerSyncer *sync.WaitGroup, wg *sync.WaitGroup, errCh chan error, doneCh chan struct{}) chan *chunk.Chunk { diff --git a/pkg/executor/join/hash_table_v1.go b/pkg/executor/join/hash_table_v1.go index 5e676e7972e8c..cba4df90d41fa 100644 --- a/pkg/executor/join/hash_table_v1.go +++ b/pkg/executor/join/hash_table_v1.go @@ -532,6 +532,18 @@ func (c *hashRowContainer) Len() uint64 { return c.hashTable.Len() } +func (c *hashRowContainer) hashStateRows() uint64 { + rows := c.Len() + if c.hashNANullBucket != nil { + nullRows := uint64(len(c.hashNANullBucket.entries)) + if ^uint64(0)-rows < nullRows { + return ^uint64(0) + } + rows += nullRows + } + return rows +} + func (c *hashRowContainer) Close() error { failpoint.Inject("issue60926", nil) diff --git a/pkg/executor/join/hash_table_v1_test.go b/pkg/executor/join/hash_table_v1_test.go index 775cb87a02be1..d769c8efaa27d 100644 --- a/pkg/executor/join/hash_table_v1_test.go +++ b/pkg/executor/join/hash_table_v1_test.go @@ -84,6 +84,12 @@ func (h hashCollision) Size() int { panic("not implement func (h hashCollision) BlockSize() int { panic("not implemented") } func TestHashRowContainer(t *testing.T) { + stateContainer := &hashRowContainer{ + hashTable: &unsafeHashTable{length: 2}, + hashNANullBucket: &hashNANullBucket{entries: []*naEntry{{}, {}, {}}}, + } + require.Equal(t, uint64(5), stateContainer.hashStateRows()) + hashFunc := fnv.New64 rowContainer, copiedRC := testHashRowContainer(t, hashFunc, false) require.Equal(t, int64(0), rowContainer.stat.probeCollision) diff --git a/pkg/executor/join/join_stats_test.go b/pkg/executor/join/join_stats_test.go index cba47f81e19fb..63927ff79fe12 100644 --- a/pkg/executor/join/join_stats_test.go +++ b/pkg/executor/join/join_stats_test.go @@ -15,6 +15,8 @@ package join import ( + "math" + "sync/atomic" "testing" "time" @@ -22,6 +24,16 @@ import ( ) func TestHashJoinRuntimeStats(t *testing.T) { + overflowRows := int64(1) + addHashTableRows(&overflowRows, math.MaxInt64) + require.Equal(t, int64(-1), overflowRows) + addHashTableRows(&overflowRows, 1) + require.Equal(t, int64(-1), overflowRows) + + rowData := &rowTable{segments: []*rowTableSegment{{validJoinKeyPos: []int{0, 2}}, {validJoinKeyPos: []int{1}}}} + hashCtx := &hashTableContext{hashTable: &hashTableV2{tables: []*subTable{{rowData: rowData}}}} + require.Equal(t, uint64(3), hashCtx.hashStateRows()) + stats := &hashJoinRuntimeStats{ fetchAndBuildHashTable: 2 * time.Second, hashStat: hashStatistic{ @@ -37,6 +49,18 @@ func TestHashJoinRuntimeStats(t *testing.T) { require.Equal(t, stats.Clone().String(), stats.String()) stats.Merge(stats.Clone()) require.Equal(t, "build_hash_table:{total:4s, fetch:3.8s, build:200ms}, probe:{concurrency:4, total:10s, max:2s, probe:8s, fetch and wait:2s, probe_collision:2}", stats.String()) + atomic.StoreInt64(&stats.hashTableRows, 7) + cloned := stats.Clone().(*hashJoinRuntimeStats) + require.Equal(t, int64(7), cloned.HashTableRows()) + stats.Merge(cloned) + require.Equal(t, int64(14), stats.HashTableRows()) + + statsV2 := &hashJoinRuntimeStatsV2{} + atomic.StoreInt64(&statsV2.hashTableRows, 11) + clonedV2 := statsV2.Clone().(*hashJoinRuntimeStatsV2) + require.Equal(t, int64(11), clonedV2.HashTableRows()) + statsV2.Merge(clonedV2) + require.Equal(t, int64(22), statsV2.HashTableRows()) } func TestIndexJoinRuntimeStats(t *testing.T) { diff --git a/pkg/executor/stmtsummary.go b/pkg/executor/stmtsummary.go index 990a1808a66f3..12c2b7a71115e 100644 --- a/pkg/executor/stmtsummary.go +++ b/pkg/executor/stmtsummary.go @@ -159,7 +159,13 @@ func (e *stmtSummaryRetriever) initSummaryRowsReader(sctx sessionctx.Context) (* } var rows [][]types.Datum - if isCumulativeTable(e.table.Name.O) { + if kind, ok := readBillingDemoTableKind(e.table.Name.O); ok { + if isCurrentTable(e.table.Name.O) { + rows = reader.GetReadBillingDemoCurrentRows(kind) + } else if isHistoryTable(e.table.Name.O) { + rows = reader.GetReadBillingDemoHistoryRows(kind) + } + } else if isCumulativeTable(e.table.Name.O) { rows = reader.GetStmtSummaryCumulativeRows() } else if isCurrentTable(e.table.Name.O) { rows = reader.GetStmtSummaryCurrentRows() @@ -254,8 +260,14 @@ func (r *stmtSummaryRetrieverV2) initSummaryRowsReader(ctx context.Context, sctx return nil, err } - mem := stmtsummaryv2.NewMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges) - memRows := mem.Rows() + var memRows [][]types.Datum + var readBillingDemoKind stmtsummary.ReadBillingDemoTableKind + readBillingDemoKind, isReadBillingDemoTable := readBillingDemoTableKind(r.table.Name.O) + if isReadBillingDemoTable { + memRows = stmtsummaryv2.NewReadBillingDemoMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges, readBillingDemoKind).Rows() + } else { + memRows = stmtsummaryv2.NewMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges).Rows() + } var rowsReader *rowsReader if isCurrentTable(r.table.Name.O) { @@ -264,7 +276,13 @@ func (r *stmtSummaryRetrieverV2) initSummaryRowsReader(ctx context.Context, sctx if isHistoryTable(r.table.Name.O) { // history table should return all rows including mem and disk concurrent := sctx.GetSessionVars().Concurrency.DistSQLScanConcurrency() - history, err := stmtsummaryv2.NewHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent) + var history rowsPuller + var err error + if isReadBillingDemoTable { + history, err = stmtsummaryv2.NewReadBillingDemoHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent, readBillingDemoKind) + } else { + history, err = stmtsummaryv2.NewHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent) + } if err != nil { return nil, err } @@ -347,6 +365,10 @@ func isClusterTable(originalTableName string) bool { case infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory, infoschema.ClusterTableStatementsSummaryEvicted, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus, infoschema.ClusterTableTiDBStatementsStats: return true } @@ -367,7 +389,11 @@ func isCumulativeTable(originalTableName string) bool { func isCurrentTable(originalTableName string) bool { switch originalTableName { case infoschema.TableStatementsSummary, - infoschema.ClusterTableStatementsSummary: + infoschema.ClusterTableStatementsSummary, + infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus: return true } @@ -377,13 +403,34 @@ func isCurrentTable(originalTableName string) bool { func isHistoryTable(originalTableName string) bool { switch originalTableName { case infoschema.TableStatementsSummaryHistory, - infoschema.ClusterTableStatementsSummaryHistory: + infoschema.ClusterTableStatementsSummaryHistory, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: return true } return false } +func readBillingDemoTableKind(originalTableName string) (stmtsummary.ReadBillingDemoTableKind, bool) { + switch originalTableName { + case infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits: + return stmtsummary.ReadBillingDemoTableBaseUnits, true + case infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: + return stmtsummary.ReadBillingDemoTableStatus, true + default: + return 0, false + } +} + func isEvictedTable(originalTableName string) bool { switch originalTableName { case infoschema.TableStatementsSummaryEvicted, diff --git a/pkg/infoschema/cluster.go b/pkg/infoschema/cluster.go index 6ed27e538af87..9bc4934a9ccff 100644 --- a/pkg/infoschema/cluster.go +++ b/pkg/infoschema/cluster.go @@ -43,6 +43,14 @@ const ( ClusterTableStatementsSummaryHistory = "CLUSTER_STATEMENTS_SUMMARY_HISTORY" // ClusterTableStatementsSummaryEvicted is the string constant of cluster statement summary evict table. ClusterTableStatementsSummaryEvicted = "CLUSTER_STATEMENTS_SUMMARY_EVICTED" + // ClusterTableStatementsSummaryReadBillingDemoBaseUnits is the string constant of cluster read billing demo base-unit summary table. + ClusterTableStatementsSummaryReadBillingDemoBaseUnits = "CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS" + // ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits is the string constant of cluster read billing demo history base-unit summary table. + ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits = "CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS" + // ClusterTableStatementsSummaryReadBillingDemoStatus is the string constant of cluster read billing demo status summary table. + ClusterTableStatementsSummaryReadBillingDemoStatus = "CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS" + // ClusterTableStatementsSummaryHistoryReadBillingDemoStatus is the string constant of cluster read billing demo history status summary table. + ClusterTableStatementsSummaryHistoryReadBillingDemoStatus = "CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS" // ClusterTableTiDBStatementsStats is the string constant of the cluster statement stats table. ClusterTableTiDBStatementsStats = "CLUSTER_TIDB_STATEMENTS_STATS" // ClusterTableTiDBTrx is the string constant of cluster transaction running table. @@ -63,19 +71,23 @@ const ( // memTableToAllTiDBClusterTables means add memory table to cluster table that will send cop request to all TiDB nodes. var memTableToAllTiDBClusterTables = map[string]string{ - TableSlowQuery: ClusterTableSlowLog, - TableProcesslist: ClusterTableProcesslist, - TableStatementsSummary: ClusterTableStatementsSummary, - TableStatementsSummaryHistory: ClusterTableStatementsSummaryHistory, - TableStatementsSummaryEvicted: ClusterTableStatementsSummaryEvicted, - TableTiDBStatementsStats: ClusterTableTiDBStatementsStats, - TableTiDBTrx: ClusterTableTiDBTrx, - TableDeadlocks: ClusterTableDeadlocks, - TableTrxSummary: ClusterTableTrxSummary, - TableMemoryUsage: ClusterTableMemoryUsage, - TableMemoryUsageOpsHistory: ClusterTableMemoryUsageOpsHistory, - TableTiDBIndexUsage: ClusterTableTiDBIndexUsage, - TableTiDBPlanCache: ClusterTableTiDBPlanCache, + TableSlowQuery: ClusterTableSlowLog, + TableProcesslist: ClusterTableProcesslist, + TableStatementsSummary: ClusterTableStatementsSummary, + TableStatementsSummaryHistory: ClusterTableStatementsSummaryHistory, + TableStatementsSummaryEvicted: ClusterTableStatementsSummaryEvicted, + TableStatementsSummaryReadBillingDemoBaseUnits: ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + TableStatementsSummaryHistoryReadBillingDemoBaseUnits: ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + TableStatementsSummaryReadBillingDemoStatus: ClusterTableStatementsSummaryReadBillingDemoStatus, + TableStatementsSummaryHistoryReadBillingDemoStatus: ClusterTableStatementsSummaryHistoryReadBillingDemoStatus, + TableTiDBStatementsStats: ClusterTableTiDBStatementsStats, + TableTiDBTrx: ClusterTableTiDBTrx, + TableDeadlocks: ClusterTableDeadlocks, + TableTrxSummary: ClusterTableTrxSummary, + TableMemoryUsage: ClusterTableMemoryUsage, + TableMemoryUsageOpsHistory: ClusterTableMemoryUsageOpsHistory, + TableTiDBIndexUsage: ClusterTableTiDBIndexUsage, + TableTiDBPlanCache: ClusterTableTiDBPlanCache, } var memTableToAllTiDBClusterTablesWithLowerCase = make(map[string]string) diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index 3072d29e07c87..a439a67057de9 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -170,6 +170,14 @@ const ( TableStatementsSummaryHistory = "STATEMENTS_SUMMARY_HISTORY" // TableStatementsSummaryEvicted is the string constant of statements summary evicted table. TableStatementsSummaryEvicted = "STATEMENTS_SUMMARY_EVICTED" + // TableStatementsSummaryReadBillingDemoBaseUnits is the string constant of read billing demo base-unit summary table. + TableStatementsSummaryReadBillingDemoBaseUnits = "STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS" + // TableStatementsSummaryHistoryReadBillingDemoBaseUnits is the string constant of read billing demo history base-unit summary table. + TableStatementsSummaryHistoryReadBillingDemoBaseUnits = "STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS" + // TableStatementsSummaryReadBillingDemoStatus is the string constant of read billing demo status summary table. + TableStatementsSummaryReadBillingDemoStatus = "STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS" + // TableStatementsSummaryHistoryReadBillingDemoStatus is the string constant of read billing demo history status summary table. + TableStatementsSummaryHistoryReadBillingDemoStatus = "STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS" // TableTiDBStatementsStats is the string constant of the TiDB statement stats table. TableTiDBStatementsStats = "TIDB_STATEMENTS_STATS" // TableStorageStats is a table that contains all tables disk usage @@ -319,41 +327,49 @@ var tableIDMap = map[string]int64{ TableTiFlashSegments: autoid.InformationSchemaDBID + 65, // Removed, see https://github.com/pingcap/tidb/issues/28890 //TablePlacementPolicy: autoid.InformationSchemaDBID + 66, - TableClientErrorsSummaryGlobal: autoid.InformationSchemaDBID + 67, - TableClientErrorsSummaryByUser: autoid.InformationSchemaDBID + 68, - TableClientErrorsSummaryByHost: autoid.InformationSchemaDBID + 69, - TableTiDBTrx: autoid.InformationSchemaDBID + 70, - ClusterTableTiDBTrx: autoid.InformationSchemaDBID + 71, - TableDeadlocks: autoid.InformationSchemaDBID + 72, - ClusterTableDeadlocks: autoid.InformationSchemaDBID + 73, - TableDataLockWaits: autoid.InformationSchemaDBID + 74, - TableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 75, - ClusterTableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 76, - TableAttributes: autoid.InformationSchemaDBID + 77, - TableTiDBHotRegionsHistory: autoid.InformationSchemaDBID + 78, - TablePlacementPolicies: autoid.InformationSchemaDBID + 79, - TableTrxSummary: autoid.InformationSchemaDBID + 80, - ClusterTableTrxSummary: autoid.InformationSchemaDBID + 81, - TableVariablesInfo: autoid.InformationSchemaDBID + 82, - TableUserAttributes: autoid.InformationSchemaDBID + 83, - TableMemoryUsage: autoid.InformationSchemaDBID + 84, - TableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 85, - ClusterTableMemoryUsage: autoid.InformationSchemaDBID + 86, - ClusterTableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 87, - TableResourceGroups: autoid.InformationSchemaDBID + 88, - TableRunawayWatches: autoid.InformationSchemaDBID + 89, - TableCheckConstraints: autoid.InformationSchemaDBID + 90, - TableTiDBCheckConstraints: autoid.InformationSchemaDBID + 91, - TableKeywords: autoid.InformationSchemaDBID + 92, - TableTiDBIndexUsage: autoid.InformationSchemaDBID + 93, - ClusterTableTiDBIndexUsage: autoid.InformationSchemaDBID + 94, - TableTiFlashIndexes: autoid.InformationSchemaDBID + 95, - TableTiDBPlanCache: autoid.InformationSchemaDBID + 96, - ClusterTableTiDBPlanCache: autoid.InformationSchemaDBID + 97, - TableTiDBStatementsStats: autoid.InformationSchemaDBID + 98, - ClusterTableTiDBStatementsStats: autoid.InformationSchemaDBID + 99, - TableKeyspaceMeta: autoid.InformationSchemaDBID + 100, - TableSchemataExtensions: autoid.InformationSchemaDBID + 101, + TableClientErrorsSummaryGlobal: autoid.InformationSchemaDBID + 67, + TableClientErrorsSummaryByUser: autoid.InformationSchemaDBID + 68, + TableClientErrorsSummaryByHost: autoid.InformationSchemaDBID + 69, + TableTiDBTrx: autoid.InformationSchemaDBID + 70, + ClusterTableTiDBTrx: autoid.InformationSchemaDBID + 71, + TableDeadlocks: autoid.InformationSchemaDBID + 72, + ClusterTableDeadlocks: autoid.InformationSchemaDBID + 73, + TableDataLockWaits: autoid.InformationSchemaDBID + 74, + TableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 75, + ClusterTableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 76, + TableAttributes: autoid.InformationSchemaDBID + 77, + TableTiDBHotRegionsHistory: autoid.InformationSchemaDBID + 78, + TablePlacementPolicies: autoid.InformationSchemaDBID + 79, + TableTrxSummary: autoid.InformationSchemaDBID + 80, + ClusterTableTrxSummary: autoid.InformationSchemaDBID + 81, + TableVariablesInfo: autoid.InformationSchemaDBID + 82, + TableUserAttributes: autoid.InformationSchemaDBID + 83, + TableMemoryUsage: autoid.InformationSchemaDBID + 84, + TableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 85, + ClusterTableMemoryUsage: autoid.InformationSchemaDBID + 86, + ClusterTableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 87, + TableResourceGroups: autoid.InformationSchemaDBID + 88, + TableRunawayWatches: autoid.InformationSchemaDBID + 89, + TableCheckConstraints: autoid.InformationSchemaDBID + 90, + TableTiDBCheckConstraints: autoid.InformationSchemaDBID + 91, + TableKeywords: autoid.InformationSchemaDBID + 92, + TableTiDBIndexUsage: autoid.InformationSchemaDBID + 93, + ClusterTableTiDBIndexUsage: autoid.InformationSchemaDBID + 94, + TableTiFlashIndexes: autoid.InformationSchemaDBID + 95, + TableTiDBPlanCache: autoid.InformationSchemaDBID + 96, + ClusterTableTiDBPlanCache: autoid.InformationSchemaDBID + 97, + TableTiDBStatementsStats: autoid.InformationSchemaDBID + 98, + ClusterTableTiDBStatementsStats: autoid.InformationSchemaDBID + 99, + TableKeyspaceMeta: autoid.InformationSchemaDBID + 100, + TableSchemataExtensions: autoid.InformationSchemaDBID + 101, + TableStatementsSummaryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 102, + TableStatementsSummaryHistoryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 103, + ClusterTableStatementsSummaryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 104, + ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 105, + TableStatementsSummaryReadBillingDemoStatus: autoid.InformationSchemaDBID + 106, + TableStatementsSummaryHistoryReadBillingDemoStatus: autoid.InformationSchemaDBID + 107, + ClusterTableStatementsSummaryReadBillingDemoStatus: autoid.InformationSchemaDBID + 108, + ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: autoid.InformationSchemaDBID + 109, } // columnInfo represents the basic column information of all kinds of INFORMATION_SCHEMA tables @@ -1429,6 +1445,9 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.AvgQueuedRcTimeStr, tp: mysql.TypeLonglong, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of waiting for available request-units"}, {name: stmtsummary.MaxRequestUnitV2Str, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Max request-unit v2 cost of these statements"}, {name: stmtsummary.AvgRequestUnitV2Str, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Average request-unit v2 cost of these statements"}, + {name: stmtsummary.SumReadBillingDemoFixedEventsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Legacy v3 read billing demo fixed-event units; zero for v4"}, + {name: stmtsummary.SumReadBillingDemoInputRowsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Legacy v3 read billing demo input-row units; zero for v4"}, + {name: stmtsummary.SumReadBillingDemoInputBytesStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Legacy v3 read billing demo input-byte units; zero for v4"}, {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, {name: stmtsummary.PlanCacheUnqualifiedStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag, comment: "The number of times that these statements are not supported by the plan cache"}, {name: stmtsummary.PlanCacheUnqualifiedLastReasonStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "The last reason why the statement is not supported by the plan cache"}, @@ -1444,6 +1463,50 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.StorageMPPStr, tp: mysql.TypeTiny, size: 1, flag: mysql.NotNullFlag, comment: "Whether the last statement read data from TiFlash"}, } +var tableStatementsSummaryReadBillingDemoBaseUnitCols = []columnInfo{ + {name: stmtsummary.SummaryBeginTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "Begin time of this summary"}, + {name: stmtsummary.SummaryEndTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "End time of this summary"}, + {name: stmtsummary.StmtTypeStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"}, + {name: stmtsummary.SchemaNameStr, tp: mysql.TypeVarchar, size: 64, comment: "Current schema"}, + {name: stmtsummary.DigestStr, tp: mysql.TypeVarchar, size: 64}, + {name: stmtsummary.DigestTextStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, flag: mysql.NotNullFlag, comment: "Normalized statement"}, + {name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"}, + {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, + {name: stmtsummary.ReadBillingDemoModelVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo model version"}, + {name: stmtsummary.ReadBillingDemoWeightVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo weight version"}, + {name: stmtsummary.ReadBillingDemoSiteStr, tp: mysql.TypeVarchar, size: 16, flag: mysql.NotNullFlag, comment: "Read billing execution site"}, + {name: stmtsummary.ReadBillingDemoOpClassStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Read billing operator class"}, + {name: stmtsummary.ReadBillingDemoOperatorKindStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Physical operator kind"}, + {name: stmtsummary.ReadBillingDemoUnitStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Coefficient-free base unit"}, + {name: stmtsummary.ReadBillingDemoInputSourceStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Runtime input evidence source"}, + {name: stmtsummary.ReadBillingDemoInputSideStr, tp: mysql.TypeVarchar, size: 16, flag: mysql.NotNullFlag, comment: "Operator input side"}, + {name: stmtsummary.ReadBillingDemoRowWidthSource, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Row-width evidence source"}, + {name: stmtsummary.ReadBillingDemoValueStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Aggregated base-unit value"}, + {name: stmtsummary.ReadBillingDemoSampleCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Number of unit samples"}, + {name: stmtsummary.ReadBillingDemoRowWidthSumStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Sum of row-width samples"}, + {name: stmtsummary.ReadBillingDemoAvgRowWidthStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average row-width sample"}, + {name: stmtsummary.ReadBillingDemoDMLKindStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Diagnostic DML kind; not part of the preview weight key"}, +} + +var tableStatementsSummaryReadBillingDemoStatusCols = []columnInfo{ + {name: stmtsummary.SummaryBeginTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "Begin time of this summary"}, + {name: stmtsummary.SummaryEndTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "End time of this summary"}, + {name: stmtsummary.StmtTypeStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"}, + {name: stmtsummary.SchemaNameStr, tp: mysql.TypeVarchar, size: 64, comment: "Current schema"}, + {name: stmtsummary.DigestStr, tp: mysql.TypeVarchar, size: 64}, + {name: stmtsummary.DigestTextStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, flag: mysql.NotNullFlag, comment: "Normalized statement"}, + {name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"}, + {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, + {name: stmtsummary.ReadBillingDemoModelVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo model version"}, + {name: stmtsummary.ReadBillingDemoWeightVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo weight version"}, + {name: stmtsummary.ReadBillingDemoSiteStr, tp: mysql.TypeVarchar, size: 16, flag: mysql.NotNullFlag, comment: "Read billing execution site"}, + {name: stmtsummary.ReadBillingDemoOpClassStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Read billing operator class"}, + {name: stmtsummary.ReadBillingDemoOperatorKindStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Physical operator kind"}, + {name: stmtsummary.ReadBillingDemoStatusStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing accounting status"}, + {name: stmtsummary.ReadBillingDemoReasonStr, tp: mysql.TypeVarchar, size: 128, flag: mysql.NotNullFlag, comment: "Read billing accounting status reason"}, + {name: stmtsummary.ReadBillingDemoCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Number of status samples"}, +} + var tableTiDBStatementsStatsCols = []columnInfo{ {name: stmtsummary.StmtTypeStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"}, {name: stmtsummary.SchemaNameStr, tp: mysql.TypeVarchar, size: 64, comment: "Current schema"}, @@ -2516,32 +2579,36 @@ var tableNameToColumns = map[string][]columnInfo{ TableStatementsSummary: tableStatementsSummaryCols, TableStatementsSummaryHistory: tableStatementsSummaryCols, TableStatementsSummaryEvicted: tableStatementsSummaryEvictedCols, - TableStorageStats: tableStorageStatsCols, - TableTiDBStatementsStats: tableTiDBStatementsStatsCols, - TableTiFlashTables: tableTableTiFlashTablesCols, - TableTiFlashSegments: tableTableTiFlashSegmentsCols, - TableTiFlashIndexes: tableTiFlashIndexesCols, - TableClientErrorsSummaryGlobal: tableClientErrorsSummaryGlobalCols, - TableClientErrorsSummaryByUser: tableClientErrorsSummaryByUserCols, - TableClientErrorsSummaryByHost: tableClientErrorsSummaryByHostCols, - TableTiDBTrx: tableTiDBTrxCols, - TableDeadlocks: tableDeadlocksCols, - TableDataLockWaits: tableDataLockWaitsCols, - TableAttributes: tableAttributesCols, - TablePlacementPolicies: tablePlacementPoliciesCols, - TableTrxSummary: tableTrxSummaryCols, - TableVariablesInfo: tableVariablesInfoCols, - TableUserAttributes: tableUserAttributesCols, - TableMemoryUsage: tableMemoryUsageCols, - TableMemoryUsageOpsHistory: tableMemoryUsageOpsHistoryCols, - TableResourceGroups: tableResourceGroupsCols, - TableRunawayWatches: tableRunawayWatchListCols, - TableCheckConstraints: tableCheckConstraintsCols, - TableTiDBCheckConstraints: tableTiDBCheckConstraintsCols, - TableKeywords: tableKeywords, - TableTiDBIndexUsage: tableTiDBIndexUsage, - TableTiDBPlanCache: tablePlanCache, - TableKeyspaceMeta: tableKeyspaceMetaCols, + TableStatementsSummaryReadBillingDemoBaseUnits: tableStatementsSummaryReadBillingDemoBaseUnitCols, + TableStatementsSummaryHistoryReadBillingDemoBaseUnits: tableStatementsSummaryReadBillingDemoBaseUnitCols, + TableStatementsSummaryReadBillingDemoStatus: tableStatementsSummaryReadBillingDemoStatusCols, + TableStatementsSummaryHistoryReadBillingDemoStatus: tableStatementsSummaryReadBillingDemoStatusCols, + TableStorageStats: tableStorageStatsCols, + TableTiDBStatementsStats: tableTiDBStatementsStatsCols, + TableTiFlashTables: tableTableTiFlashTablesCols, + TableTiFlashSegments: tableTableTiFlashSegmentsCols, + TableTiFlashIndexes: tableTiFlashIndexesCols, + TableClientErrorsSummaryGlobal: tableClientErrorsSummaryGlobalCols, + TableClientErrorsSummaryByUser: tableClientErrorsSummaryByUserCols, + TableClientErrorsSummaryByHost: tableClientErrorsSummaryByHostCols, + TableTiDBTrx: tableTiDBTrxCols, + TableDeadlocks: tableDeadlocksCols, + TableDataLockWaits: tableDataLockWaitsCols, + TableAttributes: tableAttributesCols, + TablePlacementPolicies: tablePlacementPoliciesCols, + TableTrxSummary: tableTrxSummaryCols, + TableVariablesInfo: tableVariablesInfoCols, + TableUserAttributes: tableUserAttributesCols, + TableMemoryUsage: tableMemoryUsageCols, + TableMemoryUsageOpsHistory: tableMemoryUsageOpsHistoryCols, + TableResourceGroups: tableResourceGroupsCols, + TableRunawayWatches: tableRunawayWatchListCols, + TableCheckConstraints: tableCheckConstraintsCols, + TableTiDBCheckConstraints: tableTiDBCheckConstraintsCols, + TableKeywords: tableKeywords, + TableTiDBIndexUsage: tableTiDBIndexUsage, + TableTiDBPlanCache: tablePlanCache, + TableKeyspaceMeta: tableKeyspaceMetaCols, } func createInfoSchemaTable(_ autoid.Allocators, _ func() (pools.Resource, error), meta *model.TableInfo) (table.Table, error) { diff --git a/pkg/metrics/BUILD.bazel b/pkg/metrics/BUILD.bazel index 16b3aaa56e43e..44dba62738a42 100644 --- a/pkg/metrics/BUILD.bazel +++ b/pkg/metrics/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "distsql.go", "domain.go", "executor.go", + "explain_ru.go", "external_workload.go", "gc_worker.go", "globalsort.go", @@ -69,7 +70,7 @@ go_test( ], embed = [":metrics"], flaky = True, - shard_count = 9, + shard_count = 11, deps = [ "//pkg/parser/terror", "//pkg/statistics/handle/cache", diff --git a/pkg/metrics/explain_ru.go b/pkg/metrics/explain_ru.go new file mode 100644 index 0000000000000..553e811cc3dc7 --- /dev/null +++ b/pkg/metrics/explain_ru.go @@ -0,0 +1,218 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + metricscommon "github.com/pingcap/tidb/pkg/metrics/common" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + explainRULabelStatus = "status" + explainRULabelComponentSnapshotStatus = "component_snapshot_status" + explainRULabelWeightVersion = "weight_version" + readBillingDemoLabelModelVersion = "model_version" +) + +// EXPLAIN FORMAT='RU' demo metrics. +var ( + ExplainRUPreviewRUCounter *prometheus.CounterVec + ExplainRUWorkRowsCounter *prometheus.CounterVec + ExplainRUWorkBytesCounter *prometheus.CounterVec + ExplainRURowWidthHistogram *prometheus.HistogramVec + ExplainRUStatementsCounter *prometheus.CounterVec + ExplainRURenderDurationHistogram *prometheus.HistogramVec + ExplainRUComponentSnapshotCounter *prometheus.CounterVec + + ReadBillingDemoStatementsCounter *prometheus.CounterVec + ReadBillingDemoOperatorStatusCounter *prometheus.CounterVec + ReadBillingDemoBaseUnitsCounter *prometheus.CounterVec + ReadBillingDemoRowWidthHistogram *prometheus.HistogramVec +) + +// InitExplainRUMetrics initializes metrics for EXPLAIN ANALYZE FORMAT='RU'. +func InitExplainRUMetrics() { + // Keep these labels bounded during demo calibration. SQL text, digest, + // plan ID, table name, and index name belong in the SQL result, not metrics. + ExplainRUPreviewRUCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "preview_ru_total", + Help: "Counter of read billing preview RU values generated by EXPLAIN ANALYZE FORMAT='RU'.", + }, []string{"section", "component", "operator", "source", explainRULabelWeightVersion}, + ) + ExplainRUWorkRowsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "work_rows_total", + Help: "Counter of row-count model inputs generated by EXPLAIN ANALYZE FORMAT='RU'.", + }, []string{"section", "component", "operator", "source"}, + ) + ExplainRUWorkBytesCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "work_bytes_total", + Help: "Counter of byte-shaped model inputs generated by EXPLAIN ANALYZE FORMAT='RU'.", + }, []string{"section", "component", "operator", "source"}, + ) + ExplainRUStatementsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "statements_total", + Help: "Counter of EXPLAIN ANALYZE FORMAT='RU' statement status.", + }, []string{explainRULabelStatus}, + ) + ExplainRURenderDurationHistogram = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "render_duration_seconds", + Help: "Histogram of EXPLAIN ANALYZE FORMAT='RU' render duration.", + Buckets: prometheus.DefBuckets, + }, []string{explainRULabelStatus}, + ) + ExplainRURowWidthHistogram = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "row_width_bytes", + Help: "Histogram of row-width factors generated by EXPLAIN ANALYZE FORMAT='RU'.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), + }, []string{"component", "operator", "source"}, + ) + ExplainRUComponentSnapshotCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "component_snapshot_total", + Help: "Counter of EXPLAIN ANALYZE FORMAT='RU' component snapshot status.", + }, []string{explainRULabelComponentSnapshotStatus}, + ) + ReadBillingDemoStatementsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "statements_total", + Help: "Counter of read billing demo statement status.", + }, []string{"status", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, + ) + ReadBillingDemoOperatorStatusCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "operator_status_total", + Help: "Counter of read billing demo operator status.", + }, []string{"site", "op_class", "operator_kind", "status", "reason", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, + ) + ReadBillingDemoBaseUnitsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "base_units_total", + Help: "Counter of coefficient-free read billing demo base units.", + }, []string{"site", "op_class", "operator_kind", "unit", "input_source", "input_side", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, + ) + ReadBillingDemoRowWidthHistogram = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "row_width_bytes", + Help: "Histogram of row-width factors used by read billing demo.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), + }, []string{"site", "op_class", "operator_kind", "row_width_source", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, + ) +} + +// RecordExplainRUStatus records a bounded FORMAT='RU' statement status. +func RecordExplainRUStatus(status string) { + if ExplainRUStatementsCounter == nil || status == "" { + return + } + ExplainRUStatementsCounter.WithLabelValues(status).Inc() +} + +// ObserveExplainRURenderDuration records a bounded FORMAT='RU' render duration. +func ObserveExplainRURenderDuration(status string, seconds float64) { + if ExplainRURenderDurationHistogram == nil || status == "" || seconds < 0 { + return + } + ExplainRURenderDurationHistogram.WithLabelValues(status).Observe(seconds) +} + +// RecordExplainRUComponentSnapshot records component snapshot availability. +func RecordExplainRUComponentSnapshot(status string) { + if ExplainRUComponentSnapshotCounter == nil || status == "" { + return + } + ExplainRUComponentSnapshotCounter.WithLabelValues(status).Inc() +} + +// ObserveExplainRURow records bounded numeric samples from generated FORMAT='RU' rows. +func ObserveExplainRURow(section, component, operator, source, rowWidthSource, weightVersion string, previewRU, workRows, workBytes, rowWidth float64) { + // Negative values are sentinels for "not present" so callers can use one + // path for summary and plan rows without emitting fake zeros. + if ExplainRUPreviewRUCounter != nil && previewRU >= 0 && weightVersion != "" { + ExplainRUPreviewRUCounter.WithLabelValues(section, component, operator, source, weightVersion).Add(previewRU) + } + if ExplainRUWorkRowsCounter != nil && workRows >= 0 { + ExplainRUWorkRowsCounter.WithLabelValues(section, component, operator, source).Add(workRows) + } + if ExplainRUWorkBytesCounter != nil && workBytes >= 0 { + ExplainRUWorkBytesCounter.WithLabelValues(section, component, operator, source).Add(workBytes) + } + if ExplainRURowWidthHistogram != nil && section == "plan" && rowWidth > 0 && operator != "" { + ExplainRURowWidthHistogram.WithLabelValues(component, operator, rowWidthSource).Observe(rowWidth) + } +} + +// RecordReadBillingDemoStatement records a bounded read billing demo statement status. +func RecordReadBillingDemoStatement(status, modelVersion, weightVersion string) { + if ReadBillingDemoStatementsCounter == nil || status == "" || modelVersion == "" || weightVersion == "" { + return + } + ReadBillingDemoStatementsCounter.WithLabelValues(status, modelVersion, weightVersion).Inc() +} + +// RecordReadBillingDemoOperatorStatus records a bounded read billing demo operator status. +func RecordReadBillingDemoOperatorStatus(site, opClass, operatorKind, status, reason, modelVersion, weightVersion string) { + if ReadBillingDemoOperatorStatusCounter == nil || + site == "" || opClass == "" || operatorKind == "" || status == "" || reason == "" || modelVersion == "" || weightVersion == "" { + return + } + ReadBillingDemoOperatorStatusCounter.WithLabelValues(site, opClass, operatorKind, status, reason, modelVersion, weightVersion).Inc() +} + +// AddReadBillingDemoBaseUnits records coefficient-free read billing demo base units. +func AddReadBillingDemoBaseUnits(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion, weightVersion string, value float64) { + if ReadBillingDemoBaseUnitsCounter == nil || + site == "" || opClass == "" || operatorKind == "" || unit == "" || inputSource == "" || inputSide == "" || modelVersion == "" || weightVersion == "" || + value < 0 { + return + } + ReadBillingDemoBaseUnitsCounter.WithLabelValues(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion, weightVersion).Add(value) +} + +// ObserveReadBillingDemoRowWidth records row-width factors used by read billing demo. +func ObserveReadBillingDemoRowWidth(site, opClass, operatorKind, rowWidthSource, modelVersion, weightVersion string, rowWidth float64) { + if ReadBillingDemoRowWidthHistogram == nil || + site == "" || opClass == "" || operatorKind == "" || rowWidthSource == "" || modelVersion == "" || weightVersion == "" || rowWidth <= 0 { + return + } + ReadBillingDemoRowWidthHistogram.WithLabelValues(site, opClass, operatorKind, rowWidthSource, modelVersion, weightVersion).Observe(rowWidth) +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a71cc5637d540..dac36c01a43e2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -102,6 +102,7 @@ func InitMetrics() { InitServerMetrics() InitSessionMetrics() InitRUV2Metrics() + InitExplainRUMetrics() InitSliMetrics() InitStatsMetrics() InitTelemetryMetrics() @@ -352,6 +353,7 @@ func RegisterMetrics() { prometheus.MustRegister(RUV2ResourceManagerWriteCnt) prometheus.MustRegister(RUV2WriteKeys) prometheus.MustRegister(RUV2WriteSize) + prometheus.MustRegister(RUV2PrewriteRegionNum) prometheus.MustRegister(RUV2SessionParserTotal) prometheus.MustRegister(RUV2TxnCnt) prometheus.MustRegister(RUV2TiKVKVEngineCacheMiss) @@ -361,6 +363,17 @@ func RegisterMetrics() { prometheus.MustRegister(RUV2TiKVStorageProcessedKeysBatchGet) prometheus.MustRegister(RUV2TiKVStorageProcessedKeysGet) prometheus.MustRegister(RUV2TiKVCoprocessorWorkTotal) + prometheus.MustRegister(ExplainRUPreviewRUCounter) + prometheus.MustRegister(ExplainRUWorkRowsCounter) + prometheus.MustRegister(ExplainRUWorkBytesCounter) + prometheus.MustRegister(ExplainRURowWidthHistogram) + prometheus.MustRegister(ExplainRUStatementsCounter) + prometheus.MustRegister(ExplainRURenderDurationHistogram) + prometheus.MustRegister(ExplainRUComponentSnapshotCounter) + prometheus.MustRegister(ReadBillingDemoStatementsCounter) + prometheus.MustRegister(ReadBillingDemoOperatorStatusCounter) + prometheus.MustRegister(ReadBillingDemoBaseUnitsCounter) + prometheus.MustRegister(ReadBillingDemoRowWidthHistogram) prometheus.MustRegister(NetworkTransmissionStats) diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index a0fd8e1b66f3b..f2ce1ee4d3808 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -95,6 +95,123 @@ func TestRUV2ExecutorCounterReturnsCachedKnownLabels(t *testing.T) { } } +func TestExplainRUMetrics(t *testing.T) { + InitExplainRUMetrics() + + RecordExplainRUStatus("success") + ObserveExplainRURenderDuration("success", 0.01) + RecordExplainRUComponentSnapshot("ok") + ObserveExplainRURow("plan", "", "projection", "read_billing_model", "runtime_chunk_avg", "v2", 1.25, 3.5, 24, 8) + RecordReadBillingDemoStatement("success", "v2", "weights-v2") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2", "weights-v2") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "weights-v2", 3) + AddReadBillingDemoBaseUnits("tikv", "agg_hash", "hashagg", "output_rows", "runtime_operator_act_rows", "all", "v3", "weights-v3", 0) + AddReadBillingDemoBaseUnits("tidb", "kv_mutation", "memdb_mutation", "cpu_work", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated", 3.5) + AddReadBillingDemoBaseUnits("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated", 2) + AddReadBillingDemoBaseUnits("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated", 15) + AddReadBillingDemoBaseUnits("tidb", "reader_transport", "mixed_reader", "read_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated", 4) + AddReadBillingDemoBaseUnits("tikv", "kv_write", "txn_write", "write_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated", 2) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", "weights-v2", 8) + + require.Equal(t, 1.0, readCounterValue(t, ExplainRUStatementsCounter.WithLabelValues("success"))) + require.Equal(t, 1.0, readCounterValue(t, ExplainRUComponentSnapshotCounter.WithLabelValues("ok"))) + require.Equal(t, 1.25, readCounterValue(t, ExplainRUPreviewRUCounter.WithLabelValues("plan", "", "projection", "read_billing_model", "v2"))) + require.Equal(t, 3.5, readCounterValue(t, ExplainRUWorkRowsCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) + require.Equal(t, 24.0, readCounterValue(t, ExplainRUWorkBytesCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2", "weights-v2"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v2", "weights-v2"))) + require.Equal(t, 3.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "weights-v2"))) + require.Equal(t, 3.5, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "cpu_work", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 2.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 15.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 4.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "reader_transport", "mixed_reader", "read_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 2.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_write", "write_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated"))) + + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(ExplainRUPreviewRUCounter)) + require.NoError(t, registry.Register(ExplainRUWorkRowsCounter)) + require.NoError(t, registry.Register(ExplainRUWorkBytesCounter)) + require.NoError(t, registry.Register(ExplainRURowWidthHistogram)) + require.NoError(t, registry.Register(ExplainRUStatementsCounter)) + require.NoError(t, registry.Register(ExplainRURenderDurationHistogram)) + require.NoError(t, registry.Register(ExplainRUComponentSnapshotCounter)) + require.NoError(t, registry.Register(ReadBillingDemoStatementsCounter)) + require.NoError(t, registry.Register(ReadBillingDemoOperatorStatusCounter)) + require.NoError(t, registry.Register(ReadBillingDemoBaseUnitsCounter)) + require.NoError(t, registry.Register(ReadBillingDemoRowWidthHistogram)) + families, err := registry.Gather() + require.NoError(t, err) + previewRUFamily := findMetricFamily(families, "tidb_explain_ru_preview_ru_total") + require.NotNil(t, previewRUFamily) + requireMetricFamilyHasLabels(t, previewRUFamily, "section", "component", "operator", "source", "weight_version") + require.True(t, metricHasLabelValue(previewRUFamily.GetMetric()[0], "weight_version", "v2")) + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_work_rows_total")) + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_work_bytes_total")) + rowWidthFamily := findMetricFamily(families, "tidb_explain_ru_row_width_bytes") + require.NotNil(t, rowWidthFamily) + requireMetricFamilyHasLabels(t, rowWidthFamily, "component", "operator", "source") + require.True(t, metricHasLabelValue(rowWidthFamily.GetMetric()[0], "source", "runtime_chunk_avg")) + statementFamily := findMetricFamily(families, "tidb_explain_ru_statements_total") + require.NotNil(t, statementFamily) + requireMetricFamilyHasLabels(t, statementFamily, "status") + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_render_duration_seconds")) + componentSnapshotFamily := findMetricFamily(families, "tidb_explain_ru_component_snapshot_total") + require.NotNil(t, componentSnapshotFamily) + requireMetricFamilyHasLabels(t, componentSnapshotFamily, "component_snapshot_status") + readBillingStatementFamily := findMetricFamily(families, "tidb_read_billing_demo_statements_total") + require.NotNil(t, readBillingStatementFamily) + requireMetricFamilyHasLabels(t, readBillingStatementFamily, "status", "model_version", "weight_version") + readBillingOperatorFamily := findMetricFamily(families, "tidb_read_billing_demo_operator_status_total") + require.NotNil(t, readBillingOperatorFamily) + requireMetricFamilyHasLabels(t, readBillingOperatorFamily, "site", "op_class", "operator_kind", "status", "reason", "model_version", "weight_version") + readBillingBaseUnitFamily := findMetricFamily(families, "tidb_read_billing_demo_base_units_total") + require.NotNil(t, readBillingBaseUnitFamily) + requireMetricFamilyHasLabels(t, readBillingBaseUnitFamily, "site", "op_class", "operator_kind", "unit", "input_source", "input_side", "model_version", "weight_version") + var zeroOutputRows *dto.Metric + for _, metric := range readBillingBaseUnitFamily.GetMetric() { + if metricHasLabelValue(metric, "unit", "output_rows") { + zeroOutputRows = metric + break + } + } + require.NotNil(t, zeroOutputRows) + require.Zero(t, zeroOutputRows.GetCounter().GetValue()) + readBillingRowWidthFamily := findMetricFamily(families, "tidb_read_billing_demo_row_width_bytes") + require.NotNil(t, readBillingRowWidthFamily) + requireMetricFamilyHasLabels(t, readBillingRowWidthFamily, "site", "op_class", "operator_kind", "row_width_source", "model_version", "weight_version") +} + +func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { + InitExplainRUMetrics() + + RecordExplainRUStatus("") + ObserveExplainRURenderDuration("", 0.01) + RecordExplainRUComponentSnapshot("") + ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", "", 1, -1, -1, -1) + ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", "v2", 0, -1, -1, -1) + ObserveExplainRURow("plan", "", "", "read_billing_model", "runtime_chunk_avg", "v2", -1, -1, -1, 32) + RecordReadBillingDemoStatement("", "v2", "weights-v2") + RecordReadBillingDemoStatement("success", "v2", "") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v2", "weights-v2") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2", "") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "weights-v2", -1) + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "", 1) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", "weights-v2", 0) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", "", 8) + + require.Equal(t, 0, countCollectedMetrics(ExplainRUStatementsCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRURenderDurationHistogram)) + require.Equal(t, 0, countCollectedMetrics(ExplainRUComponentSnapshotCounter)) + require.Equal(t, 1, countCollectedMetrics(ExplainRUPreviewRUCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRUWorkRowsCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRUWorkBytesCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRURowWidthHistogram)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoStatementsCounter)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoOperatorStatusCounter)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoBaseUnitsCounter)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoRowWidthHistogram)) +} + func TestStmtSummaryMetricLabels(t *testing.T) { InitStmtSummaryMetrics() require.Equal(t, 0, countCollectedMetrics(StmtSummaryWindowRecordCount)) @@ -218,3 +335,16 @@ func metricHasLabelValue(metric *dto.Metric, name string, value string) bool { } return false } + +func requireMetricFamilyHasLabels(t *testing.T, family *dto.MetricFamily, names ...string) { + t.Helper() + require.NotEmpty(t, family.GetMetric()) + labels := make(map[string]struct{}, len(family.GetMetric()[0].GetLabel())) + for _, label := range family.GetMetric()[0].GetLabel() { + labels[label.GetName()] = struct{}{} + } + for _, name := range names { + _, ok := labels[name] + require.Truef(t, ok, "metric %s missing label %s", family.GetName(), name) + } +} diff --git a/pkg/metrics/ru_v2.go b/pkg/metrics/ru_v2.go index 0712db292beba..ed6bf13a14ccc 100644 --- a/pkg/metrics/ru_v2.go +++ b/pkg/metrics/ru_v2.go @@ -32,6 +32,7 @@ var ( RUV2ResourceManagerWriteCnt prometheus.Counter RUV2WriteKeys prometheus.Counter RUV2WriteSize prometheus.Counter + RUV2PrewriteRegionNum prometheus.Counter RUV2SessionParserTotal prometheus.Counter RUV2TxnCnt prometheus.Counter @@ -183,6 +184,15 @@ func InitRUV2Metrics() { }, ) + RUV2PrewriteRegionNum = metricscommon.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "ruv2", + Name: "prewrite_region_num", + Help: "Diagnostic counter of commit prewrite region fanout for RU v2.", + }, + ) + RUV2SessionParserTotal = metricscommon.NewCounter( prometheus.CounterOpts{ Namespace: "tidb", diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 2bf15ab229081..05a570b9a0d45 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "core_init.go", "encode.go", "exhaust_physical_plans.go", + "explain_ru.go", "expression_codec_fn.go", "expression_rewriter.go", "find_best_task.go", @@ -193,6 +194,7 @@ go_library( "//pkg/util/size", "//pkg/util/slice", "//pkg/util/sqlexec", + "//pkg/util/stmtsummary", "//pkg/util/stringutil", "//pkg/util/syncutil", "//pkg/util/texttree", @@ -207,6 +209,9 @@ go_library( "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//util", + "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap", ], @@ -297,6 +302,7 @@ go_test( "//pkg/session/sessionapi", "//pkg/session/sessmgr", "//pkg/sessionctx", + "//pkg/sessionctx/stmtctx", "//pkg/sessionctx/vardef", "//pkg/sessionctx/variable", "//pkg/sessiontxn", @@ -321,17 +327,22 @@ go_test( "//pkg/util/context", "//pkg/util/dbterror", "//pkg/util/dbterror/plannererrors", + "//pkg/util/execdetails", "//pkg/util/hint", "//pkg/util/logutil", "//pkg/util/mock", "//pkg/util/plancodec", "//pkg/util/ranger", "//pkg/util/sem/v2:sem", + "//pkg/util/stmtsummary", "@com_github_docker_go_units//:go-units", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_tipb//go-tipb", "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//util", + "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_goleak//:goleak", "@org_uber_go_zap//:zap", "@org_uber_go_zap//zaptest/observer", diff --git a/pkg/planner/core/common_plans.go b/pkg/planner/core/common_plans.go index ae124a35819e5..0fb06300b2094 100644 --- a/pkg/planner/core/common_plans.go +++ b/pkg/planner/core/common_plans.go @@ -651,8 +651,9 @@ type Explain struct { ExecStmt ast.StmtNode RuntimeStatsColl *execdetails.RuntimeStatsColl - Rows [][]string - BriefBinaryPlan string + Rows [][]string + BriefBinaryPlan string + ruStatusRecorded bool } // GetBriefBinaryPlan returns the binary plan of the plan for explainfor. @@ -735,6 +736,8 @@ func (e *Explain) prepareSchema() error { fieldNames = []string{"binary plan"} case format == types.ExplainFormatTiDBJSON: fieldNames = []string{"TiDB_JSON"} + case format == types.ExplainFormatRU && e.Analyze: + fieldNames = []string{"section", "id", "component", "operatorClass", "actRows", "inputRows", "outputRows", "rowWidth", "rowWidthSource", "workRows", "workBytes", "unit", "count", "weight", "previewRU", "source", "note"} case e.Explore: fieldNames = []string{"statement", "binding_hint", "plan", "plan_digest", "avg_latency", "exec_times", "avg_scan_rows", "avg_returned_rows", "latency_per_returned_row", "scan_rows_per_returned_row", "recommend", "reason", @@ -932,6 +935,11 @@ func (e *Explain) RenderResult() error { return err } e.Rows = append(e.Rows, []string{str}) + case types.ExplainFormatRU: + if err := e.renderRUExplain(); err != nil { + e.recordExplainRUStatus(explainRUStatusError) + return err + } default: return errors.Errorf("explain format '%s' is not supported now", e.Format) } diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index cec90eef77648..f6af045a54da5 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -15,11 +15,32 @@ package core import ( + "math" "testing" + "time" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/expression/aggregation" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" + "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" + "github.com/pingcap/tidb/pkg/planner/property" + plannerutil "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tidb/pkg/util/stmtsummary" + "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikvrpc" + tikvutil "github.com/tikv/client-go/v2/util" + rmclient "github.com/tikv/pd/client/resource_group/controller" ) func TestNewLineFieldsInfo(t *testing.T) { @@ -114,3 +135,2451 @@ func TestNewLineFieldsInfo(t *testing.T) { require.Equal(t, c.expected, lineFieldsInfo) } } + +func TestExplainRUSelectGateStatus(t *testing.T) { + cases := []struct { + sql string + expected explainRUStatus + }{ + {"explain analyze format='ru' select 1", explainRUStatusSuccess}, + {"explain analyze format='ru' with cte as (select 1) select * from cte", explainRUStatusSuccess}, + {"explain analyze format='ru' select rand(), uuid()", explainRUStatusSuccess}, + {"explain analyze format='ru' select last_insert_id()", explainRUStatusSuccess}, + {"explain analyze format='ru' insert into t values (1)", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' table t", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' select 1 union table t", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' select 1 into outfile '/tmp/explain_ru.csv'", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select @a := 1", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select 1 union select @a := 2", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' with cte as (select get_lock('x', 0)) select * from cte", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select release_lock('x')", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select release_all_locks()", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select last_insert_id(1)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select nextval(seq)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select setval(seq, 1)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select sleep(1)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select * from t for update skip locked", explainRUStatusUnsupportedLockingSelect}, + {"explain analyze format='ru' select * from t for share skip locked", explainRUStatusUnsupportedLockingSelect}, + {"explain analyze format='ru' select 1 union select * from t for update", explainRUStatusUnsupportedLockingSelect}, + } + p := parser.New() + for _, tc := range cases { + stmt, err := p.ParseOneStmt(tc.sql, "", "") + require.NoError(t, err, tc.sql) + explain := stmt.(*ast.ExplainStmt) + require.Equal(t, tc.expected, explainRUSelectGateStatus(explain.Stmt), tc.sql) + } + require.Equal(t, explainRUStatusUnsupportedNonSelect, explainRUSelectGateStatus(&ast.SelectStmt{Kind: ast.SelectStmtKindValues})) + require.Equal(t, explainRUStatusUnsupportedNonSelect, explainRUValidateSetOprSelectList(&ast.SetOprSelectList{ + Selects: []ast.Node{&ast.SelectStmt{Kind: ast.SelectStmtKindValues}}, + })) +} + +func TestExplainRUTargetGateStatus(t *testing.T) { + cases := []struct { + sql string + expected explainRUStatus + }{ + {"explain analyze format='ru' select 1", explainRUStatusSuccess}, + {"explain analyze format='ru' insert into t values (1)", explainRUStatusSuccess}, + {"explain analyze format='ru' insert ignore into t values (1)", explainRUStatusSuccess}, + {"explain analyze format='ru' insert into t values (1) on duplicate key update a = values(a)", explainRUStatusSuccess}, + {"explain analyze format='ru' update t set a = 2 where a = 1", explainRUStatusSuccess}, + {"explain analyze format='ru' delete from t where a = 1", explainRUStatusSuccess}, + {"explain analyze format='ru' replace into t values (1)", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' select @a := 1", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' table t", explainRUStatusUnsupportedNonSelect}, + } + p := parser.New() + for _, tc := range cases { + stmt, err := p.ParseOneStmt(tc.sql, "", "") + require.NoError(t, err, tc.sql) + explain := stmt.(*ast.ExplainStmt) + require.Equal(t, tc.expected, explainRUTargetGateStatus(explain.Stmt), tc.sql) + } + + for sql, expectedKind := range map[string]string{ + "insert into t values (1)": "insert", + "insert ignore into t values (1)": "insert_ignore", + "insert into t values (1) on duplicate key update a = values(a)": "upsert", + "update t set a = 2 where a = 1": "update", + "delete from t where a = 1": "delete", + } { + stmt, err := parser.New().ParseOneStmt(sql, "", "") + require.NoError(t, err) + kind, ok := explainRUWriteDMLKind(stmt) + require.True(t, ok) + require.Equal(t, expectedKind, kind) + } +} + +func TestExplainRURowFormatting(t *testing.T) { + row := explainRURow{ + section: explainRUSectionPlan, + id: "Projection_1", + component: "projection", + operatorClass: "tidb/projection_eval", + actRows: 1, + hasActRows: true, + inputRows: 2, + hasInputRows: true, + outputRows: 1, + hasOutputRows: true, + rowWidth: 8, + hasRowWidth: true, + rowWidthSource: explainRUWidthSourceRuntimeChunkAvg, + workRows: 2, + hasWorkRows: true, + unit: readBillingDemoUnitInputRows, + count: 2, + hasCount: true, + weight: 0.25, + hasWeight: true, + previewRU: 6, + hasPreviewRU: true, + source: readBillingDemoInputSourceRuntimeChunkBytes, + note: "input_side=all,weight_version=v2", + } + require.Equal(t, []string{ + "plan", "Projection_1", "projection", "tidb/projection_eval", "1", "2", "1", "8.000000", "runtime_chunk_avg", "2", "", "input_rows", "2", "0.250000", "6.000000", "runtime_chunk_bytes", "input_side=all,weight_version=v2", + }, row.toStrings()) +} + +const ( + readBillingDemoWriteKeyWeight = 0.6 + readBillingDemoWriteByteWeight = 0.00002 +) + +func buildWriteBillingDemoResultFromDetails(dmlKind string, _ *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + return readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + operators: buildTiKVWriteBillingDemoOperators(dmlKind, ruv2Metrics, false), + } +} + +func readBillingDemoWriteDiagnosticStatus(dmlKind, reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "txn_write@statement", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassKVWrite, + operatorKind: readBillingDemoOperatorTxnWrite, + dmlKind: dmlKind, + scope: readBillingDemoScopeTxnPrewritePayload, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + +type legacyReadBillingDemoWeights struct { + fixedEvent, row, byte, orderWork float64 + mutationCount, mutationByte float64 + writeKey, writeByte float64 + writeRPC, region float64 +} + +func (legacyReadBillingDemoWeights) valid() bool { return true } + +func (weights legacyReadBillingDemoWeights) unitWeight(unit string) (float64, bool) { + switch unit { + case readBillingDemoUnitFixedEvents: + return weights.fixedEvent, true + case readBillingDemoUnitInputRows: + return weights.row, true + case readBillingDemoUnitInputBytes: + return weights.byte, true + case readBillingDemoUnitOrderWork: + return weights.orderWork, true + case readBillingDemoUnitEncodedMutationCount: + return weights.mutationCount, true + case readBillingDemoUnitEncodedMutationBytes: + return weights.mutationByte, true + case readBillingDemoUnitWriteKeys: + return weights.writeKey, true + case readBillingDemoUnitWriteByte: + return weights.writeByte, true + case readBillingDemoUnitPrewriteRegionNum: + return weights.region, true + case readBillingDemoUnitTiKVWriteRPCCount: + return weights.writeRPC, true + default: + return 0, false + } +} + +func readBillingDemoResolveWeights(site, opClass, version string) (legacyReadBillingDemoWeights, bool) { + if version != readBillingDemoWeightVersion || opClass == readBillingDemoOpClassPointLookup && site != readBillingDemoSiteTiKV { + return legacyReadBillingDemoWeights{}, false + } + w := legacyReadBillingDemoWeights{ + fixedEvent: 0.1, row: 0.2, byte: 0.3, orderWork: 0.4, + } + if opClass == readBillingDemoOpClassTopN { + w.row = 0 + } + if opClass == readBillingDemoOpClassKVWrite { + w.writeKey = readBillingDemoWriteKeyWeight + w.writeByte = readBillingDemoWriteByteWeight + } + return w, true +} + +type readBillingDemoRPCStatsForTest struct { + counts map[tikvrpc.CmdType]int64 + tp int +} + +func (*readBillingDemoRPCStatsForTest) String() string { + return "" +} + +func (s *readBillingDemoRPCStatsForTest) Merge(other execdetails.RuntimeStats) { + otherStats, ok := other.(*readBillingDemoRPCStatsForTest) + if !ok { + return + } + for cmd, count := range otherStats.counts { + s.counts[cmd] += count + } +} + +func (s *readBillingDemoRPCStatsForTest) Clone() execdetails.RuntimeStats { + cloned := &readBillingDemoRPCStatsForTest{ + counts: make(map[tikvrpc.CmdType]int64, len(s.counts)), + tp: s.tp, + } + for cmd, count := range s.counts { + cloned.counts[cmd] = count + } + return cloned +} + +func (s *readBillingDemoRPCStatsForTest) Tp() int { + if s.tp != 0 { + return s.tp + } + return execdetails.TpRuntimeStatsWithSnapshot +} + +func (s *readBillingDemoRPCStatsForTest) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 { + return s.counts[cmd] +} + +func TestReadBillingDemoV4FormulaContract(t *testing.T) { + weights := readBillingDemoWeights{ + Version: "test-v4-calibrated", + CPUWeight: 2, ScanWeight: 3, NetWeight: 5, ReadRequestWeight: 7, WriteRequestWeight: 17, + HashTableWeight: 11, JoinWeight: 13, MutationBytesPerCPUUnit: 10, Calibrated: true, + } + for _, tc := range []struct { + unit string + value float64 + weight float64 + }{ + {readBillingDemoUnitCPUWork, 4, 2}, {readBillingDemoUnitScanBytes, 6, 3}, + {readBillingDemoUnitNetBytes, 8, 5}, {readBillingDemoUnitReadRequestCount, 10, 7}, + {readBillingDemoUnitWriteRequestCount, 2, 17}, + {readBillingDemoUnitHashStateRows, 12, 11}, {readBillingDemoUnitJoinOutputRows, 14, 13}, + } { + weight, ru, ok := readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: tc.unit, value: tc.value}, weights) + require.True(t, ok) + require.Equal(t, tc.weight, weight) + require.Equal(t, tc.value*tc.weight, ru) + } + for _, invalid := range []float64{-1, math.NaN(), math.Inf(1)} { + _, _, ok := readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, value: invalid}, weights) + require.False(t, ok) + } + for _, invalidWeights := range []readBillingDemoWeights{ + {}, + {MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: readBillingDemoWeightVersion, MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", CPUWeight: -1, MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", CPUWeight: math.NaN(), MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", CPUWeight: math.Inf(1), MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", ReadRequestWeight: -1, MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", WriteRequestWeight: math.NaN(), MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", MutationBytesPerCPUUnit: math.NaN(), Calibrated: true}, + {Version: "test", MutationBytesPerCPUUnit: math.Inf(1), Calibrated: true}, + } { + require.False(t, readBillingDemoWeightsValid(invalidWeights)) + } + + oldWeights := readBillingDemoV4Weights + readBillingDemoV4Weights = weights + t.Cleanup(func() { readBillingDemoV4Weights = oldWeights }) + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone, operators: []readBillingDemoOperatorResult{{ + id: "formula", site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassProjection, + operatorKind: "projection", status: readBillingDemoStatusOperatorOK, + units: []readBillingDemoUnit{ + {unit: readBillingDemoUnitCPUWork, value: 4}, {unit: readBillingDemoUnitScanBytes, value: 6}, + {unit: readBillingDemoUnitNetBytes, value: 8}, {unit: readBillingDemoUnitReadRequestCount, value: 10}, + {unit: readBillingDemoUnitWriteRequestCount, value: 2}, + {unit: readBillingDemoUnitHashStateRows, value: 12}, {unit: readBillingDemoUnitJoinOutputRows, value: 14}, + }, + }}} + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.True(t, rows[0].hasPreviewRU) + require.Equal(t, 484.0, rows[0].previewRU) + require.Contains(t, rows[0].note, "weight_version=test-v4-calibrated") + require.Equal(t, "test-v4-calibrated", buildReadBillingDemoStatementStats(result).WeightVersion) + overflowWeights := weights + overflowWeights.CPUWeight = 1 + readBillingDemoV4Weights = overflowWeights + overflowResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, operators: []readBillingDemoOperatorResult{{ + status: readBillingDemoStatusOperatorOK, opClass: readBillingDemoOpClassProjection, + units: []readBillingDemoUnit{{unit: readBillingDemoUnitCPUWork, value: math.MaxFloat64}, {unit: readBillingDemoUnitCPUWork, value: math.MaxFloat64}}, + }}} + overflowRows := explainRUBuildReadBillingRows(overflowResult, explainRUComponentSnapshotOK) + require.False(t, overflowRows[0].hasPreviewRU) + readBillingDemoV4Weights = weights + + readBillingDemoV4Weights = readBillingDemoWeights{} + rows = explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.False(t, rows[0].hasPreviewRU) + require.Contains(t, rows[0].note, readBillingDemoReasonUncalibratedWeights) + stats := buildReadBillingDemoStatementStats(result) + require.Equal(t, "v4", stats.ModelVersion) + require.Equal(t, "v3-resource-formula-uncalibrated", stats.WeightVersion) + require.Equal(t, stmtsummary.ReadBillingDemoBaseUnitSummary{}, stats.Totals) + + t.Run("reader transport is emitted once and fails closed", func(t *testing.T) { + ctx := mock.NewContext() + reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + flat := &FlatPhysicalPlan{Main: FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, StoreType: kv.TiKV}, + }} + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + basic := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + basic.Record(time.Millisecond, 0) + metrics := execdetails.NewRUV2Metrics() + metrics.AddResourceManagerReadCnt(3) + metrics.AddTiKVCoprocessorResponseBytes(128) + op, present := readBillingDemoReaderTransport(flat, runtimeStats, metrics, false) + require.True(t, present) + require.Equal(t, readBillingDemoStatusOperatorOK, op.status) + require.Equal(t, 128.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitNetBytes, readBillingDemoInputSideAll)) + require.Equal(t, 3.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + + runtimeStats.RecordExpectedCopTasks([]int{scan.ID()}) + op, _ = readBillingDemoReaderTransport(flat, runtimeStats, execdetails.NewRUV2Metrics(), false) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, op.reason) + + runtimeStats.RegisterStats(reader.ID(), &readBillingDemoRPCStatsForTest{ + counts: map[tikvrpc.CmdType]int64{ + tikvrpc.CmdCop: 2, + tikvrpc.CmdCopStream: 1, + }, + tp: execdetails.TpSelectResultRuntimeStats, + }) + dmlMetrics := execdetails.NewRUV2Metrics() + dmlMetrics.AddResourceManagerReadCnt(99) + dmlMetrics.AddTiKVCoprocessorResponseBytes(128) + op, _ = readBillingDemoReaderTransport(flat, runtimeStats, dmlMetrics, true) + require.Equal(t, readBillingDemoStatusOperatorOK, op.status) + require.Equal(t, 128.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitNetBytes, readBillingDemoInputSideAll)) + require.Equal(t, 3.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceDistSQLRuntimeStats, op.units[1].source) + }) + + t.Run("point lookup transport is rpc only and emitted once", func(t *testing.T) { + ctx := mock.NewContext() + stats := &property.StatsInfo{RowCount: 1} + tblInfo := &model.TableInfo{} + pointPlan := physicalop.PointGetPlan{TblInfo: tblInfo} + pointPlan.SetSchema(expression.NewSchema()) + point := pointPlan.Init(ctx, stats, 0) + batch := (&physicalop.BatchPointGetPlan{TblInfo: tblInfo}).Init(ctx, stats, expression.NewSchema(), nil, 0) + metrics := execdetails.NewRUV2Metrics() + metrics.AddResourceManagerReadCnt(3) + + testCases := []struct { + name string + flat *FlatPhysicalPlan + kind string + }{ + { + name: "point get", + flat: &FlatPhysicalPlan{Main: FlatPlanTree{{Origin: point, IsRoot: true, StoreType: kv.TiDB}}}, + kind: "point_get", + }, + { + name: "batch point get", + flat: &FlatPhysicalPlan{Main: FlatPlanTree{{Origin: batch, IsRoot: true, StoreType: kv.TiDB}}}, + kind: "batch_point_get", + }, + { + name: "mixed point lookup", + flat: &FlatPhysicalPlan{ + Main: FlatPlanTree{{Origin: point, IsRoot: true, StoreType: kv.TiDB}}, + ScalarSubQueries: []FlatPlanTree{{{Origin: batch, IsRoot: true, StoreType: kv.TiDB}}}, + }, + kind: "mixed_point_lookup", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + op, present := readBillingDemoPointLookupTransport(tc.flat, nil, metrics, false) + require.True(t, present) + require.Equal(t, "point_lookup@statement", op.id) + require.Equal(t, readBillingDemoStatusOperatorOK, op.status) + require.Equal(t, readBillingDemoSiteTiKV, op.site) + require.Equal(t, readBillingDemoOpClassPointLookup, op.opClass) + require.Equal(t, tc.kind, op.operatorKind) + require.Len(t, op.units, 1) + require.Equal(t, 3.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.True(t, readBillingDemoOperatorBillable(op)) + }) + } + + zeroOp, present := readBillingDemoPointLookupTransport(testCases[0].flat, nil, execdetails.NewRUV2Metrics(), false) + require.True(t, present) + require.Equal(t, readBillingDemoStatusOperatorOK, zeroOp.status) + require.Zero(t, readBillingDemoUnitValue(zeroOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + + missingOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, nil, false) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, missingOp.reason) + bypassedMetrics := execdetails.NewRUV2Metrics() + bypassedMetrics.SetBypass(true) + bypassedOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, bypassedMetrics, false) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, bypassedOp.reason) + + dmlRuntimeStats := execdetails.NewRuntimeStatsColl(nil) + dmlRuntimeStats.RegisterStats(point.ID(), &readBillingDemoRPCStatsForTest{ + counts: map[tikvrpc.CmdType]int64{tikvrpc.CmdGet: 3}, + }) + dmlRuntimeStats.RegisterStats(batch.ID(), &readBillingDemoRPCStatsForTest{ + counts: map[tikvrpc.CmdType]int64{tikvrpc.CmdBatchGet: 3}, + }) + for i, expectedRequests := range []int64{3, 3, 6} { + dmlOp, present := readBillingDemoPointLookupTransport(testCases[i].flat, dmlRuntimeStats, nil, true) + require.True(t, present) + require.Equal(t, readBillingDemoStatusOperatorOK, dmlOp.status) + require.Equal(t, float64(expectedRequests), readBillingDemoUnitValue(dmlOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceSnapshotRuntimeStats, dmlOp.units[0].source) + } + missingDMLStatsOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, metrics, true) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, missingDMLStatsOp.reason) + mismatchedDMLMetrics := execdetails.NewRUV2Metrics() + mismatchedDMLMetrics.AddResourceManagerReadCnt(4) + mismatchedDMLOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, dmlRuntimeStats, mismatchedDMLMetrics, true) + require.Equal(t, readBillingDemoStatusOperatorOK, mismatchedDMLOp.status) + require.Equal(t, 3.0, readBillingDemoUnitValue(mismatchedDMLOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceSnapshotRuntimeStats, mismatchedDMLOp.units[0].source) + + point.Lock = true + lockOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, metrics, false) + require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, lockOp.reason) + dmlLockOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, dmlRuntimeStats, nil, true) + require.Equal(t, readBillingDemoStatusOperatorOK, dmlLockOp.status) + point.Lock = false + reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + mixedReaderFlat := &FlatPhysicalPlan{ + Main: testCases[0].flat.Main, + ScalarSubQueries: []FlatPlanTree{{{Origin: reader, IsRoot: true, StoreType: kv.TiDB}}}, + } + mixedReaderOp, _ := readBillingDemoPointLookupTransport(mixedReaderFlat, nil, metrics, false) + require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, mixedReaderOp.reason) + mixedReaderDMLOp, _ := readBillingDemoPointLookupTransport(mixedReaderFlat, dmlRuntimeStats, nil, true) + require.Equal(t, readBillingDemoStatusOperatorOK, mixedReaderDMLOp.status) + require.Equal(t, 3.0, readBillingDemoUnitValue(mixedReaderDMLOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + + physicalOp, supported, reason := readBillingDemoClassifyOperator(testCases[0].flat.Main[0]) + physicalOp.id = point.ExplainID().String() + require.True(t, supported) + require.Empty(t, reason) + require.False(t, readBillingDemoOperatorBillable(physicalOp)) + }) +} + +func TestReadBillingDemoV4ExpressionCountsAndOrdering(t *testing.T) { + ctx := mock.NewContext() + stats := &property.StatsInfo{RowCount: 1} + col := &expression.Column{Index: 0, RetType: types.NewFieldType(mysql.TypeLonglong)} + selection := physicalop.PhysicalSelection{Conditions: []expression.Expression{col, col}}.Init(ctx, stats, 0) + projection := physicalop.PhysicalProjection{Exprs: []expression.Expression{col, col, col}}.Init(ctx, stats, 0) + hashAgg := (&physicalop.BasePhysicalAgg{GroupByItems: []expression.Expression{col}, AggFuncs: []*aggregation.AggFuncDesc{nil, nil}}).InitForHash(ctx, stats, 0, expression.NewSchema(col)) + window := physicalop.PhysicalWindow{WindowFuncDescs: []*aggregation.WindowFuncDesc{nil}, PartitionBy: []property.SortItem{{Col: col}}, OrderBy: []property.SortItem{{Col: col}}, Frame: &logicalop.WindowFrame{Start: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col}}, End: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col, col}}}}.Init(ctx, stats, 0) + for _, tc := range []struct { + plan base.Plan + want int64 + }{{selection, 2}, {projection, 3}, {hashAgg, 3}, {window, 6}} { + got, ok := readBillingDemoExpressionCount(tc.plan) + require.True(t, ok) + require.Equal(t, tc.want, got) + } + + baseJoin := physicalop.BasePhysicalJoin{ + LeftConditions: expression.CNFExprs{col}, + RightConditions: expression.CNFExprs{col}, + OtherConditions: expression.CNFExprs{col}, + LeftJoinKeys: []*expression.Column{col, col}, + RightJoinKeys: []*expression.Column{col, col}, + OuterJoinKeys: []*expression.Column{col, col}, + InnerJoinKeys: []*expression.Column{col, col}, + } + compareFilters := &physicalop.ColWithCmpFuncManager{OpType: []string{"gt", "lt"}} + joins := []struct { + name string + plan base.Plan + want int64 + }{ + {name: "hash join", plan: &physicalop.PhysicalHashJoin{BasePhysicalJoin: baseJoin, EqualConditions: []*expression.ScalarFunction{{}, {}}, NAEqualConditions: []*expression.ScalarFunction{{}}}, want: 6}, + {name: "merge join", plan: &physicalop.PhysicalMergeJoin{BasePhysicalJoin: baseJoin, CompareFuncs: []expression.CompareFunc{nil, nil}}, want: 5}, + {name: "index join", plan: &physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, CompareFilters: compareFilters}, want: 7}, + {name: "index hash join", plan: &physicalop.PhysicalIndexHashJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, OuterHashKeys: []*expression.Column{col, col, col}, InnerHashKeys: []*expression.Column{col, col, col}, CompareFilters: compareFilters}}, want: 8}, + {name: "index merge join", plan: &physicalop.PhysicalIndexMergeJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, CompareFilters: compareFilters}, CompareFuncs: []expression.CompareFunc{nil, nil}, OuterCompareFuncs: []expression.CompareFunc{nil}, NeedOuterSort: true}, want: 8}, + } + for _, tc := range joins { + t.Run(tc.name, func(t *testing.T) { + got, ok := readBillingDemoExpressionCount(tc.plan) + require.True(t, ok) + require.Equal(t, tc.want, got) + }) + } + invalidHashJoin := &physicalop.PhysicalHashJoin{BasePhysicalJoin: baseJoin} + invalidHashJoin.RightJoinKeys = invalidHashJoin.RightJoinKeys[:1] + _, ok := readBillingDemoExpressionCount(invalidHashJoin) + require.False(t, ok) + invalidIndexJoin := &physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin} + invalidIndexJoin.InnerJoinKeys = invalidIndexJoin.InnerJoinKeys[:1] + _, ok = readBillingDemoExpressionCount(invalidIndexJoin) + require.False(t, ok) + invalidIndexHashJoin := &physicalop.PhysicalIndexHashJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, OuterHashKeys: []*expression.Column{col}}} + _, ok = readBillingDemoExpressionCount(invalidIndexHashJoin) + require.False(t, ok) + invalidIndexMergeJoin := &physicalop.PhysicalIndexMergeJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin}, OuterCompareFuncs: []expression.CompareFunc{nil}} + _, ok = readBillingDemoExpressionCount(invalidIndexMergeJoin) + require.False(t, ok) + + topN := physicalop.PhysicalTopN{Offset: 3, Count: 5, ByItems: []*plannerutil.ByItems{{Expr: col}, {Expr: col}}}.Init(ctx, stats, 0) + unit, ok := readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN, 4) + require.True(t, ok) + require.Equal(t, 8.0, unit.value) + for _, tc := range []struct { + name string + rows int64 + offset, count uint64 + want float64 + }{ + {name: "zero rows", rows: 0, count: 1, want: 0}, + {name: "one row", rows: 1, count: 1, want: 1}, + {name: "bound saturates at input rows", rows: 4, offset: 3, count: 5, want: 8}, + {name: "huge bound saturates at one input row", rows: 1, offset: math.MaxUint64 - 1, count: 1, want: 1}, + {name: "zero count ignores offset", rows: 4, offset: math.MaxUint64, count: 0, want: 0}, + } { + topN.Offset, topN.Count = tc.offset, tc.count + unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN, tc.rows) + require.True(t, ok, tc.name) + require.Equal(t, tc.want, unit.value, tc.name) + } + topN.Offset, topN.Count = math.MaxUint64, 1 + _, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN, 1) + require.False(t, ok) + require.Equal(t, readBillingDemoReasonInvalidTopNBound, readBillingDemoOrderingFailureReason(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN)) + topN.Offset, topN.Count = 0, 1 + topN.ByItems = []*plannerutil.ByItems{{Expr: &expression.ScalarFunction{}}} + require.False(t, readBillingDemoOrderingMaterialized(&FlatOperator{Origin: topN, IsRoot: false}, nil)) + sort := physicalop.PhysicalSort{ByItems: []*plannerutil.ByItems{{Expr: col}}}.Init(ctx, stats, 0) + unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: sort, IsRoot: true}, readBillingDemoOpClassSort, 3) + require.True(t, ok) + require.InDelta(t, 3*math.Log2(3), unit.value, 1e-12) + + formulaWeights := readBillingDemoWeights{ + Version: "test-v4-calibrated", + CPUWeight: 2, ScanWeight: 3, NetWeight: 5, ReadRequestWeight: 7, WriteRequestWeight: 17, + HashTableWeight: 11, JoinWeight: 13, MutationBytesPerCPUUnit: 10, Calibrated: true, + } + weightedTotal := func(t *testing.T, units []readBillingDemoUnit) float64 { + t.Helper() + var total float64 + for _, unit := range units { + _, ru, ok := readBillingDemoUnitPreviewRU(unit, formulaWeights) + if _, semantic := readBillingDemoUnitWeight(formulaWeights, unit.unit); !semantic { + continue + } + require.True(t, ok, "unit=%+v", unit) + total += ru + } + return total + } + recordRoot := func(runtimeStats *execdetails.RuntimeStatsColl, planID int, rows int) { + rootStats := runtimeStats.GetBasicRuntimeStats(planID, true) + rootStats.Record(time.Millisecond, rows) + rootStats.RecordBytes(0, int64(rows*8)) + } + schema := expression.NewSchema(col) + + t.Run("root unary formulas use exact semantic terms", func(t *testing.T) { + for _, tc := range []struct { + name string + opClass string + buildPlan func() base.Plan + wantRU float64 + }{ + {name: "selection", opClass: readBillingDemoOpClassFilter, buildPlan: func() base.Plan { + return physicalop.PhysicalSelection{Conditions: []expression.Expression{col, col}}.Init(ctx, stats, 0) + }, wantRU: 16}, + {name: "projection", opClass: readBillingDemoOpClassProjection, buildPlan: func() base.Plan { + return physicalop.PhysicalProjection{Exprs: []expression.Expression{col, col, col}}.Init(ctx, stats, 0) + }, wantRU: 24}, + {name: "stream agg", opClass: readBillingDemoOpClassStreamAgg, buildPlan: func() base.Plan { + return (&physicalop.BasePhysicalAgg{GroupByItems: []expression.Expression{col}, AggFuncs: []*aggregation.AggFuncDesc{nil, nil}}).InitForStream(ctx, stats, 0, schema) + }, wantRU: 24}, + {name: "hash agg", opClass: readBillingDemoOpClassHashAgg, buildPlan: func() base.Plan { + return (&physicalop.BasePhysicalAgg{GroupByItems: []expression.Expression{col}, AggFuncs: []*aggregation.AggFuncDesc{nil, nil}}).InitForHash(ctx, stats, 0, schema) + }, wantRU: 46}, + {name: "limit", opClass: readBillingDemoOpClassLimit, buildPlan: func() base.Plan { + return physicalop.PhysicalLimit{}.Init(ctx, stats, 0) + }, wantRU: 8}, + {name: "union scan", opClass: readBillingDemoOpClassOverlayReader, buildPlan: func() base.Plan { + return physicalop.PhysicalUnionScan{}.Init(ctx, stats, 0) + }, wantRU: 8}, + {name: "window", opClass: readBillingDemoOpClassWindow, buildPlan: func() base.Plan { + return physicalop.PhysicalWindow{WindowFuncDescs: []*aggregation.WindowFuncDesc{nil}, PartitionBy: []property.SortItem{{Col: col}}, OrderBy: []property.SortItem{{Col: col}}, Frame: &logicalop.WindowFrame{Start: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col}}, End: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col, col}}}}.Init(ctx, stats, 0) + }, wantRU: 48}, + {name: "sort", opClass: readBillingDemoOpClassSort, buildPlan: func() base.Plan { + return physicalop.PhysicalSort{ByItems: []*plannerutil.ByItems{{Expr: col}, {Expr: col}}}.Init(ctx, stats, 0) + }, wantRU: 16}, + {name: "topn with offset", opClass: readBillingDemoOpClassTopN, buildPlan: func() base.Plan { + return physicalop.PhysicalTopN{Offset: 3, Count: 5, ByItems: []*plannerutil.ByItems{{Expr: col}, {Expr: col}}}.Init(ctx, stats, 0) + }, wantRU: 16}, + } { + t.Run(tc.name, func(t *testing.T) { + plan := tc.buildPlan() + child := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + child.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: plan, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRoot(runtimeStats, plan.ID(), 2) + recordRoot(runtimeStats, child.ID(), 4) + operator := readBillingDemoOperatorResult{id: plan.ExplainID().String(), opClass: tc.opClass} + require.True(t, readBillingDemoOperatorBillable(operator)) + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], operator) + require.True(t, ok, reason) + require.Empty(t, reason) + require.Equal(t, tc.wantRU, weightedTotal(t, units)) + if tc.opClass == readBillingDemoOpClassOverlayReader { + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeChildActRows, readBillingDemoUnitSource(units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, -1.0, readBillingDemoUnitValue(units, readBillingDemoUnitExpressionCount, readBillingDemoInputSideAll)) + } + }) + } + }) + + t.Run("ordering rejects columns outside the child schema", func(t *testing.T) { + plan := physicalop.PhysicalSort{ByItems: []*plannerutil.ByItems{{Expr: col}}}.Init(ctx, stats, 0) + child := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + child.SetSchema(expression.NewSchema()) + tree := FlatPlanTree{ + {Origin: plan, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRoot(runtimeStats, plan.ID(), 2) + recordRoot(runtimeStats, child.ID(), 4) + _, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], readBillingDemoOperatorResult{opClass: readBillingDemoOpClassSort}) + require.False(t, ok) + require.Equal(t, readBillingDemoReasonMissingOrderingProjection, reason) + }) + + t.Run("root join formulas use both inputs and subtype expressions", func(t *testing.T) { + joinBase := physicalop.BasePhysicalJoin{ + OtherConditions: expression.CNFExprs{col}, + LeftJoinKeys: []*expression.Column{col}, RightJoinKeys: []*expression.Column{col}, + OuterJoinKeys: []*expression.Column{col}, InnerJoinKeys: []*expression.Column{col}, + } + for _, tc := range []struct { + name string + opClass string + buildPlan func() base.Plan + hashRows int64 + wantRU float64 + }{ + {name: "merge join", opClass: readBillingDemoOpClassMergeJoin, buildPlan: func() base.Plan { + return physicalop.PhysicalMergeJoin{BasePhysicalJoin: joinBase, CompareFuncs: []expression.CompareFunc{nil}}.Init(ctx, stats, 0) + }, wantRU: 66}, + {name: "hash join", opClass: readBillingDemoOpClassHashJoin, buildPlan: func() base.Plan { + return physicalop.PhysicalHashJoin{BasePhysicalJoin: joinBase, EqualConditions: []*expression.ScalarFunction{{}}}.Init(ctx, stats, 0) + }, hashRows: 3, wantRU: 99}, + {name: "index join", opClass: readBillingDemoOpClassLookupJoin, buildPlan: func() base.Plan { + return physicalop.PhysicalIndexJoin{BasePhysicalJoin: joinBase}.Init(ctx, stats, 0) + }, wantRU: 66}, + {name: "index hash join", opClass: readBillingDemoOpClassLookupJoin, buildPlan: func() base.Plan { + indexJoin := physicalop.PhysicalIndexJoin{BasePhysicalJoin: joinBase, OuterHashKeys: []*expression.Column{col}, InnerHashKeys: []*expression.Column{col}}.Init(ctx, stats, 0) + return physicalop.PhysicalIndexHashJoin{PhysicalIndexJoin: *indexJoin}.Init(ctx) + }, wantRU: 66}, + {name: "index merge join", opClass: readBillingDemoOpClassLookupJoin, buildPlan: func() base.Plan { + indexJoin := physicalop.PhysicalIndexJoin{BasePhysicalJoin: joinBase}.Init(ctx, stats, 0) + return physicalop.PhysicalIndexMergeJoin{PhysicalIndexJoin: *indexJoin, CompareFuncs: []expression.CompareFunc{nil}}.Init(ctx) + }, wantRU: 66}, + } { + t.Run(tc.name, func(t *testing.T) { + plan := tc.buildPlan() + left := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + right := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + left.SetSchema(schema) + right.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: plan, ChildrenIdx: []int{1, 2}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: left, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB, Label: BuildSide}, + {Origin: right, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB, Label: ProbeSide}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRoot(runtimeStats, plan.ID(), 2) + recordRoot(runtimeStats, left.ID(), 4) + recordRoot(runtimeStats, right.ID(), 6) + if tc.opClass == readBillingDemoOpClassHashJoin { + runtimeStats.RegisterStats(plan.ID(), &readBillingDemoHashStatsForTest{rows: tc.hashRows}) + } + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], readBillingDemoOperatorResult{opClass: tc.opClass}) + require.True(t, ok, reason) + require.Empty(t, reason) + require.Equal(t, tc.wantRU, weightedTotal(t, units)) + }) + } + }) + + t.Run("cop selection uses exact child rows without logical byte width", func(t *testing.T) { + reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + summary := func(rows uint64) *tipb.ExecutorExecutionSummary { + return &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &rows, + NumIterations: &one, + } + } + runtimeStats.RecordExpectedCopTasks([]int{scan.ID(), selection.ID()}) + runtimeStats.RecordOneCopTask(scan.ID(), kv.TiKV, summary(4)) + runtimeStats.RecordCopStats(selection.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, tikvutil.TimeDetail{}, summary(2)) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + outcome := readBillingDemoCopUnits(estimator, 1, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassFilter, operatorKind: "selection"}) + require.True(t, outcome.success, "%+v", outcome.failure) + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitExpressionCount, readBillingDemoInputSideAll)) + require.Equal(t, 8.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, -1.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + + scanOutcome := readBillingDemoCopUnits(estimator, 2, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, scanOutcome.success, "%+v", scanOutcome.failure) + require.Equal(t, 40.0, readBillingDemoUnitValue(scanOutcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + + selection.SetChildren(scan) + reader.TablePlan = selection + recordRoot(runtimeStats, reader.ID(), 2) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + metrics := execdetails.NewRUV2Metrics() + metrics.AddResourceManagerReadCnt(1) + metrics.AddTiKVCoprocessorResponseBytes(40) + result := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, metrics) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + seenScanBytes := false + seenSelectionCPU := false + for _, sample := range buildReadBillingDemoStatementStats(result).BaseUnits { + switch { + case sample.Site == readBillingDemoSiteTiKV && sample.OpClass == readBillingDemoOpClassRangeScan && sample.Unit == readBillingDemoUnitScanBytes: + seenScanBytes = true + require.Equal(t, 40.0, sample.Value) + case sample.Site == readBillingDemoSiteTiKV && sample.OpClass == readBillingDemoOpClassFilter && sample.Unit == readBillingDemoUnitCPUWork: + seenSelectionCPU = true + require.Equal(t, 8.0, sample.Value) + } + } + require.True(t, seenScanBytes) + require.True(t, seenSelectionCPU) + + runtimeStats.RecordExpectedCopTasks([]int{scan.ID(), selection.ID()}) + failedResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, metrics) + require.Equal(t, readBillingDemoStatusUnknownInput, failedResult.status) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, failedResult.reason) + require.Empty(t, buildReadBillingDemoStatementStats(failedResult).BaseUnits) + }) + + t.Run("cop scan detail provenance stays fail closed", func(t *testing.T) { + buildEstimator := func(detail *tikvutil.ScanDetail, responses, scanSummaries, detailRecords int, holderSummaries bool) (*readBillingDemoCopEstimator, int) { + localReader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + localSelection := physicalop.PhysicalSelection{Conditions: []expression.Expression{col}}.Init(ctx, stats, 0) + localScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tree := FlatPlanTree{ + {Origin: localReader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: localSelection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + {Origin: localScan, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + zero := uint64(0) + summary := &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &zero, + NumIterations: &one, + } + for range responses { + runtimeStats.RecordExpectedCopTasks([]int{localScan.ID(), localSelection.ID()}) + } + for range scanSummaries { + runtimeStats.RecordOneCopTask(localScan.ID(), kv.TiKV, summary) + } + for range detailRecords { + var holderSummary *tipb.ExecutorExecutionSummary + if holderSummaries { + holderSummary = summary + } + runtimeStats.RecordCopStats(localSelection.ID(), kv.TiKV, detail, tikvutil.TimeDetail{}, holderSummary) + } + return newReadBillingDemoCopEstimator(tree, runtimeStats), 2 + } + + estimator, scanIdx := buildEstimator(&tikvutil.ScanDetail{}, 1, 1, 1, true) + outcome := readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, outcome.success, "%+v", outcome.failure) + require.Zero(t, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 2, 2, 2, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, outcome.success, "%+v", outcome.failure) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 1, 1, 1, false) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, outcome.success, "%+v", outcome.failure) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4}, 1, 1, 1, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonMissingScanWidthEvidence, outcome.failure.reason) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 2, 1, 2, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, outcome.failure.reason) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 2, 2, 1, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, outcome.failure.reason) + + estimator, scanIdx = buildEstimator(nil, 1, 1, 0, false) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonMissingScanWidthEvidence, outcome.failure.reason) + + localReader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + localSelection := physicalop.PhysicalSelection{Conditions: []expression.Expression{col}}.Init(ctx, stats, 0) + localScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tree := FlatPlanTree{ + {Origin: localReader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: localSelection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + {Origin: localScan, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + zero := uint64(0) + summary := &tipb.ExecutorExecutionSummary{TimeProcessedNs: &one, NumProducedRows: &zero, NumIterations: &one} + detail := &tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40} + runtimeStats.RecordExpectedCopTasks([]int{localScan.ID(), localSelection.ID()}) + runtimeStats.RecordCopStats(localScan.ID(), kv.TiKV, detail, tikvutil.TimeDetail{}, summary) + runtimeStats.RecordCopStats(localSelection.ID(), kv.TiKV, detail, tikvutil.TimeDetail{}, summary) + outcome = readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 2, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonAmbiguousCopScanWidth, outcome.failure.reason) + }) +} + +func TestReadBillingDemoV4WriteRequests(t *testing.T) { + oldWeights := readBillingDemoV4Weights + readBillingDemoV4Weights = readBillingDemoWeights{ + Version: "test-v4-calibrated", CPUWeight: 2, MutationBytesPerCPUUnit: 10, Calibrated: true, + } + t.Cleanup(func() { readBillingDemoV4Weights = oldWeights }) + ctx := mock.NewContext() + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordSet(5, 7) + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordDelete(3) + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&result, ctx, "update") + mutation := result.operators[0] + require.Equal(t, readBillingDemoOpClassKVMutation, mutation.opClass) + require.Equal(t, readBillingDemoOperatorMemDBMutation, mutation.operatorKind) + require.Equal(t, 3.5, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitEncodedMutationCount, readBillingDemoInputSideAll)) + require.Equal(t, 15.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitEncodedMutationBytes, readBillingDemoInputSideAll)) + require.Equal(t, 1.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitSetCount, readBillingDemoInputSideAll)) + require.Equal(t, 1.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitDeleteCount, readBillingDemoInputSideAll)) + require.Equal(t, 8.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitKeyBytes, readBillingDemoInputSideAll)) + require.Equal(t, 7.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitValueBytes, readBillingDemoInputSideAll)) + + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.True(t, rows[0].hasPreviewRU) + require.Equal(t, 7.0, rows[0].previewRU) + var mutationCPUWorkRows int + for _, row := range rows { + if row.operatorClass == "tidb/kv_mutation" && row.component == readBillingDemoOperatorMemDBMutation && row.unit == readBillingDemoUnitCPUWork { + mutationCPUWorkRows++ + require.Equal(t, 3.5, row.workRows) + require.Equal(t, readBillingDemoInputSourceStmtMemDBMutation, row.source) + } + } + require.Equal(t, 1, mutationCPUWorkRows) + + stats := buildReadBillingDemoStatementStats(result) + var mutationCPUWorkSamples int + for _, sample := range stats.BaseUnits { + if sample.Unit == readBillingDemoUnitCPUWork { + mutationCPUWorkSamples++ + require.Equal(t, readBillingDemoSiteTiDB, sample.Site) + require.Equal(t, readBillingDemoOpClassKVMutation, sample.OpClass) + require.Equal(t, readBillingDemoOperatorMemDBMutation, sample.OperatorKind) + require.Equal(t, readBillingDemoInputSourceStmtMemDBMutation, sample.InputSource) + require.Equal(t, readBillingDemoInputSideAll, sample.InputSide) + require.Equal(t, 3.5, sample.Value) + } + } + require.Equal(t, 1, mutationCPUWorkSamples) + + readBillingDemoV4Weights = readBillingDemoWeights{MutationBytesPerCPUUnit: 10} + uncalibratedResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&uncalibratedResult, ctx, "update") + require.Len(t, uncalibratedResult.operators[0].units, 6) + require.Equal(t, -1.0, readBillingDemoUnitValue(uncalibratedResult.operators[0].units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(uncalibratedResult.operators[0].units, readBillingDemoUnitEncodedMutationCount, readBillingDemoInputSideAll)) + require.Equal(t, 15.0, readBillingDemoUnitValue(uncalibratedResult.operators[0].units, readBillingDemoUnitEncodedMutationBytes, readBillingDemoInputSideAll)) + require.Len(t, uncalibratedResult.operators, 2) + require.Equal(t, readBillingDemoReasonUncalibratedMutation, uncalibratedResult.operators[1].reason) + + dmlMetrics := execdetails.NewRUV2Metrics() + dmlMetrics.AddResourceManagerWriteCnt(7) + dml := buildTiKVWriteBillingDemoOperators("update", dmlMetrics, false) + require.Equal(t, 7.0, readBillingDemoUnitValue(dml[0].units, readBillingDemoUnitWriteRequestCount, readBillingDemoInputSideAll)) + commitMetrics := execdetails.NewRUV2Metrics() + commitMetrics.AddResourceManagerWriteCnt(2) + commit := buildTiKVWriteBillingDemoOperators("", commitMetrics, false) + require.Equal(t, 2.0, readBillingDemoUnitValue(commit[0].units, readBillingDemoUnitWriteRequestCount, readBillingDemoInputSideAll)) + require.Empty(t, commit[0].dmlKind) + + pipelinedDML := buildTiKVWriteBillingDemoOperators("update", dmlMetrics, true) + require.Len(t, pipelinedDML, 1) + require.Equal(t, readBillingDemoStatusPartial, pipelinedDML[0].status) + require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedDML[0].reason) + require.Empty(t, pipelinedDML[0].units) + + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.MarkPipelined() + pipelinedCommit := buildTxnCommitBillingDemoResult(ctx, commitMetrics, nil) + require.Len(t, pipelinedCommit.operators, 1) + require.Equal(t, readBillingDemoStatusPartial, pipelinedCommit.operators[0].status) + require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedCommit.operators[0].reason) + require.Empty(t, pipelinedCommit.operators[0].units) +} + +func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { + t.Skip("v3 opclass-weight expectations are superseded by TestReadBillingDemoV4FormulaContract") + require.Equal(t, "v3-resource-formula-uncalibrated", readBillingDemoWeightVersion) + tidbWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) + require.True(t, ok) + tikvWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) + require.True(t, ok) + require.NotEqual(t, tidbWeights, tikvWeights) + _, ok = readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion) + require.True(t, ok) + _, ok = readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion) + require.False(t, ok) + + weight, previewRU, ok := readBillingDemoUnitPreviewRU( + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, value: 4096}, + tidbWeights, + ) + require.True(t, ok) + require.Equal(t, tidbWeights.byte, weight) + require.Equal(t, 4096*tidbWeights.byte, previewRU) + topNWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassTopN, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, topNWeights.row) + require.NotZero(t, topNWeights.orderWork) + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitOrderWork, value: 12}, topNWeights) + require.True(t, ok) + require.Equal(t, topNWeights.orderWork, weight) + require.Equal(t, 12*topNWeights.orderWork, previewRU) + tikvTopNWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, tikvTopNWeights.row) + require.NotZero(t, tikvTopNWeights.orderWork) + + _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: "scan_total_keys", value: 4}, tidbWeights) + require.False(t, ok) + for _, outputUnit := range []string{readBillingDemoUnitOutputRows, readBillingDemoUnitOutputBytes} { + _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: outputUnit, value: 4}, tidbWeights) + require.False(t, ok) + require.True(t, readBillingDemoUnitDiagnosticOnly(outputUnit)) + } + for _, outputClass := range []string{ + readBillingDemoOpClassHashAgg, + readBillingDemoOpClassStreamAgg, + readBillingDemoOpClassHashJoin, + readBillingDemoOpClassMergeJoin, + readBillingDemoOpClassLookupJoin, + } { + require.True(t, readBillingDemoOperatorHasOutputShadows(outputClass)) + } + require.False(t, readBillingDemoOperatorHasOutputShadows(readBillingDemoOpClassProjection)) + + writeWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion) + require.True(t, ok) + require.InEpsilon(t, readBillingDemoWriteKeyWeight, writeWeights.writeKey, 0.000001) + require.Equal(t, readBillingDemoWriteByteWeight, writeWeights.writeByte) + require.Zero(t, writeWeights.region) + require.Zero(t, writeWeights.writeRPC) + + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitWriteKeys, value: 3}, writeWeights) + require.True(t, ok) + require.Equal(t, writeWeights.writeKey, weight) + require.Equal(t, 3*writeWeights.writeKey, previewRU) + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitWriteByte, value: 4096}, writeWeights) + require.True(t, ok) + require.Equal(t, writeWeights.writeByte, weight) + require.Equal(t, 4096*writeWeights.writeByte, previewRU) + for _, diagnosticUnit := range []string{readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount} { + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: diagnosticUnit, value: 99}, writeWeights) + require.True(t, ok) + require.Zero(t, weight) + require.Zero(t, previewRU) + } + + mutationWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassKVMutation, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, mutationWeights.mutationCount) + require.Zero(t, mutationWeights.mutationByte) + require.NotEqual(t, writeWeights.writeKey, mutationWeights.mutationCount) + require.NotEqual(t, writeWeights.writeByte, mutationWeights.mutationByte) + for _, mutationUnit := range []string{readBillingDemoUnitEncodedMutationCount, readBillingDemoUnitEncodedMutationBytes} { + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: mutationUnit, value: 99}, mutationWeights) + require.True(t, ok) + require.Zero(t, weight) + require.Zero(t, previewRU) + } + + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 5} + for _, tc := range []struct { + name string + site string + opClass string + op func() *FlatOperator + }{ + { + name: "range scan", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassRangeScan, + op: func() *FlatOperator { + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + return &FlatOperator{Origin: scan, IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "filter", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassFilter, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalSelection{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "projection", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassProjection, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalProjection{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "limit", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassLimit, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalLimit{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "topn", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassTopN, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalTopN{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "hash agg", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassHashAgg, + op: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "stream agg", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassStreamAgg, + op: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForStream(ctx, stats, 0, schema), IsRoot: false, StoreType: kv.TiKV} + }, + }, + } { + t.Run(tc.site+" "+tc.name, func(t *testing.T) { + requireReadBillingDemoClass(t, tc.op(), tc.site, tc.opClass, true, "") + }) + } + requireReadBillingDemoClass(t, &FlatOperator{ + Origin: physicalop.PhysicalIndexScan{}.Init(ctx, 0), + IsRoot: false, + StoreType: kv.TiFlash, + }, readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, false, readBillingDemoReasonUnsupportedTiFlash) + requireReadBillingDemoClass(t, &FlatOperator{ + Origin: physicalop.PhysicalExchangeReceiver{}.Init(ctx, stats), + IsRoot: true, + }, readBillingDemoSiteTiDB, readBillingDemoOpClassReaderReceive, false, readBillingDemoReasonUnsupportedMPP) + + t.Run("root sort and topn preserve algorithmic work", func(t *testing.T) { + child := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + child.SetSchema(schema) + sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + topN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0) + topN.SetSchema(schema) + for _, tc := range []struct { + name string + plan *FlatOperator + expectedClass string + inputRows int + inputBytes int64 + expectedWork float64 + expectedWorkText string + }{ + {name: "sort n log n preserves fractional work", plan: &FlatOperator{Origin: sort, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, expectedClass: readBillingDemoOpClassSort, inputRows: 3, inputBytes: 30, expectedWork: 4.754887502163469, expectedWorkText: "4.754887502163469"}, + {name: "topn n log k", plan: &FlatOperator{Origin: topN, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, expectedClass: readBillingDemoOpClassTopN, inputRows: 8, inputBytes: 80, expectedWork: 16, expectedWorkText: "16"}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := FlatPlanTree{ + tc.plan, + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + for _, planID := range []int{tc.plan.Origin.ID(), child.ID()} { + basic := runtimeStats.GetBasicRuntimeStats(planID, true) + basic.Record(time.Millisecond, tc.inputRows) + basic.RecordBytes(0, tc.inputBytes) + } + operator, supported, reason := readBillingDemoClassifyOperator(tree[0]) + require.True(t, supported) + require.Empty(t, reason) + require.Equal(t, tc.expectedClass, operator.opClass) + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], operator) + require.True(t, ok) + require.Empty(t, reason) + require.Equal(t, float64(tc.inputRows), readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.InDelta(t, tc.expectedWork, readBillingDemoUnitValue(units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll), 1e-12) + require.Equal(t, readBillingDemoInputSourceRuntimeOrderingWork, readBillingDemoUnitSource(units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) + operator.status = readBillingDemoStatusOperatorOK + operator.units = units + operator.id = tc.plan.ExplainID().String() + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + operators: []readBillingDemoOperatorResult{operator}, + } + weights, ok := readBillingDemoResolveWeights(operator.site, operator.opClass, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, weights.row) + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.InEpsilon(t, weights.fixedEvent+float64(tc.inputBytes)*weights.byte+tc.expectedWork*weights.orderWork, rows[0].previewRU, 0.000001) + var orderRows int + for _, row := range rows { + if row.unit == readBillingDemoUnitOrderWork { + orderRows++ + require.InDelta(t, tc.expectedWork, row.workRows, 1e-12) + require.Equal(t, tc.expectedWorkText, row.toStrings()[9]) + } + } + require.Equal(t, 1, orderRows) + + stats := buildReadBillingDemoStatementStats(result) + require.Equal(t, "v3-resource-formula-uncalibrated", stats.WeightVersion) + var orderSamples int + for _, sample := range stats.BaseUnits { + if sample.Unit == readBillingDemoUnitOrderWork { + orderSamples++ + require.InDelta(t, tc.expectedWork, sample.Value, 1e-12) + require.Equal(t, readBillingDemoInputSourceRuntimeOrderingWork, sample.InputSource) + } + } + require.Equal(t, 1, orderSamples) + }) + } + }) + + t.Run("root aggregations expose diagnostic output units without changing formula", func(t *testing.T) { + child := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + child.SetSchema(schema) + for _, tc := range []struct { + name string + opClass string + agg func() *FlatOperator + }{ + {name: "hash agg", opClass: readBillingDemoOpClassHashAgg, agg: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema), ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB} + }}, + {name: "stream agg", opClass: readBillingDemoOpClassStreamAgg, agg: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForStream(ctx, stats, 0, schema), ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := FlatPlanTree{ + tc.agg(), + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + aggStats := runtimeStats.GetBasicRuntimeStats(tree[0].Origin.ID(), true) + aggStats.Record(time.Millisecond, 2) + aggStats.RecordBytes(80, 30) + childStats := runtimeStats.GetBasicRuntimeStats(child.ID(), true) + childStats.Record(time.Millisecond, 8) + childStats.RecordBytes(0, 80) + + operator, supported, reason := readBillingDemoClassifyOperator(tree[0]) + require.True(t, supported) + require.Empty(t, reason) + require.Equal(t, tc.opClass, operator.opClass) + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], operator) + require.True(t, ok) + require.Empty(t, reason) + require.Equal(t, 8.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 30.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + + operator.id = tree[0].ExplainID().String() + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + operator.actRows = 2 + operator.hasActRows = true + operator.units = units + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone, operators: []readBillingDemoOperatorResult{operator}} + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + weights, ok := readBillingDemoResolveWeights(operator.site, operator.opClass, readBillingDemoWeightVersion) + require.True(t, ok) + require.InEpsilon(t, weights.fixedEvent+8*weights.row+80*weights.byte, rows[0].previewRU, 0.000001) + seenOutputUnits := make(map[string]bool) + for _, row := range rows { + if row.unit != readBillingDemoUnitOutputRows && row.unit != readBillingDemoUnitOutputBytes { + continue + } + seenOutputUnits[row.unit] = true + require.False(t, row.hasWeight) + require.False(t, row.hasPreviewRU) + require.Contains(t, row.note, "diagnostic_only=true") + if row.unit == readBillingDemoUnitOutputRows { + require.Equal(t, 2.0, row.workRows) + require.Equal(t, int64(2), row.count) + } else { + require.Equal(t, 30.0, row.workBytes) + } + rendered := row.toStrings() + require.Empty(t, rendered[13]) + require.Empty(t, rendered[14]) + } + require.Equal(t, map[string]bool{readBillingDemoUnitOutputRows: true, readBillingDemoUnitOutputBytes: true}, seenOutputUnits) + + statementStats := buildReadBillingDemoStatementStats(result) + seenOutputUnits = make(map[string]bool) + for _, sample := range statementStats.BaseUnits { + if sample.Unit == readBillingDemoUnitOutputRows || sample.Unit == readBillingDemoUnitOutputBytes { + seenOutputUnits[sample.Unit] = true + } + } + require.Equal(t, map[string]bool{readBillingDemoUnitOutputRows: true, readBillingDemoUnitOutputBytes: true}, seenOutputUnits) + }) + } + }) + + t.Run("ordering work boundaries fail only on invalid evidence", func(t *testing.T) { + sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + sortOp := &FlatOperator{Origin: sort, IsRoot: true, StoreType: kv.TiDB} + unit, ok := readBillingDemoOrderingWorkUnit(sortOp, readBillingDemoOpClassSort, 0) + require.True(t, ok) + require.Zero(t, unit.value) + _, ok = readBillingDemoOrderingWorkUnit(sortOp, readBillingDemoOpClassSort, -1) + require.False(t, ok) + + maxInt64 := int64(^uint64(0) >> 1) + unit, ok = readBillingDemoOrderingWorkUnit(sortOp, readBillingDemoOpClassSort, maxInt64/2) + require.True(t, ok) + require.Greater(t, unit.value, float64(maxInt64)) + + hugeTopN := physicalop.PhysicalTopN{Offset: math.MaxUint64 - 1, Count: 1}.Init(ctx, stats, 0) + unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: hugeTopN, IsRoot: true, StoreType: kv.TiDB}, readBillingDemoOpClassTopN, 1) + require.True(t, ok) + require.Equal(t, 1.0, unit.value) + + malformedCopTopN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0) + _, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: malformedCopTopN, IsRoot: false, StoreType: kv.TiKV}, readBillingDemoOpClassTopN, 8) + require.False(t, ok) + }) + indexMerge := &physicalop.PhysicalIndexMergeReader{} + indexMerge.BasePhysicalPlan = physicalop.NewBasePhysicalPlan(ctx, "IndexMerge", indexMerge, 0) + requireReadBillingDemoClass(t, &FlatOperator{ + Origin: indexMerge, + IsRoot: true, + }, readBillingDemoSiteTiDB, readBillingDemoOpClassLookupReader, false, readBillingDemoReasonUnsupportedIndexMerge) +} + +func requireReadBillingDemoClass(t *testing.T, op *FlatOperator, site, opClass string, supported bool, reason string) { + t.Helper() + operator, ok, actualReason := readBillingDemoClassifyOperator(op) + require.Equal(t, supported, ok) + require.Equal(t, reason, actualReason) + require.Equal(t, site, operator.site) + require.Equal(t, opClass, operator.opClass) + if supported && readBillingDemoOperatorBillable(operator) { + _, hasWeights := readBillingDemoResolveWeights(operator.site, operator.opClass, readBillingDemoWeightVersion) + require.True(t, hasWeights, "missing read billing demo weights for %s/%s", operator.site, operator.opClass) + } +} + +func TestExplainRUComponentSnapshotStatusAndWeights(t *testing.T) { + require.Equal(t, explainRUComponentSnapshotMissing, extractExplainRUTestSnapshotStatus(nil)) + require.Equal(t, explainRUComponentSnapshotMissing, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{})) + require.Equal(t, explainRUComponentSnapshotNonV2, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV1, + Metrics: &execdetails.RUV2Metrics{}, + })) + require.Equal(t, explainRUComponentSnapshotNilMetrics, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV2, + })) + + bypassedMetrics := &execdetails.RUV2Metrics{} + bypassedMetrics.SetBypass(true) + require.Equal(t, explainRUComponentSnapshotBypassed, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV2, + Metrics: bypassedMetrics, + })) + + okStats := &execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV2, + Metrics: &execdetails.RUV2Metrics{}, + } + snapshot, status := extractExplainRUTestSnapshot(okStats) + require.Equal(t, explainRUComponentSnapshotOK, status) + require.Same(t, okStats, snapshot) +} + +func TestReadBillingDemoWriteDMLResult(t *testing.T) { + t.Skip("v3 commit-detail formula expectations are superseded by TestReadBillingDemoV4WriteRequests") + ctx := mock.NewContext() + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordSet(5, 7) + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordDelete(3) + for _, dmlKind := range []string{"insert", "update", "delete", "upsert"} { + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&result, ctx, dmlKind) + require.NotEmpty(t, result.operators) + op := result.operators[0] + require.Equal(t, readBillingDemoSiteTiDB, op.site) + require.Equal(t, readBillingDemoOpClassKVMutation, op.opClass) + require.Equal(t, readBillingDemoOperatorMemDBMutation, op.operatorKind) + require.Equal(t, dmlKind, op.dmlKind) + require.Equal(t, readBillingDemoScopeStatementAttempted, op.scope) + units := make(map[string]readBillingDemoUnit) + for _, unit := range op.units { + units[unit.unit] = unit + } + require.Equal(t, 2.0, units[readBillingDemoUnitEncodedMutationCount].value) + require.Equal(t, 15.0, units[readBillingDemoUnitEncodedMutationBytes].value) + weights, ok := readBillingDemoResolveWeights(op.site, op.opClass, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, weights.mutationCount) + require.Zero(t, weights.mutationByte) + require.Contains(t, result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonDMLAncillaryPartial)) + } + + ctx.GetSessionVars().SetInTxn(true) + ctx.GetSessionVars().TxnCtx.CouldRetry = true + retryableResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&retryableResult, ctx, "update") + require.Contains(t, retryableResult.operators, readBillingDemoMutationDiagnostic("update", readBillingDemoReasonOptimisticReplayPartial)) + ctx.GetSessionVars().SetInTxn(false) + ctx.GetSessionVars().TxnCtx.CouldRetry = false + + ruv2Metrics := execdetails.NewRUV2Metrics() + ruv2Metrics.AddResourceManagerWriteCnt(7) + commitDetail := &tikvutil.CommitDetails{ + WriteKeys: 3, + WriteSize: 66, + PrewriteRegionNum: 2, + } + result := buildWriteBillingDemoResultFromDetails("insert", commitDetail, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + require.Equal(t, readBillingDemoReasonNone, result.reason) + require.Len(t, result.operators, 1) + require.Equal(t, readBillingDemoStatusOperatorOK, result.operators[0].status) + require.Equal(t, readBillingDemoOperatorTxnPrewrite, result.operators[0].operatorKind) + require.Equal(t, "insert", result.operators[0].dmlKind) + + units := make(map[string]readBillingDemoUnit) + for _, unit := range result.operators[0].units { + units[unit.unit] = unit + } + require.Equal(t, 3.0, units[readBillingDemoUnitWriteKeys].value) + require.Equal(t, 66.0, units[readBillingDemoUnitWriteByte].value) + require.Equal(t, 2.0, units[readBillingDemoUnitPrewriteRegionNum].value) + require.Equal(t, 7.0, units[readBillingDemoUnitTiKVWriteRPCCount].value) + + ctx.GetSessionVars().StmtCtx.MergeExecDetails(commitDetail) + commitResult := buildTxnCommitBillingDemoResult(ctx, ruv2Metrics, nil) + require.Equal(t, readBillingDemoStatusSuccess, commitResult.status) + require.NotEmpty(t, commitResult.operators) + require.Equal(t, readBillingDemoStatusOperatorOK, commitResult.operators[0].status) + require.Empty(t, commitResult.operators[0].dmlKind) + require.Equal(t, readBillingDemoScopeTxnPrewritePayload, commitResult.operators[0].scope) + commitUnits := make(map[string]readBillingDemoUnit) + for _, unit := range commitResult.operators[0].units { + commitUnits[unit.unit] = unit + } + require.Equal(t, 3.0, commitUnits[readBillingDemoUnitWriteKeys].value) + require.Equal(t, 66.0, commitUnits[readBillingDemoUnitWriteByte].value) + + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.InEpsilon(t, + 3*readBillingDemoWriteKeyWeight+66*readBillingDemoWriteByteWeight, + rows[0].previewRU, + 0.000001, + ) + var diagnosticRows int + for _, row := range rows[1:] { + if row.unit == readBillingDemoUnitPrewriteRegionNum || row.unit == readBillingDemoUnitTiKVWriteRPCCount { + diagnosticRows++ + require.True(t, row.hasWeight) + require.Zero(t, row.weight) + require.True(t, row.hasPreviewRU) + require.Zero(t, row.previewRU) + require.Contains(t, row.note, "diagnostic_only=true") + } + } + require.Equal(t, 2, diagnosticRows) + + stats := buildReadBillingDemoStatementStats(result) + require.Len(t, stats.BaseUnits, 4) + + partialResult := buildWriteBillingDemoResultFromDetails("delete", &tikvutil.CommitDetails{WriteKeys: 1, WriteSize: 2}, nil) + require.Equal(t, readBillingDemoStatusSuccess, partialResult.status) + require.Len(t, partialResult.operators, 3) + rows = explainRUBuildReadBillingRows(partialResult, explainRUComponentSnapshotMissing) + require.Contains(t, rows[0].note, "partial_missing_prewrite_region_num") + require.Contains(t, rows[0].note, "partial_missing_write_rpc_count") + + missingResult := buildWriteBillingDemoResultFromDetails("update", nil, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, missingResult.status) + require.Len(t, missingResult.operators, 1) + require.Equal(t, readBillingDemoStatusPartial, missingResult.operators[0].status) + require.Equal(t, readBillingDemoReasonMissingCommitDetail, missingResult.operators[0].reason) + require.True(t, missingResult.operators[0].emitStatusRow) + // Pipelined transactions expose a non-nil CommitDetails without logical + // WriteKeys/WriteSize. The incomplete payload must not become billable zero + // units merely because the detail object exists. + pipelinedResult := buildTiKVWriteBillingDemoOperators("update", ruv2Metrics, true) + require.Len(t, pipelinedResult, 1) + require.Equal(t, readBillingDemoStatusPartial, pipelinedResult[0].status) + require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedResult[0].reason) + require.True(t, pipelinedResult[0].emitStatusRow) + require.Empty(t, pipelinedResult[0].units) + require.Empty(t, buildReadBillingDemoStatementStats(readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + operators: pipelinedResult, + }).BaseUnits) + + missingWriteKeys := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{WriteSize: 2}, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, missingWriteKeys.status) + require.Contains(t, missingWriteKeys.operators, readBillingDemoWriteDiagnosticStatus("update", readBillingDemoReasonMissingWriteKeys)) + + missingWriteByte := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{WriteKeys: 1}, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, missingWriteByte.status) + require.Contains(t, missingWriteByte.operators, readBillingDemoWriteDiagnosticStatus("update", readBillingDemoReasonMissingWriteByte)) + + zeroResult := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{}, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, zeroResult.status) + require.Equal(t, readBillingDemoReasonZeroMutation, zeroResult.operators[0].reason) + require.Equal(t, 0.0, zeroResult.operators[0].units[0].value) + require.Equal(t, 0.0, zeroResult.operators[0].units[1].value) +} + +func extractExplainRUTestSnapshotStatus(stats *execdetails.RURuntimeStats) explainRUComponentSnapshotStatus { + _, status := extractExplainRUTestSnapshot(stats) + return status +} + +func extractExplainRUTestSnapshot(stats *execdetails.RURuntimeStats) (*execdetails.RURuntimeStats, explainRUComponentSnapshotStatus) { + coll := execdetails.NewRuntimeStatsColl(nil) + if stats != nil && (stats.RUVersion != 0 || stats.Metrics != nil) { + coll.RegisterStats(1, stats) + } + return explainRUExtractComponentSnapshot(coll, 1) +} + +func TestReadBillingDemoNonScanCopWithoutBytesFailsClosed(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 5} + reader := physicalop.PhysicalTableReader{}.Init(ctx, 0) + proj := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + reader.SetSchema(schema) + proj.SetSchema(schema) + scan.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: proj, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + operator, supported, reason := readBillingDemoClassifyOperator(tree[1]) + require.True(t, supported) + require.Empty(t, reason) + require.Equal(t, readBillingDemoSiteTiKV, operator.site) + require.Equal(t, readBillingDemoOpClassProjection, operator.opClass) + + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.False(t, outcome.success) + require.Nil(t, outcome.units) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, outcome.failure.reason) +} + +func TestReadBillingDemoRangeScanUsesProcessedKeyAverage(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + scan.SetSchema(schema) + reader := physicalop.PhysicalTableReader{}.Init(ctx, 0) + reader.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, + } + + rows, bytes, ok := readBillingDemoRangeScanInput(10, 5, 100) + require.True(t, ok) + require.Equal(t, int64(10), rows) + require.Equal(t, 200.0, bytes) + for _, tc := range []struct { + totalKeys int64 + processedKeys int64 + processedKeysSize int64 + }{ + {0, 5, 100}, + {10, 0, 100}, + {10, 5, 0}, + } { + _, _, ok = readBillingDemoRangeScanInput(tc.totalKeys, tc.processedKeys, tc.processedKeysSize) + require.False(t, ok) + } + + buildUnits := func(scanDetail *tikvutil.ScanDetail) readBillingDemoCopUnitOutcome { + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + zero := uint64(0) + runtimeStats.RecordExpectedCopTasks([]int{scan.ID()}) + runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, scanDetail, tikvutil.TimeDetail{}, &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &zero, + NumIterations: &one, + }) + return readBillingDemoCopUnits( + newReadBillingDemoCopEstimator(tree, runtimeStats), + 1, + readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "tablescan"}, + ) + } + + outcome := buildUnits(&tikvutil.ScanDetail{TotalKeys: 10, ProcessedKeys: 5, ProcessedKeysSize: 100}) + require.True(t, outcome.success, "%+v", outcome.failure) + units := outcome.units + require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) + require.Equal(t, 10.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 200.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 200.0, readBillingDemoUnitValue(units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceScanDetailProcessedAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + + outcome = buildUnits(&tikvutil.ScanDetail{}) + require.True(t, outcome.success) + require.Equal(t, 0.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) +} + +func TestReadBillingDemoCopInputEstimator(t *testing.T) { + t.Skip("v3 byte-width estimator expectations do not apply to the v4 resource formula") + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 10} + reader := physicalop.PhysicalTableReader{}.Init(ctx, 0) + limit := physicalop.PhysicalLimit{}.Init(ctx, stats, 0) + projection := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + selection := physicalop.PhysicalSelection{}.Init(ctx, stats, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + reader.SetSchema(schema) + limit.SetSchema(schema) + projection.SetSchema(schema) + scan.SetSchema(schema) + + recordSummary := func(runtimeStats *execdetails.RuntimeStatsColl, planID int, rows uint64, detail *tikvutil.ScanDetail) { + one := uint64(1) + runtimeStats.RecordExpectedCopTasks([]int{planID}) + runtimeStats.RecordCopStats(planID, kv.TiKV, detail, tikvutil.TimeDetail{}, &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &rows, + NumIterations: &one, + }) + } + + t.Run("selection uses direct child rows and component scan width", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + operator, supported, _ := readBillingDemoClassifyOperator(tree[1]) + require.True(t, supported) + outcome := readBillingDemoCopUnits(estimator, 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeChildActRows, readBillingDemoUnitSource(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceScanDetailProcessedEstimate, readBillingDemoUnitWidthSource(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.LessOrEqual(t, estimator.nodeVisits, 6*len(tree)) + require.LessOrEqual(t, estimator.edgeVisits, 4) + require.LessOrEqual(t, estimator.auxiliaryEntryCount(), 12*len(tree)) + }) + + t.Run("supported unary operator width propagation matrix", func(t *testing.T) { + topN := physicalop.PhysicalTopN{Count: 8}.Init(ctx, stats, 0) + topN.SetSchema(schema) + hashAgg := (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema) + streamAgg := (&physicalop.BasePhysicalAgg{}).InitForStream(ctx, stats, 0, schema) + cases := []struct { + name string + node *FlatOperator + widthState readBillingDemoCopWidthState + expectedOrderWork float64 + expectOutputUnits bool + }{ + {name: "selection", node: &FlatOperator{Origin: selection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, + {name: "limit", node: &FlatOperator{Origin: limit, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, + {name: "topn", node: &FlatOperator{Origin: topN, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: 8}, + {name: "projection", node: &FlatOperator{Origin: projection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, + {name: "hashagg", node: &FlatOperator{Origin: hashAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1, expectOutputUnits: true}, + {name: "streamagg", node: &FlatOperator{Origin: streamAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1, expectOutputUnits: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.node.ChildrenIdx = []int{2} + tc.node.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + tc.node, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, tc.node.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + operator, supported, _ := readBillingDemoClassifyOperator(tc.node) + require.True(t, supported) + outcome := readBillingDemoCopUnits(estimator, 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, tc.expectedOrderWork, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) + require.Equal(t, tc.widthState, estimator.outputEstimate(1).widthState) + if tc.expectOutputUnits { + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 30.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeOperatorActRows, readBillingDemoUnitSource(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeReaderOutput, readBillingDemoUnitSource(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceRuntimeReaderOutputChunkAvg, readBillingDemoUnitWidthSource(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + } else { + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + } + }) + } + }) + + t.Run("aggregation output shadows fail open without attributable bytes", func(t *testing.T) { + newHashAgg := func() *FlatOperator { + agg := (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema) + return &FlatOperator{Origin: agg, IsRoot: false, StoreType: kv.TiKV} + } + t.Run("reader row mismatch keeps exact output rows only", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{2} + agg.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 3) + readerStats.RecordBytes(0, 33) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("zero rows and bytes remain observed evidence", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{2} + agg.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 0) + readerStats.RecordBytes(0, 0) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 0, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.True(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.True(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Zero(t, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Zero(t, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("non top aggregation never borrows reader bytes", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{3} + agg.ChildrenEndIdx = 3 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + agg, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + recordSummary(runtimeStats, selection.ID(), 2, nil) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 2, operator) + require.True(t, outcome.success) + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("incomplete own summary omits both output units", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{2} + agg.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 2, nil) + recordSummary(runtimeStats, scan.ID(), 2, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("missing expected response summaries omit shadows without changing formula inputs", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{3} + agg.ChildrenEndIdx = 3 + tree := FlatPlanTree{ + {Origin: limit, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: reader, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + // This is the state after either a consumed response omitted every + // summary or an unconsumed task contributed only its expectation. The + // expected count must not be inferred from visible summaries. + runtimeStats.RecordExpectedCopTasks([]int{scan.ID(), agg.Origin.ID()}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 2, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + }) + + t.Run("selection output feeds projection input", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: projection, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: selection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 5, nil) + recordSummary(runtimeStats, selection.ID(), 3, nil) + recordSummary(runtimeStats, projection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(tree[1]) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 3.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 60.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("multi scan and multi detail components are ambiguous", func(t *testing.T) { + otherScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + otherScan.SetSchema(schema) + multiScanTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2, 3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: otherScan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, otherScan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(multiScanTree, runtimeStats) + require.Equal(t, readBillingDemoCopWidthAmbiguous, estimator.outputEstimate(2).widthState) + scanOperator, _, _ := readBillingDemoClassifyOperator(multiScanTree[2]) + require.Equal(t, readBillingDemoReasonAmbiguousCopScanWidth, readBillingDemoCopUnits(estimator, 2, scanOperator).failure.reason) + + multiDetailTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats = execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 3, ProcessedKeys: 3, ProcessedKeysSize: 60}) + estimator = newReadBillingDemoCopEstimator(multiDetailTree, runtimeStats) + require.Equal(t, readBillingDemoCopWidthAmbiguous, estimator.outputEstimate(2).widthState) + }) + + t.Run("projection consumes width but blocks its parent", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: limit, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: projection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, projection.ID(), 3, nil) + recordSummary(runtimeStats, limit.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + projectionOperator, _, _ := readBillingDemoClassifyOperator(tree[2]) + require.True(t, readBillingDemoCopUnits(estimator, 2, projectionOperator).success) + limitOperator, _, _ := readBillingDemoClassifyOperator(tree[1]) + limitOutcome := readBillingDemoCopUnits(estimator, 1, limitOperator) + require.False(t, limitOutcome.success) + require.Equal(t, readBillingDemoReasonUnsupportedCopWidthTransform, limitOutcome.failure.reason) + require.Equal(t, readBillingDemoCopFailureCurrent, limitOutcome.failure.kind) + }) + + t.Run("missing scan summary becomes projection cause across output edge", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: limit, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: projection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, projection.ID(), 3, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + recordSummary(runtimeStats, limit.ID(), 2, nil) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + limitOperator, _, _ := readBillingDemoClassifyOperator(tree[1]) + outcome := readBillingDemoCopUnits(estimator, 1, limitOperator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoCopFailureIntrinsicCause, outcome.failure.kind) + require.Equal(t, 2, outcome.failure.failingIdx) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, outcome.failure.reason) + }) + + t.Run("select and DML materialize projection cause without skipping scan", func(t *testing.T) { + limit.SetChildren(projection) + projection.SetChildren(scan) + reader.TablePlan = limit + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, projection.ID(), 3, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + recordSummary(runtimeStats, limit.ID(), 2, nil) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + selectResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusUnknownInput, selectResult.status) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, selectResult.reason) + require.Len(t, selectResult.operators, 1) + require.Equal(t, flat.Main[2].ExplainID().String(), selectResult.operators[0].id) + + dmlResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoDMLTree(&dmlResult, runtimeStats, flat.Main) + var dependentCount, projectionPartialCount, scanOKCount int + for _, operator := range dmlResult.operators { + switch { + case operator.id == flat.Main[1].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonDependentCopInputUnavailable: + dependentCount++ + case operator.id == flat.Main[2].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonMissingCopChildRuntimeRows: + projectionPartialCount++ + case operator.id == flat.Main[3].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + scanOKCount++ + } + } + require.Equal(t, 1, dependentCount) + require.Equal(t, 1, projectionPartialCount) + require.Equal(t, 1, scanOKCount) + }) + + t.Run("DML keeps projection units when its output summary is missing", func(t *testing.T) { + limit.SetChildren(projection) + projection.SetChildren(scan) + reader.TablePlan = limit + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, limit.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + selectResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusUnknownInput, selectResult.status) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, selectResult.reason) + require.Len(t, selectResult.operators, 1) + require.Equal(t, flat.Main[1].ExplainID().String(), selectResult.operators[0].id) + + dmlResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoDMLTree(&dmlResult, runtimeStats, flat.Main) + var limitPartial, projectionOK, scanOK int + for _, operator := range dmlResult.operators { + switch { + case operator.id == flat.Main[1].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonMissingCopChildRuntimeRows: + limitPartial++ + case operator.id == flat.Main[2].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + projectionOK++ + require.Equal(t, 4.0, readBillingDemoUnitValue(operator.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(operator.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + case operator.id == flat.Main[3].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + scanOK++ + } + } + require.Equal(t, 1, limitPartial) + require.Equal(t, 1, projectionOK) + require.Equal(t, 1, scanOK) + }) + + t.Run("unsupported descendant cause is preserved", func(t *testing.T) { + sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + limit.SetChildren(sort) + sort.SetChildren(scan) + reader.TablePlan = limit + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, sort.ID(), 3, nil) + recordSummary(runtimeStats, limit.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + selectResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusUnsupported, selectResult.status) + require.Equal(t, readBillingDemoReasonUnsupportedOperator, selectResult.reason) + require.Len(t, selectResult.operators, 1) + require.Equal(t, flat.Main[2].ExplainID().String(), selectResult.operators[0].id) + + dmlResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoDMLTree(&dmlResult, runtimeStats, flat.Main) + var limitDependent, sortUnsupported, scanOK int + for _, operator := range dmlResult.operators { + switch { + case operator.id == flat.Main[1].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonDependentCopInputUnavailable: + limitDependent++ + case operator.id == flat.Main[2].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonUnsupportedOperator: + sortUnsupported++ + case operator.id == flat.Main[3].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + scanOK++ + } + } + require.Equal(t, 1, limitDependent) + require.Equal(t, 1, sortUnsupported) + require.Equal(t, 1, scanOK) + }) + + t.Run("invalid child rows and multi child arity fail explicitly", func(t *testing.T) { + invalidTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), ^uint64(0), nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(invalidTree[1]) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(invalidTree, runtimeStats), 1, operator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoCopFailureIntrinsicCause, outcome.failure.kind) + require.Equal(t, 2, outcome.failure.failingIdx) + require.Equal(t, readBillingDemoReasonInvalidCopRuntimeRows, outcome.failure.reason) + + otherScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + otherScan.SetSchema(schema) + multiChildTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2, 3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: otherScan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + outcome = readBillingDemoCopUnits(newReadBillingDemoCopEstimator(multiChildTree, execdetails.NewRuntimeStatsColl(nil)), 1, operator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoStatusUnsupported, outcome.failure.status) + require.Equal(t, readBillingDemoReasonUnsupportedCopMultiChild, outcome.failure.reason) + }) + + t.Run("malformed references stay linear and fail structurally", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1, 1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, + } + estimator := newReadBillingDemoCopEstimator(tree, execdetails.NewRuntimeStatsColl(nil)) + failure, ok := estimator.firstTreeFailure() + require.True(t, ok) + require.Equal(t, readBillingDemoReasonUnsupportedCopStructure, failure.reason) + require.LessOrEqual(t, estimator.nodeVisits, 6*len(tree)) + require.LessOrEqual(t, estimator.edgeVisits, 4) + + const siblingCount = 128 + wideTree := make(FlatPlanTree, siblingCount+1) + children := make([]int, siblingCount) + wideTree[0] = &FlatOperator{Origin: reader, ChildrenIdx: children, ChildrenEndIdx: siblingCount, IsRoot: true, StoreType: kv.TiDB} + for i := 1; i <= siblingCount; i++ { + children[i-1] = i + wideTree[i] = &FlatOperator{Origin: scan, ChildrenEndIdx: siblingCount, IsRoot: false, StoreType: kv.TiKV} + } + estimator = newReadBillingDemoCopEstimator(wideTree, execdetails.NewRuntimeStatsColl(nil)) + failure, ok = estimator.firstTreeFailure() + require.True(t, ok) + require.Equal(t, readBillingDemoReasonUnsupportedCopStructure, failure.reason) + require.LessOrEqual(t, estimator.nodeVisits, 6*len(wideTree)) + require.LessOrEqual(t, estimator.edgeVisits, 2*siblingCount) + }) + + t.Run("special tree root may omit children end index", func(t *testing.T) { + // CTE-definition and scalar-subquery synthetic roots populate ChildrenIdx + // but intentionally leave ChildrenEndIdx at its zero value. + tree := FlatPlanTree{ + {Origin: projection, ChildrenIdx: []int{1}, IsRoot: true, StoreType: kv.TiDB, Label: SeedPart}, + {Origin: reader, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + _, failed := estimator.firstTreeFailure() + require.False(t, failed) + operator, _, _ := readBillingDemoClassifyOperator(tree[2]) + outcome := readBillingDemoCopUnits(estimator, 2, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("zero rows are observed and partial summaries fail closed", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 0, nil) + recordSummary(runtimeStats, selection.ID(), 0, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(tree[1]) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 0.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 0.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + + recordSummary(runtimeStats, selection.ID(), 0, nil) + outcome = readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, outcome.failure.reason) + }) + + t.Run("sibling components keep scan detail isolated", func(t *testing.T) { + indexScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + indexScan.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1, 2}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, + {Origin: indexScan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 50}, tikvutil.TimeDetail{}, nil) + runtimeStats.RecordCopStats(indexScan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 150}, tikvutil.TimeDetail{}, nil) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + require.Equal(t, 10.0, estimator.outputEstimate(1).avgRowWidth) + require.Equal(t, 30.0, estimator.outputEstimate(2).avgRowWidth) + }) + + t.Run("full builder wires flattened table reader component", func(t *testing.T) { + selection.SetChildren(scan) + reader.TablePlan = selection + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 3) + require.Equal(t, selection.ID(), flat.Main[1].Origin.ID()) + require.Equal(t, scan.ID(), flat.Main[2].Origin.ID()) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + var selectionResult readBillingDemoOperatorResult + for _, operator := range result.operators { + if operator.id == flat.Main[1].ExplainID().String() { + selectionResult = operator + break + } + } + require.Equal(t, readBillingDemoStatusOperatorOK, selectionResult.status) + require.Equal(t, 4.0, readBillingDemoUnitValue(selectionResult.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(selectionResult.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + statementStats := buildReadBillingDemoStatementStats(result) + require.NotEmpty(t, statementStats.BaseUnits) + for _, sample := range statementStats.BaseUnits { + require.Equal(t, "v3", sample.ModelVersion) + } + }) + + t.Run("full builder preserves pushed aggregation output shadows", func(t *testing.T) { + aggScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + aggScan.SetSchema(schema) + agg := (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema) + agg.SetChildren(aggScan) + reader.TablePlan = agg + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 3) + require.Equal(t, agg.ID(), flat.Main[1].Origin.ID()) + require.Equal(t, aggScan.ID(), flat.Main[2].Origin.ID()) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, aggScan.ID(), 4, nil) + recordSummary(runtimeStats, agg.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + var aggResult readBillingDemoOperatorResult + for _, operator := range result.operators { + if operator.site == readBillingDemoSiteTiKV && operator.opClass == readBillingDemoOpClassHashAgg { + aggResult = operator + break + } + } + require.Equal(t, readBillingDemoStatusOperatorOK, aggResult.status) + require.Equal(t, 4.0, readBillingDemoUnitValue(aggResult.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(aggResult.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 30.0, readBillingDemoUnitValue(aggResult.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + + seenStatementUnits := make(map[string]bool) + for _, sample := range buildReadBillingDemoStatementStats(result).BaseUnits { + if sample.Site == readBillingDemoSiteTiKV && sample.OpClass == readBillingDemoOpClassHashAgg && + (sample.Unit == readBillingDemoUnitOutputRows || sample.Unit == readBillingDemoUnitOutputBytes) { + seenStatementUnits[sample.Unit] = true + } + } + require.Equal(t, map[string]bool{readBillingDemoUnitOutputRows: true, readBillingDemoUnitOutputBytes: true}, seenStatementUnits) + }) + + t.Run("full builder wires root sort order work", func(t *testing.T) { + rootSort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + scan.SetStats(stats) + reader.TablePlan = scan + rootSort.SetChildren(reader) + flat := FlattenPhysicalPlan(rootSort, true) + require.Len(t, flat.Main, 3) + require.Equal(t, rootSort.ID(), flat.Main[0].Origin.ID()) + require.Equal(t, reader.ID(), flat.Main[1].Origin.ID()) + require.Equal(t, scan.ID(), flat.Main[2].Origin.ID()) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + for _, rootEvidence := range []struct { + planID int + rows int + bytes int64 + }{ + {planID: rootSort.ID(), rows: 8, bytes: 80}, + {planID: reader.ID(), rows: 8, bytes: 80}, + } { + basic := runtimeStats.GetBasicRuntimeStats(rootEvidence.planID, true) + basic.Record(time.Millisecond, rootEvidence.rows) + basic.RecordBytes(0, rootEvidence.bytes) + } + recordSummary(runtimeStats, scan.ID(), 8, &tikvutil.ScanDetail{TotalKeys: 8, ProcessedKeys: 8, ProcessedKeysSize: 160}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, rootSort, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + var sortResult readBillingDemoOperatorResult + for _, operator := range result.operators { + if operator.site == readBillingDemoSiteTiDB && operator.opClass == readBillingDemoOpClassSort { + sortResult = operator + break + } + } + require.Equal(t, readBillingDemoStatusOperatorOK, sortResult.status) + require.Equal(t, 8.0, readBillingDemoUnitValue(sortResult.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(sortResult.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 24.0, readBillingDemoUnitValue(sortResult.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) + }) + + t.Run("full builder wires root and pushed topn order work", func(t *testing.T) { + reqProp := property.NewPhysicalProperty(property.RootTaskType, nil, false, 0, false) + rootTopN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0, reqProp) + scan.SetStats(stats) + copTopN, globalTopN := getPushedDownTopN(rootTopN, scan, kv.TiKV) + require.NotNil(t, copTopN) + require.Nil(t, globalTopN) + require.Zero(t, copTopN.Offset) + require.Equal(t, uint64(4), copTopN.Count) + copTopN.SetSchema(schema) + rootTopN.SetSchema(schema) + reader.TablePlan = copTopN + rootTopN.SetChildren(reader) + flat := FlattenPhysicalPlan(rootTopN, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + for _, rootEvidence := range []struct { + planID int + rows int + bytes int64 + }{ + {planID: rootTopN.ID(), rows: 3, bytes: 30}, + {planID: reader.ID(), rows: 4, bytes: 40}, + } { + basic := runtimeStats.GetBasicRuntimeStats(rootEvidence.planID, true) + basic.Record(time.Millisecond, rootEvidence.rows) + basic.RecordBytes(0, rootEvidence.bytes) + } + recordSummary(runtimeStats, scan.ID(), 8, nil) + recordSummary(runtimeStats, copTopN.ID(), 4, &tikvutil.ScanDetail{TotalKeys: 8, ProcessedKeys: 8, ProcessedKeysSize: 160}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, rootTopN, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + orderWorkByClass := make(map[string]float64) + for _, operator := range result.operators { + if operator.opClass == readBillingDemoOpClassTopN { + orderWorkByClass[operator.site+"/"+operator.opClass] = readBillingDemoUnitValue(operator.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll) + } + } + require.Equal(t, 8.0, orderWorkByClass[readBillingDemoSiteTiDB+"/"+readBillingDemoOpClassTopN]) + require.Equal(t, 16.0, orderWorkByClass[readBillingDemoSiteTiKV+"/"+readBillingDemoOpClassTopN]) + }) + + t.Run("full builder keeps index lookup components isolated", func(t *testing.T) { + indexScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tableScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + indexScan.SetSchema(schema) + tableScan.SetSchema(schema) + lookup := (physicalop.PhysicalIndexLookUpReader{IndexPlan: indexScan, TablePlan: tableScan}).Init(ctx, 0, plannerutil.IndexLookUpPushDownNone) + flat := FlattenPhysicalPlan(lookup, true) + require.Len(t, flat.Main, 3) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(lookup.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + runtimeStats.RecordCopStats(indexScan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 50}, tikvutil.TimeDetail{}, nil) + runtimeStats.RecordCopStats(tableScan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 150}, tikvutil.TimeDetail{}, nil) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, lookup, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + bytesByID := make(map[string]float64) + for _, operator := range result.operators { + if operator.opClass == readBillingDemoOpClassRangeScan { + bytesByID[operator.id] = readBillingDemoUnitValue(operator.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll) + } + } + require.Equal(t, 50.0, bytesByID[flat.Main[1].ExplainID().String()]) + require.Equal(t, 150.0, bytesByID[flat.Main[2].ExplainID().String()]) + }) +} + +func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 10} + join := (&physicalop.PhysicalHashJoin{}).Init(ctx, stats, 0) + join.EqualConditions = []*expression.ScalarFunction{{}} + join.LeftJoinKeys = []*expression.Column{col} + join.RightJoinKeys = []*expression.Column{col} + left := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + right := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + join.SetSchema(schema) + left.SetSchema(schema) + right.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: join, ChildrenIdx: []int{1, 2}, IsRoot: true}, + {Origin: left, IsRoot: true, Label: BuildSide}, + {Origin: right, IsRoot: true, Label: ProbeSide}, + } + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRootRows := func(planID int, rows int) { + stats := runtimeStats.GetBasicRuntimeStats(planID, true) + stats.Record(time.Millisecond, rows) + stats.RecordBytes(0, int64(rows*10)) + } + recordRootRows(join.ID(), 6) + recordRootRows(left.ID(), 4) + recordRootRows(right.ID(), 6) + runtimeStats.RegisterStats(join.ID(), &readBillingDemoHashStatsForTest{rows: 3}) + + units, reason, ok := readBillingDemoRootUnits( + runtimeStats, + tree, + 0, + tree[0], + readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashJoin, operatorKind: "hashjoin"}, + ) + require.True(t, ok) + require.Empty(t, reason) + require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideBuild)) + require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideProbe)) + require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitExpressionCount, readBillingDemoInputSideAll)) + require.Equal(t, 10.0, readBillingDemoUnitValue(units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, 3.0, readBillingDemoUnitValue(units, readBillingDemoUnitHashStateRows, readBillingDemoInputSideBuild)) + require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitJoinOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 60.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceHashJoinRuntime, readBillingDemoUnitSource(units, readBillingDemoUnitHashStateRows, readBillingDemoInputSideBuild)) + require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) +} + +type readBillingDemoHashStatsForTest struct{ rows int64 } + +func (*readBillingDemoHashStatsForTest) String() string { return "" } +func (s *readBillingDemoHashStatsForTest) Clone() execdetails.RuntimeStats { + return &readBillingDemoHashStatsForTest{rows: s.rows} +} +func (*readBillingDemoHashStatsForTest) Merge(execdetails.RuntimeStats) {} +func (*readBillingDemoHashStatsForTest) Tp() int { return 1_000_000 } +func (s *readBillingDemoHashStatsForTest) HashTableRows() int64 { return s.rows } + +func readBillingDemoUnitValue(units []readBillingDemoUnit, unitName, side string) float64 { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return unit.value + } + } + return -1 +} + +func readBillingDemoUnitSource(units []readBillingDemoUnit, unitName, side string) string { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return unit.source + } + } + return "" +} + +func readBillingDemoUnitWidthSource(units []readBillingDemoUnit, unitName, side string) string { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return unit.widthSource + } + } + return "" +} + +func readBillingDemoUnitExists(units []readBillingDemoUnit, unitName, side string) bool { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return true + } + } + return false +} diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go new file mode 100644 index 0000000000000..c2065558e8677 --- /dev/null +++ b/pkg/planner/core/explain_ru.go @@ -0,0 +1,2888 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "math" + "strconv" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/metrics" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/plancodec" + "github.com/pingcap/tidb/pkg/util/stmtsummary" + "github.com/tikv/client-go/v2/tikvrpc" + tikvutil "github.com/tikv/client-go/v2/util" + rmclient "github.com/tikv/pd/client/resource_group/controller" +) + +type explainRUStatus string + +const ( + explainRUStatusSuccess explainRUStatus = "success" + explainRUStatusUnsupportedNonAnalyze explainRUStatus = "unsupported_non_analyze" + explainRUStatusUnsupportedNonSelect explainRUStatus = "unsupported_non_select" + explainRUStatusUnsupportedSideEffecting explainRUStatus = "unsupported_side_effecting_select" + explainRUStatusUnsupportedLockingSelect explainRUStatus = "unsupported_locking_select" + explainRUStatusUnsupportedForConnection explainRUStatus = "unsupported_for_connection" + explainRUStatusError explainRUStatus = "error" + explainRUComponentSnapshotOK explainRUComponentSnapshotStatus = "ok" + explainRUComponentSnapshotMissing explainRUComponentSnapshotStatus = "missing" + explainRUComponentSnapshotNonV2 explainRUComponentSnapshotStatus = "non_v2" + explainRUComponentSnapshotNilMetrics explainRUComponentSnapshotStatus = "nil_metrics" + explainRUComponentSnapshotBypassed explainRUComponentSnapshotStatus = "bypassed" + explainRUSectionSummary = "summary" + explainRUSectionPlan = "plan" + explainRUSourceSummaryTotal = "summary_total" + + explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" + explainRUWidthSourceRuntimeReaderOutputChunkAvg = "runtime_reader_output_chunk_avg" + explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" + explainRUWidthSourceScanDetailProcessedEstimate = "scan_detail_processed_key_avg_estimate" + explainRUWidthSourceNotApplicable = "not_applicable" + readBillingDemoModelVersion = "v4" + readBillingDemoWeightVersion = "v3-resource-formula-uncalibrated" + readBillingDemoStatusSuccess = "success" + readBillingDemoStatusUnsupported = "unsupported" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoStatusError = "error" + readBillingDemoStatusOperatorOK = "ok" + readBillingDemoStatusPartial = "partial" + readBillingDemoReasonNone = "none" + readBillingDemoReasonStatementError = "statement_error" + readBillingDemoReasonMissingPlan = "missing_plan" + readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" + readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" + readBillingDemoReasonMissingRuntimeBytes = "missing_runtime_bytes" + readBillingDemoReasonMissingInputBytes = "missing_input_bytes" + readBillingDemoReasonMissingScanDetail = "missing_scan_detail" + readBillingDemoReasonUnsupportedOperator = "unsupported_operator" + readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" + readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" + readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" + readBillingDemoReasonUnsupportedLock = "unsupported_lock" + readBillingDemoReasonNonBillable = "non_billable" + readBillingDemoReasonMissingCommitDetail = "missing_commit_detail" + readBillingDemoReasonMissingWriteKeys = "missing_write_keys" + readBillingDemoReasonMissingWriteByte = "missing_write_byte" + readBillingDemoReasonZeroMutation = "zero_mutation" + readBillingDemoReasonMissingPrewriteRegion = "missing_prewrite_region_num" + readBillingDemoReasonMissingWriteRPCCount = "missing_write_rpc_count" + readBillingDemoReasonMissingMutationRecorder = "missing_mutation_recorder" + readBillingDemoReasonUncalibratedMutation = "uncalibrated_mutation_weights" + readBillingDemoReasonDMLAncillaryPartial = "dml_ancillary_work_partial" + readBillingDemoReasonPipelinedWritePartial = "pipelined_tikv_payload_unsupported" + readBillingDemoReasonOptimisticReplayPartial = "optimistic_replay_attribution_unsupported" + readBillingDemoReasonMissingCopChildRuntimeRows = "missing_cop_child_runtime_rows" + readBillingDemoReasonMissingScanWidthEvidence = "missing_scan_width_evidence" + readBillingDemoReasonAmbiguousCopScanWidth = "ambiguous_cop_scan_width" + readBillingDemoReasonUnsupportedCopMultiChild = "unsupported_cop_multi_child" + readBillingDemoReasonUnsupportedCopWidthTransform = "unsupported_cop_width_transform" + readBillingDemoReasonUnsupportedCopStructure = "unsupported_cop_structure" + readBillingDemoReasonInvalidCopRuntimeRows = "invalid_cop_runtime_rows" + readBillingDemoReasonIncompleteCopRuntimeRows = "incomplete_cop_runtime_rows" + readBillingDemoReasonDependentCopInputUnavailable = "dependent_cop_input_unavailable" + readBillingDemoReasonInvalidOrderingWork = "invalid_ordering_work" + readBillingDemoReasonMissingOrderingProjection = "missing_ordering_projection" + readBillingDemoReasonMissingExpressionCount = "missing_expression_count" + readBillingDemoReasonInvalidTopNBound = "invalid_topn_bound" + readBillingDemoReasonMissingHashStateRows = "missing_hash_state_rows" + readBillingDemoReasonInvalidHashStateRows = "invalid_hash_state_rows" + readBillingDemoReasonMissingReaderTransport = "missing_reader_transport_details" + readBillingDemoReasonAmbiguousReaderTransport = "ambiguous_reader_transport_producers" + readBillingDemoReasonUncalibratedWeights = "uncalibrated_weights" + readBillingDemoReasonUnsupportedStatement = "unsupported_statement" + readBillingDemoSiteStatement = "statement" + readBillingDemoSiteTiDB = "tidb" + readBillingDemoSiteTiKV = "tikv" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOpClassFilter = "filter_eval" + readBillingDemoOpClassProjection = "projection_eval" + readBillingDemoOpClassLimit = "row_limit" + readBillingDemoOpClassTopN = "bounded_topn" + readBillingDemoOpClassSort = "full_ordering" + readBillingDemoOpClassWindow = "window_eval" + readBillingDemoOpClassHashAgg = "agg_hash" + readBillingDemoOpClassStreamAgg = "agg_stream" + readBillingDemoOpClassHashJoin = "join_hash" + readBillingDemoOpClassMergeJoin = "join_merge" + readBillingDemoOpClassLookupJoin = "join_lookup" + readBillingDemoOpClassReaderReceive = "reader_receive" + readBillingDemoOpClassLookupReader = "lookup_reader" + readBillingDemoOpClassOverlayReader = "overlay_reader" + readBillingDemoOpClassMetadataReader = "metadata_reader" + readBillingDemoOpClassPointLookup = "kv_point_lookup" + readBillingDemoOpClassRangeScan = "kv_range_scan" + readBillingDemoOpClassKVMutation = "kv_mutation" + readBillingDemoOpClassKVWrite = "kv_write" + readBillingDemoOpClassReaderTransport = "reader_transport" + readBillingDemoOpClassWrapper = "wrapper" + readBillingDemoOpClassSynthetic = "synthetic_source" + readBillingDemoOperatorStatement = "statement" + readBillingDemoOperatorMemDBMutation = "memdb_mutation" + readBillingDemoOperatorTxnPrewrite = "txn_prewrite" + readBillingDemoOperatorTxnWrite = "txn_write" + readBillingDemoUnitFixedEvents = "fixed_events" + readBillingDemoUnitInputRows = "input_rows" + readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoUnitOutputRows = "output_rows" + readBillingDemoUnitOutputBytes = "output_bytes" + readBillingDemoUnitOrderWork = "order_work" + readBillingDemoUnitEncodedMutationCount = "encoded_mutation_count" + readBillingDemoUnitEncodedMutationBytes = "encoded_mutation_bytes" + readBillingDemoUnitSetCount = "set_count" + readBillingDemoUnitDeleteCount = "delete_count" + readBillingDemoUnitKeyBytes = "key_bytes" + readBillingDemoUnitValueBytes = "value_bytes" + readBillingDemoUnitWriteKeys = "write_keys" + readBillingDemoUnitWriteByte = "write_byte" + readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" + readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" + readBillingDemoUnitCPUWork = "cpu_work" + readBillingDemoUnitExpressionCount = "expression_count" + readBillingDemoUnitScanBytes = "scan_bytes" + readBillingDemoUnitNetBytes = "net_bytes" + readBillingDemoUnitReadRequestCount = "read_request_count" + readBillingDemoUnitWriteRequestCount = "write_request_count" + readBillingDemoUnitHashStateRows = "hash_state_rows" + readBillingDemoUnitJoinOutputRows = "join_output_rows" + readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" + readBillingDemoInputSourceScanDetail = "scan_detail" + readBillingDemoInputSourceRuntimeChildActRows = "runtime_child_act_rows" + readBillingDemoInputSourceRuntimeOperatorActRows = "runtime_operator_act_rows" + readBillingDemoInputSourceRuntimeReaderOutput = "runtime_reader_output_chunks" + readBillingDemoInputSourceRuntimeOrderingWork = "runtime_ordering_work" + readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" + readBillingDemoInputSourceCommitDetail = "commit_detail" + readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" + readBillingDemoInputSourceDistSQLRuntimeStats = "distsql_runtime_stats" + readBillingDemoInputSourceSnapshotRuntimeStats = "snapshot_runtime_stats" + readBillingDemoInputSourcePhysicalPlan = "physical_plan" + readBillingDemoInputSourceHashJoinRuntime = "hash_join_runtime_stats" + readBillingDemoInputSideAll = "all" + readBillingDemoInputSideBuild = "build" + readBillingDemoInputSideProbe = "probe" + readBillingDemoInputSideLeft = "left" + readBillingDemoInputSideRight = "right" + readBillingDemoScopeStatementAttempted = "statement_attempted" + readBillingDemoScopeTxnPrewritePayload = "txn_prewrite_payload" +) + +type explainRUComponentSnapshotStatus string + +type readBillingDemoUnit struct { + unit string + source string + side string + value float64 + rowWidth float64 + widthSource string +} + +type readBillingDemoOperatorResult struct { + id string + site string + opClass string + operatorKind string + dmlKind string + scope string + uncalibrated bool + emitStatusRow bool + status string + reason string + actRows int64 + hasActRows bool + units []readBillingDemoUnit +} + +type readBillingDemoResult struct { + status string + reason string + operators []readBillingDemoOperatorResult +} + +type readBillingDemoCopWidthState uint8 + +const ( + readBillingDemoCopWidthMissing readBillingDemoCopWidthState = iota + readBillingDemoCopWidthKnown + readBillingDemoCopWidthBarrier + readBillingDemoCopWidthAmbiguous +) + +type readBillingDemoCopFailureKind uint8 + +const ( + readBillingDemoCopFailureCurrent readBillingDemoCopFailureKind = iota + readBillingDemoCopFailureIntrinsicCause +) + +type readBillingDemoCopRowsState uint8 + +const ( + readBillingDemoCopRowsMissing readBillingDemoCopRowsState = iota + readBillingDemoCopRowsObserved + readBillingDemoCopRowsInvalid +) + +type readBillingDemoCopFailure struct { + present bool + kind readBillingDemoCopFailureKind + status string + reason string + failingIdx int +} + +type readBillingDemoCopRowsEvidence struct { + state readBillingDemoCopRowsState + rows int64 + tasks int32 +} + +type readBillingDemoCopOutputEstimate struct { + widthState readBillingDemoCopWidthState + avgRowWidth float64 + widthSource string + scanPlanID int + failure readBillingDemoCopFailure +} + +type readBillingDemoCopInputEstimate struct { + rows int64 + inputBytes float64 + avgRowWidth float64 + inputSource string + widthSource string + failure readBillingDemoCopFailure +} + +type readBillingDemoCopComponentEvidence struct { + scanCount int + scanIdx int + scanObservedTasks int32 + scanExpectedTasks int32 + detailHolderCount int + scanDetail tikvutil.ScanDetail + scanDetailRecords int32 + scanDetailExpectedTasks int32 + maxSummaryTasks int32 +} + +type readBillingDemoCopEstimator struct { + tree FlatPlanTree + runtimeStats *execdetails.RuntimeStatsColl + parentIdx []int + componentID []int + components []readBillingDemoCopComponentEvidence + nodeFailures map[int]readBillingDemoCopFailure + treeFailures []readBillingDemoCopFailure + treeFailureSeen map[int]struct{} + inputMemo map[int]readBillingDemoCopInputEstimate + inputMemoSet map[int]struct{} + outputMemo map[int]readBillingDemoCopOutputEstimate + outputMemoSet map[int]struct{} + visiting map[int]bool + nodeVisits int + edgeVisits int + auxiliaryEntries int +} + +type readBillingDemoCopUnitOutcome struct { + success bool + units []readBillingDemoUnit + failure readBillingDemoCopFailure +} + +type readBillingDemoAppendOutcome struct { + success bool + status string + current readBillingDemoOperatorResult + cause readBillingDemoCopFailure +} + +type readBillingDemoWeights struct { + Version string + CPUWeight float64 + ScanWeight float64 + NetWeight float64 + ReadRequestWeight float64 + WriteRequestWeight float64 + HashTableWeight float64 + JoinWeight float64 + MutationBytesPerCPUUnit float64 + Calibrated bool +} + +type readBillingDemoWeightProvider interface { + valid() bool + unitWeight(string) (float64, bool) +} + +// Production intentionally starts without guessed coefficients. Formula tests +// inject a calibrated private value directly. +var readBillingDemoV4Weights = readBillingDemoWeights{Version: readBillingDemoWeightVersion} + +type explainRURow struct { + section string + id string + component string + operatorClass string + actRows int64 + hasActRows bool + inputRows int64 + hasInputRows bool + outputRows int64 + hasOutputRows bool + rowWidth float64 + hasRowWidth bool + rowWidthSource string + workRows float64 + hasWorkRows bool + workBytes float64 + hasWorkBytes bool + unit string + count int64 + hasCount bool + weight float64 + hasWeight bool + previewRU float64 + hasPreviewRU bool + source string + note string +} + +// RecordReadBillingDemoForStatement emits coefficient-free read billing demo +// metrics for a completed statement and returns the structured statement +// summary stats. It is intentionally independent from RU v2 billing/reporting +// and never calls resource-control reporters. +func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, stmt ast.StmtNode, execErr error) stmtsummary.ReadBillingDemoStatementStats { + if sctx == nil || sctx.GetSessionVars() == nil || !sctx.GetSessionVars().EnableReadBillingDemo { + return stmtsummary.ReadBillingDemoStatementStats{} + } + // Restricted/internal SQL is not external workload calibration input. + if sctx.GetSessionVars().InRestrictedSQL || sctx.GetSessionVars().StmtCtx.InRestrictedSQL { + return stmtsummary.ReadBillingDemoStatementStats{} + } + planCtx := readBillingDemoPlanContext(plan) + if planCtx == nil { + planCtx = sctx.GetPlanCtx() + } + result := buildReadBillingDemoResult(planCtx, plan, stmt, execErr, nil) + recordReadBillingDemoResult(result) + return buildReadBillingDemoStatementStats(result) +} + +func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + if _, ok := stmt.(*ast.CommitStmt); ok { + return buildTxnCommitBillingDemoResult(sctx, ruv2Metrics, execErr) + } + if _, ok := stmt.(*ast.RollbackStmt); ok { + return readBillingDemoFailure(readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedStatement) + } + if dmlKind, ok := explainRUWriteDMLKind(stmt); ok { + return buildWriteBillingDemoResult(sctx, plan, dmlKind, ruv2Metrics, execErr) + } + if execErr != nil { + return readBillingDemoFailure(readBillingDemoStatusError, readBillingDemoReasonStatementError) + } + if plan == nil { + return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingPlan) + } + if gateStatus := explainRUTargetGateStatus(stmt); gateStatus != explainRUStatusSuccess { + return readBillingDemoFailure(readBillingDemoStatusUnsupported, string(gateStatus)) + } + if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil { + return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingRuntimeStats) + } + flat := FlattenPhysicalPlan(plan, true) + if flat == nil || len(flat.Main) == 0 || flat.InExplain || flat.InExecute { + return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingPlan) + } + + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + planCtx := readBillingDemoPlanContext(plan) + runtimeStats := sctx.GetSessionVars().StmtCtx.RuntimeStatsColl + if status, op := appendReadBillingDemoTree(&result, planCtx, runtimeStats, flat.Main); status != readBillingDemoStatusSuccess { + return readBillingDemoFailedOperator(status, op) + } + for _, tree := range flat.CTEs { + if status, op := appendReadBillingDemoTree(&result, planCtx, runtimeStats, tree); status != readBillingDemoStatusSuccess { + return readBillingDemoFailedOperator(status, op) + } + } + for _, tree := range flat.ScalarSubQueries { + if status, op := appendReadBillingDemoTree(&result, planCtx, runtimeStats, tree); status != readBillingDemoStatusSuccess { + return readBillingDemoFailedOperator(status, op) + } + } + if ruv2Metrics == nil && sctx.GetSessionVars() != nil { + ruv2Metrics = sctx.GetSessionVars().RUV2Metrics + } + if op, present := readBillingDemoReaderTransport(flat, runtimeStats, ruv2Metrics, false); present { + if op.status != readBillingDemoStatusOperatorOK { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, op) + } + result.operators = append(result.operators, op) + } + if op, present := readBillingDemoPointLookupTransport(flat, runtimeStats, ruv2Metrics, false); present { + if op.status != readBillingDemoStatusOperatorOK { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, op) + } + result.operators = append(result.operators, op) + } + return result +} + +func buildTxnCommitBillingDemoResult(sctx base.PlanContext, ruv2Metrics *execdetails.RUV2Metrics, execErr error) readBillingDemoResult { + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + if execErr != nil { + result.status = readBillingDemoStatusError + result.reason = readBillingDemoReasonStatementError + } + + var pipelined bool + if sctx != nil && sctx.GetSessionVars() != nil { + vars := sctx.GetSessionVars() + if ruv2Metrics == nil { + ruv2Metrics = vars.RUV2Metrics + } + if vars.StmtCtx != nil && vars.StmtCtx.PreviewKVMutationRecorder != nil { + pipelined = vars.StmtCtx.PreviewKVMutationRecorder.Snapshot().Pipelined + } + } + + // COMMIT owns only the write requests in its fresh statement snapshot. Keep + // dml_kind empty because earlier DML statements own their own snapshots. + result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators("", ruv2Metrics, pipelined)...) + return result +} + +func buildWriteBillingDemoResult(sctx base.PlanContext, plan base.Plan, dmlKind string, ruv2Metrics *execdetails.RUV2Metrics, execErr error) readBillingDemoResult { + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + if execErr != nil { + result.status = readBillingDemoStatusError + result.reason = readBillingDemoReasonStatementError + } + + var pipelined bool + if sctx != nil && sctx.GetSessionVars() != nil { + if ruv2Metrics == nil { + ruv2Metrics = sctx.GetSessionVars().RUV2Metrics + } + if stmtCtx := sctx.GetSessionVars().StmtCtx; stmtCtx != nil && stmtCtx.PreviewKVMutationRecorder != nil { + pipelined = stmtCtx.PreviewKVMutationRecorder.Snapshot().Pipelined + } + } + + appendReadBillingDemoDMLPlan(&result, sctx, plan, ruv2Metrics) + appendReadBillingDemoMutation(&result, sctx, dmlKind) + result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators(dmlKind, ruv2Metrics, pipelined)...) + return result +} + +func appendReadBillingDemoDMLPlan(result *readBillingDemoResult, sctx base.PlanContext, plan base.Plan, ruv2Metrics *execdetails.RUV2Metrics) { + if plan == nil || sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx == nil { + result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingPlan)) + return + } + runtimeStats := sctx.GetSessionVars().StmtCtx.RuntimeStatsColl + if runtimeStats == nil { + result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingRuntimeStats)) + return + } + flat := FlattenPhysicalPlan(plan, true) + if flat == nil || len(flat.Main) == 0 || flat.InExplain || flat.InExecute { + result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingPlan)) + return + } + appendTree := func(tree FlatPlanTree) { + appendReadBillingDemoDMLTree(result, runtimeStats, tree) + } + appendTree(flat.Main) + for _, tree := range flat.CTEs { + appendTree(tree) + } + for _, tree := range flat.ScalarSubQueries { + appendTree(tree) + } + if op, present := readBillingDemoReaderTransport(flat, runtimeStats, ruv2Metrics, true); present { + result.operators = append(result.operators, op) + } + if op, present := readBillingDemoPointLookupTransport(flat, runtimeStats, ruv2Metrics, true); present { + result.operators = append(result.operators, op) + } +} + +func readBillingDemoPlanDiagnostic(reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "dml_plan", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassWrapper, + operatorKind: "dml_plan", + emitStatusRow: true, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + +func appendReadBillingDemoMutation(result *readBillingDemoResult, sctx base.PlanContext, dmlKind string) { + operator := readBillingDemoOperatorResult{ + id: "mutation@statement", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassKVMutation, + operatorKind: readBillingDemoOperatorMemDBMutation, + dmlKind: dmlKind, + scope: readBillingDemoScopeStatementAttempted, + uncalibrated: !readBillingDemoWeightsValid(readBillingDemoV4Weights), + } + if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx == nil || + sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder == nil { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonMissingMutationRecorder + result.operators = append(result.operators, operator) + return + } + + snapshot := sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.Snapshot() + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + if snapshot.EncodedMutationCount == 0 { + operator.reason = readBillingDemoReasonZeroMutation + } + operator.units = append(operator.units, + readBillingDemoUnit{unit: readBillingDemoUnitEncodedMutationCount, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.EncodedMutationCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitEncodedMutationBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.EncodedMutationBytes), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitSetCount, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.SetCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitDeleteCount, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.DeleteCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitKeyBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.KeyBytes), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitValueBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.ValueBytes), widthSource: explainRUWidthSourceNotApplicable}, + ) + if readBillingDemoWeightsValid(readBillingDemoV4Weights) { + normalization := readBillingDemoV4Weights.MutationBytesPerCPUUnit + work := float64(snapshot.EncodedMutationCount) + float64(snapshot.EncodedMutationBytes)/normalization + if work >= 0 && !math.IsNaN(work) && !math.IsInf(work, 0) { + operator.units = append(operator.units, readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: work, widthSource: explainRUWidthSourceNotApplicable}) + } + } + result.operators = append(result.operators, operator) + if !readBillingDemoWeightsValid(readBillingDemoV4Weights) { + result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonUncalibratedMutation)) + } + vars := sctx.GetSessionVars() + if vars.InTxn() && vars.TxnCtx != nil && vars.TxnCtx.CouldRetry { + result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonOptimisticReplayPartial)) + } +} + +func readBillingDemoMutationDiagnostic(dmlKind, reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "mutation@statement", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassKVMutation, + operatorKind: readBillingDemoOperatorMemDBMutation, + dmlKind: dmlKind, + scope: readBillingDemoScopeStatementAttempted, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + +func buildTiKVWriteBillingDemoOperators(dmlKind string, ruv2Metrics *execdetails.RUV2Metrics, pipelined bool) []readBillingDemoOperatorResult { + operator := readBillingDemoOperatorResult{ + id: "txn_write@statement", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassKVWrite, + operatorKind: readBillingDemoOperatorTxnWrite, + dmlKind: dmlKind, + scope: readBillingDemoScopeTxnPrewritePayload, + emitStatusRow: true, + } + if pipelined { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonPipelinedWritePartial + return []readBillingDemoOperatorResult{operator} + } + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonMissingWriteRPCCount + return []readBillingDemoOperatorResult{operator} + } + writeRPCCount := ruv2Metrics.ResourceManagerWriteCnt() + if writeRPCCount < 0 { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonMissingWriteRPCCount + return []readBillingDemoOperatorResult{operator} + } + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + operator.units = []readBillingDemoUnit{{unit: readBillingDemoUnitWriteRequestCount, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(writeRPCCount), widthSource: explainRUWidthSourceNotApplicable}} + return []readBillingDemoOperatorResult{operator} +} + +func readBillingDemoPlanContext(plan base.Plan) base.PlanContext { + if plan == nil { + return nil + } + return plan.SCtx() +} + +func readBillingDemoAllTrees(flat *FlatPhysicalPlan) []FlatPlanTree { + if flat == nil { + return nil + } + trees := make([]FlatPlanTree, 0, 1+len(flat.CTEs)+len(flat.ScalarSubQueries)) + trees = append(trees, flat.Main) + trees = append(trees, flat.CTEs...) + trees = append(trees, flat.ScalarSubQueries...) + return trees +} + +func readBillingDemoReaderTransport(flat *FlatPhysicalPlan, runtimeStats *execdetails.RuntimeStatsColl, ruv2Metrics *execdetails.RUV2Metrics, dml bool) (readBillingDemoOperatorResult, bool) { + op := readBillingDemoOperatorResult{ + id: "reader_transport@statement", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassReaderTransport, + operatorKind: "mixed_reader", + emitStatusRow: true, + } + kinds := make(map[string]struct{}) + openProducerSet := dml + hasTasks := false + allReaderRowsZero := true + for _, tree := range readBillingDemoAllTrees(flat) { + for _, node := range tree { + if node == nil || node.Origin == nil { + continue + } + kind := "" + switch plan := node.Origin.(type) { + case *physicalop.PhysicalTableReader: + if plan.StoreType == kv.TiKV { + kind = "table_reader" + } else { + openProducerSet = true + } + case *physicalop.PhysicalIndexReader: + kind = "index_reader" + case *physicalop.PhysicalIndexLookUpReader: + kind = "index_lookup" + case *physicalop.PhysicalIndexMergeReader: + kind = "index_merge" + case *physicalop.PointGetPlan, *physicalop.BatchPointGetPlan, *physicalop.PhysicalExchangeReceiver, *physicalop.PhysicalExchangeSender: + openProducerSet = true + } + if kind != "" { + kinds[kind] = struct{}{} + rows, ok := readBillingDemoPlanActRows(runtimeStats, node.Origin.ID()) + if !ok || rows != 0 { + allReaderRowsZero = false + } + } + if !node.IsRoot && node.StoreType == kv.TiKV && runtimeStats != nil { + stats := runtimeStats.GetCopStats(node.Origin.ID()) + if (stats != nil && stats.GetTasks() > 0) || runtimeStats.GetExpectedCopTasks(node.Origin.ID()) > 0 { + hasTasks = true + } + } + } + } + if len(kinds) == 0 { + return readBillingDemoOperatorResult{}, false + } + if len(kinds) == 1 { + for kind := range kinds { + op.operatorKind = kind + } + } + if dml { + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + requests, ok := readBillingDemoCopRPCCount(flat, runtimeStats) + if !ok { + if hasTasks || !allReaderRowsZero { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + requests = 0 + } + netBytes := ruv2Metrics.TiKVCoprocessorResponseBytes() + if netBytes < 0 || requests < 0 || (netBytes > 0 && requests == 0) { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{ + {unit: readBillingDemoUnitNetBytes, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(netBytes), widthSource: explainRUWidthSourceNotApplicable}, + {unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceDistSQLRuntimeStats, side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable}, + } + return op, true + } + if openProducerSet { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonAmbiguousReaderTransport + return op, true + } + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + netBytes := ruv2Metrics.TiKVCoprocessorResponseBytes() + requests := ruv2Metrics.ResourceManagerReadCnt() + if netBytes < 0 || requests < 0 || (netBytes > 0 && requests == 0) || (netBytes == 0 && requests == 0 && (hasTasks || !allReaderRowsZero)) { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{ + {unit: readBillingDemoUnitNetBytes, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(netBytes), widthSource: explainRUWidthSourceNotApplicable}, + {unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable}, + } + return op, true +} + +type readBillingDemoRPCStats interface { + GetCmdRPCCount(tikvrpc.CmdType) int64 +} + +func readBillingDemoCopRPCCount(flat *FlatPhysicalPlan, runtimeStats *execdetails.RuntimeStatsColl) (int64, bool) { + if flat == nil || runtimeStats == nil { + return 0, false + } + planIDs := make(map[int]struct{}) + for _, tree := range readBillingDemoAllTrees(flat) { + for _, node := range tree { + if node != nil && node.Origin != nil { + planIDs[node.Origin.ID()] = struct{}{} + } + } + } + var total int64 + found := false + for planID := range planIDs { + if !runtimeStats.ExistsRootStats(planID) { + continue + } + _, groups := runtimeStats.GetRootStats(planID).MergeStats() + for _, group := range groups { + if group.Tp() != execdetails.TpSelectResultRuntimeStats { + continue + } + rpcStats, ok := group.(readBillingDemoRPCStats) + if !ok { + return 0, false + } + found = true + for _, cmd := range []tikvrpc.CmdType{tikvrpc.CmdCop, tikvrpc.CmdCopStream} { + count := rpcStats.GetCmdRPCCount(cmd) + if count < 0 || count > math.MaxInt64-total { + return 0, false + } + total += count + } + } + } + return total, found +} + +func readBillingDemoPointLookupTransport( + flat *FlatPhysicalPlan, + runtimeStats *execdetails.RuntimeStatsColl, + ruv2Metrics *execdetails.RUV2Metrics, + dml bool, +) (readBillingDemoOperatorResult, bool) { + op := readBillingDemoOperatorResult{ + id: "point_lookup@statement", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassPointLookup, + operatorKind: "mixed_point_lookup", + emitStatusRow: true, + } + kinds := make(map[string]struct{}) + pointLookupPlans := make(map[int]tikvrpc.CmdType) + openProducerSet := dml + for _, tree := range readBillingDemoAllTrees(flat) { + for _, node := range tree { + if node == nil || node.Origin == nil { + continue + } + switch plan := node.Origin.(type) { + case *physicalop.PointGetPlan: + kinds["point_get"] = struct{}{} + pointLookupPlans[plan.ID()] = tikvrpc.CmdGet + openProducerSet = openProducerSet || plan.Lock + case *physicalop.BatchPointGetPlan: + kinds["batch_point_get"] = struct{}{} + pointLookupPlans[plan.ID()] = tikvrpc.CmdBatchGet + openProducerSet = openProducerSet || plan.Lock + case *physicalop.PhysicalTableReader: + openProducerSet = true + case *physicalop.PhysicalIndexReader, *physicalop.PhysicalIndexLookUpReader, + *physicalop.PhysicalIndexMergeReader, *physicalop.PhysicalExchangeReceiver, + *physicalop.PhysicalExchangeSender: + openProducerSet = true + } + } + } + if len(kinds) == 0 { + return readBillingDemoOperatorResult{}, false + } + if len(kinds) == 1 { + for kind := range kinds { + op.operatorKind = kind + } + } + if dml { + requests, ok := readBillingDemoPointLookupRPCCount(runtimeStats, pointLookupPlans) + if !ok { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{{ + unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceSnapshotRuntimeStats, + side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable, + }} + return op, true + } + if openProducerSet { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonAmbiguousReaderTransport + return op, true + } + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + requests := ruv2Metrics.ResourceManagerReadCnt() + if requests < 0 { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{{ + unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceRUV2Metrics, + side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable, + }} + return op, true +} + +func readBillingDemoPointLookupRPCCount(runtimeStats *execdetails.RuntimeStatsColl, plans map[int]tikvrpc.CmdType) (int64, bool) { + if runtimeStats == nil { + return 0, false + } + var total int64 + for planID, cmd := range plans { + if !runtimeStats.ExistsRootStats(planID) { + return 0, false + } + _, groups := runtimeStats.GetRootStats(planID).MergeStats() + found := false + for _, group := range groups { + rpcStats, ok := group.(readBillingDemoRPCStats) + if !ok { + continue + } + found = true + count := rpcStats.GetCmdRPCCount(cmd) + if count < 0 || count > math.MaxInt64-total { + return 0, false + } + total += count + } + if !found { + return 0, false + } + } + return total, true +} + +func readBillingDemoWeightsValid(weights readBillingDemoWeights) bool { + if weights.Version == "" || weights.Version == readBillingDemoWeightVersion || !weights.Calibrated || + weights.MutationBytesPerCPUUnit <= 0 || math.IsNaN(weights.MutationBytesPerCPUUnit) || math.IsInf(weights.MutationBytesPerCPUUnit, 0) { + return false + } + for _, weight := range []float64{weights.CPUWeight, weights.ScanWeight, weights.NetWeight, weights.ReadRequestWeight, weights.WriteRequestWeight, weights.HashTableWeight, weights.JoinWeight} { + if weight < 0 || math.IsNaN(weight) || math.IsInf(weight, 0) { + return false + } + } + return true +} + +func readBillingDemoActiveWeightVersion() string { + if readBillingDemoV4Weights.Version != "" { + return readBillingDemoV4Weights.Version + } + return readBillingDemoWeightVersion +} + +func (weights readBillingDemoWeights) valid() bool { + return readBillingDemoWeightsValid(weights) +} + +func (weights readBillingDemoWeights) unitWeight(unit string) (float64, bool) { + return readBillingDemoUnitWeight(weights, unit) +} + +func readBillingDemoUnitWeight(weights readBillingDemoWeights, unit string) (float64, bool) { + switch unit { + case readBillingDemoUnitCPUWork: + return weights.CPUWeight, true + case readBillingDemoUnitScanBytes: + return weights.ScanWeight, true + case readBillingDemoUnitNetBytes: + return weights.NetWeight, true + case readBillingDemoUnitReadRequestCount: + return weights.ReadRequestWeight, true + case readBillingDemoUnitWriteRequestCount: + return weights.WriteRequestWeight, true + case readBillingDemoUnitHashStateRows: + return weights.HashTableWeight, true + case readBillingDemoUnitJoinOutputRows: + return weights.JoinWeight, true + default: + return 0, false + } +} + +func readBillingDemoUnitPreviewRU(unit readBillingDemoUnit, weights readBillingDemoWeightProvider) (float64, float64, bool) { + if !weights.valid() || unit.value < 0 || math.IsNaN(unit.value) || math.IsInf(unit.value, 0) { + return 0, 0, false + } + weight, ok := weights.unitWeight(unit.unit) + if !ok { + return 0, 0, false + } + ru := unit.value * weight + if ru < 0 || math.IsNaN(ru) || math.IsInf(ru, 0) { + return 0, 0, false + } + return weight, ru, true +} + +func readBillingDemoFailure(status, reason string) readBillingDemoResult { + return readBillingDemoResult{ + status: status, + reason: reason, + operators: []readBillingDemoOperatorResult{{ + site: readBillingDemoSiteStatement, + opClass: readBillingDemoOpClassStatement, + operatorKind: readBillingDemoOperatorStatement, + status: status, + reason: reason, + }}, + } +} + +func readBillingDemoFailedOperator(status string, op readBillingDemoOperatorResult) readBillingDemoResult { + op.status = status + if op.reason == "" { + op.reason = readBillingDemoReasonUnsupportedOperator + } + return readBillingDemoResult{ + status: status, + reason: op.reason, + operators: []readBillingDemoOperatorResult{op}, + } +} + +func summarizeReadBillingDemoBaseUnits(result readBillingDemoResult) stmtsummary.ReadBillingDemoBaseUnitSummary { + // The three convenience totals are a v3 schema. V4 detail is preserved in + // the versioned base-unit table and must not be projected into those fields. + return stmtsummary.ReadBillingDemoBaseUnitSummary{} +} + +func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummary.ReadBillingDemoStatementStats { + stats := stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), + } + status := result.status + if status == "" { + status = readBillingDemoStatusUnknownInput + } + reason := result.reason + if reason == "" { + reason = readBillingDemoReasonNone + } + stats.Statuses = append(stats.Statuses, stmtsummary.ReadBillingDemoStatusSample{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), + Site: readBillingDemoSiteStatement, + OpClass: readBillingDemoOpClassStatement, + OperatorKind: readBillingDemoOperatorStatement, + Status: status, + Reason: reason, + }) + for _, op := range result.operators { + opStatus := op.status + if opStatus == "" { + opStatus = status + } + opReason := op.reason + if opReason == "" { + opReason = reason + } + if opReason == "" { + opReason = readBillingDemoReasonNone + } + if !(op.site == readBillingDemoSiteStatement && + op.opClass == readBillingDemoOpClassStatement && + op.operatorKind == readBillingDemoOperatorStatement && + opStatus == status && + opReason == reason) { + stats.Statuses = append(stats.Statuses, stmtsummary.ReadBillingDemoStatusSample{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), + Site: op.site, + OpClass: op.opClass, + OperatorKind: op.operatorKind, + Status: opStatus, + Reason: opReason, + }) + } + if opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + continue + } + for _, unit := range op.units { + sample := stmtsummary.ReadBillingDemoBaseUnitSample{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), + Site: op.site, + OpClass: op.opClass, + OperatorKind: op.operatorKind, + DMLKind: op.dmlKind, + Unit: unit.unit, + InputSource: unit.source, + InputSide: unit.side, + RowWidthSource: unit.widthSource, + Value: unit.value, + RowWidth: unit.rowWidth, + } + stats.BaseUnits = append(stats.BaseUnits, sample) + } + } + return stats +} + +func readBillingDemoCopFailureAt(idx int, kind readBillingDemoCopFailureKind, status, reason string) readBillingDemoCopFailure { + return readBillingDemoCopFailure{ + present: true, + kind: kind, + status: status, + reason: reason, + failingIdx: idx, + } +} + +func readBillingDemoPromoteCopFailure(failure readBillingDemoCopFailure) readBillingDemoCopFailure { + if failure.present { + failure.kind = readBillingDemoCopFailureIntrinsicCause + } + return failure +} + +func readBillingDemoExactCopRowsEvidence(runtimeStats *execdetails.RuntimeStatsColl, planID int) readBillingDemoCopRowsEvidence { + if runtimeStats == nil { + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsMissing} + } + stats := runtimeStats.GetCopStats(planID) + if stats == nil || stats.GetTasks() <= 0 { + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsMissing} + } + rows := stats.GetActRows() + if rows < 0 { + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsInvalid, rows: rows, tasks: stats.GetTasks()} + } + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsObserved, rows: rows, tasks: stats.GetTasks()} +} + +func newReadBillingDemoCopEstimator(tree FlatPlanTree, runtimeStats *execdetails.RuntimeStatsColl) *readBillingDemoCopEstimator { + estimator := &readBillingDemoCopEstimator{ + tree: tree, + runtimeStats: runtimeStats, + parentIdx: make([]int, len(tree)), + componentID: make([]int, len(tree)), + nodeFailures: make(map[int]readBillingDemoCopFailure), + treeFailureSeen: make(map[int]struct{}), + inputMemo: make(map[int]readBillingDemoCopInputEstimate), + inputMemoSet: make(map[int]struct{}), + outputMemo: make(map[int]readBillingDemoCopOutputEstimate), + outputMemoSet: make(map[int]struct{}), + visiting: make(map[int]bool), + } + for i := range tree { + estimator.parentIdx[i] = -1 + estimator.componentID[i] = -1 + } + addStructuralFailure := func(idx int) { + if idx < 0 || idx >= len(tree) { + return + } + failure := readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + estimator.nodeFailures[idx] = failure + if _, ok := estimator.treeFailureSeen[idx]; !ok { + estimator.treeFailureSeen[idx] = struct{}{} + estimator.treeFailures = append(estimator.treeFailures, failure) + } + } + + // Build the reverse direct-edge index and reject malformed references. This + // pass is O(n+m), where m is the number of explicit ChildrenIdx entries. + for idx, node := range tree { + estimator.nodeVisits++ + if node == nil || node.Origin == nil { + continue + } + previousChild := idx + for _, childIdx := range node.ChildrenIdx { + estimator.edgeVisits++ + if childIdx <= previousChild || childIdx <= idx || childIdx >= len(tree) { + addStructuralFailure(idx) + continue + } + previousChild = childIdx + if tree[childIdx] == nil || tree[childIdx].Origin == nil { + addStructuralFailure(idx) + continue + } + if previousParent := estimator.parentIdx[childIdx]; previousParent >= 0 { + addStructuralFailure(previousParent) + addStructuralFailure(idx) + addStructuralFailure(childIdx) + continue + } + estimator.parentIdx[childIdx] = idx + } + } + + componentRoots := make([]int, 0) + for idx, node := range tree { + estimator.nodeVisits++ + if !readBillingDemoIsTiKVCopNode(node) { + continue + } + parentIdx := estimator.parentIdx[idx] + if parentIdx < 0 || parentIdx >= len(tree) || tree[parentIdx] == nil || tree[parentIdx].Origin == nil { + addStructuralFailure(idx) + continue + } + parent := tree[parentIdx] + if parent.IsRoot { + componentRoots = append(componentRoots, idx) + continue + } + if !readBillingDemoIsTiKVCopNode(parent) { + addStructuralFailure(idx) + addStructuralFailure(parentIdx) + } + } + + lastComponentRoot := -1 + lastComponentEnd := -1 + for _, rootIdx := range componentRoots { + rootEnd := tree[rootIdx].ChildrenEndIdx + if rootEnd < rootIdx || rootEnd >= len(tree) { + addStructuralFailure(rootIdx) + continue + } + // Component roots are discovered in preorder. Validate their intervals + // with one ordered sweep so malformed, overlapping roots cannot make us + // rescan the same suffix once per sibling component. + if rootIdx <= lastComponentEnd { + addStructuralFailure(lastComponentRoot) + addStructuralFailure(rootIdx) + continue + } + lastComponentRoot = rootIdx + lastComponentEnd = rootEnd + estimator.validateReadBillingDemoCopComponent(rootIdx, addStructuralFailure) + } + for idx, node := range tree { + estimator.nodeVisits++ + if readBillingDemoIsTiKVCopNode(node) && estimator.componentID[idx] < 0 { + addStructuralFailure(idx) + } + } + + // Gather component evidence and intrinsic invalid-row failures in one + // exact-plan-ID pass. Detail ownership and row ownership intentionally stay + // separate because distsql attaches response ScanDetail to the last plan ID. + for idx, node := range tree { + estimator.nodeVisits++ + if !readBillingDemoIsTiKVCopNode(node) { + continue + } + if evidence := readBillingDemoExactCopRowsEvidence(runtimeStats, node.Origin.ID()); evidence.state == readBillingDemoCopRowsInvalid { + if _, structural := estimator.nodeFailures[idx]; !structural { + estimator.nodeFailures[idx] = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidCopRuntimeRows) + } + } + componentID := estimator.componentID[idx] + if componentID < 0 || componentID >= len(estimator.components) { + continue + } + component := &estimator.components[componentID] + operator, supported, _ := readBillingDemoClassifyOperator(node) + if supported && operator.opClass == readBillingDemoOpClassRangeScan { + component.scanCount++ + component.scanIdx = idx + } + if runtimeStats == nil { + continue + } + detail, detailRecords, observedTasks, expectedTasks := runtimeStats.GetCopScanDetailAndCoverage(node.Origin.ID()) + if observedTasks > component.maxSummaryTasks { + component.maxSummaryTasks = observedTasks + } + if supported && operator.opClass == readBillingDemoOpClassRangeScan { + component.scanObservedTasks = observedTasks + component.scanExpectedTasks = expectedTasks + } + if detailRecords > 0 { + component.detailHolderCount++ + component.scanDetail = detail + component.scanDetailRecords = detailRecords + component.scanDetailExpectedTasks = expectedTasks + } + } + estimator.auxiliaryEntries = len(estimator.parentIdx) + len(estimator.componentID) + len(estimator.components) + len(estimator.nodeFailures) + len(estimator.treeFailures) + return estimator +} + +func readBillingDemoIsTiKVCopNode(node *FlatOperator) bool { + return node != nil && node.Origin != nil && !node.IsRoot && node.StoreType == kv.TiKV +} + +func (e *readBillingDemoCopEstimator) validateReadBillingDemoCopComponent(rootIdx int, addStructuralFailure func(int)) { + if rootIdx < 0 || rootIdx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[rootIdx]) { + addStructuralFailure(rootIdx) + return + } + rootEnd := e.tree[rootIdx].ChildrenEndIdx + if rootEnd < rootIdx || rootEnd >= len(e.tree) { + addStructuralFailure(rootIdx) + return + } + componentID := len(e.components) + e.components = append(e.components, readBillingDemoCopComponentEvidence{scanIdx: -1}) + visited := make(map[int]struct{}) + + var visit func(int, int) int + visit = func(idx, expectedParent int) int { + e.nodeVisits++ + if idx < rootIdx || idx > rootEnd || idx >= len(e.tree) { + addStructuralFailure(rootIdx) + return idx + } + node := e.tree[idx] + if !readBillingDemoIsTiKVCopNode(node) { + addStructuralFailure(idx) + return idx + } + if _, duplicate := visited[idx]; duplicate { + addStructuralFailure(idx) + return node.ChildrenEndIdx + } + visited[idx] = struct{}{} + if expectedParent >= 0 && e.parentIdx[idx] != expectedParent { + addStructuralFailure(expectedParent) + addStructuralFailure(idx) + } + if previousComponent := e.componentID[idx]; previousComponent >= 0 && previousComponent != componentID { + addStructuralFailure(idx) + return idx + } + e.componentID[idx] = componentID + nodeEnd := node.ChildrenEndIdx + if nodeEnd < idx || nodeEnd > rootEnd { + addStructuralFailure(idx) + return idx + } + if len(node.ChildrenIdx) == 0 { + if nodeEnd != idx { + addStructuralFailure(idx) + } + return nodeEnd + } + expectedChild := idx + 1 + for _, childIdx := range node.ChildrenIdx { + e.edgeVisits++ + if childIdx != expectedChild { + addStructuralFailure(idx) + } + childEnd := visit(childIdx, idx) + if childEnd < childIdx { + addStructuralFailure(childIdx) + childEnd = childIdx + } + expectedChild = childEnd + 1 + } + if expectedChild-1 != nodeEnd { + addStructuralFailure(idx) + } + return nodeEnd + } + + if visit(rootIdx, e.parentIdx[rootIdx]) != rootEnd { + addStructuralFailure(rootIdx) + } + for idx := rootIdx; idx <= rootEnd; idx++ { + e.nodeVisits++ + if e.componentID[idx] != componentID { + addStructuralFailure(idx) + } + } +} + +func (e *readBillingDemoCopEstimator) firstTreeFailure() (readBillingDemoCopFailure, bool) { + if len(e.treeFailures) == 0 { + return readBillingDemoCopFailure{}, false + } + failure := e.treeFailures[0] + for _, candidate := range e.treeFailures[1:] { + if candidate.failingIdx < failure.failingIdx { + failure = candidate + } + } + return failure, true +} + +func (e *readBillingDemoCopEstimator) componentOutputWidth(idx int) readBillingDemoCopOutputEstimate { + if idx < 0 || idx >= len(e.componentID) { + return readBillingDemoCopOutputEstimate{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} + } + componentID := e.componentID[idx] + if componentID < 0 || componentID >= len(e.components) { + return readBillingDemoCopOutputEstimate{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} + } + component := e.components[componentID] + if component.scanCount > 1 || component.detailHolderCount > 1 { + return readBillingDemoCopOutputEstimate{widthState: readBillingDemoCopWidthAmbiguous} + } + if component.scanCount != 1 || component.detailHolderCount != 1 { + return readBillingDemoCopOutputEstimate{widthState: readBillingDemoCopWidthMissing} + } + rowWidth := float64(component.scanDetail.ProcessedKeysSize) / float64(component.scanDetail.ProcessedKeys) + if rowWidth <= 0 || math.IsNaN(rowWidth) || math.IsInf(rowWidth, 0) { + return readBillingDemoCopOutputEstimate{widthState: readBillingDemoCopWidthMissing} + } + return readBillingDemoCopOutputEstimate{ + widthState: readBillingDemoCopWidthKnown, + avgRowWidth: rowWidth, + widthSource: explainRUWidthSourceScanDetailProcessedEstimate, + scanPlanID: e.tree[component.scanIdx].Origin.ID(), + } +} + +func (e *readBillingDemoCopEstimator) directCopChild(idx int) (int, readBillingDemoCopFailure, bool) { + if idx < 0 || idx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[idx]) { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure), false + } + children := e.tree[idx].ChildrenIdx + if len(children) == 0 { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure), false + } + if len(children) > 1 { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopMultiChild), false + } + childIdx := children[0] + if childIdx < 0 || childIdx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[childIdx]) || e.componentID[childIdx] != e.componentID[idx] { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure), false + } + return childIdx, readBillingDemoCopFailure{}, true +} + +func readBillingDemoCopOperatorPreservesWidth(opClass string) bool { + switch opClass { + case readBillingDemoOpClassFilter, readBillingDemoOpClassLimit, readBillingDemoOpClassTopN: + return true + default: + return false + } +} + +func (e *readBillingDemoCopEstimator) inputEstimate(idx int) readBillingDemoCopInputEstimate { + if _, ok := e.inputMemoSet[idx]; ok { + return e.inputMemo[idx] + } + estimate := readBillingDemoCopInputEstimate{} + defer func() { + e.inputMemo[idx] = estimate + e.inputMemoSet[idx] = struct{}{} + }() + if failure, ok := e.nodeFailures[idx]; ok { + estimate.failure = failure + return estimate + } + if idx < 0 || idx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[idx]) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + operator, supported, reason := readBillingDemoClassifyOperator(e.tree[idx]) + if !supported { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, reason) + return estimate + } + if operator.opClass == readBillingDemoOpClassRangeScan { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + childIdx, failure, ok := e.directCopChild(idx) + if !ok { + estimate.failure = failure + return estimate + } + childOutput := e.outputEstimate(childIdx) + if childOutput.failure.present { + estimate.failure = readBillingDemoPromoteCopFailure(childOutput.failure) + return estimate + } + rowsEvidence := readBillingDemoExactCopRowsEvidence(e.runtimeStats, e.tree[childIdx].Origin.ID()) + switch rowsEvidence.state { + case readBillingDemoCopRowsInvalid: + estimate.failure = readBillingDemoCopFailureAt(childIdx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidCopRuntimeRows) + return estimate + case readBillingDemoCopRowsMissing: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingCopChildRuntimeRows) + return estimate + } + componentID := e.componentID[idx] + if componentID < 0 || componentID >= len(e.components) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + if maxTasks := e.components[componentID].maxSummaryTasks; maxTasks > 0 && rowsEvidence.tasks < maxTasks { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows) + return estimate + } + switch childOutput.widthState { + case readBillingDemoCopWidthBarrier: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonUnsupportedCopWidthTransform) + return estimate + case readBillingDemoCopWidthAmbiguous: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonAmbiguousCopScanWidth) + return estimate + case readBillingDemoCopWidthMissing: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence) + return estimate + case readBillingDemoCopWidthKnown: + default: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence) + return estimate + } + inputBytes := float64(rowsEvidence.rows) * childOutput.avgRowWidth + if inputBytes < 0 || math.IsNaN(inputBytes) || math.IsInf(inputBytes, 0) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence) + return estimate + } + estimate.rows = rowsEvidence.rows + estimate.inputBytes = inputBytes + estimate.avgRowWidth = childOutput.avgRowWidth + estimate.inputSource = readBillingDemoInputSourceRuntimeChildActRows + estimate.widthSource = childOutput.widthSource + return estimate +} + +func (e *readBillingDemoCopEstimator) outputEstimate(idx int) readBillingDemoCopOutputEstimate { + if _, ok := e.outputMemoSet[idx]; ok { + return e.outputMemo[idx] + } + estimate := readBillingDemoCopOutputEstimate{} + defer func() { + e.outputMemo[idx] = estimate + e.outputMemoSet[idx] = struct{}{} + }() + if e.visiting[idx] { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + e.visiting[idx] = true + defer delete(e.visiting, idx) + if failure, ok := e.nodeFailures[idx]; ok { + estimate.failure = failure + return estimate + } + if idx < 0 || idx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[idx]) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + operator, supported, reason := readBillingDemoClassifyOperator(e.tree[idx]) + if !supported { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, reason) + return estimate + } + if operator.opClass == readBillingDemoOpClassRangeScan { + if len(e.tree[idx].ChildrenIdx) != 0 { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + estimate = e.componentOutputWidth(idx) + return estimate + } + input := e.inputEstimate(idx) + if input.failure.present { + estimate.failure = readBillingDemoPromoteCopFailure(input.failure) + return estimate + } + if readBillingDemoCopOperatorPreservesWidth(operator.opClass) { + estimate.widthState = readBillingDemoCopWidthKnown + estimate.avgRowWidth = input.avgRowWidth + estimate.widthSource = input.widthSource + childIdx, _, ok := e.directCopChild(idx) + if ok { + estimate.scanPlanID = e.outputEstimate(childIdx).scanPlanID + } + return estimate + } + estimate.widthState = readBillingDemoCopWidthBarrier + return estimate +} + +func (e *readBillingDemoCopEstimator) auxiliaryEntryCount() int { + return e.auxiliaryEntries + len(e.inputMemo) + len(e.inputMemoSet) + len(e.outputMemo) + len(e.outputMemoSet) + len(e.visiting) +} + +func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) (string, readBillingDemoOperatorResult) { + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + if failure, ok := estimator.firstTreeFailure(); ok { + return failure.status, readBillingDemoMaterializeCopFailure(runtimeStats, tree, failure) + } + for i := range tree { + outcome := appendReadBillingDemoOperator(result, runtimeStats, tree, estimator, i) + if outcome.success { + continue + } + if outcome.cause.present { + return outcome.cause.status, readBillingDemoMaterializeCopFailure(runtimeStats, tree, outcome.cause) + } + return outcome.status, outcome.current + } + return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} +} + +// appendReadBillingDemoDMLTree keeps every independently usable plan operator. +// A missing or unsupported node makes only that node partial; it must not hide +// supported descendants from the DML read/compute tree. +func appendReadBillingDemoDMLTree(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) { + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + type diagnosticKey struct { + idx int + reason string + } + reported := make(map[diagnosticKey]struct{}) + appendDiagnostic := func(idx int, operator readBillingDemoOperatorResult, reason string) { + key := diagnosticKey{idx: idx, reason: reason} + if _, ok := reported[key]; ok { + return + } + reported[key] = struct{}{} + operator.emitStatusRow = true + operator.status = readBillingDemoStatusPartial + operator.reason = reason + result.operators = append(result.operators, operator) + } + for _, failure := range estimator.treeFailures { + appendDiagnostic(failure.failingIdx, readBillingDemoMaterializeCopFailure(runtimeStats, tree, failure), failure.reason) + } + for i := range tree { + outcome := appendReadBillingDemoOperator(result, runtimeStats, tree, estimator, i) + if outcome.success { + continue + } + if outcome.cause.present && outcome.cause.failingIdx != i { + appendDiagnostic(i, outcome.current, readBillingDemoReasonDependentCopInputUnavailable) + appendDiagnostic( + outcome.cause.failingIdx, + readBillingDemoMaterializeCopFailure(runtimeStats, tree, outcome.cause), + outcome.cause.reason, + ) + continue + } + appendDiagnostic(i, outcome.current, outcome.current.reason) + } +} + +func appendReadBillingDemoOperator(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, estimator *readBillingDemoCopEstimator, idx int) readBillingDemoAppendOutcome { + op := tree[idx] + if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { + return readBillingDemoAppendOutcome{success: true, status: readBillingDemoStatusSuccess} + } + if failure, ok := estimator.nodeFailures[idx]; ok { + operator := readBillingDemoMaterializeCopFailure(runtimeStats, tree, failure) + return readBillingDemoAppendOutcome{status: failure.status, current: operator, cause: failure} + } + operator, supported, reason := readBillingDemoClassifyOperator(op) + operator.id = op.ExplainID().String() + if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, op); ok { + operator.actRows = actRows + operator.hasActRows = true + } + if !supported { + return readBillingDemoAppendOutcome{status: readBillingDemoStatusUnsupported, current: operator.withReason(reason)} + } + if !readBillingDemoOperatorBillable(operator) { + operator.status = readBillingDemoStatusOperatorOK + result.operators = append(result.operators, operator.withReason(readBillingDemoReasonNonBillable)) + return readBillingDemoAppendOutcome{success: true, status: readBillingDemoStatusSuccess} + } + var units []readBillingDemoUnit + var missingReason string + var ok bool + if op.IsRoot { + units, missingReason, ok = readBillingDemoRootUnits(runtimeStats, tree, idx, op, operator) + } else { + outcome := readBillingDemoCopUnits(estimator, idx, operator) + if outcome.success { + units, ok = outcome.units, true + } else { + failure := outcome.failure + currentReason := failure.reason + cause := readBillingDemoCopFailure{} + if failure.kind == readBillingDemoCopFailureIntrinsicCause && failure.failingIdx != idx { + currentReason = readBillingDemoReasonDependentCopInputUnavailable + cause = failure + } + return readBillingDemoAppendOutcome{ + status: failure.status, + current: operator.withReason(currentReason), + cause: cause, + } + } + } + if !ok { + if missingReason == "" { + if op.IsRoot { + missingReason = readBillingDemoReasonMissingRuntimeBytes + } else { + missingReason = readBillingDemoReasonMissingScanDetail + } + } + return readBillingDemoAppendOutcome{status: readBillingDemoStatusUnknownInput, current: operator.withReason(missingReason)} + } + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + operator.units = units + result.operators = append(result.operators, operator) + return readBillingDemoAppendOutcome{success: true, status: readBillingDemoStatusSuccess} +} + +func readBillingDemoMaterializeCopFailure(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, failure readBillingDemoCopFailure) readBillingDemoOperatorResult { + if failure.failingIdx < 0 || failure.failingIdx >= len(tree) || tree[failure.failingIdx] == nil || tree[failure.failingIdx].Origin == nil { + return readBillingDemoOperatorResult{ + id: "cop_structure", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassRangeScan, + operatorKind: "cop_structure", + reason: failure.reason, + } + } + op := tree[failure.failingIdx] + operator, _, _ := readBillingDemoClassifyOperator(op) + operator.id = op.ExplainID().String() + operator.reason = failure.reason + if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, op); ok { + operator.actRows = actRows + operator.hasActRows = true + } + return operator +} + +func (op readBillingDemoOperatorResult) withReason(reason string) readBillingDemoOperatorResult { + op.reason = reason + return op +} + +func readBillingDemoOperatorBillable(op readBillingDemoOperatorResult) bool { + switch op.opClass { + case readBillingDemoOpClassFilter, readBillingDemoOpClassProjection, readBillingDemoOpClassLimit, + readBillingDemoOpClassTopN, readBillingDemoOpClassSort, readBillingDemoOpClassWindow, + readBillingDemoOpClassHashAgg, readBillingDemoOpClassStreamAgg, readBillingDemoOpClassHashJoin, + readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin, readBillingDemoOpClassRangeScan, + readBillingDemoOpClassReaderTransport, readBillingDemoOpClassOverlayReader, + readBillingDemoOpClassKVMutation, readBillingDemoOpClassKVWrite: + return true + case readBillingDemoOpClassPointLookup: + return op.id == "point_lookup@statement" + default: + return false + } +} + +func readBillingDemoOperatorActRows(runtimeStats *execdetails.RuntimeStatsColl, op *FlatOperator) (int64, bool) { + if op == nil { + return 0, false + } + if op.IsRoot { + return readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) + } + evidence := readBillingDemoExactCopRowsEvidence(runtimeStats, op.Origin.ID()) + if evidence.state == readBillingDemoCopRowsObserved { + return evidence.rows, true + } + return 0, false +} + +func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorResult, bool, string) { + operatorKind := strings.ToLower(op.Origin.TP()) + if !op.IsRoot { + if op.StoreType == kv.TiFlash { + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedTiFlash + } + switch op.Origin.TP() { + case plancodec.TypeTableScan, plancodec.TypeIdxScan, + plancodec.TypeTableFullScan, plancodec.TypeTableRangeScan, plancodec.TypeTableRowIDScan, + plancodec.TypeIndexFullScan, plancodec.TypeIndexRangeScan: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, true, "" + case plancodec.TypeSel: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassFilter, operatorKind: operatorKind}, true, "" + case plancodec.TypeProj: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassProjection, operatorKind: operatorKind}, true, "" + case plancodec.TypeLimit: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassLimit, operatorKind: operatorKind}, true, "" + case plancodec.TypeTopN: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassTopN, operatorKind: operatorKind}, true, "" + case plancodec.TypeHashAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassHashAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeStreamAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassStreamAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeSort, plancodec.TypeHashJoin, plancodec.TypeMergeJoin, plancodec.TypeIndexJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedOperator + default: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedOperator + } + } + + switch op.Origin.TP() { + case plancodec.TypeInsert, plancodec.TypeUpdate, plancodec.TypeDelete: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassWrapper, operatorKind: operatorKind}, true, "" + case plancodec.TypeExchangeReceiver, plancodec.TypeExchangeSender: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedMPP + case plancodec.TypeIndexMerge: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeLock: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedLock + case plancodec.TypePointGet, plancodec.TypeBatchPointGet: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassPointLookup, operatorKind: operatorKind}, true, "" + case plancodec.TypeSel: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassFilter, operatorKind: operatorKind}, true, "" + case plancodec.TypeProj: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassProjection, operatorKind: operatorKind}, true, "" + case plancodec.TypeLimit: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLimit, operatorKind: operatorKind}, true, "" + case plancodec.TypeTopN: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassTopN, operatorKind: operatorKind}, true, "" + case plancodec.TypeSort: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassSort, operatorKind: operatorKind}, true, "" + case plancodec.TypeWindow: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassWindow, operatorKind: operatorKind}, true, "" + case plancodec.TypeHashAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeStreamAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassStreamAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeHashJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashJoin, operatorKind: operatorKind}, true, "" + case plancodec.TypeMergeJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassMergeJoin, operatorKind: operatorKind}, true, "" + case plancodec.TypeIndexJoin, plancodec.TypeIndexHashJoin, plancodec.TypeIndexMergeJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupJoin, operatorKind: operatorKind}, true, "" + case plancodec.TypeTableReader, plancodec.TypeIndexReader: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, true, "" + case plancodec.TypeIndexLookUp, plancodec.TypeLocalIndexLookUp: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeUnionScan: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassOverlayReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeMemTableScan, plancodec.TypeClusterMemTableReader: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassMetadataReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeUnion: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassWrapper, operatorKind: operatorKind}, true, "" + case plancodec.TypeDual: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassSynthetic, operatorKind: operatorKind}, true, "" + default: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedOperator + } +} + +func readBillingDemoJoinConditionCount(join *physicalop.BasePhysicalJoin) int64 { + return int64(len(join.LeftConditions) + len(join.RightConditions) + len(join.OtherConditions)) +} + +func readBillingDemoCompareFilterCount(filters *physicalop.ColWithCmpFuncManager) int64 { + if filters == nil { + return 0 + } + return int64(len(filters.OpType)) +} + +func readBillingDemoExpressionCount(plan base.Plan) (int64, bool) { + switch p := plan.(type) { + case *physicalop.PhysicalSelection: + return int64(len(p.Conditions)), true + case *physicalop.PhysicalProjection: + return int64(len(p.Exprs)), true + case *physicalop.PhysicalHashAgg: + return int64(len(p.GroupByItems) + len(p.AggFuncs)), true + case *physicalop.PhysicalStreamAgg: + return int64(len(p.GroupByItems) + len(p.AggFuncs)), true + case *physicalop.PhysicalHashJoin: + if len(p.LeftJoinKeys) != len(p.RightJoinKeys) || len(p.LeftNAJoinKeys) != len(p.RightNAJoinKeys) { + return 0, false + } + return int64(len(p.EqualConditions)+len(p.NAEqualConditions)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin), true + case *physicalop.PhysicalMergeJoin: + if len(p.LeftJoinKeys) != len(p.RightJoinKeys) { + return 0, false + } + return int64(len(p.CompareFuncs)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin), true + case *physicalop.PhysicalIndexHashJoin: + if len(p.OuterHashKeys) != len(p.InnerHashKeys) { + return 0, false + } + return int64(len(p.OuterHashKeys)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin) + readBillingDemoCompareFilterCount(p.CompareFilters), true + case *physicalop.PhysicalIndexMergeJoin: + if p.NeedOuterSort != (len(p.OuterCompareFuncs) > 0) { + return 0, false + } + return int64(len(p.CompareFuncs)+len(p.OuterCompareFuncs)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin) + readBillingDemoCompareFilterCount(p.CompareFilters), true + case *physicalop.PhysicalIndexJoin: + if len(p.OuterJoinKeys) != len(p.InnerJoinKeys) { + return 0, false + } + return int64(len(p.OuterJoinKeys)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin) + readBillingDemoCompareFilterCount(p.CompareFilters), true + case *physicalop.PhysicalWindow: + count := len(p.WindowFuncDescs) + len(p.PartitionBy) + len(p.OrderBy) + if p.Frame != nil { + if p.Frame.Start != nil { + count += len(p.Frame.Start.CalcFuncs) + } + if p.Frame.End != nil { + count += len(p.Frame.End.CalcFuncs) + } + } + return int64(count), true + default: + return 0, false + } +} + +func readBillingDemoOrderingMaterialized(op, child *FlatOperator) bool { + if op == nil || op.Origin == nil || child == nil || child.Origin == nil { + return false + } + childSchema := child.Origin.Schema() + if projection, ok := child.Origin.(*physicalop.PhysicalProjection); ok && + (childSchema == nil || childSchema.Len() != len(projection.Exprs)) { + return false + } + checkExpr := func(expr expression.Expression) bool { + if expr == nil { + return false + } + _, scalar := expr.(*expression.ScalarFunction) + if scalar { + return false + } + if col, ok := expr.(*expression.Column); ok { + return childSchema != nil && childSchema.ColumnIndex(col) >= 0 + } + return true + } + switch p := op.Origin.(type) { + case *physicalop.PhysicalSort: + for _, item := range p.ByItems { + if item == nil || !checkExpr(item.Expr) { + return false + } + } + return true + case *physicalop.PhysicalTopN: + for _, item := range p.ByItems { + if item == nil || !checkExpr(item.Expr) { + return false + } + } + return true + default: + return false + } +} + +func readBillingDemoCheckedWork(rows int64, multiplier float64) (float64, bool) { + if rows < 0 || multiplier < 0 || math.IsNaN(multiplier) || math.IsInf(multiplier, 0) { + return 0, false + } + work := float64(rows) * multiplier + return work, work >= 0 && !math.IsNaN(work) && !math.IsInf(work, 0) +} + +func readBillingDemoHashStateRows(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, bool) { + if runtimeStats == nil || !runtimeStats.ExistsRootStats(planID) { + return 0, false + } + _, groups := runtimeStats.GetRootStats(planID).MergeStats() + for _, group := range groups { + if stats, ok := group.(execdetails.HashTableRuntimeStats); ok { + return stats.HashTableRows(), true + } + } + return 0, false +} + +func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { + outputRows, hasOutputRows := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) + if !hasOutputRows || outputRows < 0 { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + var units []readBillingDemoUnit + appendExpressionCPU := func(rows int64) (string, bool) { + exprCount, ok := readBillingDemoExpressionCount(op.Origin) + if !ok || exprCount < 0 { + return readBillingDemoReasonMissingExpressionCount, false + } + work, ok := readBillingDemoCheckedWork(rows, float64(exprCount)) + if !ok { + return readBillingDemoReasonMissingExpressionCount, false + } + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitExpressionCount, source: readBillingDemoInputSourcePhysicalPlan, side: readBillingDemoInputSideAll, value: float64(exprCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: work, widthSource: explainRUWidthSourceNotApplicable}, + ) + return "", true + } + switch operator.opClass { + case readBillingDemoOpClassHashJoin, readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: + if idx < 0 || idx >= len(tree) || len(tree[idx].ChildrenIdx) != 2 { + return nil, readBillingDemoReasonMissingExpressionCount, false + } + var inputRows int64 + for _, childIdx := range tree[idx].ChildrenIdx { + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + rows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + if !ok || rows < 0 || (rows > 0 && inputRows > math.MaxInt64-rows) { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + inputRows += rows + } + if reason, ok := appendExpressionCPU(inputRows); !ok { + return nil, reason, false + } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitJoinOutputRows, source: readBillingDemoInputSourceRuntimeOperatorActRows, side: readBillingDemoInputSideAll, value: float64(outputRows), widthSource: explainRUWidthSourceNotApplicable}) + if operator.opClass == readBillingDemoOpClassHashJoin { + stateRows, ok := readBillingDemoHashStateRows(runtimeStats, op.Origin.ID()) + if !ok { + return nil, readBillingDemoReasonMissingHashStateRows, false + } + if stateRows < 0 { + return nil, readBillingDemoReasonInvalidHashStateRows, false + } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitHashStateRows, source: readBillingDemoInputSourceHashJoinRuntime, side: readBillingDemoInputSideBuild, value: float64(stateRows), widthSource: explainRUWidthSourceNotApplicable}) + } + default: + if idx < 0 || idx >= len(tree) || len(tree[idx].ChildrenIdx) != 1 { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + childIdx := tree[idx].ChildrenIdx[0] + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + inputRows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + if !ok || inputRows < 0 { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + orderWork, ok := readBillingDemoOrderingWorkUnit(op, operator.opClass, inputRows) + if !ok { + return nil, readBillingDemoOrderingFailureReason(op, operator.opClass), false + } + if orderWork.unit != "" { + if !readBillingDemoOrderingMaterialized(op, tree[childIdx]) { + return nil, readBillingDemoReasonMissingOrderingProjection, false + } + orderWork.unit = readBillingDemoUnitCPUWork + units = append(units, orderWork) + } else if operator.opClass == readBillingDemoOpClassLimit || operator.opClass == readBillingDemoOpClassOverlayReader { + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(inputRows), widthSource: explainRUWidthSourceNotApplicable}) + } else if reason, ok := appendExpressionCPU(inputRows); !ok { + return nil, reason, false + } + if operator.opClass == readBillingDemoOpClassHashAgg { + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitHashStateRows, source: readBillingDemoInputSourceRuntimeOperatorActRows, side: readBillingDemoInputSideAll, value: float64(outputRows), widthSource: explainRUWidthSourceNotApplicable}) + } + } + if _, outputBytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, op.Origin.ID()); ok && readBillingDemoOperatorHasOutputShadows(operator.opClass) { + units = append(units, readBillingDemoRuntimeChunkOutputUnits(outputRows, outputBytes)...) + } + return units, "", true +} + +func readBillingDemoRuntimeChunkOutputUnits(rows, bytes int64) []readBillingDemoUnit { + if rows < 0 || bytes < 0 { + return nil + } + return []readBillingDemoUnit{ + {unit: readBillingDemoUnitOutputRows, source: readBillingDemoInputSourceRuntimeChunkBytes, side: readBillingDemoInputSideAll, value: float64(rows), widthSource: explainRUWidthSourceNotApplicable}, + {unit: readBillingDemoUnitOutputBytes, source: readBillingDemoInputSourceRuntimeChunkBytes, side: readBillingDemoInputSideAll, value: float64(bytes), rowWidth: readBillingDemoAverageRowWidth(rows, float64(bytes)), widthSource: explainRUWidthSourceRuntimeChunkAvg}, + } +} + +func readBillingDemoIsAggClass(opClass string) bool { + return opClass == readBillingDemoOpClassHashAgg || opClass == readBillingDemoOpClassStreamAgg +} + +func readBillingDemoOperatorHasOutputShadows(opClass string) bool { + if readBillingDemoIsAggClass(opClass) { + return true + } + switch opClass { + case readBillingDemoOpClassHashJoin, readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: + return true + default: + return false + } +} + +func readBillingDemoAverageRowWidth(rows int64, bytes float64) float64 { + if rows <= 0 || bytes <= 0 { + return 0 + } + return bytes / float64(rows) +} + +func readBillingDemoOrderingWorkUnit(op *FlatOperator, opClass string, inputRows int64) (readBillingDemoUnit, bool) { + if opClass != readBillingDemoOpClassTopN && opClass != readBillingDemoOpClassSort { + return readBillingDemoUnit{}, true + } + if op == nil || op.Origin == nil || inputRows < 0 { + return readBillingDemoUnit{}, false + } + + logWidth := max(float64(inputRows), 2) + if opClass == readBillingDemoOpClassTopN { + topN, ok := op.Origin.(*physicalop.PhysicalTopN) + if !ok { + return readBillingDemoUnit{}, false + } + if topN.Count == 0 { + return readBillingDemoUnit{ + unit: readBillingDemoUnitOrderWork, + source: readBillingDemoInputSourceRuntimeOrderingWork, + side: readBillingDemoInputSideAll, + value: 0, + widthSource: explainRUWidthSourceNotApplicable, + }, true + } + if topN.Count > math.MaxUint64-topN.Offset { + return readBillingDemoUnit{}, false + } + effectiveK := min(uint64(inputRows), topN.Offset+topN.Count) + logWidth = max(float64(effectiveK), 2) + } + work := float64(inputRows) * math.Log2(logWidth) + if work < 0 || math.IsNaN(work) || math.IsInf(work, 0) { + return readBillingDemoUnit{}, false + } + return readBillingDemoUnit{ + unit: readBillingDemoUnitOrderWork, + source: readBillingDemoInputSourceRuntimeOrderingWork, + side: readBillingDemoInputSideAll, + value: work, + widthSource: explainRUWidthSourceNotApplicable, + }, true +} + +func readBillingDemoOrderingFailureReason(op *FlatOperator, opClass string) string { + if opClass == readBillingDemoOpClassTopN && op != nil { + if topN, ok := op.Origin.(*physicalop.PhysicalTopN); ok && topN.Count > math.MaxUint64-topN.Offset { + return readBillingDemoReasonInvalidTopNBound + } + } + return readBillingDemoReasonInvalidOrderingWork +} + +func readBillingDemoPlanActRows(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, bool) { + if runtimeStats == nil || !runtimeStats.ExistsRootStats(planID) { + return 0, false + } + return runtimeStats.GetPlanActRows(planID), true +} + +func readBillingDemoRootOutputRowsAndBytes(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, int64, bool) { + if runtimeStats == nil || !runtimeStats.ExistsRootStats(planID) { + return 0, 0, false + } + basic := runtimeStats.GetBasicRuntimeStats(planID, false) + if basic == nil || !basic.HasBytes() { + return 0, 0, false + } + return basic.GetActRows(), basic.GetOutputBytes(), true +} + +func (e *readBillingDemoCopEstimator) aggOutputShadowUnits(idx int, opClass string) []readBillingDemoUnit { + if !readBillingDemoIsAggClass(opClass) || idx < 0 || idx >= len(e.tree) || e.tree[idx] == nil || e.tree[idx].Origin == nil { + return nil + } + rowsEvidence := readBillingDemoExactCopRowsEvidence(e.runtimeStats, e.tree[idx].Origin.ID()) + if rowsEvidence.state != readBillingDemoCopRowsObserved { + return nil + } + expectedTasks := e.runtimeStats.GetExpectedCopTasks(e.tree[idx].Origin.ID()) + if expectedTasks <= 0 || rowsEvidence.tasks != expectedTasks { + return nil + } + componentID := e.componentID[idx] + if componentID < 0 || componentID >= len(e.components) { + return nil + } + if maxTasks := e.components[componentID].maxSummaryTasks; maxTasks > 0 && rowsEvidence.tasks < maxTasks { + return nil + } + units := []readBillingDemoUnit{{ + unit: readBillingDemoUnitOutputRows, + source: readBillingDemoInputSourceRuntimeOperatorActRows, + side: readBillingDemoInputSideAll, + value: float64(rowsEvidence.rows), + widthSource: explainRUWidthSourceNotApplicable, + }} + + parentIdx := e.parentIdx[idx] + if parentIdx < 0 || parentIdx >= len(e.tree) { + return units + } + parent := e.tree[parentIdx] + if parent == nil || parent.Origin == nil || !parent.IsRoot || len(parent.ChildrenIdx) != 1 || parent.ChildrenIdx[0] != idx { + return units + } + parentOperator, supported, _ := readBillingDemoClassifyOperator(parent) + if !supported || parentOperator.opClass != readBillingDemoOpClassReaderReceive { + return units + } + switch parent.Origin.TP() { + case plancodec.TypeTableReader, plancodec.TypeIndexReader: + default: + return units + } + readerRows, readerBytes, ok := readBillingDemoRootOutputRowsAndBytes(e.runtimeStats, parent.Origin.ID()) + if !ok || readerRows != rowsEvidence.rows || readerRows < 0 || readerBytes < 0 { + return units + } + return append(units, readBillingDemoUnit{ + unit: readBillingDemoUnitOutputBytes, + source: readBillingDemoInputSourceRuntimeReaderOutput, + side: readBillingDemoInputSideAll, + value: float64(readerBytes), + rowWidth: readBillingDemoAverageRowWidth(readerRows, float64(readerBytes)), + widthSource: explainRUWidthSourceRuntimeReaderOutputChunkAvg, + }) +} + +func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, operator readBillingDemoOperatorResult) readBillingDemoCopUnitOutcome { + if failure, ok := estimator.nodeFailures[idx]; ok { + return readBillingDemoCopUnitOutcome{failure: failure} + } + if operator.opClass == readBillingDemoOpClassRangeScan { + if idx < 0 || idx >= len(estimator.tree) || len(estimator.tree[idx].ChildrenIdx) != 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} + } + component := estimator.components[estimator.componentID[idx]] + if component.scanCount != 1 || component.detailHolderCount > 1 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonAmbiguousCopScanWidth)} + } + if component.detailHolderCount != 1 || component.scanObservedTasks <= 0 || component.scanExpectedTasks <= 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} + } + if component.scanObservedTasks != component.scanExpectedTasks || + component.scanDetailExpectedTasks != component.scanExpectedTasks || + component.scanDetailRecords != component.scanDetailExpectedTasks { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} + } + scanDetail := component.scanDetail + scanInputRows, scanInputBytes, ok := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeys, scanDetail.ProcessedKeysSize) + if !ok { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} + } + rowWidth := readBillingDemoAverageRowWidth(scanInputRows, scanInputBytes) + units := []readBillingDemoUnit{ + readBillingDemoUnit{unit: readBillingDemoUnitScanBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedEstimate}, + } + return readBillingDemoCopUnitOutcome{success: true, units: units} + } + childIdx, failure, ok := estimator.directCopChild(idx) + if !ok { + return readBillingDemoCopUnitOutcome{failure: failure} + } + rowsEvidence := readBillingDemoExactCopRowsEvidence(estimator.runtimeStats, estimator.tree[childIdx].Origin.ID()) + if rowsEvidence.state == readBillingDemoCopRowsMissing { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingCopChildRuntimeRows)} + } + if rowsEvidence.state == readBillingDemoCopRowsInvalid { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(childIdx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidCopRuntimeRows)} + } + component := estimator.components[estimator.componentID[idx]] + if component.maxSummaryTasks > 0 && rowsEvidence.tasks < component.maxSummaryTasks { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} + } + var units []readBillingDemoUnit + orderWork, ok := readBillingDemoOrderingWorkUnit(estimator.tree[idx], operator.opClass, rowsEvidence.rows) + if !ok { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoOrderingFailureReason(estimator.tree[idx], operator.opClass))} + } + if orderWork.unit != "" { + if !readBillingDemoOrderingMaterialized(estimator.tree[idx], estimator.tree[childIdx]) { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingOrderingProjection)} + } + orderWork.unit = readBillingDemoUnitCPUWork + units = append(units, orderWork) + } else if operator.opClass == readBillingDemoOpClassLimit { + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(rowsEvidence.rows), widthSource: explainRUWidthSourceNotApplicable}) + } else { + exprCount, ok := readBillingDemoExpressionCount(estimator.tree[idx].Origin) + if !ok || exprCount < 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingExpressionCount)} + } + work, ok := readBillingDemoCheckedWork(rowsEvidence.rows, float64(exprCount)) + if !ok { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingExpressionCount)} + } + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitExpressionCount, source: readBillingDemoInputSourcePhysicalPlan, side: readBillingDemoInputSideAll, value: float64(exprCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: work, widthSource: explainRUWidthSourceNotApplicable}, + ) + } + aggUnits := estimator.aggOutputShadowUnits(idx, operator.opClass) + if operator.opClass == readBillingDemoOpClassHashAgg { + if len(aggUnits) == 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} + } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitHashStateRows, source: readBillingDemoInputSourceRuntimeOperatorActRows, side: readBillingDemoInputSideAll, value: aggUnits[0].value, widthSource: explainRUWidthSourceNotApplicable}) + } + units = append(units, aggUnits...) + return readBillingDemoCopUnitOutcome{success: true, units: units} +} + +func readBillingDemoRangeScanInput(totalKeys, processedKeys, processedKeysSize int64) (int64, float64, bool) { + if totalKeys == 0 && processedKeys == 0 && processedKeysSize == 0 { + return 0, 0, true + } + if totalKeys <= 0 || processedKeys <= 0 || processedKeysSize <= 0 { + return 0, 0, false + } + inputBytes := float64(processedKeysSize) / float64(processedKeys) * float64(totalKeys) + if inputBytes < 0 || math.IsNaN(inputBytes) || math.IsInf(inputBytes, 0) { + return 0, 0, false + } + return totalKeys, inputBytes, true +} + +func recordReadBillingDemoResult(result readBillingDemoResult) { + status := result.status + if status == "" { + status = readBillingDemoStatusUnknownInput + } + weightVersion := readBillingDemoActiveWeightVersion() + metrics.RecordReadBillingDemoStatement(status, readBillingDemoModelVersion, weightVersion) + for _, op := range result.operators { + opStatus := op.status + if opStatus == "" { + opStatus = status + } + reason := op.reason + if reason == "" { + reason = result.reason + } + if reason == "" { + reason = readBillingDemoReasonNone + } + metrics.RecordReadBillingDemoOperatorStatus(op.site, op.opClass, op.operatorKind, opStatus, reason, readBillingDemoModelVersion, weightVersion) + if opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + continue + } + for _, unit := range op.units { + metrics.AddReadBillingDemoBaseUnits(op.site, op.opClass, op.operatorKind, unit.unit, unit.source, unit.side, readBillingDemoModelVersion, weightVersion, unit.value) + metrics.ObserveReadBillingDemoRowWidth(op.site, op.opClass, op.operatorKind, unit.widthSource, readBillingDemoModelVersion, weightVersion, unit.rowWidth) + } + } +} + +func explainRUError(status explainRUStatus) error { + return errors.NewNoStackErrorf("EXPLAIN ANALYZE FORMAT='RU' is not supported for this target: %s", status) +} + +func recordExplainRUStatus(status explainRUStatus) { + metrics.RecordExplainRUStatus(string(status)) +} + +func (e *Explain) recordExplainRUStatus(status explainRUStatus) { + if e == nil || e.ruStatusRecorded { + return + } + e.ruStatusRecorded = true + recordExplainRUStatus(status) +} + +// explainRUTargetGateStatus is the pre-execution safety gate for FORMAT='RU'. +// SELECT keeps the side-effect-free checks from the read-side demo; write DML +// is limited to foreground statements with statement-local mutation recording. +// TiKV commit detail availability is handled independently after execution. +func explainRUTargetGateStatus(stmt ast.StmtNode) explainRUStatus { + if _, ok := explainRUWriteDMLKind(stmt); ok { + return explainRUStatusSuccess + } + return explainRUSelectGateStatus(stmt) +} + +func explainRUWriteDMLKind(stmt ast.StmtNode) (string, bool) { + switch x := stmt.(type) { + case *ast.InsertStmt: + if x == nil || x.IsReplace { + return "", false + } + if len(x.OnDuplicate) > 0 { + return "upsert", true + } + if x.IgnoreErr { + return "insert_ignore", true + } + return "insert", true + case *ast.UpdateStmt: + if x == nil { + return "", false + } + return "update", true + case *ast.DeleteStmt: + if x == nil { + return "", false + } + return "delete", true + default: + return "", false + } +} + +// explainRUSelectGateStatus is the first-demo pre-execution safety gate. It +// accepts only SELECT keyword surfaces and set operations whose leaves can be +// checked before EXPLAIN ANALYZE can run the target statement. +func explainRUSelectGateStatus(stmt ast.StmtNode) explainRUStatus { + switch x := stmt.(type) { + case *ast.SelectStmt: + return explainRUValidateSelectNode(x) + case *ast.SetOprStmt: + if x == nil || x.SelectList == nil { + return explainRUStatusUnsupportedNonSelect + } + return explainRUValidateSetOprSelectList(x.SelectList) + default: + return explainRUStatusUnsupportedNonSelect + } +} + +func explainRUValidateSetOprSelectList(list *ast.SetOprSelectList) explainRUStatus { + if list == nil || len(list.Selects) == 0 { + return explainRUStatusUnsupportedNonSelect + } + for _, sel := range list.Selects { + switch x := sel.(type) { + case *ast.SelectStmt: + if status := explainRUValidateSelectNode(x); status != explainRUStatusSuccess { + return status + } + case *ast.SetOprSelectList: + if status := explainRUValidateSetOprSelectList(x); status != explainRUStatusSuccess { + return status + } + default: + // SetOprSelectList is documented as SELECT/TABLE/VALUES capable. Fail + // closed until non-SELECT leaves have an explicit attribution design. + return explainRUStatusUnsupportedNonSelect + } + } + visitor := &explainRUSideEffectVisitor{status: explainRUStatusSuccess} + list.Accept(visitor) + return visitor.status +} + +func explainRUValidateSelectNode(sel *ast.SelectStmt) explainRUStatus { + if sel == nil || sel.Kind != ast.SelectStmtKindSelect { + return explainRUStatusUnsupportedNonSelect + } + if sel.SelectIntoOpt != nil { + return explainRUStatusUnsupportedSideEffecting + } + if sel.LockInfo != nil && sel.LockInfo.LockType != ast.SelectLockNone { + return explainRUStatusUnsupportedLockingSelect + } + visitor := &explainRUSideEffectVisitor{status: explainRUStatusSuccess} + sel.Accept(visitor) + return visitor.status +} + +type explainRUSideEffectVisitor struct { + status explainRUStatus +} + +func (v *explainRUSideEffectVisitor) Enter(n ast.Node) (ast.Node, bool) { + if v.status != explainRUStatusSuccess { + return n, true + } + switch x := n.(type) { + case *ast.SelectStmt: + if x.Kind != ast.SelectStmtKindSelect { + v.status = explainRUStatusUnsupportedNonSelect + return n, true + } + if x.SelectIntoOpt != nil { + v.status = explainRUStatusUnsupportedSideEffecting + return n, true + } + if x.LockInfo != nil && x.LockInfo.LockType != ast.SelectLockNone { + v.status = explainRUStatusUnsupportedLockingSelect + return n, true + } + case *ast.VariableExpr: + // User-variable assignment is syntactically a SELECT expression but + // mutates session state, so it is outside the side-effect-free demo scope. + if x.Value != nil { + v.status = explainRUStatusUnsupportedSideEffecting + return n, true + } + case *ast.FuncCallExpr: + if explainRUFuncCallHasSideEffect(x) { + v.status = explainRUStatusUnsupportedSideEffecting + return n, true + } + } + return n, false +} + +func (v *explainRUSideEffectVisitor) Leave(n ast.Node) (ast.Node, bool) { + return n, v.status == explainRUStatusSuccess +} + +func explainRUFuncCallHasSideEffect(fn *ast.FuncCallExpr) bool { + if fn == nil { + return false + } + switch strings.ToLower(fn.FnName.L) { + case ast.GetLock, ast.ReleaseLock, ast.ReleaseAllLocks, ast.NextVal, ast.SetVal, ast.Sleep: + return true + case ast.LastInsertId: + return len(fn.Args) > 0 + default: + return false + } +} + +func (e *Explain) renderRUExplain() (err error) { + start := time.Now() + status := explainRUStatusError + defer func() { + metrics.ObserveExplainRURenderDuration(string(status), time.Since(start).Seconds()) + e.recordExplainRUStatus(status) + }() + + if !e.Analyze { + status = explainRUStatusUnsupportedNonAnalyze + return explainRUError(explainRUStatusUnsupportedNonAnalyze) + } + if gateStatus := explainRUTargetGateStatus(e.ExecStmt); gateStatus != explainRUStatusSuccess { + status = gateStatus + return explainRUError(gateStatus) + } + flat := FlattenPhysicalPlan(e.TargetPlan, true) + if flat == nil || len(flat.Main) == 0 || flat.InExplain { + return errors.NewNoStackError("EXPLAIN ANALYZE FORMAT='RU' cannot render an empty target plan") + } + runtimeStats := e.RuntimeStatsColl + if runtimeStats == nil && e.SCtx() != nil && e.SCtx().GetSessionVars() != nil { + runtimeStats = e.SCtx().GetSessionVars().StmtCtx.RuntimeStatsColl + } + // The snapshot belongs to the target statement execution. Returning this + // EXPLAIN result can add more result-chunk counters later, so render output + // and Demo Metrics are derived from this frozen input and the generated rows. + snapshot, snapshotStatus := explainRUExtractComponentSnapshot(runtimeStats, e.TargetPlan.ID()) + metrics.RecordExplainRUComponentSnapshot(string(snapshotStatus)) + var snapshotRUV2Metrics *execdetails.RUV2Metrics + if snapshot != nil { + snapshotRUV2Metrics = snapshot.Metrics + } + result := buildReadBillingDemoResult(e.SCtx(), e.TargetPlan, e.ExecStmt, nil, snapshotRUV2Metrics) + if result.status != readBillingDemoStatusSuccess { + operator := "" + if len(result.operators) > 0 { + op := result.operators[0] + operator = " operator=" + op.site + "/" + op.opClass + "/" + op.operatorKind + } + status = explainRUStatusError + return errors.NewNoStackErrorf( + "EXPLAIN ANALYZE FORMAT='RU' cannot render a complete preview RU model result: status=%s reason=%s%s", + result.status, + result.reason, + operator, + ) + } + rows := explainRUBuildReadBillingRows(result, snapshotStatus) + + e.Rows = make([][]string, 0, len(rows)) + for _, row := range rows { + e.Rows = append(e.Rows, row.toStrings()) + explainRUObserveRow(row) + } + status = explainRUStatusSuccess + return nil +} + +func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus explainRUComponentSnapshotStatus) []explainRURow { + rows := []explainRURow{{ + section: explainRUSectionSummary, + component: "total_preview_ru", + source: explainRUSourceSummaryTotal, + note: explainRUReadBillingSummaryNote(snapshotStatus, result), + }} + totalPreviewRU := 0.0 + weightsReady := readBillingDemoWeightsValid(readBillingDemoV4Weights) + completeTotal := weightsReady && result.status == readBillingDemoStatusSuccess + for _, op := range result.operators { + if op.status != readBillingDemoStatusOperatorOK { + if readBillingDemoOperatorBillable(op) { + completeTotal = false + } + if op.emitStatusRow { + rows = append(rows, explainRUReadBillingStatusRow(op)) + } + continue + } + if !readBillingDemoOperatorBillable(op) { + continue + } + for _, unit := range op.units { + row := explainRUReadBillingUnitRow(op, unit) + if _, semantic := readBillingDemoUnitWeight(readBillingDemoV4Weights, unit.unit); semantic { + if weight, previewRU, ok := readBillingDemoUnitPreviewRU(unit, readBillingDemoV4Weights); ok { + row.weight = weight + row.hasWeight = true + row.previewRU = previewRU + row.hasPreviewRU = true + nextTotal := totalPreviewRU + previewRU + if nextTotal < 0 || math.IsNaN(nextTotal) || math.IsInf(nextTotal, 0) { + completeTotal = false + } else { + totalPreviewRU = nextTotal + } + } else { + completeTotal = false + } + } + rows = append(rows, row) + } + } + if completeTotal { + rows[0].previewRU = totalPreviewRU + rows[0].hasPreviewRU = true + } + return rows +} + +func explainRUReadBillingStatusRow(op readBillingDemoOperatorResult) explainRURow { + row := explainRURow{ + section: explainRUSectionPlan, + id: op.id, + component: op.operatorKind, + operatorClass: op.site + "/" + op.opClass, + note: "weight_version=" + readBillingDemoActiveWeightVersion(), + } + row.note = appendExplainRUNote(row.note, "status="+op.status) + if op.reason != "" { + row.note = appendExplainRUNote(row.note, "reason="+op.reason) + } + if op.scope != "" { + row.note = appendExplainRUNote(row.note, "scope="+op.scope) + } + if op.dmlKind != "" { + row.note = appendExplainRUNote(row.note, "dml_kind="+op.dmlKind) + } + if op.hasActRows { + row.actRows = op.actRows + row.hasActRows = true + row.outputRows = op.actRows + row.hasOutputRows = true + } + return row +} + +func explainRUReadBillingSummaryNote(snapshotStatus explainRUComponentSnapshotStatus, result readBillingDemoResult) string { + note := "weight_version=" + readBillingDemoActiveWeightVersion() + if !readBillingDemoWeightsValid(readBillingDemoV4Weights) { + note = appendExplainRUNote(note, readBillingDemoReasonUncalibratedWeights) + } + if snapshotStatus != explainRUComponentSnapshotOK { + note = appendExplainRUNote(note, "component_snapshot_"+string(snapshotStatus)) + } + for _, op := range result.operators { + if op.uncalibrated { + note = appendExplainRUNote(note, "mutation_weights_uncalibrated=true") + } + if op.status == readBillingDemoStatusPartial && op.reason != "" { + note = appendExplainRUNote(note, "partial_"+op.reason) + } + } + return note +} + +func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBillingDemoUnit) explainRURow { + row := explainRURow{ + section: explainRUSectionPlan, + id: op.id, + component: op.operatorKind, + operatorClass: op.site + "/" + op.opClass, + rowWidth: unit.rowWidth, + hasRowWidth: unit.rowWidth > 0, + rowWidthSource: unit.widthSource, + unit: unit.unit, + source: unit.source, + note: "input_side=" + unit.side + ",weight_version=" + readBillingDemoActiveWeightVersion(), + } + if op.scope != "" { + row.note = appendExplainRUNote(row.note, "scope="+op.scope) + } + if op.dmlKind != "" { + row.note = appendExplainRUNote(row.note, "dml_kind="+op.dmlKind) + } + if op.uncalibrated { + row.note = appendExplainRUNote(row.note, "uncalibrated=true") + } + if unit.unit == readBillingDemoUnitWriteByte { + row.note = appendExplainRUNote(row.note, "semantic_name=write_bytes") + } + if readBillingDemoUnitDiagnosticOnly(unit.unit) { + row.note = appendExplainRUNote(row.note, "diagnostic_only=true") + } + if op.hasActRows { + row.actRows = op.actRows + row.hasActRows = true + row.outputRows = op.actRows + row.hasOutputRows = true + } + switch unit.unit { + case readBillingDemoUnitFixedEvents: + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitInputRows: + row.inputRows = int64(unit.value) + row.hasInputRows = true + row.workRows = unit.value + row.hasWorkRows = true + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitInputBytes: + row.workBytes = unit.value + row.hasWorkBytes = true + case readBillingDemoUnitOutputRows: + row.outputRows = int64(unit.value) + row.hasOutputRows = true + row.workRows = unit.value + row.hasWorkRows = true + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitOutputBytes: + row.workBytes = unit.value + row.hasWorkBytes = true + case readBillingDemoUnitOrderWork: + row.workRows = unit.value + row.hasWorkRows = true + case readBillingDemoUnitCPUWork: + row.workRows = unit.value + row.hasWorkRows = true + case readBillingDemoUnitScanBytes, readBillingDemoUnitNetBytes: + row.workBytes = unit.value + row.hasWorkBytes = true + case readBillingDemoUnitExpressionCount, readBillingDemoUnitReadRequestCount, readBillingDemoUnitWriteRequestCount, + readBillingDemoUnitHashStateRows, readBillingDemoUnitJoinOutputRows: + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitEncodedMutationCount, readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, + readBillingDemoUnitWriteKeys, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitEncodedMutationBytes, readBillingDemoUnitKeyBytes, readBillingDemoUnitValueBytes, + readBillingDemoUnitWriteByte: + row.workBytes = unit.value + row.hasWorkBytes = true + } + return row +} + +func readBillingDemoUnitDiagnosticOnly(unit string) bool { + switch unit { + case readBillingDemoUnitFixedEvents, readBillingDemoUnitInputRows, readBillingDemoUnitInputBytes, + readBillingDemoUnitOrderWork, readBillingDemoUnitExpressionCount, readBillingDemoUnitEncodedMutationCount, + readBillingDemoUnitEncodedMutationBytes, readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitKeyBytes, + readBillingDemoUnitValueBytes, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount, + readBillingDemoUnitWriteKeys, readBillingDemoUnitWriteByte, readBillingDemoUnitOutputRows, readBillingDemoUnitOutputBytes: + return true + default: + return false + } +} + +func appendExplainRUNote(note, extra string) string { + if note == "" { + return extra + } + if extra == "" { + return note + } + return note + "," + extra +} + +func explainRUExtractComponentSnapshot(runtimeStats *execdetails.RuntimeStatsColl, targetPlanID int) (*execdetails.RURuntimeStats, explainRUComponentSnapshotStatus) { + // GetRootStats creates an empty entry for a missing plan ID; check + // ExistsRootStats first so "missing snapshot" stays observable. + if runtimeStats == nil || !runtimeStats.ExistsRootStats(targetPlanID) { + return nil, explainRUComponentSnapshotMissing + } + _, groups := runtimeStats.GetRootStats(targetPlanID).MergeStats() + for _, group := range groups { + ruStats, ok := group.(*execdetails.RURuntimeStats) + if !ok { + continue + } + if ruStats.RUVersion != rmclient.RUVersionV2 { + return ruStats, explainRUComponentSnapshotNonV2 + } + if ruStats.Metrics == nil { + return ruStats, explainRUComponentSnapshotNilMetrics + } + if ruStats.Metrics.Bypass() { + return ruStats, explainRUComponentSnapshotBypassed + } + return ruStats, explainRUComponentSnapshotOK + } + return nil, explainRUComponentSnapshotMissing +} + +func (row explainRURow) toStrings() []string { + return []string{ + row.section, + row.id, + row.component, + row.operatorClass, + formatOptionalInt(row.actRows, row.hasActRows), + formatOptionalInt(row.inputRows, row.hasInputRows), + formatOptionalInt(row.outputRows, row.hasOutputRows), + formatOptionalFloat(row.rowWidth, row.hasRowWidth), + row.rowWidthSource, + formatOptionalCompactFloat(row.workRows, row.hasWorkRows), + formatOptionalFloat(row.workBytes, row.hasWorkBytes), + row.unit, + formatOptionalInt(row.count, row.hasCount), + formatOptionalFloat(row.weight, row.hasWeight), + formatOptionalFloat(row.previewRU, row.hasPreviewRU), + row.source, + row.note, + } +} + +func formatOptionalInt(v int64, ok bool) string { + if !ok { + return "" + } + return strconv.FormatInt(v, 10) +} + +func formatOptionalFloat(v float64, ok bool) string { + if !ok { + return "" + } + return strconv.FormatFloat(v, 'f', 6, 64) +} + +func formatOptionalCompactFloat(v float64, ok bool) string { + if !ok { + return "" + } + return strconv.FormatFloat(v, 'f', -1, 64) +} + +func explainRUObserveRow(row explainRURow) { + // Metrics are emitted from rendered rows so the Prometheus view matches the + // SQL output and avoids reading live counters after render-side accounting. + previewRU := -1.0 + if row.hasPreviewRU { + previewRU = row.previewRU + } + workRows := -1.0 + if row.hasWorkRows { + workRows = row.workRows + } + workBytes := -1.0 + if row.hasWorkBytes { + workBytes = row.workBytes + } + rowWidth := -1.0 + if row.section == explainRUSectionPlan && row.hasRowWidth { + rowWidth = row.rowWidth + } + component, operator := explainRUMetricComponentOperator(row) + metrics.ObserveExplainRURow(row.section, component, operator, row.source, row.rowWidthSource, readBillingDemoActiveWeightVersion(), previewRU, workRows, workBytes, rowWidth) +} + +func explainRUMetricComponentOperator(row explainRURow) (component, operator string) { + switch row.section { + case explainRUSectionPlan: + return "", row.component + default: + return row.component, "" + } +} + +func explainRUUnsupportedFormatError(format string) error { + return errors.Errorf("'explain format=%v' cannot work without 'analyze', please use 'explain analyze format=%v'", format, format) +} + +func isExplainRUFormat(format string) bool { + return strings.ToLower(format) == types.ExplainFormatRU +} diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index d37b0d418f2ee..1c974681aabd3 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -5439,7 +5439,15 @@ func (b *PlanBuilder) buildMemTable(_ context.Context, dbName ast.CIStr, tableIn p.Extractor = &TableStorageStatsExtractor{} case infoschema.TableTiFlashTables, infoschema.TableTiFlashSegments, infoschema.TableTiFlashIndexes: p.Extractor = &TiFlashSystemTableExtractor{} - case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, infoschema.TableTiDBStatementsStats: + case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, infoschema.TableTiDBStatementsStats, + infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: p.Extractor = &StatementsSummaryExtractor{} case infoschema.TableTiKVRegionPeers: p.Extractor = &TikvRegionPeersExtractor{} diff --git a/pkg/planner/core/operator/logicalop/logical_mem_table.go b/pkg/planner/core/operator/logicalop/logical_mem_table.go index f9854098e301b..2aeb5260b0465 100644 --- a/pkg/planner/core/operator/logicalop/logical_mem_table.go +++ b/pkg/planner/core/operator/logicalop/logical_mem_table.go @@ -81,10 +81,18 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column) (bas case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, infoschema.TableTiDBStatementsStats, + infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, infoschema.TableSlowQuery, infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory, infoschema.ClusterTableTiDBStatementsStats, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus, infoschema.ClusterTableSlowLog, infoschema.TableTiDBTrx, infoschema.ClusterTableTiDBTrx, diff --git a/pkg/planner/core/pb_to_plan.go b/pkg/planner/core/pb_to_plan.go index e8588cc9fc047..659d5ac0d92f4 100644 --- a/pkg/planner/core/pb_to_plan.go +++ b/pkg/planner/core/pb_to_plan.go @@ -146,7 +146,11 @@ func (b *PBPlanBuilder) pbToTableScan(e *tipb.Executor) (base.PhysicalPlan, erro } } p.Extractor = extractor - case infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory: + case infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: p.Extractor = &StatementsSummaryExtractor{} case infoschema.ClusterTableTiDBIndexUsage: p.Extractor = NewInfoSchemaTiDBIndexUsageExtractor() diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index efc9dbbe2498b..c0f199f2e201c 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -5712,6 +5712,18 @@ func (b *PlanBuilder) buildTrace(trace *ast.TraceStmt) (base.Plan, error) { func (b *PlanBuilder) buildExplainPlan(targetPlan base.Plan, format string, explainBriefBinary string, analyze, explore bool, execStmt ast.StmtNode, runtimeStats *execdetails.RuntimeStatsColl, sqlDigest, replayerFile string) (base.Plan, error) { format = strings.ToLower(format) + if format == types.ExplainFormatRU && !analyze { + recordExplainRUStatus(explainRUStatusUnsupportedNonAnalyze) + return nil, explainRUUnsupportedFormatError(format) + } + if format == types.ExplainFormatRU && analyze { + // Secondary guard for callers that already have a target plan. The main + // SQL path below rejects before optimization or no-delay execution. + if status := explainRUTargetGateStatus(execStmt); status != explainRUStatusSuccess { + recordExplainRUStatus(status) + return nil, explainRUError(status) + } + } if format == types.ExplainFormatTrueCardCost && !analyze { return nil, errors.Errorf("'explain format=%v' cannot work without 'analyze', please use 'explain analyze format=%v'", format, format) } @@ -5748,6 +5760,12 @@ func (b *PlanBuilder) buildExplainFor(explainFor *ast.ExplainForStmt) (base.Plan targetPlan, ok := processInfo.Plan.(base.Plan) explainForFormat := strings.ToLower(explainFor.Format) + // Check RU before the missing-plan branch so idle and running connections + // return the same unsupported status. + if explainForFormat == types.ExplainFormatRU { + recordExplainRUStatus(explainRUStatusUnsupportedForConnection) + return nil, errors.Errorf("explain format '%s' for connection is not supported now: %s", explainForFormat, explainRUStatusUnsupportedForConnection) + } if !ok || targetPlan == nil { return &Explain{Format: explainForFormat}, nil } @@ -5801,6 +5819,23 @@ func (b *PlanBuilder) buildExplain(ctx context.Context, explain *ast.ExplainStmt } explain.Stmt = hintedStmt } + if isExplainRUFormat(explain.Format) { + // This is the primary FORMAT='RU' safety boundary. It runs after plan + // digest expansion but before target optimization/execution, which keeps + // DML no-delay paths and side-effecting SELECT forms from running first. + if !explain.Analyze { + recordExplainRUStatus(explainRUStatusUnsupportedNonAnalyze) + return nil, explainRUUnsupportedFormatError(strings.ToLower(explain.Format)) + } + if explain.Explore { + recordExplainRUStatus(explainRUStatusUnsupportedNonSelect) + return nil, explainRUError(explainRUStatusUnsupportedNonSelect) + } + if status := explainRUTargetGateStatus(explain.Stmt); status != explainRUStatusSuccess { + recordExplainRUStatus(status) + return nil, explainRUError(status) + } + } sctx, err := AsSctx(b.ctx) if err != nil { diff --git a/pkg/session/BUILD.bazel b/pkg/session/BUILD.bazel index 3dbd454337571..87e9dcd4d6d40 100644 --- a/pkg/session/BUILD.bazel +++ b/pkg/session/BUILD.bazel @@ -126,6 +126,8 @@ go_library( "//pkg/util/sli", "//pkg/util/sqlescape", "//pkg/util/sqlexec", + "//pkg/util/stmtsummary", + "//pkg/util/stmtsummary/v2:stmtsummary", "//pkg/util/syncutil", "//pkg/util/timeutil", "//pkg/util/topsql", @@ -188,6 +190,7 @@ go_test( "//pkg/parser/mysql", "//pkg/session/sessionapi", "//pkg/sessionctx", + "//pkg/sessionctx/stmtctx", "//pkg/sessionctx/vardef", "//pkg/sessionctx/variable", "//pkg/statistics", diff --git a/pkg/session/session.go b/pkg/session/session.go index a414b90cd3941..54ce556716920 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -136,6 +136,8 @@ import ( "github.com/pingcap/tidb/pkg/util/sli" "github.com/pingcap/tidb/pkg/util/sqlescape" "github.com/pingcap/tidb/pkg/util/sqlexec" + "github.com/pingcap/tidb/pkg/util/stmtsummary" + stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2" "github.com/pingcap/tidb/pkg/util/syncutil" "github.com/pingcap/tidb/pkg/util/timeutil" "github.com/pingcap/tidb/pkg/util/topsql" @@ -985,6 +987,7 @@ func (s *session) CommitTxn(ctx context.Context) error { defer r.End() s.setLastTxnInfoBeforeTxnEnd() + s.markPreviewKVMutationTxnPipelined() var commitDetail *tikvutil.CommitDetails ctx = context.WithValue(ctx, tikvutil.CommitDetailCtxKey, &commitDetail) err := s.doCommitWithRetry(ctx) @@ -1020,6 +1023,16 @@ func (s *session) CommitTxn(ctx context.Context) error { return err } +func (s *session) markPreviewKVMutationTxnPipelined() { + // Pipelined transactions currently do not populate CommitDetails write + // keys/bytes. Preserve that fact on the COMMIT statement recorder before + // doCommitWithRetry invalidates the transaction. + if recorder := s.sessionVars.StmtCtx.PreviewKVMutationRecorder; recorder != nil && + s.txn.Valid() && s.txn.IsPipelined() { + recorder.MarkPipelined() + } +} + func (s *session) RollbackTxn(ctx context.Context) { r, ctx := tracing.StartRegionEx(ctx, "session.RollbackTxn") defer r.End() @@ -2172,7 +2185,7 @@ func (s *session) ExecRestrictedStmt(ctx context.Context, stmtNode ast.StmtNode, startTime := time.Now() metrics.SessionRestrictedSQLCounter.Inc() ctx = execdetails.ContextWithInitializedExecDetails(ctx) - rs, err := se.ExecuteStmt(ctx, stmtNode) + rs, err := se.ExecuteInternalStmt(ctx, stmtNode) if err != nil { se.sessionVars.StmtCtx.AppendError(err) } @@ -2424,10 +2437,23 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex return rs, err } -func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (sqlexec.RecordSet, error) { +func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (recordSet sqlexec.RecordSet, err error) { r, ctx := tracing.StartRegionEx(ctx, "session.ExecuteStmt") defer r.End() ctx = execdetails.ContextWithMissingExecDetailsInitialized(ctx) + // Prevent a recorder from the previous statement from observing any setup + // work before this statement receives a fresh StatementContext. + s.txn.setPreviewKVMutationCollection(nil, false) + readBillingDemoHandledByStmt := false + readBillingDemoLegacyGeneralLogRegistered := false + defer func() { + if err != nil && !readBillingDemoHandledByStmt { + readBillingDemoStats := s.recordReadBillingDemoEarlyError(stmtNode, err) + if readBillingDemoLegacyGeneralLogRegistered { + executor.LogReadBillingDemoGeneralLog(s, readBillingDemoStats) + } + } + }() if err := s.PrepareTxnCtx(ctx, stmtNode); err != nil { return nil, err @@ -2444,6 +2470,14 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s if err := executor.ResetContextOfStmt(s, stmtNode); err != nil { return nil, err } + previewKVMutationEnabled := !s.sessionVars.InRestrictedSQL && + (s.sessionVars.EnableReadBillingDemo || + (s.sessionVars.StmtCtx.InExplainStmt && s.sessionVars.StmtCtx.InExplainAnalyzeStmt && + strings.EqualFold(s.sessionVars.StmtCtx.ExplainFormat, "ru"))) + if previewKVMutationEnabled { + s.sessionVars.StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + } + s.txn.setPreviewKVMutationCollection(s.sessionVars, previewKVMutationEnabled) // ResetContextOfStmt clears SQLKiller, so honor a canceled caller before executing the next statement. if err := ctx.Err(); err != nil { return nil, err @@ -2572,7 +2606,6 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s } var stmt *executor.ExecStmt - var err error { // Transform abstract syntax tree to a physical plan(stored in executor.ExecStmt). @@ -2632,6 +2665,7 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s // Execute the physical plan. defer logStmt(stmt, s) // defer until txnStartTS is set + readBillingDemoLegacyGeneralLogRegistered = true if sessVars.MemArbitrator.WaitAverse == variable.MemArbitratorNolimit { metrics.GlobalMemArbitratorSubTasks.NoLimit.Inc() @@ -2678,7 +2712,6 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s }() } - var recordSet sqlexec.RecordSet if stmt.PsStmt != nil { // point plan short path ctx, prevTraceID := resetStmtTraceID(ctx, s) @@ -2711,6 +2744,7 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s s.setLastTxnInfoBeforeTxnEnd() s.txn.changeToInvalid() } else { + readBillingDemoHandledByStmt = true recordSet, err = runStmt(ctx, s, stmt) } @@ -2766,6 +2800,47 @@ func resolvePreparedStmt(stmt ast.StmtNode, vars *variable.SessionVars) (ast.Stm return prepareStmt.PreparedAst.Stmt, nil } +func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err error) stmtsummary.ReadBillingDemoStatementStats { + if err == nil { + return stmtsummary.ReadBillingDemoStatementStats{} + } + metricsStmt := stmtNode + if resolvedStmt, resolveErr := resolvePreparedStmt(stmtNode, s.sessionVars); resolveErr == nil && resolvedStmt != nil { + metricsStmt = resolvedStmt + } + readBillingDemoStats := plannercore.RecordReadBillingDemoForStatement(s, nil, metricsStmt, err) + if readBillingDemoStats.IsEmpty() || s.sessionVars == nil || s.sessionVars.StmtCtx == nil { + return stmtsummary.ReadBillingDemoStatementStats{} + } + userString := "" + if s.sessionVars.User != nil { + userString = s.sessionVars.User.Username + } + stmtCtx := s.sessionVars.StmtCtx + if stmtCtx.StmtType == "" { + stmtCtx.StmtType = stmtctx.GetStmtLabel(context.Background(), metricsStmt) + } + normalizedSQL, digest := stmtCtx.SQLDigest() + digestString := "" + if digest != nil { + digestString = digest.String() + } + isInternalSQL := (s.sessionVars.InRestrictedSQL || len(userString) == 0) && !s.sessionVars.InExplainExplore + stmtsummaryv2.AddReadBillingDemoStatusOnly(&stmtsummary.StmtExecInfo{ + SchemaName: strings.ToLower(s.sessionVars.CurrentDB), + NormalizedSQL: normalizedSQL, + Digest: digestString, + User: userString, + StmtCtx: stmtCtx, + StartTime: s.sessionVars.StartTime, + IsInternal: isInternalSQL, + Succeed: false, + ResourceGroupName: stmtCtx.ResourceGroupName, + ReadBillingDemoStats: readBillingDemoStats, + }) + return readBillingDemoStats +} + func shouldBypass(ctx context.Context, stmtNode ast.StmtNode, sessVars *variable.SessionVars) bool { switch kv.GetInternalSourceType(ctx) { case kv.InternalTxnOthers: @@ -2992,6 +3067,8 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. origTxnCtx := sessVars.TxnCtx err = se.checkTxnAborted(s) if err != nil { + readBillingDemoStats := se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) + executor.LogReadBillingDemoGeneralLog(se, readBillingDemoStats) return nil, err } if sessVars.TxnCtx.CouldRetry && !s.IsReadOnly(sessVars) { @@ -2999,6 +3076,8 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. // otherwise, the stmt won't be add into stmt history, and also don't need check. // About `stmt-count-limit`, see more in https://docs.pingcap.com/tidb/stable/tidb-configuration-file#stmt-count-limit if err := checkStmtLimit(ctx, se, false); err != nil { + readBillingDemoStats := se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) + executor.LogReadBillingDemoGeneralLog(se, readBillingDemoStats) return nil, err } } @@ -3024,6 +3103,10 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. if !sessVars.InTxn() { se.StmtCommit(ctx) if err := se.CommitTxn(ctx); err != nil { + if closeErr := executor.CloseRecordSetWithError(rs, err); closeErr != nil { + logutil.Logger(ctx).Error("close EXPLAIN ANALYZE DML record set after commit error failed", + zap.Error(closeErr), zap.NamedError("commitError", err)) + } return nil, err } } diff --git a/pkg/session/tidb_test.go b/pkg/session/tidb_test.go index b72430bfed24d..d738f5634bbdc 100644 --- a/pkg/session/tidb_test.go +++ b/pkg/session/tidb_test.go @@ -16,13 +16,16 @@ package session import ( "context" + "errors" "testing" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/vardef" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/store/mockstore" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/execdetails" @@ -30,6 +33,138 @@ import ( "github.com/stretchr/testify/require" ) +type previewMutationTestBuffer struct { + kv.MemBuffer + setErr error + setWithFlagsErr error + deleteErr error + deleteWithFlagsErr error + nextStaging kv.StagingHandle +} + +func (b *previewMutationTestBuffer) Set(_ kv.Key, _ []byte) error { + return b.setErr +} + +func (b *previewMutationTestBuffer) SetWithFlags(_ kv.Key, _ []byte, _ ...kv.FlagsOp) error { + return b.setWithFlagsErr +} + +func (b *previewMutationTestBuffer) Delete(_ kv.Key) error { + return b.deleteErr +} + +func (b *previewMutationTestBuffer) DeleteWithFlags(_ kv.Key, _ ...kv.FlagsOp) error { + return b.deleteWithFlagsErr +} + +func (*previewMutationTestBuffer) UpdateFlags(_ kv.Key, _ ...kv.FlagsOp) {} + +func (b *previewMutationTestBuffer) Staging() kv.StagingHandle { + b.nextStaging++ + return b.nextStaging +} + +func (*previewMutationTestBuffer) Release(kv.StagingHandle) {} + +func (*previewMutationTestBuffer) Cleanup(kv.StagingHandle) {} + +type previewMutationTestTxn struct { + kv.Transaction + buffer kv.MemBuffer + pipelined bool +} + +func (t *previewMutationTestTxn) Set(key kv.Key, value []byte) error { + return t.buffer.Set(key, value) +} + +func (t *previewMutationTestTxn) Delete(key kv.Key) error { + return t.buffer.Delete(key) +} + +func (t *previewMutationTestTxn) GetMemBuffer() kv.MemBuffer { + return t.buffer +} + +func (t *previewMutationTestTxn) IsPipelined() bool { + return t.pipelined +} + +func (*previewMutationTestTxn) Valid() bool { + return true +} + +func TestPreviewKVMutationRecorder(t *testing.T) { + vars := variable.NewSessionVars(nil) + vars.StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + buffer := &previewMutationTestBuffer{} + innerTxn := &previewMutationTestTxn{buffer: buffer} + txn := &LazyTxn{Transaction: innerTxn} + txn.setPreviewKVMutationCollection(vars, true) + + memBuffer := txn.GetMemBuffer() + require.NoError(t, memBuffer.Set(kv.Key("a"), []byte("12"))) + require.NoError(t, memBuffer.SetWithFlags(kv.Key("bb"), []byte("3"), kv.SetNeedLocked)) + require.NoError(t, memBuffer.Delete(kv.Key("ccc"))) + require.NoError(t, memBuffer.DeleteWithFlags(kv.Key("d"), kv.SetNeedLocked)) + memBuffer.UpdateFlags(kv.Key("flags-only"), kv.SetNeedLocked) + + stage := memBuffer.Staging() + require.NoError(t, memBuffer.Set(kv.Key("same"), []byte("v1"))) + require.NoError(t, memBuffer.Set(kv.Key("same"), []byte("v2"))) + memBuffer.Cleanup(stage) + + // Direct Transaction calls use the same recorder but do not pass through + // the MemBuffer wrapper, so each attempted mutation is counted once. + require.NoError(t, txn.Set(kv.Key("z"), []byte("vv"))) + require.NoError(t, txn.Delete(kv.Key("zz"))) + + buffer.setErr = errors.New("injected set failure") + require.Error(t, memBuffer.Set(kv.Key("fail"), []byte("x"))) + buffer.setWithFlagsErr = errors.New("injected set-with-flags failure") + require.Error(t, memBuffer.SetWithFlags(kv.Key("fail-flags"), []byte("vv"), kv.SetNeedLocked)) + buffer.deleteErr = errors.New("injected delete failure") + require.Error(t, memBuffer.Delete(kv.Key("fail-delete"))) + buffer.deleteWithFlagsErr = errors.New("injected delete-with-flags failure") + require.Error(t, memBuffer.DeleteWithFlags(kv.Key("fail-delete-flags"), kv.SetNeedLocked)) + + // A pessimistic statement retry resets execution state but must retain the + // attempted mutation work from the previous attempt. + vars.StmtCtx.ResetForRetry() + buffer.setErr = nil + buffer.setWithFlagsErr = nil + buffer.deleteErr = nil + buffer.deleteWithFlagsErr = nil + require.NoError(t, memBuffer.Set(kv.Key("retry"), []byte("v"))) + + innerTxn.pipelined = true + require.NoError(t, txn.GetMemBuffer().Set(kv.Key("p"), []byte("v"))) + + snapshot := vars.StmtCtx.PreviewKVMutationRecorder.Snapshot() + require.Equal(t, uint64(14), snapshot.EncodedMutationCount) + require.Equal(t, uint64(80), snapshot.EncodedMutationBytes) + require.Equal(t, uint64(9), snapshot.SetCount) + require.Equal(t, uint64(5), snapshot.DeleteCount) + require.Equal(t, uint64(66), snapshot.KeyBytes) + require.Equal(t, uint64(14), snapshot.ValueBytes) + require.True(t, snapshot.Pipelined) + + // The explicit COMMIT statement has no mutation call through which to + // observe pipelined mode. Preserve the transaction mode before commit + // invalidates the transaction so its TiKV payload is reported partial. + vars.StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + se := &session{sessionVars: vars, txn: *txn} + se.markPreviewKVMutationTxnPipelined() + commitSnapshot := vars.StmtCtx.PreviewKVMutationRecorder.Snapshot() + require.True(t, commitSnapshot.Pipelined) + + // Disabling preview collection restores the original hot path. + txn.setPreviewKVMutationCollection(nil, false) + require.NoError(t, txn.GetMemBuffer().Set(kv.Key("ignored"), []byte("ignored"))) + require.Equal(t, commitSnapshot, vars.StmtCtx.PreviewKVMutationRecorder.Snapshot()) +} + func TestDomapHandleNil(t *testing.T) { // this is required for enterprise plugins // ref: https://github.com/pingcap/tidb/issues/37319 @@ -262,6 +397,41 @@ func TestRUV2MetricsIsolatedPerStatementInExplicitTxn(t *testing.T) { }) } +func TestRUV2MetricsWriteRequestsInPessimisticTxn(t *testing.T) { + store, dom := CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + defer dom.Close() + + se, err := createSession(store) + require.NoError(t, err) + MustExec(t, se, "use test") + MustExec(t, se, "drop table if exists ruv2_pessimistic_requests") + MustExec(t, se, "create table ruv2_pessimistic_requests (id int primary key, v int)") + MustExec(t, se, "insert into ruv2_pessimistic_requests values (1, 1)") + MustExec(t, se, "set @@session.tidb_txn_mode = 'pessimistic'") + + MustExec(t, se, "begin pessimistic") + MustExec(t, se, "update ruv2_pessimistic_requests set v = v + 1 where id = 1") + dmlMetrics := se.sessionVars.RUV2Metrics + require.NotNil(t, dmlMetrics) + // MockStore does not produce resource-manager RPC detail. Inject the + // finalized DML observation into the same statement snapshot to prove that + // the following COMMIT cannot inherit, drain, or overwrite it. + dmlMetrics.AddResourceManagerWriteCnt(7) + dmlWriteRequests := dmlMetrics.ResourceManagerWriteCnt() + require.Equal(t, int64(7), dmlWriteRequests) + + MustExec(t, se, "commit") + commitMetrics := se.sessionVars.RUV2Metrics + require.NotNil(t, commitMetrics) + require.NotSame(t, dmlMetrics, commitMetrics) + require.Zero(t, commitMetrics.ResourceManagerWriteCnt(), "COMMIT must start with a fresh statement snapshot") + commitMetrics.AddResourceManagerWriteCnt(2) + commitWriteRequests := commitMetrics.ResourceManagerWriteCnt() + require.Equal(t, int64(2), commitWriteRequests) + require.Equal(t, dmlWriteRequests, dmlMetrics.ResourceManagerWriteCnt(), "COMMIT must not drain or mutate the completed DML snapshot") +} + func TestSchemaCacheSizeVar(t *testing.T) { store, err := mockstore.NewMockStore(mockstore.WithStoreType(mockstore.EmbedUnistore)) require.NoError(t, err) diff --git a/pkg/session/txn.go b/pkg/session/txn.go index 2295510b2cd00..6b3c578cc3176 100644 --- a/pkg/session/txn.go +++ b/pkg/session/txn.go @@ -32,6 +32,8 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/session/txninfo" "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/sessiontxn" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/util/logutil" @@ -57,6 +59,10 @@ type LazyTxn struct { initCnt int stagingHandle kv.StagingHandle writeSLI sli.TxnWriteThroughputSLI + // previewMutationVars is set only while preview RU collection is enabled. + // Reading the current StmtCtx through SessionVars keeps optimistic replay + // mutations attributed to the statement being replayed. + previewMutationVars *variable.SessionVars enterFairLockingOnValid bool @@ -76,6 +82,80 @@ type LazyTxn struct { lastCommitTS uint64 } +type previewMutationMemBuffer struct { + kv.MemBuffer + recorder *stmtctx.PreviewKVMutationRecorder +} + +func (b *previewMutationMemBuffer) Set(key kv.Key, value []byte) error { + b.recorder.RecordSet(len(key), len(value)) + return b.MemBuffer.Set(key, value) +} + +func (b *previewMutationMemBuffer) SetWithFlags(key kv.Key, value []byte, flags ...kv.FlagsOp) error { + b.recorder.RecordSet(len(key), len(value)) + return b.MemBuffer.SetWithFlags(key, value, flags...) +} + +func (b *previewMutationMemBuffer) Delete(key kv.Key) error { + b.recorder.RecordDelete(len(key)) + return b.MemBuffer.Delete(key) +} + +func (b *previewMutationMemBuffer) DeleteWithFlags(key kv.Key, flags ...kv.FlagsOp) error { + b.recorder.RecordDelete(len(key)) + return b.MemBuffer.DeleteWithFlags(key, flags...) +} + +func (txn *LazyTxn) setPreviewKVMutationCollection(vars *variable.SessionVars, enabled bool) { + if !enabled { + txn.previewMutationVars = nil + return + } + txn.previewMutationVars = vars +} + +func (txn *LazyTxn) previewKVMutationRecorder() *stmtctx.PreviewKVMutationRecorder { + vars := txn.previewMutationVars + if vars == nil || vars.StmtCtx == nil || vars.StmtCtx.PreviewKVMutationRecorder == nil { + return nil + } + recorder := vars.StmtCtx.PreviewKVMutationRecorder + if txn.Transaction != nil && txn.Transaction.IsPipelined() { + recorder.MarkPipelined() + } + return recorder +} + +// GetMemBuffer overrides the embedded transaction method so all foreground +// table record/index Set/Delete calls share one preview recorder. The normal +// production path returns the original MemBuffer directly. +func (txn *LazyTxn) GetMemBuffer() kv.MemBuffer { + buffer := txn.Transaction.GetMemBuffer() + if recorder := txn.previewKVMutationRecorder(); recorder != nil { + return &previewMutationMemBuffer{MemBuffer: buffer, recorder: recorder} + } + return buffer +} + +// Set overrides the embedded transaction method for the few foreground paths +// that write through Transaction rather than GetMemBuffer. +func (txn *LazyTxn) Set(key kv.Key, value []byte) error { + if recorder := txn.previewKVMutationRecorder(); recorder != nil { + recorder.RecordSet(len(key), len(value)) + } + return txn.Transaction.Set(key, value) +} + +// Delete overrides the embedded transaction method for record deletes and +// other foreground paths that write through Transaction directly. +func (txn *LazyTxn) Delete(key kv.Key) error { + if recorder := txn.previewKVMutationRecorder(); recorder != nil { + recorder.RecordDelete(len(key)) + } + return txn.Transaction.Delete(key) +} + // GetTableInfo returns the cached index name. func (txn *LazyTxn) GetTableInfo(id int64) *model.TableInfo { return txn.Transaction.GetTableInfo(id) diff --git a/pkg/sessionctx/stmtctx/stmtctx.go b/pkg/sessionctx/stmtctx/stmtctx.go index 42982b6842ed1..74c6d802f1dad 100644 --- a/pkg/sessionctx/stmtctx/stmtctx.go +++ b/pkg/sessionctx/stmtctx/stmtctx.go @@ -89,6 +89,86 @@ type LogicalPlanBuildState struct { planCacheAlwaysWarn bool } +// PreviewKVMutationSnapshot is a statement-local snapshot of attempted encoded +// mutations sent to the foreground transaction MemDB. It deliberately counts +// calls rather than the final distinct write set, so staging cleanup and +// same-key overwrites do not subtract from these values. +type PreviewKVMutationSnapshot struct { + EncodedMutationCount uint64 + EncodedMutationBytes uint64 + SetCount uint64 + DeleteCount uint64 + KeyBytes uint64 + ValueBytes uint64 + Pipelined bool +} + +// PreviewKVMutationRecorder records attempted transaction MemDB mutation calls +// for the preview RU model. A recorder is allocated only for statements that +// explicitly enable preview collection. +type PreviewKVMutationRecorder struct { + encodedMutationCount atomic.Uint64 + encodedMutationBytes atomic.Uint64 + setCount atomic.Uint64 + deleteCount atomic.Uint64 + keyBytes atomic.Uint64 + valueBytes atomic.Uint64 + pipelined atomic.Bool +} + +// RecordSet records one attempted Set or SetWithFlags call before the MemDB is +// invoked. Recording before the call preserves work when the MemDB returns an +// error. +func (r *PreviewKVMutationRecorder) RecordSet(keyBytes, valueBytes int) { + if r == nil { + return + } + keySize, valueSize := uint64(keyBytes), uint64(valueBytes) + r.encodedMutationCount.Add(1) + r.encodedMutationBytes.Add(keySize + valueSize) + r.setCount.Add(1) + r.keyBytes.Add(keySize) + r.valueBytes.Add(valueSize) +} + +// RecordDelete records one attempted Delete or DeleteWithFlags call before the +// MemDB is invoked. +func (r *PreviewKVMutationRecorder) RecordDelete(keyBytes int) { + if r == nil { + return + } + keySize := uint64(keyBytes) + r.encodedMutationCount.Add(1) + r.encodedMutationBytes.Add(keySize) + r.deleteCount.Add(1) + r.keyBytes.Add(keySize) +} + +// MarkPipelined records that at least one mutation used a pipelined MemDB. The +// TiDB mutation units remain valid, while the current TiKV commit payload source +// is incomplete for this path. +func (r *PreviewKVMutationRecorder) MarkPipelined() { + if r != nil { + r.pipelined.Store(true) + } +} + +// Snapshot returns the current statement-local attempted mutation counters. +func (r *PreviewKVMutationRecorder) Snapshot() PreviewKVMutationSnapshot { + if r == nil { + return PreviewKVMutationSnapshot{} + } + return PreviewKVMutationSnapshot{ + EncodedMutationCount: r.encodedMutationCount.Load(), + EncodedMutationBytes: r.encodedMutationBytes.Load(), + SetCount: r.setCount.Load(), + DeleteCount: r.deleteCount.Load(), + KeyBytes: r.keyBytes.Load(), + ValueBytes: r.valueBytes.Load(), + Pipelined: r.pipelined.Load(), + } +} + // ReferenceCount indicates the reference count of StmtCtx. type ReferenceCount int32 @@ -437,6 +517,10 @@ type StatementContext struct { // always created during each statement execution. KvExecCounter *stmtstats.KvExecCounter + // PreviewKVMutationRecorder is non-nil only while the preview RU model needs + // statement-local foreground transaction MemDB mutation counters. + PreviewKVMutationRecorder *PreviewKVMutationRecorder + // WeakConsistency is true when read consistency is weak and in a read statement and not in a transaction. WeakConsistency bool diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index cbe8896a9c9cd..79d8eb50ea491 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -768,6 +768,9 @@ const ( // TiDBEnableCollectExecutionInfo indicates that whether execution info is collected. TiDBEnableCollectExecutionInfo = "tidb_enable_collect_execution_info" + // TiDBEnableReadBillingDemo enables preview RU base-unit metrics for foreground statements. + TiDBEnableReadBillingDemo = "tidb_enable_read_billing_demo" + // TiDBExecutorConcurrency is used for controlling the concurrency of all types of executors. TiDBExecutorConcurrency = "tidb_executor_concurrency" @@ -1594,6 +1597,7 @@ const ( DefTiDBFoundInPlanCache = false DefTiDBFoundInBinding = false DefTiDBEnableCollectExecutionInfo = true + DefTiDBEnableReadBillingDemo = false DefTiDBAllowAutoRandExplicitInsert = false DefTiDBEnableClusteredIndex = ClusteredIndexDefModeOn DefTiDBRedactLog = Off diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 718703e440cfd..ab40040ee2c25 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1231,6 +1231,9 @@ type SessionVars struct { // AllowProjectionPushDown enables pushdown projection on TiKV. AllowProjectionPushDown bool + // EnableReadBillingDemo indicates whether preview RU base-unit metrics are emitted for foreground statements. + EnableReadBillingDemo bool + // EnableStrictNotNullCheck enables strict not-null check for single-row insert in non-strict mode. EnableStrictNotNullCheck bool @@ -2491,6 +2494,7 @@ func NewSessionVars(hctx HookContext) *SessionVars { OptOrderingIdxSelRatio: vardef.DefTiDBOptOrderingIdxSelRatio, RegardNULLAsPoint: vardef.DefTiDBRegardNULLAsPoint, AllowProjectionPushDown: vardef.DefOptEnableProjectionPushDown, + EnableReadBillingDemo: vardef.DefTiDBEnableReadBillingDemo, SkipMissingPartitionStats: vardef.DefTiDBSkipMissingPartitionStats, IndexLookUpPushDownPolicy: vardef.DefTiDBIndexLookUpPushDownPolicy, OptPartialOrderedIndexForTopN: vardef.DefTiDBOptPartialOrderedIndexForTopN, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index cc4c97f600e67..843f91caeb6b1 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -609,6 +609,10 @@ var defaultSysVars = []*SysVar{ }, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return BoolToOnOff(config.GetGlobalConfig().Instance.EnableCollectExecutionInfo.Load()), nil }}, + {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableReadBillingDemo, Value: BoolToOnOff(vardef.DefTiDBEnableReadBillingDemo), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableReadBillingDemo = TiDBOptOn(val) + return nil + }}, {Scope: vardef.ScopeInstance, Name: vardef.PluginLoad, Value: "", ReadOnly: true, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return config.GetGlobalConfig().Instance.PluginLoad, nil }}, diff --git a/pkg/types/explain_format.go b/pkg/types/explain_format.go index 67aabe4b27287..95a8dd07385c5 100644 --- a/pkg/types/explain_format.go +++ b/pkg/types/explain_format.go @@ -41,6 +41,8 @@ var ( ExplainFormatPlanCache = "plan_cache" // ExplainFormatPlanTree displays the plan in a tree structure format ExplainFormatPlanTree = "plan_tree" + // ExplainFormatRU displays read billing model attribution for EXPLAIN ANALYZE. + ExplainFormatRU = "ru" // ExplainFormats stores the valid formats for explain statement, used by validator. ExplainFormats = []string{ @@ -57,5 +59,6 @@ var ( ExplainFormatCostTrace, ExplainFormatPlanCache, ExplainFormatPlanTree, + ExplainFormatRU, } ) diff --git a/pkg/util/chunk/chunk.go b/pkg/util/chunk/chunk.go index 4065cc9285408..19ffd8e805e19 100644 --- a/pkg/util/chunk/chunk.go +++ b/pkg/util/chunk/chunk.go @@ -192,6 +192,41 @@ func (c *Chunk) MemoryUsage() (sum int64) { return } +// LogicalLiveBytes returns bytes for the logical live rows in this chunk. +// It counts payload bytes plus the logical null bitmap and offset arrays, not +// retained allocation capacity. +func (c *Chunk) LogicalLiveBytes() (sum int64) { + if c == nil { + return 0 + } + rows := c.NumRows() + if rows <= 0 || len(c.columns) == 0 { + return 0 + } + for _, col := range c.columns { + sum += col.logicalLiveBytes(c.sel, rows) + } + return sum +} + +func (c *Column) logicalLiveBytes(sel []int, rows int) int64 { + if c == nil || rows <= 0 { + return 0 + } + sum := int64((rows + 7) >> 3) + if c.IsFixed() { + return sum + int64(rows*len(c.elemBuf)) + } + sum += int64((rows + 1) * 8) + if sel == nil { + return sum + int64(c.offsets[rows]-c.offsets[0]) + } + for _, rowIdx := range sel { + sum += int64(c.offsets[rowIdx+1] - c.offsets[rowIdx]) + } + return sum +} + // RequiredRows returns how many rows is considered full. func (c *Chunk) RequiredRows() int { return c.requiredRows diff --git a/pkg/util/chunk/chunk_test.go b/pkg/util/chunk/chunk_test.go index 14194699ee811..395fadf91d110 100644 --- a/pkg/util/chunk/chunk_test.go +++ b/pkg/util/chunk/chunk_test.go @@ -610,6 +610,32 @@ func TestChunkMemoryUsage(t *testing.T) { require.Equal(t, int64(expectedUsage), chk.MemoryUsage()) } +func TestChunkLogicalLiveBytes(t *testing.T) { + fieldTypes := []*types.FieldType{ + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeVarchar), + } + chk := NewChunkWithCapacity(fieldTypes, 8) + chk.AppendInt64(0, 1) + chk.AppendString(1, "abc") + chk.AppendNull(0) + chk.AppendString(1, "d") + + require.Equal(t, int64(46), chk.LogicalLiveBytes()) + require.Greater(t, chk.MemoryUsage(), chk.LogicalLiveBytes()) + + chk.SetSel([]int{1}) + require.Equal(t, int64(27), chk.LogicalLiveBytes()) + + chk.SetSel(nil) + chk.MakeRef(0, 1) + require.Equal(t, int64(34), chk.LogicalLiveBytes()) + + virtualRows := NewChunkWithCapacity(nil, 8) + virtualRows.SetNumVirtualRows(3) + require.Equal(t, int64(0), virtualRows.LogicalLiveBytes()) +} + func TestSwapColumn(t *testing.T) { fieldTypes := make([]*types.FieldType, 0, 2) fieldTypes = append(fieldTypes, types.NewFieldType(mysql.TypeFloat)) diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index 2369dda4cb37e..3ae49866b2bf9 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -358,10 +358,17 @@ func TestCopRuntimeStats(t *testing.T) { tableScanID := 1 aggID := 2 tableReaderID := 3 + stats.RecordExpectedCopTasks([]int{tableScanID, aggID}) + stats.RecordExpectedCopTasks([]int{tableScanID, aggID}) stats.RecordOneCopTask(tableScanID, kv.TiKV, mockExecutorExecutionSummary(1, 1, 1)) stats.RecordOneCopTask(tableScanID, kv.TiKV, mockExecutorExecutionSummary(2, 2, 2)) stats.RecordOneCopTask(aggID, kv.TiKV, mockExecutorExecutionSummary(3, 3, 3)) stats.RecordOneCopTask(aggID, kv.TiKV, mockExecutorExecutionSummary(4, 4, 4)) + detail, detailRecords, observedTasks, expectedTasks := stats.GetCopScanDetailAndCoverage(tableScanID) + require.Zero(t, detail) + require.Zero(t, detailRecords) + require.Equal(t, int32(2), observedTasks) + require.Equal(t, int32(2), expectedTasks) scanDetail := &util.ScanDetail{ TotalKeys: 15, ProcessedKeys: 10, @@ -373,6 +380,17 @@ func TestCopRuntimeStats(t *testing.T) { RocksdbBlockReadByte: 100, } stats.RecordCopStats(tableScanID, kv.TiKV, scanDetail, util.TimeDetail{}, nil) + stats.RecordCopStats(tableScanID, kv.TiKV, &util.ScanDetail{}, util.TimeDetail{}, nil) + stats.RecordCopStats(aggID, kv.TiKV, nil, util.TimeDetail{}, nil) + detail, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(tableScanID) + require.Equal(t, *scanDetail, detail) + require.Equal(t, int32(2), detailRecords) + require.Equal(t, int32(2), observedTasks) + require.Equal(t, int32(2), expectedTasks) + _, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(aggID) + require.Zero(t, detailRecords) + require.Equal(t, int32(2), observedTasks) + require.Equal(t, int32(2), expectedTasks) require.True(t, stats.ExistsCopStats(tableScanID)) cop := stats.GetCopStats(tableScanID) @@ -400,6 +418,22 @@ func TestCopRuntimeStats(t *testing.T) { require.Equal(t, "", zeroScanDetail.String()) require.Equal(t, "", zeroTimeDetail.String()) require.Equal(t, "", zeroCopStats.String()) + + originalPlanID := 4 + remappedPlanID := 5 + executorID := "selection_" + strconv.Itoa(remappedPlanID) + remappedSummary := mockExecutorExecutionSummary(1, 1, 1) + remappedSummary.ExecutorId = &executorID + stats.RecordExpectedCopTasks([]int{originalPlanID}) + stats.RecordCopStats(originalPlanID, kv.TiFlash, &zeroScanDetail, util.TimeDetail{}, remappedSummary) + _, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(originalPlanID) + require.Equal(t, int32(1), detailRecords) + require.Zero(t, observedTasks) + require.Equal(t, int32(1), expectedTasks) + _, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(remappedPlanID) + require.Zero(t, detailRecords) + require.Equal(t, int32(1), observedTasks) + require.Zero(t, expectedTasks) } func TestRUV2MetricsSnapshotCalculateRUValues(t *testing.T) { @@ -417,6 +451,7 @@ func TestRUV2MetricsSnapshotCalculateRUValues(t *testing.T) { metrics.AddResourceManagerWriteCnt(31) metrics.AddWriteKeys(3) metrics.AddWriteSize(66) + metrics.AddPrewriteRegionNum(4) metrics.AddSessionParserTotal(37) metrics.AddTxnCnt(41) metrics.AddTiKVKVEngineCacheMiss(43) @@ -438,6 +473,7 @@ func TestRUV2MetricsSnapshotCalculateRUValues(t *testing.T) { require.InEpsilon(t, 181980.2851783309, totalRU, 0.01) require.Equal(t, int64(3), metrics.WriteKeys()) require.Equal(t, int64(66), metrics.WriteSize()) + require.Equal(t, int64(4), metrics.PrewriteRegionNum()) t.Run("zero scale stays zero", func(t *testing.T) { zeroScaleWeights := weights @@ -480,26 +516,31 @@ func TestUpdateRUV2MetricsFromCommitDetails(t *testing.T) { beforeRU := metrics.CalculateRUValues(weights) UpdateRUV2MetricsFromCommitDetails(metrics, &util.CommitDetails{ - WriteKeys: 3, - WriteSize: 66, + WriteKeys: 3, + WriteSize: 66, + PrewriteRegionNum: 4, }) require.Equal(t, int64(3), metrics.WriteKeys()) require.Equal(t, int64(66), metrics.WriteSize()) + require.Equal(t, int64(4), metrics.PrewriteRegionNum()) require.InEpsilon(t, beforeRU+float64(3)*weights.WriteKeys*weights.RUScale, metrics.CalculateRUValues(weights), 0.01) detail := FormatRUV2Metrics(metrics, weights, 0, 0) require.Contains(t, detail, "write_keys:3") require.Contains(t, detail, "write_size:66") + require.Contains(t, detail, "prewrite_region_num:4") bypassed := NewRUV2Metrics() bypassed.SetBypass(true) UpdateRUV2MetricsFromCommitDetails(bypassed, &util.CommitDetails{ - WriteKeys: 1, - WriteSize: 2, + WriteKeys: 1, + WriteSize: 2, + PrewriteRegionNum: 3, }) require.Zero(t, bypassed.WriteKeys()) require.Zero(t, bypassed.WriteSize()) + require.Zero(t, bypassed.PrewriteRegionNum()) } func TestRUV2MetricsSnapshotFreezesRUValues(t *testing.T) { @@ -1001,6 +1042,23 @@ func TestRootRuntimeStats(t *testing.T) { require.Equal(t, expect, stats.String()) } +func TestBasicRuntimeStatsBytes(t *testing.T) { + stats := &BasicRuntimeStats{} + require.False(t, stats.HasBytes()) + + stats.RecordBytes(0, 0) + require.True(t, stats.HasBytes()) + require.Equal(t, int64(0), stats.GetInputBytes()) + require.Equal(t, int64(0), stats.GetOutputBytes()) + + other := &BasicRuntimeStats{} + other.RecordBytes(11, 22) + stats.Merge(other) + require.True(t, stats.HasBytes()) + require.Equal(t, int64(11), stats.GetInputBytes()) + require.Equal(t, int64(22), stats.GetOutputBytes()) +} + func TestFormatDurationForExplain(t *testing.T) { cases := []struct { t string diff --git a/pkg/util/execdetails/runtime_stats.go b/pkg/util/execdetails/runtime_stats.go index 7dbf875c4faab..affb8f6720130 100644 --- a/pkg/util/execdetails/runtime_stats.go +++ b/pkg/util/execdetails/runtime_stats.go @@ -80,6 +80,14 @@ type RuntimeStats interface { Tp() int } +// HashTableRuntimeStats exposes the number of rows admitted into a hash join's +// lookup state. It is deliberately narrow so preview resource accounting does +// not depend on executor-private runtime-stat implementations. +type HashTableRuntimeStats interface { + RuntimeStats + HashTableRows() int64 +} + type basicCopRuntimeStats struct { loop int32 rows int64 @@ -215,10 +223,11 @@ type CopRuntimeStats struct { // have many region leaders, several coprocessor tasks can be sent to the // same tikv-server instance. We have to use a list to maintain all tasks // executed on each instance. - stats basicCopRuntimeStats - scanDetail util.ScanDetail - timeDetail util.TimeDetail - storeType kv.StoreType + stats basicCopRuntimeStats + scanDetail util.ScanDetail + scanDetailRecords int32 + timeDetail util.TimeDetail + storeType kv.StoreType } // GetActRows return total rows of CopRuntimeStats. @@ -231,6 +240,14 @@ func (crs *CopRuntimeStats) GetTasks() int32 { return int32(crs.stats.procTimes.size) } +// GetScanDetail returns a copy of the coprocessor scan detail collected for this plan. +func (crs *CopRuntimeStats) GetScanDetail() util.ScanDetail { + if crs == nil { + return util.ScanDetail{} + } + return crs.scanDetail +} + var zeroTimeDetail = util.TimeDetail{} func (crs *CopRuntimeStats) String() string { @@ -323,6 +340,13 @@ type BasicRuntimeStats struct { close atomic.Int64 // executor return row count. rows atomic.Int64 + // executor input bytes counted from child output chunks. + inputBytes atomic.Int64 + // executor output bytes counted from returned chunks. + outputBytes atomic.Int64 + // runtime byte accounting records, used to distinguish zero bytes from + // missing byte evidence. + byteRecords atomic.Int64 } // GetActRows return total rows of BasicRuntimeStats. @@ -330,6 +354,21 @@ func (e *BasicRuntimeStats) GetActRows() int64 { return e.rows.Load() } +// GetInputBytes returns total input bytes of BasicRuntimeStats. +func (e *BasicRuntimeStats) GetInputBytes() int64 { + return e.inputBytes.Load() +} + +// GetOutputBytes returns total output bytes of BasicRuntimeStats. +func (e *BasicRuntimeStats) GetOutputBytes() int64 { + return e.outputBytes.Load() +} + +// HasBytes returns whether runtime byte accounting has recorded evidence. +func (e *BasicRuntimeStats) HasBytes() bool { + return e != nil && e.byteRecords.Load() > 0 +} + // Clone implements the RuntimeStats interface. // BasicRuntimeStats shouldn't implement Clone interface because all executors with the same executor_id // should share the same BasicRuntimeStats, duplicated BasicRuntimeStats are easy to cause mistakes. @@ -348,6 +387,9 @@ func (e *BasicRuntimeStats) Merge(rs RuntimeStats) { e.open.Add(tmp.open.Load()) e.close.Add(tmp.close.Load()) e.rows.Add(tmp.rows.Load()) + e.inputBytes.Add(tmp.inputBytes.Load()) + e.outputBytes.Add(tmp.outputBytes.Load()) + e.byteRecords.Add(tmp.byteRecords.Load()) } // Tp implements the RuntimeStats interface. @@ -402,6 +444,20 @@ func (e *BasicRuntimeStats) Record(d time.Duration, rowNum int) { e.rows.Add(int64(rowNum)) } +// RecordBytes records executor input and output bytes for one Next call. +func (e *BasicRuntimeStats) RecordBytes(inputBytes, outputBytes int64) { + if e == nil { + return + } + e.byteRecords.Add(1) + if inputBytes > 0 { + e.inputBytes.Add(inputBytes) + } + if outputBytes > 0 { + e.outputBytes.Add(outputBytes) + } +} + // RecordOpen records executor's open time. func (e *BasicRuntimeStats) RecordOpen(d time.Duration) { e.consume.Add(int64(d)) @@ -455,10 +511,11 @@ func (e *BasicRuntimeStats) GetTime() int64 { // RuntimeStatsColl collects executors's execution info. type RuntimeStatsColl struct { - rootStats map[int]*RootRuntimeStats - copStats map[int]*CopRuntimeStats - stmtCopStats StmtCopRuntimeStats - mu sync.Mutex + rootStats map[int]*RootRuntimeStats + copStats map[int]*CopRuntimeStats + expectedCopTasks map[int]int32 + stmtCopStats StmtCopRuntimeStats + mu sync.Mutex } // NewRuntimeStatsColl creates new executor collector. @@ -475,11 +532,15 @@ func NewRuntimeStatsColl(reuse *RuntimeStatsColl) *RuntimeStatsColl { for k := range reuse.copStats { delete(reuse.copStats, k) } + for k := range reuse.expectedCopTasks { + delete(reuse.expectedCopTasks, k) + } return reuse } return &RuntimeStatsColl{ - rootStats: make(map[int]*RootRuntimeStats), - copStats: make(map[int]*CopRuntimeStats), + rootStats: make(map[int]*RootRuntimeStats), + copStats: make(map[int]*CopRuntimeStats), + expectedCopTasks: make(map[int]int32), } } @@ -581,6 +642,44 @@ func (e *RuntimeStatsColl) GetCopCountAndRows(planID int) (int32, int64) { return copStats.GetTasks(), copStats.GetActRows() } +// RecordExpectedCopTasks records that one received TiKV cop response is expected +// to carry an execution summary for every listed plan. +func (e *RuntimeStatsColl) RecordExpectedCopTasks(planIDs []int) { + e.mu.Lock() + defer e.mu.Unlock() + if e.expectedCopTasks == nil { + e.expectedCopTasks = make(map[int]int32) + } + for _, planID := range planIDs { + if planID > 0 { + e.expectedCopTasks[planID]++ + } + } +} + +// GetExpectedCopTasks returns the number of received TiKV cop responses that +// were expected to carry an execution summary for the specified plan. +func (e *RuntimeStatsColl) GetExpectedCopTasks(planID int) int32 { + e.mu.Lock() + defer e.mu.Unlock() + return e.expectedCopTasks[planID] +} + +// GetCopScanDetailAndCoverage returns one consistent snapshot of the scan +// detail attachment and task coverage recorded for planID. detailRecords +// counts RecordCopStats calls with a non-nil scan detail; summary-only +// RecordOneCopTask calls do not contribute to it. +func (e *RuntimeStatsColl) GetCopScanDetailAndCoverage(planID int) (detail util.ScanDetail, detailRecords, observedTasks, expectedTasks int32) { + e.mu.Lock() + defer e.mu.Unlock() + expectedTasks = e.expectedCopTasks[planID] + copStats, ok := e.copStats[planID] + if !ok { + return detail, 0, 0, expectedTasks + } + return copStats.scanDetail, copStats.scanDetailRecords, copStats.GetTasks(), expectedTasks +} + func getPlanIDFromExecutionSummary(summary *tipb.ExecutorExecutionSummary) (int, bool) { if summary.GetExecutorId() != "" { strs := strings.Split(summary.GetExecutorId(), "_") @@ -601,16 +700,14 @@ func (e *RuntimeStatsColl) RecordCopStats(planID int, storeType kv.StoreType, sc timeDetail: time, storeType: storeType, } - if scan != nil { - copStats.scanDetail = *scan - } e.copStats[planID] = copStats } else { - if scan != nil { - copStats.scanDetail.Merge(scan) - } copStats.timeDetail.Merge(&time) } + if scan != nil { + copStats.scanDetailRecords++ + copStats.scanDetail.Merge(scan) + } if summary != nil { // for TiFlash cop response, ExecutorExecutionSummary contains executor id, so if there is a valid executor id in // summary, use it overwrite the planID diff --git a/pkg/util/execdetails/ruv2_metrics.go b/pkg/util/execdetails/ruv2_metrics.go index 0f6f861cd0f47..08ae039bd2416 100644 --- a/pkg/util/execdetails/ruv2_metrics.go +++ b/pkg/util/execdetails/ruv2_metrics.go @@ -160,6 +160,9 @@ func UpdateRUV2MetricsFromCommitDetails(metrics *RUV2Metrics, commitDetails *tik if commitDetails.WriteSize != 0 { metrics.AddWriteSize(int64(commitDetails.WriteSize)) } + if prewriteRegionNum := atomic.LoadInt32(&commitDetails.PrewriteRegionNum); prewriteRegionNum != 0 { + metrics.AddPrewriteRegionNum(int64(prewriteRegionNum)) + } } // RUV2Metrics stores statement-level RUv2 metrics. @@ -193,6 +196,7 @@ type ruv2MetricsExtra struct { resourceManagerWriteCnt int64 writeKeys int64 writeSize int64 + prewriteRegionNum int64 tikvCoprocessorExecutorIterations int64 tikvCoprocessorResponseBytes int64 @@ -400,6 +404,15 @@ func (m *RUV2Metrics) AddWriteSize(delta int64) { atomic.AddInt64(&m.ensureExtra().writeSize, delta) } +// AddPrewriteRegionNum records commit prewrite region fanout for RUv2 diagnostics. +func (m *RUV2Metrics) AddPrewriteRegionNum(delta int64) { + if m.Bypass() { + return + } + metrics.RUV2PrewriteRegionNum.Add(float64(delta)) + atomic.AddInt64(&m.ensureExtra().prewriteRegionNum, delta) +} + // AddTiKVKVEngineCacheMiss records TiKV kv_engine_cache_miss counters from ExecDetailsV2. func (m *RUV2Metrics) AddTiKVKVEngineCacheMiss(delta int64) { if m.Bypass() { @@ -680,6 +693,7 @@ func cloneRUV2MetricsExtra(dst, src *ruv2MetricsExtra) { addRUV2FixedCounter(&dst.resourceManagerWriteCnt, atomic.LoadInt64(&src.resourceManagerWriteCnt)) addRUV2FixedCounter(&dst.writeKeys, atomic.LoadInt64(&src.writeKeys)) addRUV2FixedCounter(&dst.writeSize, atomic.LoadInt64(&src.writeSize)) + addRUV2FixedCounter(&dst.prewriteRegionNum, atomic.LoadInt64(&src.prewriteRegionNum)) addRUV2FixedCounter(&dst.tikvCoprocessorExecutorIterations, atomic.LoadInt64(&src.tikvCoprocessorExecutorIterations)) addRUV2FixedCounter(&dst.tikvCoprocessorResponseBytes, atomic.LoadInt64(&src.tikvCoprocessorResponseBytes)) addRUV2FixedCounter(&dst.tikvRaftstoreStoreWriteTriggerWB, atomic.LoadInt64(&src.tikvRaftstoreStoreWriteTriggerWB)) @@ -793,6 +807,15 @@ func (m *RUV2Metrics) WriteSize() int64 { return atomic.LoadInt64(&extra.writeSize) } +// PrewriteRegionNum returns commit prewrite region fanout for RUv2 diagnostics. +func (m *RUV2Metrics) PrewriteRegionNum() int64 { + extra := m.loadExtra() + if extra == nil { + return 0 + } + return atomic.LoadInt64(&extra.prewriteRegionNum) +} + // TiKVKVEngineCacheMiss returns TiKV kv_engine_cache_miss counters from ExecDetailsV2. func (m *RUV2Metrics) TiKVKVEngineCacheMiss() int64 { if m == nil { @@ -869,6 +892,7 @@ func (m *RUV2Metrics) IsZero() bool { m.ResourceManagerWriteCnt() == 0 && m.WriteKeys() == 0 && m.WriteSize() == 0 && + m.PrewriteRegionNum() == 0 && m.TiKVCoprocessorExecutorIterations() == 0 && m.TiKVCoprocessorResponseBytes() == 0 && m.TiKVRaftstoreStoreWriteTriggerWB() == 0 && @@ -949,6 +973,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas resourceManagerWriteCnt int64 writeKeys int64 writeSize int64 + prewriteRegionNum int64 tiKVKVEngineCacheMiss int64 tiKVCoprocessorExecutorIterations int64 tiKVCoprocessorResponseBytes int64 @@ -969,6 +994,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas resourceManagerWriteCnt = atomic.LoadInt64(&extra.resourceManagerWriteCnt) writeKeys = atomic.LoadInt64(&extra.writeKeys) writeSize = atomic.LoadInt64(&extra.writeSize) + prewriteRegionNum = atomic.LoadInt64(&extra.prewriteRegionNum) tiKVCoprocessorExecutorIterations = atomic.LoadInt64(&extra.tikvCoprocessorExecutorIterations) tiKVCoprocessorResponseBytes = atomic.LoadInt64(&extra.tikvCoprocessorResponseBytes) tiKVRaftstoreStoreWriteTriggerWB = atomic.LoadInt64(&extra.tikvRaftstoreStoreWriteTriggerWB) @@ -996,6 +1022,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas resourceManagerWriteCnt == 0 && writeKeys == 0 && writeSize == 0 && + prewriteRegionNum == 0 && tiKVKVEngineCacheMiss == 0 && tiKVCoprocessorExecutorIterations == 0 && tiKVCoprocessorResponseBytes == 0 && @@ -1046,6 +1073,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas appendInt("resource_manager_write_cnt", resourceManagerWriteCnt) appendInt("write_keys", writeKeys) appendInt("write_size", writeSize) + appendInt("prewrite_region_num", prewriteRegionNum) appendInt("tikv_kv_engine_cache_miss", tiKVKVEngineCacheMiss) appendInt("tikv_coprocessor_executor_iterations", tiKVCoprocessorExecutorIterations) appendInt("tikv_coprocessor_response_bytes", tiKVCoprocessorResponseBytes) diff --git a/pkg/util/stmtsummary/BUILD.bazel b/pkg/util/stmtsummary/BUILD.bazel index f98a374797061..9a9d6f68ccbf0 100644 --- a/pkg/util/stmtsummary/BUILD.bazel +++ b/pkg/util/stmtsummary/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "stmtsummary", srcs = [ "evicted.go", + "read_billing.go", "reader.go", "statement_summary.go", ], @@ -36,11 +37,12 @@ go_test( srcs = [ "evicted_test.go", "main_test.go", + "read_billing_test.go", "statement_summary_test.go", ], embed = [":stmtsummary"], flaky = True, - shard_count = 28, + shard_count = 33, deps = [ "//pkg/meta/model", "//pkg/metrics", diff --git a/pkg/util/stmtsummary/evicted.go b/pkg/util/stmtsummary/evicted.go index fec9e5f9399a6..4c77021ac2529 100644 --- a/pkg/util/stmtsummary/evicted.go +++ b/pkg/util/stmtsummary/evicted.go @@ -390,6 +390,18 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem addTo.sumErrors += addWith.sumErrors addTo.StmtRUSummary.Merge(&addWith.StmtRUSummary) + var acceptedSummary ReadBillingDemoBaseUnitSummary + addTo.ReadBillingDemoBaseUnitAggs, addTo.ReadBillingDemoStatusAggs, acceptedSummary = MergeReadBillingDemoAggMaps( + addTo.ReadBillingDemoBaseUnitAggs, + addTo.ReadBillingDemoStatusAggs, + addWith.ReadBillingDemoBaseUnitAggs, + addWith.ReadBillingDemoStatusAggs, + ) + if len(addWith.ReadBillingDemoBaseUnitAggs) == 0 && len(addWith.ReadBillingDemoStatusAggs) == 0 { + addTo.ReadBillingDemoBaseUnitSummary.Merge(&addWith.ReadBillingDemoBaseUnitSummary) + } else { + addTo.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) + } // resourceGroupName might not be inited because when it is a evicted item. addTo.resourceGroupName = addWith.resourceGroupName } diff --git a/pkg/util/stmtsummary/read_billing.go b/pkg/util/stmtsummary/read_billing.go new file mode 100644 index 0000000000000..503386affc16f --- /dev/null +++ b/pkg/util/stmtsummary/read_billing.go @@ -0,0 +1,710 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "cmp" + "slices" +) + +const ( + // MaxReadBillingDemoBaseUnitKeysPerRecord bounds read billing dimensions per + // statement summary digest/window record. + MaxReadBillingDemoBaseUnitKeysPerRecord = 256 + // MaxReadBillingDemoStatusKeysPerRecord bounds non-reserved read billing + // status dimensions per statement summary digest/window record. + MaxReadBillingDemoStatusKeysPerRecord = 128 +) + +// Read billing demo column names used by INFORMATION_SCHEMA table definitions +// and statement-summary readers. +const ( + ReadBillingDemoModelVersionStr = "MODEL_VERSION" + ReadBillingDemoWeightVersionStr = "WEIGHT_VERSION" + ReadBillingDemoSiteStr = "SITE" + ReadBillingDemoOpClassStr = "OP_CLASS" + ReadBillingDemoOperatorKindStr = "OPERATOR_KIND" + ReadBillingDemoDMLKindStr = "DML_KIND" + ReadBillingDemoUnitStr = "UNIT" + ReadBillingDemoInputSourceStr = "INPUT_SOURCE" + ReadBillingDemoInputSideStr = "INPUT_SIDE" + ReadBillingDemoRowWidthSource = "ROW_WIDTH_SOURCE" + ReadBillingDemoValueStr = "VALUE" + ReadBillingDemoSampleCountStr = "SAMPLE_COUNT" + ReadBillingDemoRowWidthSumStr = "ROW_WIDTH_SUM" + ReadBillingDemoAvgRowWidthStr = "AVG_ROW_WIDTH" + ReadBillingDemoStatusStr = "STATUS" + ReadBillingDemoReasonStr = "REASON" + ReadBillingDemoCountStr = "COUNT" +) + +const ( + readBillingDemoSiteStatement = "statement" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOperatorStatement = "statement" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoReasonAggregation = "aggregation_overflow" + readBillingDemoReasonStatusAggregation = "status_aggregation_overflow" +) + +// ReadBillingDemoTableKind identifies which read billing demo table is being read. +type ReadBillingDemoTableKind int + +const ( + // ReadBillingDemoTableBaseUnits expands coefficient-free base-unit aggregates. + ReadBillingDemoTableBaseUnits ReadBillingDemoTableKind = iota + // ReadBillingDemoTableStatus expands statement/operator status aggregates. + ReadBillingDemoTableStatus +) + +// ReadBillingDemoStatementStats is the structured read billing demo snapshot +// attached to one statement execution before statement summary aggregation. +type ReadBillingDemoStatementStats struct { + ModelVersion string + WeightVersion string + Statuses []ReadBillingDemoStatusSample + BaseUnits []ReadBillingDemoBaseUnitSample + Totals ReadBillingDemoBaseUnitSummary +} + +// IsEmpty returns whether the snapshot contains no read billing data. +func (s *ReadBillingDemoStatementStats) IsEmpty() bool { + if s == nil { + return true + } + return len(s.Statuses) == 0 && len(s.BaseUnits) == 0 && + s.Totals.SumReadBillingDemoFixedEvents == 0 && + s.Totals.SumReadBillingDemoInputRows == 0 && + s.Totals.SumReadBillingDemoInputBytes == 0 +} + +// StatusOnly returns the status portion of a read billing snapshot. +func (s ReadBillingDemoStatementStats) StatusOnly() ReadBillingDemoStatementStats { + return ReadBillingDemoStatementStats{ + ModelVersion: s.ModelVersion, + WeightVersion: s.WeightVersion, + Statuses: append([]ReadBillingDemoStatusSample(nil), s.Statuses...), + } +} + +// ReadBillingDemoStatusOnlyExecInfo returns a shallow copy that is safe for the +// status-only statement summary path. +func ReadBillingDemoStatusOnlyExecInfo(sei *StmtExecInfo) (*StmtExecInfo, bool) { + if sei == nil { + return nil, false + } + statusOnly := sei.ReadBillingDemoStats.StatusOnly() + if statusOnly.IsEmpty() { + return nil, false + } + copied := *sei + copied.ReadBillingDemoStats = statusOnly + copied.ReadBillingDemoBaseUnits = ReadBillingDemoBaseUnitSummary{} + return &copied, true +} + +// ReadBillingDemoBaseUnitSample is a coefficient-free read billing unit sample. +type ReadBillingDemoBaseUnitSample struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + DMLKind string + Unit string + InputSource string + InputSide string + RowWidthSource string + Value float64 + RowWidth float64 +} + +// ReadBillingDemoStatusSample is a statement/operator read billing status sample. +type ReadBillingDemoStatusSample struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + Status string + Reason string +} + +// ReadBillingDemoBaseUnitKey is the dimension key for statement-summary base-unit aggregation. +type ReadBillingDemoBaseUnitKey struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + DMLKind string + Unit string + InputSource string + InputSide string + RowWidthSource string +} + +// ReadBillingDemoBaseUnitAgg is the aggregate for one base-unit key. +type ReadBillingDemoBaseUnitAgg struct { + Value float64 + SampleCount uint64 + RowWidthSum float64 +} + +// ReadBillingDemoBaseUnitAggEntry is a JSON-stable base-unit aggregate entry. +type ReadBillingDemoBaseUnitAggEntry struct { + ModelVersion string `json:"model_version"` + WeightVersion string `json:"weight_version"` + Site string `json:"site"` + OpClass string `json:"op_class"` + OperatorKind string `json:"operator_kind"` + DMLKind string `json:"dml_kind,omitempty"` + Unit string `json:"unit"` + InputSource string `json:"input_source"` + InputSide string `json:"input_side"` + RowWidthSource string `json:"row_width_source"` + Value float64 `json:"value"` + SampleCount uint64 `json:"sample_count"` + RowWidthSum float64 `json:"row_width_sum"` +} + +// ReadBillingDemoStatusKey is the dimension key for statement-summary status aggregation. +type ReadBillingDemoStatusKey struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + Status string + Reason string +} + +// ReadBillingDemoStatusAgg is the aggregate for one status key. +type ReadBillingDemoStatusAgg struct { + Count uint64 +} + +// ReadBillingDemoStatusAggEntry is a JSON-stable status aggregate entry. +type ReadBillingDemoStatusAggEntry struct { + ModelVersion string `json:"model_version"` + WeightVersion string `json:"weight_version"` + Site string `json:"site"` + OpClass string `json:"op_class"` + OperatorKind string `json:"operator_kind"` + Status string `json:"status"` + Reason string `json:"reason"` + Count uint64 `json:"count"` +} + +func makeReadBillingDemoBaseUnitKey(sample ReadBillingDemoBaseUnitSample) ReadBillingDemoBaseUnitKey { + return ReadBillingDemoBaseUnitKey{ + ModelVersion: sample.ModelVersion, + WeightVersion: sample.WeightVersion, + Site: sample.Site, + OpClass: sample.OpClass, + OperatorKind: sample.OperatorKind, + DMLKind: sample.DMLKind, + Unit: sample.Unit, + InputSource: sample.InputSource, + InputSide: sample.InputSide, + RowWidthSource: sample.RowWidthSource, + } +} + +func makeReadBillingDemoStatusKey(sample ReadBillingDemoStatusSample) ReadBillingDemoStatusKey { + return ReadBillingDemoStatusKey{ + ModelVersion: sample.ModelVersion, + WeightVersion: sample.WeightVersion, + Site: sample.Site, + OpClass: sample.OpClass, + OperatorKind: sample.OperatorKind, + Status: sample.Status, + Reason: sample.Reason, + } +} + +func (e ReadBillingDemoBaseUnitAggEntry) key() ReadBillingDemoBaseUnitKey { + return ReadBillingDemoBaseUnitKey{ + ModelVersion: e.ModelVersion, + WeightVersion: e.WeightVersion, + Site: e.Site, + OpClass: e.OpClass, + OperatorKind: e.OperatorKind, + DMLKind: e.DMLKind, + Unit: e.Unit, + InputSource: e.InputSource, + InputSide: e.InputSide, + RowWidthSource: e.RowWidthSource, + } +} + +func (e ReadBillingDemoStatusAggEntry) key() ReadBillingDemoStatusKey { + return ReadBillingDemoStatusKey{ + ModelVersion: e.ModelVersion, + WeightVersion: e.WeightVersion, + Site: e.Site, + OpClass: e.OpClass, + OperatorKind: e.OperatorKind, + Status: e.Status, + Reason: e.Reason, + } +} + +func (a *ReadBillingDemoBaseUnitAgg) addSample(sample ReadBillingDemoBaseUnitSample) { + a.Value += sample.Value + a.SampleCount++ + a.RowWidthSum += sample.RowWidth +} + +func (a *ReadBillingDemoBaseUnitAgg) addEntry(entry ReadBillingDemoBaseUnitAggEntry) { + a.Value += entry.Value + a.SampleCount += entry.SampleCount + a.RowWidthSum += entry.RowWidthSum +} + +func (a *ReadBillingDemoStatusAgg) addSample() { + a.Count++ +} + +func (a *ReadBillingDemoStatusAgg) addEntry(entry ReadBillingDemoStatusAggEntry) { + a.Count += entry.Count +} + +func (s *ReadBillingDemoBaseUnitSummary) addSample(sample ReadBillingDemoBaseUnitSample) { + if sample.ModelVersion != "v3" { + return + } + switch sample.Unit { + case "fixed_events": + s.SumReadBillingDemoFixedEvents += sample.Value + case "input_rows": + s.SumReadBillingDemoInputRows += sample.Value + case "input_bytes": + s.SumReadBillingDemoInputBytes += sample.Value + } +} + +func (s *ReadBillingDemoBaseUnitSummary) addEntry(entry ReadBillingDemoBaseUnitAggEntry) { + if entry.ModelVersion != "v3" { + return + } + switch entry.Unit { + case "fixed_events": + s.SumReadBillingDemoFixedEvents += entry.Value + case "input_rows": + s.SumReadBillingDemoInputRows += entry.Value + case "input_bytes": + s.SumReadBillingDemoInputBytes += entry.Value + } +} + +func readBillingDemoBaseUnitEntry(key ReadBillingDemoBaseUnitKey, agg ReadBillingDemoBaseUnitAgg) ReadBillingDemoBaseUnitAggEntry { + return ReadBillingDemoBaseUnitAggEntry{ + ModelVersion: key.ModelVersion, + WeightVersion: key.WeightVersion, + Site: key.Site, + OpClass: key.OpClass, + OperatorKind: key.OperatorKind, + DMLKind: key.DMLKind, + Unit: key.Unit, + InputSource: key.InputSource, + InputSide: key.InputSide, + RowWidthSource: key.RowWidthSource, + Value: agg.Value, + SampleCount: agg.SampleCount, + RowWidthSum: agg.RowWidthSum, + } +} + +func readBillingDemoStatusEntry(key ReadBillingDemoStatusKey, agg ReadBillingDemoStatusAgg) ReadBillingDemoStatusAggEntry { + return ReadBillingDemoStatusAggEntry{ + ModelVersion: key.ModelVersion, + WeightVersion: key.WeightVersion, + Site: key.Site, + OpClass: key.OpClass, + OperatorKind: key.OperatorKind, + Status: key.Status, + Reason: key.Reason, + Count: agg.Count, + } +} + +// ReadBillingDemoBaseUnitEntriesFromMap converts map aggregates into deterministic entries. +func ReadBillingDemoBaseUnitEntriesFromMap(aggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg) []ReadBillingDemoBaseUnitAggEntry { + entries := make([]ReadBillingDemoBaseUnitAggEntry, 0, len(aggs)) + for key, agg := range aggs { + entries = append(entries, readBillingDemoBaseUnitEntry(key, agg)) + } + sortReadBillingDemoBaseUnitEntries(entries) + return entries +} + +// ReadBillingDemoStatusEntriesFromMap converts map aggregates into deterministic entries. +func ReadBillingDemoStatusEntriesFromMap(aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) []ReadBillingDemoStatusAggEntry { + entries := make([]ReadBillingDemoStatusAggEntry, 0, len(aggs)) + for key, agg := range aggs { + entries = append(entries, readBillingDemoStatusEntry(key, agg)) + } + sortReadBillingDemoStatusEntries(entries) + return entries +} + +// AddReadBillingDemoStatementStatsToMaps merges one statement snapshot into v1 maps. +func AddReadBillingDemoStatementStatsToMaps( + baseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + statusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + stats *ReadBillingDemoStatementStats, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary + if stats == nil { + return baseUnitAggs, statusAggs, acceptedSummary + } + for _, status := range stats.Statuses { + statusAggs, _ = addReadBillingDemoStatusSampleToMap(statusAggs, status, false) + } + for _, sample := range stats.BaseUnits { + var overflow bool + baseUnitAggs, overflow = addReadBillingDemoBaseUnitSampleToMap(baseUnitAggs, sample) + if overflow { + statusAggs = addReadBillingDemoReservedStatusToMap(statusAggs, stats.ModelVersion, stats.WeightVersion, readBillingDemoReasonAggregation) + continue + } + acceptedSummary.addSample(sample) + } + return baseUnitAggs, statusAggs, acceptedSummary +} + +// MergeReadBillingDemoAggMaps merges source maps into destination maps with the same caps used on write. +func MergeReadBillingDemoAggMaps( + dstBase map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + dstStatus map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + srcBase map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + srcStatus map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary + for key, agg := range srcStatus { + entry := readBillingDemoStatusEntry(key, agg) + dstStatus, _ = addReadBillingDemoStatusEntryToMap(dstStatus, entry, readBillingDemoStatusReasonReserved(entry.Reason)) + } + for key, agg := range srcBase { + var overflow bool + dstBase, overflow = addReadBillingDemoBaseUnitEntryToMap(dstBase, readBillingDemoBaseUnitEntry(key, agg)) + if overflow { + dstStatus = addReadBillingDemoReservedStatusToMap(dstStatus, key.ModelVersion, key.WeightVersion, readBillingDemoReasonAggregation) + continue + } + acceptedSummary.addEntry(readBillingDemoBaseUnitEntry(key, agg)) + } + return dstBase, dstStatus, acceptedSummary +} + +// AddReadBillingDemoStatementStatsToEntries merges one statement snapshot into v2 entries. +func AddReadBillingDemoStatementStatsToEntries( + baseUnitEntries []ReadBillingDemoBaseUnitAggEntry, + statusEntries []ReadBillingDemoStatusAggEntry, + stats *ReadBillingDemoStatementStats, +) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary + if stats == nil { + return baseUnitEntries, statusEntries, acceptedSummary + } + for _, status := range stats.Statuses { + statusEntries, _ = addReadBillingDemoStatusSampleToEntries(statusEntries, status, false) + } + for _, sample := range stats.BaseUnits { + var overflow bool + baseUnitEntries, overflow = addReadBillingDemoBaseUnitSampleToEntries(baseUnitEntries, sample) + if overflow { + statusEntries = addReadBillingDemoReservedStatusToEntries(statusEntries, stats.ModelVersion, stats.WeightVersion, readBillingDemoReasonAggregation) + continue + } + acceptedSummary.addSample(sample) + } + return baseUnitEntries, statusEntries, acceptedSummary +} + +// MergeReadBillingDemoEntrySlices merges source entries into destination entries with caps. +func MergeReadBillingDemoEntrySlices( + dstBase []ReadBillingDemoBaseUnitAggEntry, + dstStatus []ReadBillingDemoStatusAggEntry, + srcBase []ReadBillingDemoBaseUnitAggEntry, + srcStatus []ReadBillingDemoStatusAggEntry, +) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary + for _, entry := range srcStatus { + dstStatus, _ = addReadBillingDemoStatusEntryToEntries(dstStatus, entry, readBillingDemoStatusReasonReserved(entry.Reason)) + } + for _, entry := range srcBase { + var overflow bool + dstBase, overflow = addReadBillingDemoBaseUnitEntryToEntries(dstBase, entry) + if overflow { + dstStatus = addReadBillingDemoReservedStatusToEntries(dstStatus, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonAggregation) + continue + } + acceptedSummary.addEntry(entry) + } + return dstBase, dstStatus, acceptedSummary +} + +func addReadBillingDemoBaseUnitSampleToMap( + aggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + sample ReadBillingDemoBaseUnitSample, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, bool) { + key := makeReadBillingDemoBaseUnitKey(sample) + if aggs == nil { + aggs = make(map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg) + } + agg, exists := aggs[key] + if !exists && len(aggs) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return aggs, true + } + agg.addSample(sample) + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoBaseUnitEntryToMap( + aggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + entry ReadBillingDemoBaseUnitAggEntry, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, bool) { + key := entry.key() + if aggs == nil { + aggs = make(map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg) + } + agg, exists := aggs[key] + if !exists && len(aggs) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return aggs, true + } + agg.addEntry(entry) + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoStatusSampleToMap( + aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + sample ReadBillingDemoStatusSample, + reserved bool, +) (map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, bool) { + key := makeReadBillingDemoStatusKey(sample) + if aggs == nil { + aggs = make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) + } + agg, exists := aggs[key] + if !exists && !reserved && readBillingDemoNonReservedStatusKeyCount(aggs) >= MaxReadBillingDemoStatusKeysPerRecord { + aggs = addReadBillingDemoReservedStatusToMap(aggs, sample.ModelVersion, sample.WeightVersion, readBillingDemoReasonStatusAggregation) + return aggs, true + } + agg.addSample() + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoStatusEntryToMap( + aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + entry ReadBillingDemoStatusAggEntry, + reserved bool, +) (map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, bool) { + key := entry.key() + if aggs == nil { + aggs = make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) + } + agg, exists := aggs[key] + if !exists && !reserved && readBillingDemoNonReservedStatusKeyCount(aggs) >= MaxReadBillingDemoStatusKeysPerRecord { + aggs = addReadBillingDemoReservedStatusToMap(aggs, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonStatusAggregation) + return aggs, true + } + agg.addEntry(entry) + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoReservedStatusToMap( + aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + modelVersion string, + weightVersion string, + reason string, +) map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg { + sample := readBillingDemoReservedStatusSample(modelVersion, weightVersion, reason) + aggs, _ = addReadBillingDemoStatusSampleToMap(aggs, sample, true) + return aggs +} + +func readBillingDemoNonReservedStatusKeyCount(aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) int { + count := 0 + for key := range aggs { + if readBillingDemoStatusReasonReserved(key.Reason) { + continue + } + count++ + } + return count +} + +func addReadBillingDemoBaseUnitSampleToEntries( + entries []ReadBillingDemoBaseUnitAggEntry, + sample ReadBillingDemoBaseUnitSample, +) ([]ReadBillingDemoBaseUnitAggEntry, bool) { + key := makeReadBillingDemoBaseUnitKey(sample) + for i := range entries { + if entries[i].key() == key { + entries[i].Value += sample.Value + entries[i].SampleCount++ + entries[i].RowWidthSum += sample.RowWidth + return entries, false + } + } + if len(entries) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return entries, true + } + entries = append(entries, readBillingDemoBaseUnitEntry(key, ReadBillingDemoBaseUnitAgg{ + Value: sample.Value, + SampleCount: 1, + RowWidthSum: sample.RowWidth, + })) + sortReadBillingDemoBaseUnitEntries(entries) + return entries, false +} + +func addReadBillingDemoBaseUnitEntryToEntries( + entries []ReadBillingDemoBaseUnitAggEntry, + entry ReadBillingDemoBaseUnitAggEntry, +) ([]ReadBillingDemoBaseUnitAggEntry, bool) { + key := entry.key() + for i := range entries { + if entries[i].key() == key { + entries[i].Value += entry.Value + entries[i].SampleCount += entry.SampleCount + entries[i].RowWidthSum += entry.RowWidthSum + return entries, false + } + } + if len(entries) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return entries, true + } + entries = append(entries, entry) + sortReadBillingDemoBaseUnitEntries(entries) + return entries, false +} + +func addReadBillingDemoStatusSampleToEntries( + entries []ReadBillingDemoStatusAggEntry, + sample ReadBillingDemoStatusSample, + reserved bool, +) ([]ReadBillingDemoStatusAggEntry, bool) { + key := makeReadBillingDemoStatusKey(sample) + for i := range entries { + if entries[i].key() == key { + entries[i].Count++ + return entries, false + } + } + if !reserved && readBillingDemoNonReservedStatusEntryCount(entries) >= MaxReadBillingDemoStatusKeysPerRecord { + entries = addReadBillingDemoReservedStatusToEntries(entries, sample.ModelVersion, sample.WeightVersion, readBillingDemoReasonStatusAggregation) + return entries, true + } + entries = append(entries, readBillingDemoStatusEntry(key, ReadBillingDemoStatusAgg{Count: 1})) + sortReadBillingDemoStatusEntries(entries) + return entries, false +} + +func addReadBillingDemoStatusEntryToEntries( + entries []ReadBillingDemoStatusAggEntry, + entry ReadBillingDemoStatusAggEntry, + reserved bool, +) ([]ReadBillingDemoStatusAggEntry, bool) { + key := entry.key() + for i := range entries { + if entries[i].key() == key { + entries[i].Count += entry.Count + return entries, false + } + } + if !reserved && readBillingDemoNonReservedStatusEntryCount(entries) >= MaxReadBillingDemoStatusKeysPerRecord { + entries = addReadBillingDemoReservedStatusToEntries(entries, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonStatusAggregation) + return entries, true + } + entries = append(entries, entry) + sortReadBillingDemoStatusEntries(entries) + return entries, false +} + +func addReadBillingDemoReservedStatusToEntries( + entries []ReadBillingDemoStatusAggEntry, + modelVersion string, + weightVersion string, + reason string, +) []ReadBillingDemoStatusAggEntry { + sample := readBillingDemoReservedStatusSample(modelVersion, weightVersion, reason) + entries, _ = addReadBillingDemoStatusSampleToEntries(entries, sample, true) + return entries +} + +func readBillingDemoReservedStatusSample(modelVersion string, weightVersion string, reason string) ReadBillingDemoStatusSample { + return ReadBillingDemoStatusSample{ + ModelVersion: modelVersion, + WeightVersion: weightVersion, + Site: readBillingDemoSiteStatement, + OpClass: readBillingDemoOpClassStatement, + OperatorKind: readBillingDemoOperatorStatement, + Status: readBillingDemoStatusUnknownInput, + Reason: reason, + } +} + +func readBillingDemoNonReservedStatusEntryCount(entries []ReadBillingDemoStatusAggEntry) int { + count := 0 + for _, entry := range entries { + if readBillingDemoStatusReasonReserved(entry.Reason) { + continue + } + count++ + } + return count +} + +func readBillingDemoStatusReasonReserved(reason string) bool { + return reason == readBillingDemoReasonAggregation || reason == readBillingDemoReasonStatusAggregation +} + +func sortReadBillingDemoBaseUnitEntries(entries []ReadBillingDemoBaseUnitAggEntry) { + slices.SortFunc(entries, func(i, j ReadBillingDemoBaseUnitAggEntry) int { + return cmp.Or( + cmp.Compare(i.ModelVersion, j.ModelVersion), + cmp.Compare(i.WeightVersion, j.WeightVersion), + cmp.Compare(i.Site, j.Site), + cmp.Compare(i.OpClass, j.OpClass), + cmp.Compare(i.OperatorKind, j.OperatorKind), + cmp.Compare(i.DMLKind, j.DMLKind), + cmp.Compare(i.Unit, j.Unit), + cmp.Compare(i.InputSource, j.InputSource), + cmp.Compare(i.InputSide, j.InputSide), + cmp.Compare(i.RowWidthSource, j.RowWidthSource), + ) + }) +} + +func sortReadBillingDemoStatusEntries(entries []ReadBillingDemoStatusAggEntry) { + slices.SortFunc(entries, func(i, j ReadBillingDemoStatusAggEntry) int { + return cmp.Or( + cmp.Compare(i.ModelVersion, j.ModelVersion), + cmp.Compare(i.WeightVersion, j.WeightVersion), + cmp.Compare(i.Site, j.Site), + cmp.Compare(i.OpClass, j.OpClass), + cmp.Compare(i.OperatorKind, j.OperatorKind), + cmp.Compare(i.Status, j.Status), + cmp.Compare(i.Reason, j.Reason), + ) + }) +} diff --git a/pkg/util/stmtsummary/read_billing_test.go b/pkg/util/stmtsummary/read_billing_test.go new file mode 100644 index 0000000000000..3265a8b39cf4f --- /dev/null +++ b/pkg/util/stmtsummary/read_billing_test.go @@ -0,0 +1,219 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadBillingDemoAggregationCaps(t *testing.T) { + stats := ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v1", + } + for i := 0; i < MaxReadBillingDemoBaseUnitKeysPerRecord+2; i++ { + stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ + ModelVersion: "v3", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Unit: "input_rows", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: float64(i + 1), + RowWidth: 8, + }) + } + + baseAggs, statusAggs, acceptedSummary := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) + require.Len(t, baseAggs, MaxReadBillingDemoBaseUnitKeysPerRecord) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonAggregation).Count) + require.Equal(t, float64(MaxReadBillingDemoBaseUnitKeysPerRecord*(MaxReadBillingDemoBaseUnitKeysPerRecord+1)/2), acceptedSummary.SumReadBillingDemoInputRows) + + v4Stats := ReadBillingDemoStatementStats{ + ModelVersion: "v4", + WeightVersion: "v3-resource-formula-uncalibrated", + BaseUnits: []ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v4", + WeightVersion: "v3-resource-formula-uncalibrated", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "input_rows", + InputSource: "runtime_child_act_rows", + InputSide: "all", + Value: 10, + }}, + } + _, _, v4Summary := AddReadBillingDemoStatementStatsToMaps(nil, nil, &v4Stats) + require.Zero(t, v4Summary.SumReadBillingDemoFixedEvents) + require.Zero(t, v4Summary.SumReadBillingDemoInputRows) + require.Zero(t, v4Summary.SumReadBillingDemoInputBytes) + + baseEntries, statusEntries, acceptedSummary := AddReadBillingDemoStatementStatsToEntries(nil, nil, &stats) + require.Len(t, baseEntries, MaxReadBillingDemoBaseUnitKeysPerRecord) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonAggregation).Count) + require.Equal(t, float64(MaxReadBillingDemoBaseUnitKeysPerRecord*(MaxReadBillingDemoBaseUnitKeysPerRecord+1)/2), acceptedSummary.SumReadBillingDemoInputRows) + + statusOnly := ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + } + for i := 0; i < MaxReadBillingDemoStatusKeysPerRecord+2; i++ { + statusOnly.Statuses = append(statusOnly.Statuses, ReadBillingDemoStatusSample{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Status: "unsupported", + Reason: "unsupported_operator", + }) + } + + _, statusAggs, acceptedSummary = AddReadBillingDemoStatementStatsToMaps(nil, nil, &statusOnly) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusKeyCount(statusAggs)) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonStatusAggregation).Count) + require.Zero(t, acceptedSummary.SumReadBillingDemoFixedEvents) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputRows) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputBytes) + + _, statusEntries, acceptedSummary = AddReadBillingDemoStatementStatsToEntries(nil, nil, &statusOnly) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusEntryCount(statusEntries)) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonStatusAggregation).Count) + require.Zero(t, acceptedSummary.SumReadBillingDemoFixedEvents) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputRows) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputBytes) +} + +func TestReadBillingDemoDMLKindAggregation(t *testing.T) { + stats := ReadBillingDemoStatementStats{ModelVersion: "v4", WeightVersion: "test-v4-calibrated"} + for _, dmlKind := range []string{"insert", "update"} { + for _, unit := range []string{"cpu_work", "encoded_mutation_count"} { + stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ + ModelVersion: "v4", + WeightVersion: "test-v4-calibrated", + Site: "tidb", + OpClass: "kv_mutation", + OperatorKind: "memdb_mutation", + DMLKind: dmlKind, + Unit: unit, + InputSource: "stmt_memdb_mutation_calls", + InputSide: "all", + RowWidthSource: "not_applicable", + Value: 1, + }) + } + } + stats.BaseUnits = append(stats.BaseUnits, + ReadBillingDemoBaseUnitSample{ + ModelVersion: "v4", WeightVersion: "test-v4-calibrated", Site: "tidb", OpClass: "reader_transport", OperatorKind: "mixed_reader", + Unit: "read_request_count", InputSource: "ruv2_metrics", InputSide: "all", RowWidthSource: "not_applicable", Value: 4, + }, + ReadBillingDemoBaseUnitSample{ + ModelVersion: "v4", WeightVersion: "test-v4-calibrated", Site: "tikv", OpClass: "kv_write", OperatorKind: "txn_write", DMLKind: "update", + Unit: "write_request_count", InputSource: "ruv2_metrics", InputSide: "all", RowWidthSource: "not_applicable", Value: 2, + }, + ) + + aggs, _, _ := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) + require.Len(t, aggs, 6) + entries := ReadBillingDemoBaseUnitEntriesFromMap(aggs) + seenUnits := make(map[string]int) + for _, entry := range entries { + require.Equal(t, "all", entry.InputSide) + switch entry.Unit { + case "cpu_work", "encoded_mutation_count": + require.Equal(t, "kv_mutation", entry.OpClass) + require.Equal(t, "memdb_mutation", entry.OperatorKind) + require.Equal(t, "stmt_memdb_mutation_calls", entry.InputSource) + require.Contains(t, []string{"insert", "update"}, entry.DMLKind) + case "read_request_count": + require.Equal(t, "reader_transport", entry.OpClass) + require.Equal(t, "mixed_reader", entry.OperatorKind) + require.Equal(t, "ruv2_metrics", entry.InputSource) + case "write_request_count": + require.Equal(t, "kv_write", entry.OpClass) + require.Equal(t, "txn_write", entry.OperatorKind) + require.Equal(t, "ruv2_metrics", entry.InputSource) + default: + require.Failf(t, "unexpected unit", "entry=%+v", entry) + } + seenUnits[entry.Unit]++ + } + require.Equal(t, map[string]int{"cpu_work": 2, "encoded_mutation_count": 2, "read_request_count": 1, "write_request_count": 1}, seenUnits) +} + +func TestReadBillingDemoReservedStatusMergeBypassesStatusCap(t *testing.T) { + fullStatusAggs := make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) + fullStatusEntries := make([]ReadBillingDemoStatusAggEntry, 0, MaxReadBillingDemoStatusKeysPerRecord) + for i := 0; i < MaxReadBillingDemoStatusKeysPerRecord; i++ { + key := ReadBillingDemoStatusKey{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Status: "unsupported", + Reason: "unsupported_operator", + } + fullStatusAggs[key] = ReadBillingDemoStatusAgg{Count: 1} + fullStatusEntries = append(fullStatusEntries, readBillingDemoStatusEntry(key, ReadBillingDemoStatusAgg{Count: 1})) + } + + srcStatusAggs := map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg{ + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonAggregation)): { + Count: 7, + }, + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonStatusAggregation)): { + Count: 11, + }, + } + _, mergedStatusAggs, _ := MergeReadBillingDemoAggMaps(nil, fullStatusAggs, nil, srcStatusAggs) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusKeyCount(mergedStatusAggs)) + require.Equal(t, uint64(7), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(mergedStatusAggs), readBillingDemoReasonAggregation).Count) + require.Equal(t, uint64(11), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(mergedStatusAggs), readBillingDemoReasonStatusAggregation).Count) + + srcStatusEntries := []ReadBillingDemoStatusAggEntry{ + readBillingDemoStatusEntry( + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonAggregation)), + ReadBillingDemoStatusAgg{Count: 7}, + ), + readBillingDemoStatusEntry( + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonStatusAggregation)), + ReadBillingDemoStatusAgg{Count: 11}, + ), + } + _, mergedStatusEntries, _ := MergeReadBillingDemoEntrySlices(nil, fullStatusEntries, nil, srcStatusEntries) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusEntryCount(mergedStatusEntries)) + require.Equal(t, uint64(7), requireReadBillingDemoStatusReason(t, mergedStatusEntries, readBillingDemoReasonAggregation).Count) + require.Equal(t, uint64(11), requireReadBillingDemoStatusReason(t, mergedStatusEntries, readBillingDemoReasonStatusAggregation).Count) +} + +func requireReadBillingDemoStatusReason(t *testing.T, entries []ReadBillingDemoStatusAggEntry, reason string) ReadBillingDemoStatusAggEntry { + t.Helper() + for _, entry := range entries { + if entry.Reason == reason { + return entry + } + } + require.Failf(t, "missing read billing status reason", "reason=%s entries=%v", reason, entries) + return ReadBillingDemoStatusAggEntry{} +} diff --git a/pkg/util/stmtsummary/reader.go b/pkg/util/stmtsummary/reader.go index 90edc89cbbb33..25637a3aa3a1f 100644 --- a/pkg/util/stmtsummary/reader.go +++ b/pkg/util/stmtsummary/reader.go @@ -15,7 +15,6 @@ package stmtsummary import ( - "fmt" "strings" "time" @@ -56,10 +55,9 @@ func NewStmtSummaryReader(user *auth.UserIdentity, hasProcessPriv bool, cols []* reader.columnValueFactories = make([]columnValueFactory, len(reader.columns)) for i, col := range reader.columns { factory, ok := columnValueFactoryMap[col.Name.O] - if !ok { - panic(fmt.Sprintf("should never happen, should register new column %v into columnValueFactoryMap", col.Name.O)) + if ok { + reader.columnValueFactories[i] = factory } - reader.columnValueFactories[i] = factory } return reader } @@ -135,6 +133,238 @@ func (ssr *stmtSummaryReader) GetStmtSummaryHistoryRows() [][]types.Datum { return rows } +// GetReadBillingDemoCurrentRows gets current-window read billing demo rows. +func (ssr *stmtSummaryReader) GetReadBillingDemoCurrentRows(kind ReadBillingDemoTableKind) [][]types.Datum { + ssMap := ssr.ssMap + ssMap.Lock() + values := ssMap.summaryMap.Values() + beginTime := ssMap.beginTimeForCurInterval + other := ssMap.other + ssMap.Unlock() + + rows := make([][]types.Datum, 0, len(values)) + for _, value := range values { + ssbd := value.(*stmtSummaryByDigest) + if ssr.checker != nil && !ssr.checker.isDigestValid(ssbd.digest) { + continue + } + rows = append(rows, ssr.getReadBillingDemoCurrentRows(ssbd, beginTime, kind)...) + } + if ssr.checker == nil { + rows = append(rows, ssr.getReadBillingDemoEvictedOtherRows(other, kind)...) + } + return rows +} + +// GetReadBillingDemoHistoryRows gets history-window read billing demo rows. +func (ssr *stmtSummaryReader) GetReadBillingDemoHistoryRows(kind ReadBillingDemoTableKind) [][]types.Datum { + ssMap := ssr.ssMap + ssMap.Lock() + values := ssMap.summaryMap.Values() + other := ssMap.other + ssMap.Unlock() + + historySize := ssMap.historySize() + rows := make([][]types.Datum, 0, len(values)*historySize) + for _, value := range values { + ssbd := value.(*stmtSummaryByDigest) + ssElements := ssbd.collectHistorySummaries(ssr.checker, historySize) + for _, ssElement := range ssElements { + rows = append(rows, ssr.getReadBillingDemoElementRows(ssElement, ssbd, kind)...) + } + } + if ssr.checker == nil { + rows = append(rows, ssr.getReadBillingDemoEvictedOtherHistoryRows(other, historySize, kind)...) + } + return rows +} + +func (ssr *stmtSummaryReader) getReadBillingDemoCurrentRows(ssbd *stmtSummaryByDigest, beginTimeForCurInterval int64, kind ReadBillingDemoTableKind) [][]types.Datum { + var ssElement *stmtSummaryByDigestElement + + ssbd.Lock() + if ssbd.initialized && ssbd.history.Len() > 0 { + ssElement = ssbd.history.Back().Value.(*stmtSummaryByDigestElement) + } + ssbd.Unlock() + + if ssElement == nil || ssElement.beginTime < beginTimeForCurInterval { + return nil + } + return ssr.getReadBillingDemoElementRows(ssElement, ssbd, kind) +} + +func (ssr *stmtSummaryReader) getReadBillingDemoEvictedOtherRows(ssbde *stmtSummaryByDigestEvicted, kind ReadBillingDemoTableKind) [][]types.Datum { + var seElement *stmtSummaryByDigestEvictedElement + + ssbde.Lock() + if ssbde.history.Len() > 0 { + seElement = ssbde.history.Back().Value.(*stmtSummaryByDigestEvictedElement) + } + ssbde.Unlock() + + if seElement == nil { + return nil + } + return ssr.getReadBillingDemoElementRows(seElement.otherSummary, new(stmtSummaryByDigest), kind) +} + +func (ssr *stmtSummaryReader) getReadBillingDemoEvictedOtherHistoryRows(ssbde *stmtSummaryByDigestEvicted, historySize int, kind ReadBillingDemoTableKind) [][]types.Datum { + ssbde.Lock() + seElements := ssbde.collectHistorySummaries(historySize) + ssbde.Unlock() + rows := make([][]types.Datum, 0, len(seElements)) + + ssbd := new(stmtSummaryByDigest) + for _, seElement := range seElements { + rows = append(rows, ssr.getReadBillingDemoElementRows(seElement.otherSummary, ssbd, kind)...) + } + return rows +} + +func (ssr *stmtSummaryReader) getReadBillingDemoElementRows(ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, kind ReadBillingDemoTableKind) [][]types.Datum { + ssElement.Lock() + defer ssElement.Unlock() + if !ssr.isAuthed(&ssElement.stmtSummaryStats) { + return nil + } + + switch kind { + case ReadBillingDemoTableBaseUnits: + entries := ReadBillingDemoBaseUnitEntriesFromMap(ssElement.ReadBillingDemoBaseUnitAggs) + rows := make([][]types.Datum, 0, len(entries)) + for _, entry := range entries { + rows = append(rows, ssr.readBillingDemoBaseUnitRow(ssElement, ssbd, entry)) + } + return rows + case ReadBillingDemoTableStatus: + entries := ReadBillingDemoStatusEntriesFromMap(ssElement.ReadBillingDemoStatusAggs) + rows := make([][]types.Datum, 0, len(entries)) + for _, entry := range entries { + rows = append(rows, ssr.readBillingDemoStatusRow(ssElement, ssbd, entry)) + } + return rows + default: + return nil + } +} + +func (ssr *stmtSummaryReader) readBillingDemoBaseUnitRow(ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoBaseUnitAggEntry) []types.Datum { + row := make([]types.Datum, len(ssr.columns)) + for i, col := range ssr.columns { + row[i] = types.NewDatum(ssr.readBillingDemoBaseUnitColumnValue(col.Name.O, ssElement, ssbd, entry)) + } + return row +} + +func (ssr *stmtSummaryReader) readBillingDemoStatusRow(ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoStatusAggEntry) []types.Datum { + row := make([]types.Datum, len(ssr.columns)) + for i, col := range ssr.columns { + row[i] = types.NewDatum(ssr.readBillingDemoStatusColumnValue(col.Name.O, ssElement, ssbd, entry)) + } + return row +} + +func (ssr *stmtSummaryReader) readBillingDemoCommonColumnValue(col string, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest) (any, bool) { + switch col { + case ClusterTableInstanceColumnNameStr: + return ssr.instanceAddr, true + case SummaryBeginTimeStr: + beginTime := time.Unix(ssElement.beginTime, 0) + if beginTime.Location() != ssr.tz { + beginTime = beginTime.In(ssr.tz) + } + return types.NewTime(types.FromGoTime(beginTime), mysql.TypeTimestamp, 0), true + case SummaryEndTimeStr: + endTime := time.Unix(ssElement.endTime, 0) + if endTime.Location() != ssr.tz { + endTime = endTime.In(ssr.tz) + } + return types.NewTime(types.FromGoTime(endTime), mysql.TypeTimestamp, 0), true + case StmtTypeStr: + return ssbd.stmtType, true + case SchemaNameStr: + return convertEmptyToNil(ssbd.schemaName), true + case DigestStr: + return convertEmptyToNil(ssbd.digest), true + case DigestTextStr: + return ssbd.normalizedSQL, true + case PlanDigestStr: + return convertEmptyToNil(ssbd.planDigest), true + case ResourceGroupName: + return ssElement.resourceGroupName, true + default: + return nil, false + } +} + +func (ssr *stmtSummaryReader) readBillingDemoBaseUnitColumnValue(col string, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoBaseUnitAggEntry) any { + if value, ok := ssr.readBillingDemoCommonColumnValue(col, ssElement, ssbd); ok { + return value + } + switch col { + case ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case ReadBillingDemoSiteStr: + return entry.Site + case ReadBillingDemoOpClassStr: + return entry.OpClass + case ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case ReadBillingDemoDMLKindStr: + return entry.DMLKind + case ReadBillingDemoUnitStr: + return entry.Unit + case ReadBillingDemoInputSourceStr: + return entry.InputSource + case ReadBillingDemoInputSideStr: + return entry.InputSide + case ReadBillingDemoRowWidthSource: + return entry.RowWidthSource + case ReadBillingDemoValueStr: + return entry.Value + case ReadBillingDemoSampleCountStr: + return entry.SampleCount + case ReadBillingDemoRowWidthSumStr: + return entry.RowWidthSum + case ReadBillingDemoAvgRowWidthStr: + if entry.SampleCount == 0 { + return float64(0) + } + return entry.RowWidthSum / float64(entry.SampleCount) + default: + return nil + } +} + +func (ssr *stmtSummaryReader) readBillingDemoStatusColumnValue(col string, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoStatusAggEntry) any { + if value, ok := ssr.readBillingDemoCommonColumnValue(col, ssElement, ssbd); ok { + return value + } + switch col { + case ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case ReadBillingDemoSiteStr: + return entry.Site + case ReadBillingDemoOpClassStr: + return entry.OpClass + case ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case ReadBillingDemoStatusStr: + return entry.Status + case ReadBillingDemoReasonStr: + return entry.Reason + case ReadBillingDemoCountStr: + return entry.Count + default: + return nil + } +} + func (ssr *stmtSummaryReader) SetChecker(checker *stmtSummaryChecker) { ssr.checker = checker } @@ -153,6 +383,9 @@ func (ssr *stmtSummaryReader) getStmtByDigestCumulativeRow(ssbd *stmtSummaryByDi if !ssr.isAuthed(&ssbd.cumulative) { return nil } + if ssbd.cumulative.isReadBillingDemoStatusOnly() { + return nil + } datums := make([]types.Datum, len(ssr.columnValueFactories)) for i, factory := range ssr.columnValueFactories { @@ -184,6 +417,9 @@ func (ssr *stmtSummaryReader) getStmtByDigestElementRow(ssElement *stmtSummaryBy if !ssr.isAuthed(&ssElement.stmtSummaryStats) { return nil } + if ssElement.stmtSummaryStats.isReadBillingDemoStatusOnly() { + return nil + } datums := make([]types.Datum, len(ssr.columnValueFactories)) for i, factory := range ssr.columnValueFactories { @@ -368,6 +604,9 @@ const ( MaxQueuedRcTimeStr = "MAX_QUEUED_RC_TIME" AvgRequestUnitV2Str = "AVG_REQUEST_UNIT_V2" MaxRequestUnitV2Str = "MAX_REQUEST_UNIT_V2" + SumReadBillingDemoFixedEventsStr = "SUM_READ_BILLING_DEMO_FIXED_EVENTS" + SumReadBillingDemoInputRowsStr = "SUM_READ_BILLING_DEMO_INPUT_ROWS" + SumReadBillingDemoInputBytesStr = "SUM_READ_BILLING_DEMO_INPUT_BYTES" ResourceGroupName = "RESOURCE_GROUP" SumUnpackedBytesSentTiKVTotalStr = "SUM_UNPACKED_BYTES_SENT_TIKV_TOTAL" SumUnpackedBytesReceivedTiKVTotalStr = "SUM_UNPACKED_BYTES_RECEIVED_TIKV_TOTAL" @@ -925,6 +1164,15 @@ var columnValueFactoryMap = map[string]columnValueFactory{ MaxRequestUnitV2Str: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { return ssStats.MaxRUV2 }, + SumReadBillingDemoFixedEventsStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.SumReadBillingDemoFixedEvents + }, + SumReadBillingDemoInputRowsStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.SumReadBillingDemoInputRows + }, + SumReadBillingDemoInputBytesStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.SumReadBillingDemoInputBytes + }, ResourceGroupName: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { return ssStats.resourceGroupName }, diff --git a/pkg/util/stmtsummary/statement_summary.go b/pkg/util/stmtsummary/statement_summary.go index d81e1a72c2275..b74fc046b5184 100644 --- a/pkg/util/stmtsummary/statement_summary.go +++ b/pkg/util/stmtsummary/statement_summary.go @@ -254,6 +254,9 @@ type stmtSummaryStats struct { // request-units resourceGroupName string StmtRUSummary + ReadBillingDemoBaseUnitSummary + ReadBillingDemoBaseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg + ReadBillingDemoStatusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg StmtNetworkTrafficSummary planCacheUnqualifiedCount int64 @@ -296,15 +299,17 @@ type StmtExecInfo struct { WriteSQLRespDuration time.Duration - ResultRows int64 - TiKVExecDetails *util.ExecDetails - Prepared bool - KeyspaceName string - KeyspaceID uint32 - ResourceGroupName string - RUDetail *util.RUDetails - TotalRUV2 float64 - CPUUsages ppcpuusage.CPUUsages + ResultRows int64 + TiKVExecDetails *util.ExecDetails + Prepared bool + KeyspaceName string + KeyspaceID uint32 + ResourceGroupName string + RUDetail *util.RUDetails + TotalRUV2 float64 + ReadBillingDemoBaseUnits ReadBillingDemoBaseUnitSummary + ReadBillingDemoStats ReadBillingDemoStatementStats + CPUUsages ppcpuusage.CPUUsages PlanCacheUnqualified string @@ -432,6 +437,75 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { ssMap.updateMetricsLocked() } +// AddReadBillingDemoStatusOnly records read billing demo statuses that happen +// before the normal statement finish path can build a full StmtExecInfo. +func (ssMap *stmtSummaryByDigestMap) AddReadBillingDemoStatusOnly(sei *StmtExecInfo) { + var ok bool + sei, ok = ReadBillingDemoStatusOnlyExecInfo(sei) + if !ok { + return + } + now := time.Now().Unix() + failpoint.Inject("mockTimeForStatementsSummary", func(val failpoint.Value) { + if unixTimeStr, ok := val.(string); ok { + unixTime, err := strconv.ParseInt(unixTimeStr, 10, 64) + if err != nil { + panic(err.Error()) + } + now = unixTime + } + }) + + intervalSeconds := ssMap.refreshInterval() + historySize := 0 + if ssMap.historyEnabled() { + historySize = ssMap.historySize() + } + + key := StmtDigestKeyPool.Get().(*StmtDigestKey) + + ssMap.Lock() + defer ssMap.Unlock() + + userForKey := "" + if ssMap.optGroupByUser.Load() { + userForKey = sei.User + } + key.Init(sei.SchemaName, sei.Digest, sei.PrevSQLDigest, sei.PlanDigest, sei.ResourceGroupName, userForKey) + + if !ssMap.Enabled() { + StmtDigestKeyPool.Put(key) + return + } + if sei.IsInternal && !ssMap.EnabledInternal() { + StmtDigestKeyPool.Put(key) + return + } + + if ssMap.beginTimeForCurInterval+intervalSeconds <= now { + ssMap.beginTimeForCurInterval = now / intervalSeconds * intervalSeconds + ssMap.currentWindowEvictedCount = 0 + } + + beginTime := ssMap.beginTimeForCurInterval + value, exist := ssMap.summaryMap.Get(key) + var summary *stmtSummaryByDigest + if !exist { + summary = new(stmtSummaryByDigest) + ssMap.summaryMap.Put(key, summary) + } else { + summary = value.(*stmtSummaryByDigest) + } + summary.isInternal = summary.isInternal && sei.IsInternal + if summary != nil { + summary.addReadBillingDemoStatusOnly(sei, beginTime, intervalSeconds, historySize) + } + if exist { + StmtDigestKeyPool.Put(key) + } + ssMap.updateMetricsLocked() +} + // Clear removes all statement summaries. func (ssMap *stmtSummaryByDigestMap) Clear() { ssMap.Lock() @@ -692,6 +766,59 @@ func (ssbd *stmtSummaryByDigest) add(sei *StmtExecInfo, beginTime int64, interva } } +func (ssbd *stmtSummaryByDigest) initReadBillingDemoStatusOnly(sei *StmtExecInfo) { + ssbd.cumulative = *newReadBillingDemoStatusOnlyStats(sei) + ssbd.schemaName = sei.SchemaName + ssbd.digest = sei.Digest + ssbd.planDigest = sei.PlanDigest + if sei.StmtCtx != nil { + ssbd.stmtType = sei.StmtCtx.StmtType + } + ssbd.normalizedSQL = formatSQL(sei.NormalizedSQL) + ssbd.history = list.New() + ssbd.initialized = true +} + +func (ssbd *stmtSummaryByDigest) addReadBillingDemoStatusOnly(sei *StmtExecInfo, beginTime int64, intervalSeconds int64, historySize int) { + ssElement, isElementNew := func() (*stmtSummaryByDigestElement, bool) { + ssbd.Lock() + defer ssbd.Unlock() + + initializedNow := false + if !ssbd.initialized { + ssbd.initReadBillingDemoStatusOnly(sei) + initializedNow = true + } + if !initializedNow { + ssbd.cumulative.addReadBillingDemoStatementStats(sei.User, &sei.ReadBillingDemoStats) + } + + var ssElement *stmtSummaryByDigestElement + isElementNew := true + if ssbd.history.Len() > 0 { + lastElement := ssbd.history.Back().Value.(*stmtSummaryByDigestElement) + if lastElement.beginTime >= beginTime { + ssElement = lastElement + isElementNew = false + } else { + lastElement.onExpire(intervalSeconds) + } + } + if isElementNew { + ssElement = newReadBillingDemoStatusOnlyElement(sei, beginTime) + ssbd.history.PushBack(ssElement) + } + for ssbd.history.Len() > historySize && ssbd.history.Len() > 1 { + ssbd.history.Remove(ssbd.history.Front()) + } + return ssElement, isElementNew + }() + + if !isElementNew { + ssElement.addReadBillingDemoStatusOnly(sei) + } +} + // collectHistorySummaries puts at most `historySize` summaries to an array. func (ssbd *stmtSummaryByDigest) collectHistorySummaries(checker *stmtSummaryChecker, historySize int) []*stmtSummaryByDigestElement { ssbd.Lock() @@ -754,6 +881,28 @@ func newStmtSummaryStats(sei *StmtExecInfo) *stmtSummaryStats { } } +func newReadBillingDemoStatusOnlyStats(sei *StmtExecInfo) *stmtSummaryStats { + sei, ok := ReadBillingDemoStatusOnlyExecInfo(sei) + if !ok { + return &stmtSummaryStats{} + } + startTime := sei.StartTime + if startTime.IsZero() { + startTime = time.Now() + } + stats := &stmtSummaryStats{ + backoffTypes: make(map[string]int), + authUsers: make(map[string]struct{}), + minLatency: time.Duration(math.MaxInt64), + minResultRows: math.MaxInt64, + firstSeen: startTime, + lastSeen: startTime, + resourceGroupName: sei.ResourceGroupName, + } + stats.addReadBillingDemoStatementStats(sei.User, &sei.ReadBillingDemoStats) + return stats +} + func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalSeconds int64, warningCount int, affectedRows uint64) *stmtSummaryByDigestElement { ssElement := &stmtSummaryByDigestElement{ beginTime: beginTime, @@ -763,6 +912,13 @@ func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalS return ssElement } +func newReadBillingDemoStatusOnlyElement(sei *StmtExecInfo, beginTime int64) *stmtSummaryByDigestElement { + return &stmtSummaryByDigestElement{ + beginTime: beginTime, + stmtSummaryStats: *newReadBillingDemoStatusOnlyStats(sei), + } +} + // onExpire is called when this element expires to history. func (ssElement *stmtSummaryByDigestElement) onExpire(intervalSeconds int64) { ssElement.Lock() @@ -781,6 +937,35 @@ func (ssElement *stmtSummaryByDigestElement) onExpire(intervalSeconds int64) { } } +func (ssStats *stmtSummaryStats) addReadBillingDemoStatementStats(user string, stats *ReadBillingDemoStatementStats) { + if stats == nil { + return + } + if len(user) > 0 { + if ssStats.authUsers == nil { + ssStats.authUsers = make(map[string]struct{}) + } + ssStats.authUsers[user] = struct{}{} + } + var acceptedSummary ReadBillingDemoBaseUnitSummary + ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs, acceptedSummary = AddReadBillingDemoStatementStatsToMaps( + ssStats.ReadBillingDemoBaseUnitAggs, + ssStats.ReadBillingDemoStatusAggs, + stats, + ) + ssStats.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) +} + +func (ssStats *stmtSummaryStats) isReadBillingDemoStatusOnly() bool { + if ssStats == nil || ssStats.execCount != 0 { + return false + } + return len(ssStats.ReadBillingDemoBaseUnitAggs) > 0 || len(ssStats.ReadBillingDemoStatusAggs) > 0 || + ssStats.SumReadBillingDemoFixedEvents != 0 || + ssStats.SumReadBillingDemoInputRows != 0 || + ssStats.SumReadBillingDemoInputBytes != 0 +} + func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affectedRows uint64) { // add user to auth users set if len(sei.User) > 0 { @@ -997,6 +1182,17 @@ func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affect // request-units ssStats.StmtRUSummary.Add(sei.RUDetail, sei.TotalRUV2) + if !sei.ReadBillingDemoStats.IsEmpty() { + var acceptedSummary ReadBillingDemoBaseUnitSummary + ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs, acceptedSummary = AddReadBillingDemoStatementStatsToMaps( + ssStats.ReadBillingDemoBaseUnitAggs, + ssStats.ReadBillingDemoStatusAggs, + &sei.ReadBillingDemoStats, + ) + ssStats.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) + } else { + ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoBaseUnits) + } ssStats.storageKV = sei.StmtCtx.IsTiKV.Load() ssStats.storageMPP = sei.StmtCtx.IsTiFlash.Load() @@ -1011,6 +1207,13 @@ func (ssElement *stmtSummaryByDigestElement) add(sei *StmtExecInfo, intervalSeco ssElement.stmtSummaryStats.add(sei, warningCount, affectedRows) } +func (ssElement *stmtSummaryByDigestElement) addReadBillingDemoStatusOnly(sei *StmtExecInfo) { + ssElement.Lock() + defer ssElement.Unlock() + + ssElement.stmtSummaryStats.addReadBillingDemoStatementStats(sei.User, &sei.ReadBillingDemoStats) +} + // Truncate SQL to maxSQLLength. func formatSQL(sql string) string { maxSQLLength := StmtSummaryByDigestMap.maxSQLLength() @@ -1141,6 +1344,28 @@ func (s *StmtRUSummary) Merge(other *StmtRUSummary) { } } +// ReadBillingDemoBaseUnitSummary is the read billing demo base-unit summary for each type of statements. +type ReadBillingDemoBaseUnitSummary struct { + SumReadBillingDemoFixedEvents float64 `json:"sum_read_billing_demo_fixed_events"` + SumReadBillingDemoInputRows float64 `json:"sum_read_billing_demo_input_rows"` + SumReadBillingDemoInputBytes float64 `json:"sum_read_billing_demo_input_bytes"` +} + +// Add adds one read billing demo base-unit sample to the summary. +func (s *ReadBillingDemoBaseUnitSummary) Add(other *ReadBillingDemoBaseUnitSummary) { + if other == nil { + return + } + s.SumReadBillingDemoFixedEvents += other.SumReadBillingDemoFixedEvents + s.SumReadBillingDemoInputRows += other.SumReadBillingDemoInputRows + s.SumReadBillingDemoInputBytes += other.SumReadBillingDemoInputBytes +} + +// Merge merges another read billing demo base-unit summary. +func (s *ReadBillingDemoBaseUnitSummary) Merge(other *ReadBillingDemoBaseUnitSummary) { + s.Add(other) +} + // StmtNetworkTrafficSummary is the network traffic summary for each type of statements. type StmtNetworkTrafficSummary struct { UnpackedBytesSentTiKVTotal int64 `json:"unpacked_bytes_send_tikv_total"` diff --git a/pkg/util/stmtsummary/statement_summary_test.go b/pkg/util/stmtsummary/statement_summary_test.go index bdf4fac921c78..5b15bd02f1568 100644 --- a/pkg/util/stmtsummary/statement_summary_test.go +++ b/pkg/util/stmtsummary/statement_summary_test.go @@ -1053,6 +1053,252 @@ func TestToDatum(t *testing.T) { match(t, datums[1], expectedEvictedDatum...) } +func TestReadBillingDemoBaseUnitsToDatum(t *testing.T) { + ssMap := newStmtSummaryByDigestMap() + now := time.Now().Unix() + // to disable expiration + ssMap.beginTimeForCurInterval = now + 60 + + stmtExecInfo1 := generateAnyExecInfo() + stmtExecInfo1.ReadBillingDemoBaseUnits = ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 2, + SumReadBillingDemoInputRows: 100, + SumReadBillingDemoInputBytes: 2048, + } + ssMap.AddStatement(stmtExecInfo1) + + stmtExecInfo2 := generateAnyExecInfo() + stmtExecInfo2.ReadBillingDemoBaseUnits = ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 3, + SumReadBillingDemoInputRows: 200, + SumReadBillingDemoInputBytes: 4096, + } + ssMap.AddStatement(stmtExecInfo2) + + key := &StmtDigestKey{} + key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "") + value, ok := ssMap.summaryMap.Get(key) + require.True(t, ok) + ssElement := value.(*stmtSummaryByDigest).history.Back().Value.(*stmtSummaryByDigestElement) + require.Equal(t, 5.0, ssElement.SumReadBillingDemoFixedEvents) + require.Equal(t, 300.0, ssElement.SumReadBillingDemoInputRows) + require.Equal(t, 6144.0, ssElement.SumReadBillingDemoInputBytes) + + cols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(SumReadBillingDemoFixedEventsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputRowsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputBytesStr)}, + } + reader := NewStmtSummaryReader(nil, true, cols, "", time.UTC) + reader.ssMap = ssMap + datums := reader.GetStmtSummaryCurrentRows() + require.Len(t, datums, 1) + match(t, datums[0], 5.0, 300.0, 6144.0) +} + +func TestReadBillingDemoStructuredRowsToDatum(t *testing.T) { + ssMap := newStmtSummaryByDigestMap() + now := time.Now().Unix() + // to disable expiration + ssMap.beginTimeForCurInterval = now + 60 + + stmtExecInfo := generateAnyExecInfo() + stmtExecInfo.ReadBillingDemoStats = ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []ReadBillingDemoStatusSample{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "success", + Reason: "none", + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Status: "ok", + Reason: "none", + }, + }, + BaseUnits: []ReadBillingDemoBaseUnitSample{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 2, + RowWidth: 16, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 3, + RowWidth: 24, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "join_hash", + OperatorKind: "hashjoin", + Unit: "input_rows", + InputSource: "runtime_act_rows", + InputSide: "build", + RowWidthSource: "schema_type_width", + Value: 100, + RowWidth: 32, + }, + }, + Totals: ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 5, + SumReadBillingDemoInputRows: 100, + }, + } + ssMap.AddStatement(stmtExecInfo) + + statusOnlyInfo := generateAnyExecInfo() + statusOnlyInfo.Digest = "status_digest" + statusOnlyInfo.NormalizedSQL = "status_only_sql" + statusOnlyInfo.ReadBillingDemoStats = ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + BaseUnits: []ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 999, + RowWidth: 999, + }}, + Totals: ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 999, + }, + } + ssMap.AddReadBillingDemoStatusOnly(statusOnlyInfo) + + unknownInputInfo := generateAnyExecInfo() + unknownInputInfo.Digest = "unknown_input_digest" + unknownInputInfo.NormalizedSQL = "unknown_input_sql" + unknownInputInfo.ReadBillingDemoStats = ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "unknown_input", + Reason: "missing_runtime_stats", + }}, + } + ssMap.AddReadBillingDemoStatusOnly(unknownInputInfo) + + baseUnitCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOperatorKindStr)}, + {Name: ast.NewCIStr(ReadBillingDemoUnitStr)}, + {Name: ast.NewCIStr(ReadBillingDemoInputSourceStr)}, + {Name: ast.NewCIStr(ReadBillingDemoInputSideStr)}, + {Name: ast.NewCIStr(ReadBillingDemoRowWidthSource)}, + {Name: ast.NewCIStr(ReadBillingDemoValueStr)}, + {Name: ast.NewCIStr(ReadBillingDemoSampleCountStr)}, + {Name: ast.NewCIStr(ReadBillingDemoRowWidthSumStr)}, + {Name: ast.NewCIStr(ReadBillingDemoAvgRowWidthStr)}, + } + reader := NewStmtSummaryReader(nil, true, baseUnitCols, "", time.UTC) + reader.ssMap = ssMap + baseRows := reader.GetReadBillingDemoCurrentRows(ReadBillingDemoTableBaseUnits) + require.Len(t, baseRows, 2) + expectedBaseRows := map[string]string{ + "tidb/join_hash/hashjoin/input_rows/runtime_act_rows/build/schema_type_width": "100 1 32 32", + "tidb/projection_eval/projection/fixed_events/runtime_act_rows/all/operator_helper": "5 2 40 20", + } + require.Equal(t, expectedBaseRows, readBillingDemoRowsByKey(baseRows, 7)) + require.Equal(t, expectedBaseRows, readBillingDemoRowsByKey(reader.GetReadBillingDemoHistoryRows(ReadBillingDemoTableBaseUnits), 7)) + + statusCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestTextStr)}, + {Name: ast.NewCIStr(ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOperatorKindStr)}, + {Name: ast.NewCIStr(ReadBillingDemoStatusStr)}, + {Name: ast.NewCIStr(ReadBillingDemoReasonStr)}, + {Name: ast.NewCIStr(ReadBillingDemoCountStr)}, + } + statusReader := NewStmtSummaryReader(nil, true, statusCols, "", time.UTC) + statusReader.ssMap = ssMap + statusRows := statusReader.GetReadBillingDemoCurrentRows(ReadBillingDemoTableStatus) + require.Len(t, statusRows, 4) + expectedStatusRows := map[string]string{ + "normalized_sql/statement/statement/statement/success/none": "1", + "normalized_sql/tidb/projection_eval/projection/ok/none": "1", + "status_only_sql/statement/statement/statement/error/statement_error": "1", + "unknown_input_sql/statement/statement/statement/unknown_input/missing_runtime_stats": "1", + } + require.Equal(t, expectedStatusRows, readBillingDemoRowsByKey(statusRows, 6)) + require.Equal(t, expectedStatusRows, readBillingDemoRowsByKey(statusReader.GetReadBillingDemoHistoryRows(ReadBillingDemoTableStatus), 6)) + + normalCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(ExecCountStr)}, + } + normalReader := NewStmtSummaryReader(nil, true, normalCols, "", time.UTC) + normalReader.ssMap = ssMap + normalRows := normalReader.GetStmtSummaryCurrentRows() + require.Len(t, normalRows, 1) + match(t, normalRows[0], "digest", 1) +} + +func readBillingDemoRowsByKey(rows [][]types.Datum, keyColumns int) map[string]string { + result := make(map[string]string, len(rows)) + for _, row := range rows { + keyParts := make([]string, 0, keyColumns) + for i := 0; i < keyColumns; i++ { + keyParts = append(keyParts, fmt.Sprintf("%v", row[i].GetValue())) + } + valueParts := make([]string, 0, len(row)-keyColumns) + for i := keyColumns; i < len(row); i++ { + valueParts = append(valueParts, fmt.Sprintf("%v", row[i].GetValue())) + } + result[strings.Join(keyParts, "/")] = strings.Join(valueParts, " ") + } + return result +} + // Test AddStatement and ToDatum parallel. func TestAddStatementParallel(t *testing.T) { ssMap := newStmtSummaryByDigestMap() diff --git a/pkg/util/stmtsummary/v2/BUILD.bazel b/pkg/util/stmtsummary/v2/BUILD.bazel index 2ad5781b7b0b9..a8220186c06e5 100644 --- a/pkg/util/stmtsummary/v2/BUILD.bazel +++ b/pkg/util/stmtsummary/v2/BUILD.bazel @@ -49,7 +49,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 18, + shard_count = 23, deps = [ "//pkg/config", "//pkg/meta/model", diff --git a/pkg/util/stmtsummary/v2/column.go b/pkg/util/stmtsummary/v2/column.go index 68ec4e170a88e..2ada88544d364 100644 --- a/pkg/util/stmtsummary/v2/column.go +++ b/pkg/util/stmtsummary/v2/column.go @@ -144,6 +144,9 @@ const ( MaxQueuedRcTimeStr = "MAX_QUEUED_RC_TIME" AvgRequestUnitV2 = "AVG_REQUEST_UNIT_V2" MaxRequestUnitV2 = "MAX_REQUEST_UNIT_V2" + SumReadBillingDemoFixedEventsStr = "SUM_READ_BILLING_DEMO_FIXED_EVENTS" + SumReadBillingDemoInputRowsStr = "SUM_READ_BILLING_DEMO_INPUT_ROWS" + SumReadBillingDemoInputBytesStr = "SUM_READ_BILLING_DEMO_INPUT_BYTES" ResourceGroupName = "RESOURCE_GROUP" SumUnpackedBytesSentTiKVTotalStr = "SUM_UNPACKED_BYTES_SENT_TIKV_TOTAL" SumUnpackedBytesReceivedTiKVTotalStr = "SUM_UNPACKED_BYTES_RECEIVED_TIKV_TOTAL" @@ -523,6 +526,15 @@ var columnFactoryMap = map[string]columnFactory{ MaxRequestUnitV2: func(_ columnInfo, record *StmtRecord) any { return record.MaxRUV2 }, + SumReadBillingDemoFixedEventsStr: func(_ columnInfo, record *StmtRecord) any { + return record.SumReadBillingDemoFixedEvents + }, + SumReadBillingDemoInputRowsStr: func(_ columnInfo, record *StmtRecord) any { + return record.SumReadBillingDemoInputRows + }, + SumReadBillingDemoInputBytesStr: func(_ columnInfo, record *StmtRecord) any { + return record.SumReadBillingDemoInputBytes + }, ResourceGroupName: func(_ columnInfo, record *StmtRecord) any { return record.ResourceGroupName }, diff --git a/pkg/util/stmtsummary/v2/column_test.go b/pkg/util/stmtsummary/v2/column_test.go index dd07fd5e22077..6431beaabde3b 100644 --- a/pkg/util/stmtsummary/v2/column_test.go +++ b/pkg/util/stmtsummary/v2/column_test.go @@ -37,6 +37,9 @@ func TestColumn(t *testing.T) { {Name: ast.NewCIStr(ExecCountStr)}, {Name: ast.NewCIStr(SumLatencyStr)}, {Name: ast.NewCIStr(MaxLatencyStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoFixedEventsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputRowsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputBytesStr)}, {Name: ast.NewCIStr(AvgTidbCPUTimeStr)}, {Name: ast.NewCIStr(AvgTikvCPUTimeStr)}, } @@ -69,6 +72,12 @@ func TestColumn(t *testing.T) { require.Equal(t, int64(record.SumLatency), column) case MaxLatencyStr: require.Equal(t, int64(record.MaxLatency), column) + case SumReadBillingDemoFixedEventsStr: + require.Equal(t, record.SumReadBillingDemoFixedEvents, column) + case SumReadBillingDemoInputRowsStr: + require.Equal(t, record.SumReadBillingDemoInputRows, column) + case SumReadBillingDemoInputBytesStr: + require.Equal(t, record.SumReadBillingDemoInputBytes, column) case AvgTidbCPUTimeStr: require.Equal(t, int64(record.SumTidbCPU), column) case AvgTikvCPUTimeStr: diff --git a/pkg/util/stmtsummary/v2/logger.go b/pkg/util/stmtsummary/v2/logger.go index 028b20ed11fc9..c37956f9bd5ab 100644 --- a/pkg/util/stmtsummary/v2/logger.go +++ b/pkg/util/stmtsummary/v2/logger.go @@ -61,7 +61,7 @@ func (s *stmtLogStorage) persist(w *stmtWindow, end time.Time) { r.Unlock() } w.evicted.Lock() - if w.evicted.otherForPersist.ExecCount > 0 { + if w.evicted.otherForPersist.ExecCount > 0 || w.evicted.otherForPersist.hasReadBillingDemoData() { w.evicted.otherForPersist.Begin = begin w.evicted.otherForPersist.End = end.Unix() s.log(w.evicted.otherForPersist) diff --git a/pkg/util/stmtsummary/v2/reader.go b/pkg/util/stmtsummary/v2/reader.go index 589fc4aa0a59f..bb2cef85f37a6 100644 --- a/pkg/util/stmtsummary/v2/reader.go +++ b/pkg/util/stmtsummary/v2/reader.go @@ -31,10 +31,12 @@ import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/auth" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/set" + stmtsummarybase "github.com/pingcap/tidb/pkg/util/stmtsummary" "go.uber.org/zap" ) @@ -63,6 +65,17 @@ type MemReader struct { checker *stmtChecker } +// ReadBillingDemoMemReader expands read billing demo aggregates from the +// current in-memory statement summary window. +type ReadBillingDemoMemReader struct { + s *StmtSummary + columns []*model.ColumnInfo + instanceAddr string + timeLocation *time.Location + kind stmtsummarybase.ReadBillingDemoTableKind + checker *stmtChecker +} + // NewMemReader creates a MemReader from StmtSummary and other necessary parameters. func NewMemReader(s *StmtSummary, columns []*model.ColumnInfo, @@ -87,6 +100,31 @@ func NewMemReader(s *StmtSummary, } } +// NewReadBillingDemoMemReader creates a reader for current read billing demo rows. +func NewReadBillingDemoMemReader(s *StmtSummary, + columns []*model.ColumnInfo, + instanceAddr string, + timeLocation *time.Location, + user *auth.UserIdentity, + hasProcessPriv bool, + digests set.StringSet, + timeRanges []*StmtTimeRange, + kind stmtsummarybase.ReadBillingDemoTableKind) *ReadBillingDemoMemReader { + return &ReadBillingDemoMemReader{ + s: s, + columns: columns, + instanceAddr: instanceAddr, + timeLocation: timeLocation, + kind: kind, + checker: &stmtChecker{ + user: user, + hasProcessPriv: hasProcessPriv, + digests: digests, + timeRanges: timeRanges, + }, + } +} + // Rows returns rows converted from the current window's data maintained // in memory by StmtSummary. All evicted data will be aggregated into a // single row appended at the end. @@ -116,6 +154,9 @@ func (r *MemReader) Rows() [][]types.Datum { if !r.checker.hasPrivilege(record.AuthUsers) { return } + if record.StmtRecord.isReadBillingDemoStatusOnly() { + return + } record.Begin = w.begin.Unix() record.End = end row := make([]types.Datum, len(r.columnFactories)) @@ -147,6 +188,53 @@ func (r *MemReader) Rows() [][]types.Datum { return rows } +// Rows returns expanded read billing demo rows from the current window. +func (r *ReadBillingDemoMemReader) Rows() [][]types.Datum { + if r.s == nil { + return nil + } + end := timeNow().Unix() + r.s.windowLock.Lock() + w := r.s.window + if !r.checker.isTimeValid(w.begin.Unix(), end) { + r.s.windowLock.Unlock() + return nil + } + values := w.lru.Values() + evicted := w.evicted + r.s.windowLock.Unlock() + rows := make([][]types.Datum, 0, len(values)) + for _, v := range values { + record := v.(*lockedStmtRecord) + if !r.checker.isDigestValid(record.Digest) { + continue + } + func() { + record.Lock() + defer record.Unlock() + if !r.checker.hasPrivilege(record.AuthUsers) { + return + } + record.Begin = w.begin.Unix() + record.End = end + rows = append(rows, readBillingDemoRowsFromRecord(r, r.columns, record.StmtRecord, r.kind)...) + }() + } + if r.checker.digests == nil { + func() { + evicted.Lock() + defer evicted.Unlock() + if !r.checker.hasPrivilege(evicted.other.AuthUsers) { + return + } + evicted.other.Begin = w.begin.Unix() + evicted.other.End = end + rows = append(rows, readBillingDemoRowsFromRecord(r, r.columns, evicted.other, r.kind)...) + }() + } + return rows +} + // getInstanceAddr implements columnInfo. func (r *MemReader) getInstanceAddr() string { return r.instanceAddr @@ -157,6 +245,16 @@ func (r *MemReader) getTimeLocation() *time.Location { return r.timeLocation } +// getInstanceAddr implements columnInfo. +func (r *ReadBillingDemoMemReader) getInstanceAddr() string { + return r.instanceAddr +} + +// getInstanceAddr implements columnInfo. +func (r *ReadBillingDemoMemReader) getTimeLocation() *time.Location { + return r.timeLocation +} + // HistoryReader is used to read data that has been persisted to files. type HistoryReader struct { ctx context.Context @@ -166,9 +264,12 @@ type HistoryReader struct { instanceAddr string timeLocation *time.Location + columns []*model.ColumnInfo columnFactories []columnFactory checker *stmtChecker files *stmtFiles + readBillingDemo bool + readBillingKind stmtsummarybase.ReadBillingDemoTableKind concurrent int rowsCh <-chan [][]types.Datum @@ -188,6 +289,22 @@ func NewHistoryReader( digests set.StringSet, timeRanges []*StmtTimeRange, concurrent int, +) (*HistoryReader, error) { + return newHistoryReader(ctx, columns, instanceAddr, timeLocation, user, hasProcessPriv, digests, timeRanges, concurrent, false, 0) +} + +func newHistoryReader( + ctx context.Context, + columns []*model.ColumnInfo, + instanceAddr string, + timeLocation *time.Location, + user *auth.UserIdentity, + hasProcessPriv bool, + digests set.StringSet, + timeRanges []*StmtTimeRange, + concurrent int, + readBillingDemo bool, + readBillingKind stmtsummarybase.ReadBillingDemoTableKind, ) (*HistoryReader, error) { files, err := newStmtFiles(ctx, timeRanges) if err != nil { @@ -201,23 +318,30 @@ func NewHistoryReader( errCh := make(chan error, concurrent) ctx, cancel := context.WithCancel(ctx) + var columnFactories []columnFactory + if !readBillingDemo { + columnFactories = makeColumnFactories(columns) + } r := &HistoryReader{ ctx: ctx, cancel: cancel, instanceAddr: instanceAddr, timeLocation: timeLocation, - columnFactories: makeColumnFactories(columns), + columns: columns, + columnFactories: columnFactories, checker: &stmtChecker{ user: user, hasProcessPriv: hasProcessPriv, digests: digests, timeRanges: timeRanges, }, - files: files, - concurrent: concurrent, - rowsCh: rowsCh, - errCh: errCh, + files: files, + readBillingDemo: readBillingDemo, + readBillingKind: readBillingKind, + concurrent: concurrent, + rowsCh: rowsCh, + errCh: errCh, } r.wg.Add(1) @@ -228,6 +352,22 @@ func NewHistoryReader( return r, nil } +// NewReadBillingDemoHistoryReader creates a history reader for persisted read billing demo rows. +func NewReadBillingDemoHistoryReader( + ctx context.Context, + columns []*model.ColumnInfo, + instanceAddr string, + timeLocation *time.Location, + user *auth.UserIdentity, + hasProcessPriv bool, + digests set.StringSet, + timeRanges []*StmtTimeRange, + concurrent int, + kind stmtsummarybase.ReadBillingDemoTableKind, +) (*HistoryReader, error) { + return newHistoryReader(ctx, columns, instanceAddr, timeLocation, user, hasProcessPriv, digests, timeRanges, concurrent, true, kind) +} + // Rows returns rows converted from records in files. Reading and parsing // works asynchronously. If (nil, nil) is returned, it means that the // reading has been completed. @@ -311,7 +451,10 @@ func (r *HistoryReader) scheduleTasks( instanceAddr: r.instanceAddr, timeLocation: r.timeLocation, checker: r.checker, + columns: r.columns, columnFactories: r.columnFactories, + readBillingDemo: r.readBillingDemo, + readBillingKind: r.readBillingKind, } concurrent := r.concurrent @@ -443,6 +586,141 @@ func (c *stmtChecker) needStop(curBegin int64) bool { return stop } +func readBillingDemoRowsFromRecord(info columnInfo, columns []*model.ColumnInfo, record *StmtRecord, kind stmtsummarybase.ReadBillingDemoTableKind) [][]types.Datum { + switch kind { + case stmtsummarybase.ReadBillingDemoTableBaseUnits: + rows := make([][]types.Datum, 0, len(record.ReadBillingDemoBaseUnitAggs)) + for _, entry := range record.ReadBillingDemoBaseUnitAggs { + rows = append(rows, readBillingDemoBaseUnitRow(info, columns, record, entry)) + } + return rows + case stmtsummarybase.ReadBillingDemoTableStatus: + rows := make([][]types.Datum, 0, len(record.ReadBillingDemoStatusAggs)) + for _, entry := range record.ReadBillingDemoStatusAggs { + rows = append(rows, readBillingDemoStatusRow(info, columns, record, entry)) + } + return rows + default: + return nil + } +} + +func readBillingDemoBaseUnitRow(info columnInfo, columns []*model.ColumnInfo, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoBaseUnitAggEntry) []types.Datum { + row := make([]types.Datum, len(columns)) + for i, col := range columns { + row[i] = types.NewDatum(readBillingDemoBaseUnitColumnValue(info, col.Name.O, record, entry)) + } + return row +} + +func readBillingDemoStatusRow(info columnInfo, columns []*model.ColumnInfo, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoStatusAggEntry) []types.Datum { + row := make([]types.Datum, len(columns)) + for i, col := range columns { + row[i] = types.NewDatum(readBillingDemoStatusColumnValue(info, col.Name.O, record, entry)) + } + return row +} + +func readBillingDemoCommonColumnValue(info columnInfo, col string, record *StmtRecord) (any, bool) { + switch col { + case ClusterTableInstanceColumnNameStr: + return info.getInstanceAddr(), true + case SummaryBeginTimeStr: + beginTime := time.Unix(record.Begin, 0) + if beginTime.Location() != info.getTimeLocation() { + beginTime = beginTime.In(info.getTimeLocation()) + } + return types.NewTime(types.FromGoTime(beginTime), mysql.TypeTimestamp, 0), true + case SummaryEndTimeStr: + endTime := time.Unix(record.End, 0) + if endTime.Location() != info.getTimeLocation() { + endTime = endTime.In(info.getTimeLocation()) + } + return types.NewTime(types.FromGoTime(endTime), mysql.TypeTimestamp, 0), true + case StmtTypeStr: + return record.StmtType, true + case SchemaNameStr: + return convertEmptyToNil(record.SchemaName), true + case DigestStr: + return convertEmptyToNil(record.Digest), true + case DigestTextStr: + return record.NormalizedSQL, true + case PlanDigestStr: + return convertEmptyToNil(record.PlanDigest), true + case ResourceGroupName: + return record.ResourceGroupName, true + default: + return nil, false + } +} + +func readBillingDemoBaseUnitColumnValue(info columnInfo, col string, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoBaseUnitAggEntry) any { + if value, ok := readBillingDemoCommonColumnValue(info, col, record); ok { + return value + } + switch col { + case stmtsummarybase.ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case stmtsummarybase.ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case stmtsummarybase.ReadBillingDemoSiteStr: + return entry.Site + case stmtsummarybase.ReadBillingDemoOpClassStr: + return entry.OpClass + case stmtsummarybase.ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case stmtsummarybase.ReadBillingDemoDMLKindStr: + return entry.DMLKind + case stmtsummarybase.ReadBillingDemoUnitStr: + return entry.Unit + case stmtsummarybase.ReadBillingDemoInputSourceStr: + return entry.InputSource + case stmtsummarybase.ReadBillingDemoInputSideStr: + return entry.InputSide + case stmtsummarybase.ReadBillingDemoRowWidthSource: + return entry.RowWidthSource + case stmtsummarybase.ReadBillingDemoValueStr: + return entry.Value + case stmtsummarybase.ReadBillingDemoSampleCountStr: + return entry.SampleCount + case stmtsummarybase.ReadBillingDemoRowWidthSumStr: + return entry.RowWidthSum + case stmtsummarybase.ReadBillingDemoAvgRowWidthStr: + if entry.SampleCount == 0 { + return float64(0) + } + return entry.RowWidthSum / float64(entry.SampleCount) + default: + return nil + } +} + +func readBillingDemoStatusColumnValue(info columnInfo, col string, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoStatusAggEntry) any { + if value, ok := readBillingDemoCommonColumnValue(info, col, record); ok { + return value + } + switch col { + case stmtsummarybase.ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case stmtsummarybase.ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case stmtsummarybase.ReadBillingDemoSiteStr: + return entry.Site + case stmtsummarybase.ReadBillingDemoOpClassStr: + return entry.OpClass + case stmtsummarybase.ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case stmtsummarybase.ReadBillingDemoStatusStr: + return entry.Status + case stmtsummarybase.ReadBillingDemoReasonStr: + return entry.Reason + case stmtsummarybase.ReadBillingDemoCountStr: + return entry.Count + default: + return nil + } +} + type stmtTinyRecord struct { Begin int64 `json:"begin"` End int64 `json:"end"` @@ -730,7 +1008,10 @@ type stmtParseWorker struct { instanceAddr string timeLocation *time.Location checker *stmtChecker + columns []*model.ColumnInfo columnFactories []columnFactory + readBillingDemo bool + readBillingKind stmtsummarybase.ReadBillingDemoTableKind } func (w *stmtParseWorker) run( @@ -779,8 +1060,12 @@ func (w *stmtParseWorker) handleLines( continue } - row := w.buildRow(record) - rows = append(rows, row) + if w.readBillingDemo { + rows = append(rows, readBillingDemoRowsFromRecord(w, w.columns, record, w.readBillingKind)...) + } else { + row := w.buildRow(record) + rows = append(rows, row) + } } if len(rows) > 0 { @@ -823,6 +1108,9 @@ func (w *stmtParseWorker) matchConds(record *StmtRecord) bool { if !w.checker.hasPrivilege(record.AuthUsers) { return false } + if !w.readBillingDemo && record.isReadBillingDemoStatusOnly() { + return false + } return true } diff --git a/pkg/util/stmtsummary/v2/reader_test.go b/pkg/util/stmtsummary/v2/reader_test.go index 6f24b1c3422fe..cd689e180a891 100644 --- a/pkg/util/stmtsummary/v2/reader_test.go +++ b/pkg/util/stmtsummary/v2/reader_test.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "os" + "strings" "testing" "time" @@ -28,6 +29,7 @@ import ( "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/set" + stmtsummarybase "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/stretchr/testify/require" ) @@ -253,6 +255,116 @@ func TestMemReader(t *testing.T) { require.Len(t, evicted, 3) // begin, end, count } +func TestReadBillingDemoMemReader(t *testing.T) { + timeLocation, err := time.LoadLocation("Asia/Shanghai") + require.NoError(t, err) + + ss := NewStmtSummary4Test(3) + defer ss.Close() + + info := GenerateStmtExecInfo4Test("digest_read_billing") + info.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "success", + Reason: "none", + }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + DMLKind: "insert", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 2, + RowWidth: 16, + }}, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 2, + }, + } + ss.Add(info) + + statusOnly := GenerateStmtExecInfo4Test("digest_read_billing_error") + statusOnly.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 999, + RowWidth: 999, + }}, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 999, + }, + } + ss.AddReadBillingDemoStatusOnly(statusOnly) + + baseUnitCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoDMLKindStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoUnitStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoValueStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSampleCountStr)}, + } + baseRows := NewReadBillingDemoMemReader(ss, baseUnitCols, "", timeLocation, nil, false, nil, nil, stmtsummarybase.ReadBillingDemoTableBaseUnits).Rows() + require.Len(t, baseRows, 1) + require.Equal(t, map[string]string{ + "digest_read_billing/tidb/projection_eval/insert/fixed_events": "2 1", + }, readBillingDemoRowsByKey(baseRows, 5)) + + statusCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoStatusStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoReasonStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoCountStr)}, + } + statusRows := NewReadBillingDemoMemReader(ss, statusCols, "", timeLocation, nil, false, nil, nil, stmtsummarybase.ReadBillingDemoTableStatus).Rows() + require.Len(t, statusRows, 2) + require.Equal(t, map[string]string{ + "digest_read_billing/success/none": "1", + "digest_read_billing_error/error/statement_error": "1", + }, readBillingDemoRowsByKey(statusRows, 3)) + + normalReader := NewMemReader(ss, []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(ExecCountStr)}, + }, "", timeLocation, nil, false, nil, nil) + normalRows := normalReader.Rows() + require.Len(t, normalRows, 1) + require.Equal(t, "digest_read_billing", normalRows[0][0].GetString()) + require.Equal(t, int64(1), normalRows[0][1].GetInt64()) +} + func TestHistoryReader(t *testing.T) { filename1 := "tidb-statements-2022-12-27T16-21-20.245.log" filename2 := "tidb-statements.log" @@ -409,6 +521,65 @@ func TestHistoryReader(t *testing.T) { }() } +func TestReadBillingDemoHistoryReader(t *testing.T) { + filename := "tidb-statements.log" + file, err := os.Create(filename) + require.NoError(t, err) + defer func() { + require.NoError(t, os.Remove(filename)) + }() + _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_base_unit_aggs":[{"model_version":"v1","weight_version":"v1","site":"tidb","op_class":"projection_eval","operator_kind":"projection","dml_kind":"insert","unit":"fixed_events","input_source":"runtime_act_rows","input_side":"all","row_width_source":"operator_helper","value":2,"sample_count":1,"row_width_sum":16}],"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"success","reason":"none","count":1}]}` + "\n") + require.NoError(t, err) + // The second persisted record intentionally omits dml_kind to prove older + // JSON remains readable and yields the empty diagnostic dimension. + _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing_error","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_base_unit_aggs":[{"model_version":"v1","weight_version":"v1","site":"tidb","op_class":"projection_eval","operator_kind":"projection","unit":"fixed_events","input_source":"runtime_act_rows","input_side":"all","row_width_source":"operator_helper","value":1,"sample_count":1,"row_width_sum":8}],"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"error","reason":"statement_error","count":1}]}` + "\n") + require.NoError(t, err) + require.NoError(t, file.Close()) + + timeLocation, err := time.LoadLocation("Asia/Shanghai") + require.NoError(t, err) + baseUnitCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoDMLKindStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoUnitStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoValueStr)}, + } + baseReader, err := NewReadBillingDemoHistoryReader(context.Background(), baseUnitCols, "", timeLocation, nil, false, nil, nil, 2, stmtsummarybase.ReadBillingDemoTableBaseUnits) + require.NoError(t, err) + baseRows := readAllRows(t, baseReader) + require.NoError(t, baseReader.Close()) + require.Equal(t, map[string]string{ + "digest_read_billing/tidb/projection_eval/insert/fixed_events": "2", + "digest_read_billing_error/tidb/projection_eval//fixed_events": "1", + }, readBillingDemoRowsByKey(baseRows, 5)) + + statusCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoStatusStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoReasonStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoCountStr)}, + } + statusReader, err := NewReadBillingDemoHistoryReader(context.Background(), statusCols, "", timeLocation, nil, false, nil, nil, 2, stmtsummarybase.ReadBillingDemoTableStatus) + require.NoError(t, err) + statusRows := readAllRows(t, statusReader) + require.NoError(t, statusReader.Close()) + require.Equal(t, map[string]string{ + "digest_read_billing/success/none": "1", + "digest_read_billing_error/error/statement_error": "1", + }, readBillingDemoRowsByKey(statusRows, 3)) + + normalReader, err := NewHistoryReader(context.Background(), []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(ExecCountStr)}, + }, "", timeLocation, nil, false, nil, nil, 2) + require.NoError(t, err) + normalRows := readAllRows(t, normalReader) + require.NoError(t, normalReader.Close()) + require.Empty(t, normalRows) +} + func TestHistoryReaderInvalidLine(t *testing.T) { filename := "tidb-statements.log" @@ -458,3 +629,19 @@ func readAllRows(t *testing.T, reader *HistoryReader) [][]types.Datum { } return results } + +func readBillingDemoRowsByKey(rows [][]types.Datum, keyColumns int) map[string]string { + result := make(map[string]string, len(rows)) + for _, row := range rows { + keyParts := make([]string, 0, keyColumns) + for i := 0; i < keyColumns; i++ { + keyParts = append(keyParts, fmt.Sprintf("%v", row[i].GetValue())) + } + valueParts := make([]string, 0, len(row)-keyColumns) + for i := keyColumns; i < len(row); i++ { + valueParts = append(valueParts, fmt.Sprintf("%v", row[i].GetValue())) + } + result[strings.Join(keyParts, "/")] = strings.Join(valueParts, " ") + } + return result +} diff --git a/pkg/util/stmtsummary/v2/record.go b/pkg/util/stmtsummary/v2/record.go index fdd9a2de1d471..531b2bb5e5152 100644 --- a/pkg/util/stmtsummary/v2/record.go +++ b/pkg/util/stmtsummary/v2/record.go @@ -155,6 +155,9 @@ type StmtRecord struct { // request units(RU) ResourceGroupName string `json:"resource_group_name"` stmtsummary.StmtRUSummary + stmtsummary.ReadBillingDemoBaseUnitSummary + ReadBillingDemoBaseUnitAggs []stmtsummary.ReadBillingDemoBaseUnitAggEntry `json:"read_billing_demo_base_unit_aggs,omitempty"` + ReadBillingDemoStatusAggs []stmtsummary.ReadBillingDemoStatusAggEntry `json:"read_billing_demo_status_aggs,omitempty"` PlanCacheUnqualifiedCount int64 `json:"plan_cache_unqualified_count"` PlanCacheUnqualifiedLastReason string `json:"plan_cache_unqualified_last_reason"` // the reason why this query is unqualified for the plan cache @@ -237,6 +240,44 @@ func NewStmtRecord(info *stmtsummary.StmtExecInfo) *StmtRecord { } } +// NewReadBillingDemoStatusOnlyRecord creates a minimal record for status-only +// read billing demo rows produced before the normal statement finish path. +func NewReadBillingDemoStatusOnlyRecord(info *stmtsummary.StmtExecInfo) *StmtRecord { + var ok bool + info, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(info) + if !ok { + return &StmtRecord{ + AuthUsers: make(map[string]struct{}), + BackoffTypes: make(map[string]int), + MinLatency: time.Duration(math.MaxInt64), + MinResultRows: math.MaxInt64, + } + } + startTime := info.StartTime + if startTime.IsZero() { + startTime = time.Now() + } + record := &StmtRecord{ + SchemaName: info.SchemaName, + Digest: info.Digest, + PlanDigest: info.PlanDigest, + NormalizedSQL: info.NormalizedSQL, + IsInternal: info.IsInternal, + AuthUsers: make(map[string]struct{}), + BackoffTypes: make(map[string]int), + MinLatency: time.Duration(math.MaxInt64), + MinResultRows: math.MaxInt64, + FirstSeen: startTime, + LastSeen: startTime, + ResourceGroupName: info.ResourceGroupName, + } + if info.StmtCtx != nil { + record.StmtType = info.StmtCtx.StmtType + } + record.AddReadBillingDemoStatusOnly(info) + return record +} + // Add adds the statistics of StmtExecInfo to StmtRecord. func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { r.IsInternal = r.IsInternal && info.IsInternal @@ -444,11 +485,59 @@ func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { r.StmtNetworkTrafficSummary.Add(&tikvExecDetails) // RU r.StmtRUSummary.Add(info.RUDetail, info.TotalRUV2) + if !info.ReadBillingDemoStats.IsEmpty() { + var acceptedSummary stmtsummary.ReadBillingDemoBaseUnitSummary + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, acceptedSummary = stmtsummary.AddReadBillingDemoStatementStatsToEntries( + r.ReadBillingDemoBaseUnitAggs, + r.ReadBillingDemoStatusAggs, + &info.ReadBillingDemoStats, + ) + r.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) + } else { + r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoBaseUnits) + } r.StorageKV = info.StmtCtx.IsTiKV.Load() r.StorageMPP = info.StmtCtx.IsTiFlash.Load() } +// AddReadBillingDemoStatusOnly adds only read billing demo status/base-unit +// aggregates without changing ordinary statement summary counters. +func (r *StmtRecord) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo) { + var ok bool + info, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(info) + if !ok { + return + } + if len(info.User) > 0 { + if r.AuthUsers == nil { + r.AuthUsers = make(map[string]struct{}) + } + r.AuthUsers[info.User] = struct{}{} + } + var acceptedSummary stmtsummary.ReadBillingDemoBaseUnitSummary + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, acceptedSummary = stmtsummary.AddReadBillingDemoStatementStatsToEntries( + r.ReadBillingDemoBaseUnitAggs, + r.ReadBillingDemoStatusAggs, + &info.ReadBillingDemoStats, + ) + r.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) +} + +func (r *StmtRecord) isReadBillingDemoStatusOnly() bool { + return r != nil && r.ExecCount == 0 && r.hasReadBillingDemoData() +} + +func (r *StmtRecord) hasReadBillingDemoData() bool { + if r == nil { + return false + } + return len(r.ReadBillingDemoBaseUnitAggs) > 0 || len(r.ReadBillingDemoStatusAggs) > 0 || + r.SumReadBillingDemoFixedEvents != 0 || + r.SumReadBillingDemoInputRows != 0 || + r.SumReadBillingDemoInputBytes != 0 +} + // Merge merges the statistics of another StmtRecord to this StmtRecord. func (r *StmtRecord) Merge(other *StmtRecord) { // User @@ -608,6 +697,18 @@ func (r *StmtRecord) Merge(other *StmtRecord) { r.SumTikvCPU += other.SumTikvCPU r.SumErrors += other.SumErrors r.StmtRUSummary.Merge(&other.StmtRUSummary) + var acceptedSummary stmtsummary.ReadBillingDemoBaseUnitSummary + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, acceptedSummary = stmtsummary.MergeReadBillingDemoEntrySlices( + r.ReadBillingDemoBaseUnitAggs, + r.ReadBillingDemoStatusAggs, + other.ReadBillingDemoBaseUnitAggs, + other.ReadBillingDemoStatusAggs, + ) + if len(other.ReadBillingDemoBaseUnitAggs) == 0 && len(other.ReadBillingDemoStatusAggs) == 0 { + r.ReadBillingDemoBaseUnitSummary.Merge(&other.ReadBillingDemoBaseUnitSummary) + } else { + r.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) + } } // Truncate SQL to maxSQLLength. @@ -712,10 +813,15 @@ func GenerateStmtExecInfo4Test(digest string) *stmtsummary.StmtExecInfo { ResourceGroupName: "rg1", RUDetail: util.NewRUDetailsWith(1.2, 3.4, 2*time.Millisecond), TotalRUV2: 12345, - TiKVExecDetails: &util.ExecDetails{}, - CPUUsages: ppcpuusage.CPUUsages{TidbCPUTime: time.Duration(20), TikvCPUTime: time.Duration(10000)}, - LazyInfo: &mockLazyInfo{}, - MemArbitration: 22222, + ReadBillingDemoBaseUnits: stmtsummary.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 2, + SumReadBillingDemoInputRows: 100, + SumReadBillingDemoInputBytes: 2048, + }, + TiKVExecDetails: &util.ExecDetails{}, + CPUUsages: ppcpuusage.CPUUsages{TidbCPUTime: time.Duration(20), TikvCPUTime: time.Duration(10000)}, + LazyInfo: &mockLazyInfo{}, + MemArbitration: 22222, } stmtExecInfo.StmtCtx.AddAffectedRows(10000) return stmtExecInfo diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index d35d3807a970f..6b6ca96ca79c6 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -16,9 +16,11 @@ package stmtsummary import ( "encoding/json" + "fmt" "testing" "github.com/pingcap/tidb/pkg/config" + stmtsummarybase "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/stretchr/testify/require" ) @@ -67,6 +69,9 @@ func TestStmtRecord(t *testing.T) { require.Equal(t, info.RUDetail.RUWaitDuration(), record1.SumRUWaitDuration) require.Equal(t, info.TotalRUV2, record1.MaxRUV2) require.Equal(t, info.TotalRUV2, record1.SumRUV2) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoFixedEvents, record1.SumReadBillingDemoFixedEvents) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputRows, record1.SumReadBillingDemoInputRows) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputBytes, record1.SumReadBillingDemoInputBytes) require.Equal(t, info.CPUUsages.TidbCPUTime, record1.SumTidbCPU) require.Equal(t, info.CPUUsages.TikvCPUTime, record1.SumTikvCPU) @@ -83,6 +88,9 @@ func TestStmtRecord(t *testing.T) { require.Equal(t, info.RUDetail.WRU()*2, record2.SumWRU) require.Equal(t, info.RUDetail.RUWaitDuration()*2, record2.SumRUWaitDuration) require.Equal(t, info.TotalRUV2*2, record2.SumRUV2) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoFixedEvents*2, record2.SumReadBillingDemoFixedEvents) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputRows*2, record2.SumReadBillingDemoInputRows) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputBytes*2, record2.SumReadBillingDemoInputBytes) require.Equal(t, info.CPUUsages.TidbCPUTime*2, record2.SumTidbCPU) require.Equal(t, info.CPUUsages.TikvCPUTime*2, record2.SumTikvCPU) @@ -103,6 +111,7 @@ func TestStmtRecord(t *testing.T) { require.NoError(t, json.Unmarshal(b, &items)) require.Equal(t, map[string]any{"stmt_meta_a": "value_a"}, items["additional_fields"]) require.Equal(t, record2.Digest, items["digest"]) + require.Equal(t, record2.SumReadBillingDemoInputBytes, items["sum_read_billing_demo_input_bytes"]) b, err = marshalEvictedStmtRecord(record2) require.NoError(t, err) @@ -112,3 +121,164 @@ func TestStmtRecord(t *testing.T) { require.Equal(t, true, items["evicted"]) require.Equal(t, record2.Digest, items["digest"]) } + +func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { + info := GenerateStmtExecInfo4Test("digest_read_billing") + info.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v3", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "success", + Reason: "none", + }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{ + { + ModelVersion: "v3", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + DMLKind: "insert", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 2, + RowWidth: 16, + }, + { + ModelVersion: "v3", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + DMLKind: "insert", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 4, + RowWidth: 24, + }, + }, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 6, + }, + } + + record := NewStmtRecord(info) + record.Add(info) + require.Equal(t, int64(1), record.ExecCount) + require.Equal(t, 6.0, record.SumReadBillingDemoFixedEvents) + require.Len(t, record.ReadBillingDemoBaseUnitAggs, 1) + require.Equal(t, 6.0, record.ReadBillingDemoBaseUnitAggs[0].Value) + require.Equal(t, uint64(2), record.ReadBillingDemoBaseUnitAggs[0].SampleCount) + require.Equal(t, 40.0, record.ReadBillingDemoBaseUnitAggs[0].RowWidthSum) + require.Equal(t, "insert", record.ReadBillingDemoBaseUnitAggs[0].DMLKind) + require.Len(t, record.ReadBillingDemoStatusAggs, 1) + + b, err := marshalStmtRecord(record) + require.NoError(t, err) + var restored StmtRecord + require.NoError(t, json.Unmarshal(b, &restored)) + require.Equal(t, record.ReadBillingDemoBaseUnitAggs, restored.ReadBillingDemoBaseUnitAggs) + require.Equal(t, record.ReadBillingDemoStatusAggs, restored.ReadBillingDemoStatusAggs) + + statusOnly := GenerateStmtExecInfo4Test("digest_read_billing_error") + statusOnly.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 999, + RowWidth: 999, + }}, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 999, + }, + } + statusOnlyRecord := NewReadBillingDemoStatusOnlyRecord(statusOnly) + require.Zero(t, statusOnlyRecord.ExecCount) + require.True(t, statusOnlyRecord.isReadBillingDemoStatusOnly()) + require.Len(t, statusOnlyRecord.ReadBillingDemoStatusAggs, 1) + require.Empty(t, statusOnlyRecord.ReadBillingDemoBaseUnitAggs) + require.Zero(t, statusOnlyRecord.SumReadBillingDemoFixedEvents) + require.Contains(t, statusOnlyRecord.AuthUsers, "user") +} + +func TestStmtRecordMergeReadBillingDemoReservedStatusBypassesStatusCap(t *testing.T) { + record := NewStmtRecord(GenerateStmtExecInfo4Test("digest_read_billing_dst")) + for i := 0; i < stmtsummarybase.MaxReadBillingDemoStatusKeysPerRecord; i++ { + record.ReadBillingDemoStatusAggs = append(record.ReadBillingDemoStatusAggs, stmtsummarybase.ReadBillingDemoStatusAggEntry{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Status: "unsupported", + Reason: "unsupported_operator", + Count: 1, + }) + } + + other := NewStmtRecord(GenerateStmtExecInfo4Test("digest_read_billing_src")) + other.ReadBillingDemoStatusAggs = []stmtsummarybase.ReadBillingDemoStatusAggEntry{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "unknown_input", + Reason: "aggregation_overflow", + Count: 7, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "unknown_input", + Reason: "status_aggregation_overflow", + Count: 11, + }, + } + + record.Merge(other) + require.Equal(t, uint64(7), requireReadBillingDemoStatusReason(t, record.ReadBillingDemoStatusAggs, "aggregation_overflow").Count) + require.Equal(t, uint64(11), requireReadBillingDemoStatusReason(t, record.ReadBillingDemoStatusAggs, "status_aggregation_overflow").Count) +} + +func requireReadBillingDemoStatusReason(t *testing.T, entries []stmtsummarybase.ReadBillingDemoStatusAggEntry, reason string) stmtsummarybase.ReadBillingDemoStatusAggEntry { + t.Helper() + for _, entry := range entries { + if entry.Reason == reason { + return entry + } + } + require.Failf(t, "missing read billing status reason", "reason=%s entries=%v", reason, entries) + return stmtsummarybase.ReadBillingDemoStatusAggEntry{} +} diff --git a/pkg/util/stmtsummary/v2/stmtsummary.go b/pkg/util/stmtsummary/v2/stmtsummary.go index 2eab73de1ca2f..b68e44c866d39 100644 --- a/pkg/util/stmtsummary/v2/stmtsummary.go +++ b/pkg/util/stmtsummary/v2/stmtsummary.go @@ -348,6 +348,46 @@ func (s *StmtSummary) Add(info *stmtsummary.StmtExecInfo) { } } +// AddReadBillingDemoStatusOnly adds read billing demo status-only data to the +// current statistics window without changing ordinary statement counters. +func (s *StmtSummary) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo) { + var ok bool + info, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(info) + if s.closed.Load() || !ok { + return + } + + k := stmtsummary.StmtDigestKeyPool.Get().(*stmtsummary.StmtDigestKey) + + s.windowLock.Lock() + if s.closed.Load() { + s.windowLock.Unlock() + stmtsummary.StmtDigestKeyPool.Put(k) + return + } + userForKey := "" + if s.optGroupByUser.Load() { + userForKey = info.User + } + k.Init(info.SchemaName, info.Digest, info.PrevSQLDigest, info.PlanDigest, info.ResourceGroupName, userForKey) + var record *lockedStmtRecord + v, exist := s.window.lru.Get(k) + if exist { + record = v.(*lockedStmtRecord) + } else { + record = &lockedStmtRecord{StmtRecord: NewReadBillingDemoStatusOnlyRecord(info)} + s.window.lru.Put(k, record) + } + s.windowLock.Unlock() + + if exist { + record.Lock() + record.AddReadBillingDemoStatusOnly(info) + record.Unlock() + stmtsummary.StmtDigestKeyPool.Put(k) + } +} + // Evicted returns the number of statements evicted for the current // time window. The returned type is one row consisting of three // columns: [BEGIN_TIME, END_TIME, EVICTED_COUNT]. @@ -743,6 +783,24 @@ func Add(stmtExecInfo *stmtsummary.StmtExecInfo) { } } +// AddReadBillingDemoStatusOnly records read billing demo early-error statuses. +func AddReadBillingDemoStatusOnly(stmtExecInfo *stmtsummary.StmtExecInfo) { + var ok bool + stmtExecInfo, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(stmtExecInfo) + if !ok { + return + } + if config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent { + if GlobalStmtSummary == nil || !GlobalStmtSummary.Enabled() || + (stmtExecInfo.IsInternal && !GlobalStmtSummary.EnableInternalQuery()) { + return + } + GlobalStmtSummary.AddReadBillingDemoStatusOnly(stmtExecInfo) + return + } + stmtsummary.StmtSummaryByDigestMap.AddReadBillingDemoStatusOnly(stmtExecInfo) +} + // Enabled wraps GlobalStmtSummary.Enabled and stmtsummary.StmtSummaryByDigestMap.Enabled. func Enabled() bool { if config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent { diff --git a/pkg/util/stmtsummary/v2/stmtsummary_test.go b/pkg/util/stmtsummary/v2/stmtsummary_test.go index c395045e7b387..9e455aeb16120 100644 --- a/pkg/util/stmtsummary/v2/stmtsummary_test.go +++ b/pkg/util/stmtsummary/v2/stmtsummary_test.go @@ -195,6 +195,52 @@ func TestStmtSummaryPersistEvictedDoesNotPersistLoggedRecordsAsAggregate(t *test require.Equal(t, int64(4), totalExecCount) } +func TestStmtSummaryPersistStatusOnlyEvictedAggregate(t *testing.T) { + var logBuf bytes.Buffer + storage := &stmtLogStorage{ + logger: zap.New(zapcore.NewCore(&stmtLogEncoder{}, zapcore.AddSync(&logBuf), zapcore.InfoLevel)), + } + statusOnlyInfo := func(digest string) *stmtsummary.StmtExecInfo { + info := GenerateStmtExecInfo4Test(digest) + info.ReadBillingDemoStats = stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummary.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + } + return info + } + + ss := NewStmtSummary4Test(1) + ss.storage = storage + ss.AddReadBillingDemoStatusOnly(statusOnlyInfo("digest_status_1")) + ss.AddReadBillingDemoStatusOnly(statusOnlyInfo("digest_status_2")) // evicts digest_status_1 + ss.Close() + + type loggedRecord struct { + Digest string `json:"digest"` + StatusAggs []stmtsummary.ReadBillingDemoStatusAggEntry `json:"read_billing_demo_status_aggs"` + } + var recordsWithStatus []loggedRecord + for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { + var record loggedRecord + require.NoError(t, json.Unmarshal([]byte(line), &record)) + if len(record.StatusAggs) > 0 { + recordsWithStatus = append(recordsWithStatus, record) + } + } + require.Len(t, recordsWithStatus, 2) + require.Contains(t, []string{recordsWithStatus[0].Digest, recordsWithStatus[1].Digest}, "") + require.Contains(t, []string{recordsWithStatus[0].Digest, recordsWithStatus[1].Digest}, "digest_status_2") +} + func TestStmtSummaryGroupByUser(t *testing.T) { ss := NewStmtSummary4Test(100) defer ss.Close()