From 0f0f0c11a7b8787a9286aedb0b5d720be9534d67 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 1 Jul 2026 03:39:41 +0800 Subject: [PATCH 01/25] planner: add read billing RU model --- pkg/executor/adapter.go | 23 +- pkg/executor/compiler.go | 3 + pkg/metrics/BUILD.bazel | 3 +- pkg/metrics/explain_ru.go | 217 +++++ pkg/metrics/metrics.go | 12 + pkg/planner/core/BUILD.bazel | 5 + pkg/planner/core/common_plans.go | 12 +- pkg/planner/core/explain_ru.go | 1194 +++++++++++++++++++++++++ pkg/planner/core/planbuilder.go | 35 + pkg/session/session.go | 24 +- pkg/sessionctx/vardef/tidb_vars.go | 4 + pkg/sessionctx/variable/session.go | 4 + pkg/sessionctx/variable/sysvar.go | 4 + pkg/types/explain_format.go | 3 + pkg/util/execdetails/runtime_stats.go | 8 + 15 files changed, 1540 insertions(+), 11 deletions(-) create mode 100644 pkg/metrics/explain_ru.go create mode 100644 pkg/planner/core/explain_ru.go diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index b3a826f033e9a..7f9c4e631b317 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -366,11 +366,12 @@ type ExecStmt struct { Ctx sessionctx.Context // LowerPriority represents whether to lower the execution priority of a query. - LowerPriority bool - isPreparedStmt bool - isSelectForUpdate bool - retryCount uint - retryStartTime time.Time + LowerPriority bool + isPreparedStmt bool + isSelectForUpdate bool + resolvedPreparedStmt ast.StmtNode + retryCount uint + retryStartTime time.Time // Phase durations are splited into two parts: 1. trying to lock keys (but // failed); 2. the final iteration of the retry loop. Here we use @@ -770,11 +771,19 @@ func (a *ExecStmt) inheritContextFromExecuteStmt() { a.Ctx.SetValue(sessionctx.QueryString, executePlan.Stmt.Text()) a.OutputNames = executePlan.OutputNames() a.isPreparedStmt = true + a.resolvedPreparedStmt = executePlan.Stmt a.Plan = executePlan.Plan a.Ctx.GetSessionVars().StmtCtx.SetPlan(executePlan.Plan) } } +func (a *ExecStmt) readBillingDemoStmtNode() ast.StmtNode { + if a.resolvedPreparedStmt != nil { + return a.resolvedPreparedStmt + } + return a.StmtNode +} + func (a *ExecStmt) getSQLForProcessInfo() string { sql := a.Text() if simple, ok := a.Plan.(*plannercore.Simple); ok && simple.Statement != nil { @@ -1689,6 +1698,10 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo } a.finalizeStatementRUV2Metrics() + // Lazy SELECT metrics are emitted when the record set is closed. If a + // client abandons a result without closing/draining it, this first demo can + // miss that statement instead of guessing an incomplete status. + plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) a.updateNetworkTrafficStatsAndMetrics() // `LowSlowQuery` and `SummaryStmt` must be called before recording `PrevStmt`. a.LogSlowQuery(txnTS, succ, hasMoreResults) diff --git a/pkg/executor/compiler.go b/pkg/executor/compiler.go index b686423358f67..c478dacd5331a 100644 --- a/pkg/executor/compiler.go +++ b/pkg/executor/compiler.go @@ -153,6 +153,9 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecS OutputNames: names, Ti: &TelemetryInfo{}, } + if preparedObj != nil && preparedObj.PreparedAst != nil { + stmt.resolvedPreparedStmt = preparedObj.PreparedAst.Stmt + } // Use cached plan if possible. if preparedObj != nil && plannercore.IsSafeToReusePointGetExecutor(c.Ctx, is, preparedObj) { if exec, isExec := finalPlan.(*plannercore.Execute); isExec { diff --git a/pkg/metrics/BUILD.bazel b/pkg/metrics/BUILD.bazel index 16b3aaa56e43e..44dba62738a42 100644 --- a/pkg/metrics/BUILD.bazel +++ b/pkg/metrics/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "distsql.go", "domain.go", "executor.go", + "explain_ru.go", "external_workload.go", "gc_worker.go", "globalsort.go", @@ -69,7 +70,7 @@ go_test( ], embed = [":metrics"], flaky = True, - shard_count = 9, + shard_count = 11, deps = [ "//pkg/parser/terror", "//pkg/statistics/handle/cache", diff --git a/pkg/metrics/explain_ru.go b/pkg/metrics/explain_ru.go new file mode 100644 index 0000000000000..42d48e101beb7 --- /dev/null +++ b/pkg/metrics/explain_ru.go @@ -0,0 +1,217 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + metricscommon "github.com/pingcap/tidb/pkg/metrics/common" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + explainRULabelStatus = "status" + explainRULabelComponentSnapshotStatus = "component_snapshot_status" + readBillingDemoLabelModelVersion = "model_version" +) + +// EXPLAIN FORMAT='RU' demo metrics. +var ( + ExplainRUPreviewRUCounter *prometheus.CounterVec + ExplainRUWorkRowsCounter *prometheus.CounterVec + ExplainRUWorkBytesCounter *prometheus.CounterVec + ExplainRURowWidthHistogram *prometheus.HistogramVec + ExplainRUStatementsCounter *prometheus.CounterVec + ExplainRURenderDurationHistogram *prometheus.HistogramVec + ExplainRUComponentSnapshotCounter *prometheus.CounterVec + + ReadBillingDemoStatementsCounter *prometheus.CounterVec + ReadBillingDemoOperatorStatusCounter *prometheus.CounterVec + ReadBillingDemoBaseUnitsCounter *prometheus.CounterVec + ReadBillingDemoRowWidthHistogram *prometheus.HistogramVec +) + +// InitExplainRUMetrics initializes metrics for EXPLAIN ANALYZE FORMAT='RU'. +func InitExplainRUMetrics() { + // Keep these labels bounded during demo calibration. SQL text, digest, + // plan ID, table name, and index name belong in the SQL result, not metrics. + ExplainRUPreviewRUCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "preview_ru_total", + Help: "Counter of read billing preview RU values generated by EXPLAIN ANALYZE FORMAT='RU'.", + }, []string{"section", "component", "operator", "source"}, + ) + ExplainRUWorkRowsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "work_rows_total", + Help: "Counter of row-count model inputs generated by EXPLAIN ANALYZE FORMAT='RU'.", + }, []string{"section", "component", "operator", "source"}, + ) + ExplainRUWorkBytesCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "work_bytes_total", + Help: "Counter of byte-shaped model inputs generated by EXPLAIN ANALYZE FORMAT='RU'.", + }, []string{"section", "component", "operator", "source"}, + ) + ExplainRUStatementsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "statements_total", + Help: "Counter of EXPLAIN ANALYZE FORMAT='RU' statement status.", + }, []string{explainRULabelStatus}, + ) + ExplainRURenderDurationHistogram = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "render_duration_seconds", + Help: "Histogram of EXPLAIN ANALYZE FORMAT='RU' render duration.", + Buckets: prometheus.DefBuckets, + }, []string{explainRULabelStatus}, + ) + ExplainRURowWidthHistogram = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "row_width_bytes", + Help: "Histogram of row-width factors generated by EXPLAIN ANALYZE FORMAT='RU'.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), + }, []string{"component", "operator", "source"}, + ) + ExplainRUComponentSnapshotCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "explain_ru", + Name: "component_snapshot_total", + Help: "Counter of EXPLAIN ANALYZE FORMAT='RU' component snapshot status.", + }, []string{explainRULabelComponentSnapshotStatus}, + ) + ReadBillingDemoStatementsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "statements_total", + Help: "Counter of read billing demo statement status.", + }, []string{"status", readBillingDemoLabelModelVersion}, + ) + ReadBillingDemoOperatorStatusCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "operator_status_total", + Help: "Counter of read billing demo operator status.", + }, []string{"site", "op_class", "operator_kind", "status", "reason", readBillingDemoLabelModelVersion}, + ) + ReadBillingDemoBaseUnitsCounter = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "base_units_total", + Help: "Counter of coefficient-free read billing demo base units.", + }, []string{"site", "op_class", "operator_kind", "unit", "input_source", "input_side", readBillingDemoLabelModelVersion}, + ) + ReadBillingDemoRowWidthHistogram = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "read_billing_demo", + Name: "row_width_bytes", + Help: "Histogram of row-width factors used by read billing demo.", + Buckets: prometheus.ExponentialBuckets(1, 2, 12), + }, []string{"site", "op_class", "operator_kind", "row_width_source", readBillingDemoLabelModelVersion}, + ) +} + +// RecordExplainRUStatus records a bounded FORMAT='RU' statement status. +func RecordExplainRUStatus(status string) { + if ExplainRUStatementsCounter == nil || status == "" { + return + } + ExplainRUStatementsCounter.WithLabelValues(status).Inc() +} + +// ObserveExplainRURenderDuration records a bounded FORMAT='RU' render duration. +func ObserveExplainRURenderDuration(status string, seconds float64) { + if ExplainRURenderDurationHistogram == nil || status == "" || seconds < 0 { + return + } + ExplainRURenderDurationHistogram.WithLabelValues(status).Observe(seconds) +} + +// RecordExplainRUComponentSnapshot records component snapshot availability. +func RecordExplainRUComponentSnapshot(status string) { + if ExplainRUComponentSnapshotCounter == nil || status == "" { + return + } + ExplainRUComponentSnapshotCounter.WithLabelValues(status).Inc() +} + +// ObserveExplainRURow records bounded numeric samples from generated FORMAT='RU' rows. +func ObserveExplainRURow(section, component, operator, source, rowWidthSource string, previewRU, workRows, workBytes, rowWidth float64) { + // Negative values are sentinels for "not present" so callers can use one + // path for summary and plan rows without emitting fake zeros. + if ExplainRUPreviewRUCounter != nil && previewRU >= 0 { + ExplainRUPreviewRUCounter.WithLabelValues(section, component, operator, source).Add(previewRU) + } + if ExplainRUWorkRowsCounter != nil && workRows >= 0 { + ExplainRUWorkRowsCounter.WithLabelValues(section, component, operator, source).Add(workRows) + } + if ExplainRUWorkBytesCounter != nil && workBytes >= 0 { + ExplainRUWorkBytesCounter.WithLabelValues(section, component, operator, source).Add(workBytes) + } + if ExplainRURowWidthHistogram != nil && section == "plan" && rowWidth > 0 && operator != "" { + ExplainRURowWidthHistogram.WithLabelValues(component, operator, rowWidthSource).Observe(rowWidth) + } +} + +// RecordReadBillingDemoStatement records a bounded read billing demo statement status. +func RecordReadBillingDemoStatement(status, modelVersion string) { + if ReadBillingDemoStatementsCounter == nil || status == "" || modelVersion == "" { + return + } + ReadBillingDemoStatementsCounter.WithLabelValues(status, modelVersion).Inc() +} + +// RecordReadBillingDemoOperatorStatus records a bounded read billing demo operator status. +func RecordReadBillingDemoOperatorStatus(site, opClass, operatorKind, status, reason, modelVersion string) { + if ReadBillingDemoOperatorStatusCounter == nil || + site == "" || opClass == "" || operatorKind == "" || status == "" || reason == "" || modelVersion == "" { + return + } + ReadBillingDemoOperatorStatusCounter.WithLabelValues(site, opClass, operatorKind, status, reason, modelVersion).Inc() +} + +// AddReadBillingDemoBaseUnits records coefficient-free read billing demo base units. +func AddReadBillingDemoBaseUnits(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion string, value float64) { + if ReadBillingDemoBaseUnitsCounter == nil || + site == "" || opClass == "" || operatorKind == "" || unit == "" || inputSource == "" || inputSide == "" || modelVersion == "" || + value <= 0 { + return + } + ReadBillingDemoBaseUnitsCounter.WithLabelValues(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion).Add(value) +} + +// ObserveReadBillingDemoRowWidth records row-width factors used by read billing demo. +func ObserveReadBillingDemoRowWidth(site, opClass, operatorKind, rowWidthSource, modelVersion string, rowWidth float64) { + if ReadBillingDemoRowWidthHistogram == nil || + site == "" || opClass == "" || operatorKind == "" || rowWidthSource == "" || modelVersion == "" || rowWidth <= 0 { + return + } + ReadBillingDemoRowWidthHistogram.WithLabelValues(site, opClass, operatorKind, rowWidthSource, modelVersion).Observe(rowWidth) +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a71cc5637d540..19031cc494802 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -102,6 +102,7 @@ func InitMetrics() { InitServerMetrics() InitSessionMetrics() InitRUV2Metrics() + InitExplainRUMetrics() InitSliMetrics() InitStatsMetrics() InitTelemetryMetrics() @@ -361,6 +362,17 @@ func RegisterMetrics() { prometheus.MustRegister(RUV2TiKVStorageProcessedKeysBatchGet) prometheus.MustRegister(RUV2TiKVStorageProcessedKeysGet) prometheus.MustRegister(RUV2TiKVCoprocessorWorkTotal) + prometheus.MustRegister(ExplainRUPreviewRUCounter) + prometheus.MustRegister(ExplainRUWorkRowsCounter) + prometheus.MustRegister(ExplainRUWorkBytesCounter) + prometheus.MustRegister(ExplainRURowWidthHistogram) + prometheus.MustRegister(ExplainRUStatementsCounter) + prometheus.MustRegister(ExplainRURenderDurationHistogram) + prometheus.MustRegister(ExplainRUComponentSnapshotCounter) + prometheus.MustRegister(ReadBillingDemoStatementsCounter) + prometheus.MustRegister(ReadBillingDemoOperatorStatusCounter) + prometheus.MustRegister(ReadBillingDemoBaseUnitsCounter) + prometheus.MustRegister(ReadBillingDemoRowWidthHistogram) prometheus.MustRegister(NetworkTransmissionStats) diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 2bf15ab229081..21235f1d6403a 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "core_init.go", "encode.go", "exhaust_physical_plans.go", + "explain_ru.go", "expression_codec_fn.go", "expression_rewriter.go", "find_best_task.go", @@ -207,6 +208,7 @@ go_library( "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap", ], @@ -321,6 +323,7 @@ go_test( "//pkg/util/context", "//pkg/util/dbterror", "//pkg/util/dbterror/plannererrors", + "//pkg/util/execdetails", "//pkg/util/hint", "//pkg/util/logutil", "//pkg/util/mock", @@ -332,6 +335,8 @@ go_test( "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_tipb//go-tipb", "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//util", + "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_goleak//:goleak", "@org_uber_go_zap//:zap", "@org_uber_go_zap//zaptest/observer", diff --git a/pkg/planner/core/common_plans.go b/pkg/planner/core/common_plans.go index ae124a35819e5..0fb06300b2094 100644 --- a/pkg/planner/core/common_plans.go +++ b/pkg/planner/core/common_plans.go @@ -651,8 +651,9 @@ type Explain struct { ExecStmt ast.StmtNode RuntimeStatsColl *execdetails.RuntimeStatsColl - Rows [][]string - BriefBinaryPlan string + Rows [][]string + BriefBinaryPlan string + ruStatusRecorded bool } // GetBriefBinaryPlan returns the binary plan of the plan for explainfor. @@ -735,6 +736,8 @@ func (e *Explain) prepareSchema() error { fieldNames = []string{"binary plan"} case format == types.ExplainFormatTiDBJSON: fieldNames = []string{"TiDB_JSON"} + case format == types.ExplainFormatRU && e.Analyze: + fieldNames = []string{"section", "id", "component", "operatorClass", "actRows", "inputRows", "outputRows", "rowWidth", "rowWidthSource", "workRows", "workBytes", "unit", "count", "weight", "previewRU", "source", "note"} case e.Explore: fieldNames = []string{"statement", "binding_hint", "plan", "plan_digest", "avg_latency", "exec_times", "avg_scan_rows", "avg_returned_rows", "latency_per_returned_row", "scan_rows_per_returned_row", "recommend", "reason", @@ -932,6 +935,11 @@ func (e *Explain) RenderResult() error { return err } e.Rows = append(e.Rows, []string{str}) + case types.ExplainFormatRU: + if err := e.renderRUExplain(); err != nil { + e.recordExplainRUStatus(explainRUStatusError) + return err + } default: return errors.Errorf("explain format '%s' is not supported now", e.Format) } diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go new file mode 100644 index 0000000000000..82d6402cad408 --- /dev/null +++ b/pkg/planner/core/explain_ru.go @@ -0,0 +1,1194 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "strconv" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/metrics" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/planner/cardinality" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/plancodec" + rmclient "github.com/tikv/pd/client/resource_group/controller" +) + +type explainRUStatus string + +const ( + explainRUStatusSuccess explainRUStatus = "success" + explainRUStatusUnsupportedNonAnalyze explainRUStatus = "unsupported_non_analyze" + explainRUStatusUnsupportedNonSelect explainRUStatus = "unsupported_non_select" + explainRUStatusUnsupportedSideEffecting explainRUStatus = "unsupported_side_effecting_select" + explainRUStatusUnsupportedLockingSelect explainRUStatus = "unsupported_locking_select" + explainRUStatusUnsupportedForConnection explainRUStatus = "unsupported_for_connection" + explainRUStatusError explainRUStatus = "error" + explainRUComponentSnapshotOK explainRUComponentSnapshotStatus = "ok" + explainRUComponentSnapshotMissing explainRUComponentSnapshotStatus = "missing" + explainRUComponentSnapshotNonV2 explainRUComponentSnapshotStatus = "non_v2" + explainRUComponentSnapshotNilMetrics explainRUComponentSnapshotStatus = "nil_metrics" + explainRUComponentSnapshotBypassed explainRUComponentSnapshotStatus = "bypassed" + explainRUSectionSummary = "summary" + explainRUSectionPlan = "plan" + explainRUSourceSummaryTotal = "summary_total" + explainRUWidthSourceOperatorHelper = "operator_helper" + explainRUWidthSourcePlanStats = "plan_stats" + explainRUWidthSourceSchemaTypeWidth = "schema_type_width" + explainRUWidthSourceSchemaFallback = "schema_fallback" + + readBillingDemoModelVersion = "v1" + readBillingDemoWeightVersion = "v1" + readBillingDemoStatusSuccess = "success" + readBillingDemoStatusUnsupported = "unsupported" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoStatusError = "error" + readBillingDemoStatusOperatorOK = "ok" + readBillingDemoReasonNone = "none" + readBillingDemoReasonStatementError = "statement_error" + readBillingDemoReasonMissingPlan = "missing_plan" + readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" + readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" + readBillingDemoReasonMissingScanDetail = "missing_scan_detail" + readBillingDemoReasonUnsupportedOperator = "unsupported_operator" + readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" + readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" + readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" + readBillingDemoReasonUnsupportedLock = "unsupported_lock" + readBillingDemoReasonNonBillable = "non_billable" + readBillingDemoSiteStatement = "statement" + readBillingDemoSiteTiDB = "tidb" + readBillingDemoSiteTiKV = "tikv" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOpClassFilter = "filter_eval" + readBillingDemoOpClassProjection = "projection_eval" + readBillingDemoOpClassLimit = "row_limit" + readBillingDemoOpClassTopN = "bounded_topn" + readBillingDemoOpClassSort = "full_ordering" + readBillingDemoOpClassWindow = "window_eval" + readBillingDemoOpClassHashAgg = "agg_hash" + readBillingDemoOpClassStreamAgg = "agg_stream" + readBillingDemoOpClassHashJoin = "join_hash" + readBillingDemoOpClassMergeJoin = "join_merge" + readBillingDemoOpClassLookupJoin = "join_lookup" + readBillingDemoOpClassReaderReceive = "reader_receive" + readBillingDemoOpClassLookupReader = "lookup_reader" + readBillingDemoOpClassOverlayReader = "overlay_reader" + readBillingDemoOpClassMetadataReader = "metadata_reader" + readBillingDemoOpClassPointLookup = "kv_point_lookup" + readBillingDemoOpClassRangeScan = "kv_range_scan" + readBillingDemoOpClassWrapper = "wrapper" + readBillingDemoOpClassSynthetic = "synthetic_source" + readBillingDemoOperatorStatement = "statement" + readBillingDemoUnitFixedEvents = "fixed_events" + readBillingDemoUnitInputRows = "input_rows" + readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoInputSourceRuntimeRows = "runtime_act_rows" + readBillingDemoInputSourceScanDetail = "scan_detail" + readBillingDemoInputSideAll = "all" + readBillingDemoInputSideBuild = "build" + readBillingDemoInputSideProbe = "probe" + readBillingDemoInputSideLeft = "left" + readBillingDemoInputSideRight = "right" +) + +type explainRUComponentSnapshotStatus string + +type readBillingDemoUnit struct { + unit string + source string + side string + value float64 + rowWidth float64 + widthSource string +} + +type readBillingDemoOperatorResult struct { + id string + site string + opClass string + operatorKind string + status string + reason string + actRows int64 + hasActRows bool + units []readBillingDemoUnit +} + +type readBillingDemoResult struct { + status string + reason string + operators []readBillingDemoOperatorResult +} + +type readBillingDemoOperatorWeights struct { + fixedEvent float64 + row float64 + byte float64 +} + +type readBillingDemoWeightKey struct { + site string + opClass string + version string +} + +var readBillingDemoWeights = map[readBillingDemoWeightKey]readBillingDemoOperatorWeights{ + {readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000020}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000040, byte: 0.000006}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000006}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000008, byte: 0.000002}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000075, byte: 0.000012}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000100, byte: 0.000014}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000065, byte: 0.000010}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion}: {fixedEvent: 0.045, row: 0.000030, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000005}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000020, byte: 0.000004}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000006, byte: 0.000001}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000060, byte: 0.000010}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassSort, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000070, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassWindow, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000070, byte: 0.000010}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000085, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.050, row: 0.000055, byte: 0.000008}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassHashJoin, readBillingDemoWeightVersion}: {fixedEvent: 0.110, row: 0.000115, byte: 0.000020}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassMergeJoin, readBillingDemoWeightVersion}: {fixedEvent: 0.090, row: 0.000075, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassLookupJoin, readBillingDemoWeightVersion}: {fixedEvent: 0.120, row: 0.000120, byte: 0.000020}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassReaderReceive, readBillingDemoWeightVersion}: {fixedEvent: 0.040, row: 0.000025, byte: 0.000014}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassLookupReader, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000016}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassOverlayReader, readBillingDemoWeightVersion}: {fixedEvent: 0.050, row: 0.000035, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassMetadataReader, readBillingDemoWeightVersion}: { + fixedEvent: 0.020, + row: 0.000008, + byte: 0.000002, + }, +} + +type explainRURow struct { + section string + id string + component string + operatorClass string + actRows int64 + hasActRows bool + inputRows int64 + hasInputRows bool + outputRows int64 + hasOutputRows bool + rowWidth float64 + hasRowWidth bool + rowWidthSource string + workRows int64 + hasWorkRows bool + workBytes float64 + hasWorkBytes bool + unit string + count int64 + hasCount bool + weight float64 + hasWeight bool + previewRU float64 + hasPreviewRU bool + source string + note string +} + +// RecordReadBillingDemoForStatement emits coefficient-free read billing demo +// metrics for a completed statement. It is intentionally independent from RU +// v2 billing/reporting and never calls resource-control reporters. +func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, stmt ast.StmtNode, execErr error) { + if sctx == nil || sctx.GetSessionVars() == nil || !sctx.GetSessionVars().EnableReadBillingDemo { + return + } + // Restricted/internal SQL is not external workload calibration input. + if sctx.GetSessionVars().InRestrictedSQL || sctx.GetSessionVars().StmtCtx.InRestrictedSQL { + return + } + planCtx := readBillingDemoPlanContext(plan) + if planCtx == nil { + planCtx = sctx.GetPlanCtx() + } + result := buildReadBillingDemoResult(planCtx, plan, stmt, execErr) + recordReadBillingDemoResult(result) +} + +func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error) readBillingDemoResult { + if execErr != nil { + return readBillingDemoFailure(readBillingDemoStatusError, readBillingDemoReasonStatementError) + } + if plan == nil { + return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingPlan) + } + if gateStatus := explainRUSelectGateStatus(stmt); gateStatus != explainRUStatusSuccess { + return readBillingDemoFailure(readBillingDemoStatusUnsupported, string(gateStatus)) + } + if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil { + return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingRuntimeStats) + } + flat := FlattenPhysicalPlan(plan, true) + if flat == nil || len(flat.Main) == 0 || flat.InExplain || flat.InExecute { + return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingPlan) + } + + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + planCtx := readBillingDemoPlanContext(plan) + runtimeStats := sctx.GetSessionVars().StmtCtx.RuntimeStatsColl + if status, op := appendReadBillingDemoTree(&result, planCtx, runtimeStats, flat.Main); status != readBillingDemoStatusSuccess { + return readBillingDemoFailedOperator(status, op) + } + for _, tree := range flat.CTEs { + if status, op := appendReadBillingDemoTree(&result, planCtx, runtimeStats, tree); status != readBillingDemoStatusSuccess { + return readBillingDemoFailedOperator(status, op) + } + } + for _, tree := range flat.ScalarSubQueries { + if status, op := appendReadBillingDemoTree(&result, planCtx, runtimeStats, tree); status != readBillingDemoStatusSuccess { + return readBillingDemoFailedOperator(status, op) + } + } + return result +} + +func readBillingDemoPlanContext(plan base.Plan) base.PlanContext { + if plan == nil { + return nil + } + return plan.SCtx() +} + +func readBillingDemoResolveWeights(site, opClass, version string) (readBillingDemoOperatorWeights, bool) { + weights, ok := readBillingDemoWeights[readBillingDemoWeightKey{site: site, opClass: opClass, version: version}] + return weights, ok +} + +func readBillingDemoUnitWeight(weights readBillingDemoOperatorWeights, unit string) (float64, bool) { + switch unit { + case readBillingDemoUnitFixedEvents: + return weights.fixedEvent, true + case readBillingDemoUnitInputRows: + return weights.row, true + case readBillingDemoUnitInputBytes: + return weights.byte, true + default: + return 0, false + } +} + +func readBillingDemoUnitPreviewRU(unit readBillingDemoUnit, weights readBillingDemoOperatorWeights) (float64, float64, bool) { + weight, ok := readBillingDemoUnitWeight(weights, unit.unit) + if !ok { + return 0, 0, false + } + return weight, unit.value * weight, true +} + +func readBillingDemoFailure(status, reason string) readBillingDemoResult { + return readBillingDemoResult{ + status: status, + reason: reason, + operators: []readBillingDemoOperatorResult{{ + site: readBillingDemoSiteStatement, + opClass: readBillingDemoOpClassStatement, + operatorKind: readBillingDemoOperatorStatement, + status: status, + reason: reason, + }}, + } +} + +func readBillingDemoFailedOperator(status string, op readBillingDemoOperatorResult) readBillingDemoResult { + op.status = status + if op.reason == "" { + op.reason = readBillingDemoReasonUnsupportedOperator + } + return readBillingDemoResult{ + status: status, + reason: op.reason, + operators: []readBillingDemoOperatorResult{op}, + } +} + +func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) (string, readBillingDemoOperatorResult) { + for i, op := range tree { + if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { + continue + } + operator, supported, reason := readBillingDemoClassifyOperator(op) + if !supported { + return readBillingDemoStatusUnsupported, operator.withReason(reason) + } + operator.id = op.ExplainID().String() + if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, tree, i, op, operator); ok { + operator.actRows = actRows + operator.hasActRows = true + } + if !readBillingDemoOperatorBillable(operator) { + operator.status = readBillingDemoStatusOperatorOK + result.operators = append(result.operators, operator.withReason(readBillingDemoReasonNonBillable)) + continue + } + var units []readBillingDemoUnit + var ok bool + if op.IsRoot { + units, ok = readBillingDemoRootUnits(sctx, runtimeStats, tree, i, op, operator) + } else { + units, ok = readBillingDemoCopUnits(sctx, runtimeStats, tree, i, op, operator) + } + if !ok { + if op.IsRoot { + return readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingRuntimeRows) + } + return readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingScanDetail) + } + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + operator.units = units + result.operators = append(result.operators, operator) + } + return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} +} + +func (op readBillingDemoOperatorResult) withReason(reason string) readBillingDemoOperatorResult { + op.reason = reason + return op +} + +func readBillingDemoOperatorBillable(op readBillingDemoOperatorResult) bool { + return op.opClass != readBillingDemoOpClassWrapper && op.opClass != readBillingDemoOpClassSynthetic +} + +func readBillingDemoOperatorActRows(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) (int64, bool) { + if op == nil { + return 0, false + } + if op.IsRoot { + return readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) + } + if copStats := readBillingDemoCopStats(runtimeStats, tree, idx, operator.opClass); copStats != nil { + return copStats.GetActRows(), true + } + return 0, false +} + +func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorResult, bool, string) { + operatorKind := strings.ToLower(op.Origin.TP()) + if !op.IsRoot { + if op.StoreType == kv.TiFlash { + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedTiFlash + } + switch op.Origin.TP() { + case plancodec.TypeTableScan, plancodec.TypeIdxScan, + plancodec.TypeTableFullScan, plancodec.TypeTableRangeScan, plancodec.TypeTableRowIDScan, + plancodec.TypeIndexFullScan, plancodec.TypeIndexRangeScan: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, true, "" + case plancodec.TypeSel: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassFilter, operatorKind: operatorKind}, true, "" + case plancodec.TypeProj: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassProjection, operatorKind: operatorKind}, true, "" + case plancodec.TypeLimit: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassLimit, operatorKind: operatorKind}, true, "" + case plancodec.TypeTopN: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassTopN, operatorKind: operatorKind}, true, "" + case plancodec.TypeHashAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassHashAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeStreamAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassStreamAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeSort, plancodec.TypeHashJoin, plancodec.TypeMergeJoin, plancodec.TypeIndexJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedOperator + default: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedOperator + } + } + + switch op.Origin.TP() { + case plancodec.TypeExchangeReceiver, plancodec.TypeExchangeSender: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedMPP + case plancodec.TypeIndexMerge: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupReader, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedIndexMerge + case plancodec.TypeLock: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedLock + case plancodec.TypePointGet, plancodec.TypeBatchPointGet: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassPointLookup, operatorKind: operatorKind}, true, "" + case plancodec.TypeSel: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassFilter, operatorKind: operatorKind}, true, "" + case plancodec.TypeProj: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassProjection, operatorKind: operatorKind}, true, "" + case plancodec.TypeLimit: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLimit, operatorKind: operatorKind}, true, "" + case plancodec.TypeTopN: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassTopN, operatorKind: operatorKind}, true, "" + case plancodec.TypeSort: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassSort, operatorKind: operatorKind}, true, "" + case plancodec.TypeWindow: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassWindow, operatorKind: operatorKind}, true, "" + case plancodec.TypeHashAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeStreamAgg: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassStreamAgg, operatorKind: operatorKind}, true, "" + case plancodec.TypeHashJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashJoin, operatorKind: operatorKind}, true, "" + case plancodec.TypeMergeJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassMergeJoin, operatorKind: operatorKind}, true, "" + case plancodec.TypeIndexJoin, plancodec.TypeIndexHashJoin, plancodec.TypeIndexMergeJoin: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupJoin, operatorKind: operatorKind}, true, "" + case plancodec.TypeTableReader, plancodec.TypeIndexReader: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, true, "" + case plancodec.TypeIndexLookUp, plancodec.TypeLocalIndexLookUp: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeUnionScan: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassOverlayReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeMemTableScan, plancodec.TypeClusterMemTableReader: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassMetadataReader, operatorKind: operatorKind}, true, "" + case plancodec.TypeUnion: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassWrapper, operatorKind: operatorKind}, true, "" + case plancodec.TypeDual: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassSynthetic, operatorKind: operatorKind}, true, "" + default: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedOperator + } +} + +func readBillingDemoRootUnits(sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, bool) { + outputRows, ok := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) + if !ok { + return nil, false + } + rowWidth, rowWidthSource := explainRURowWidth(sctx, op.Origin, true) + units := []readBillingDemoUnit{{ + unit: readBillingDemoUnitFixedEvents, + source: readBillingDemoInputSourceRuntimeRows, + side: readBillingDemoInputSideAll, + value: 1, + rowWidth: rowWidth, + widthSource: rowWidthSource, + }} + switch operator.opClass { + case readBillingDemoOpClassHashJoin: + return appendReadBillingDemoJoinUnits(units, sctx, runtimeStats, tree, idx, rowWidth, rowWidthSource, true) + case readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: + return appendReadBillingDemoJoinUnits(units, sctx, runtimeStats, tree, idx, rowWidth, rowWidthSource, false) + default: + inputRows, inputRowWidth, ok := readBillingDemoDirectLocalInputRowsAndWidth(sctx, runtimeStats, tree, idx, rowWidth) + if !ok { + return nil, false + } + if inputRows == 0 && (len(tree[idx].ChildrenIdx) == 0 || readBillingDemoUseOutputRowsAsInput(operator.opClass)) { + inputRows = outputRows + inputRowWidth = rowWidth + } + inputBytes := float64(inputRows) * inputRowWidth + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: float64(inputRows), rowWidth: inputRowWidth, widthSource: rowWidthSource}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: inputBytes, rowWidth: inputRowWidth, widthSource: rowWidthSource}, + ) + return units, true + } +} + +func readBillingDemoUseOutputRowsAsInput(opClass string) bool { + switch opClass { + case readBillingDemoOpClassReaderReceive, readBillingDemoOpClassLookupReader, readBillingDemoOpClassMetadataReader, readBillingDemoOpClassPointLookup: + return true + default: + return false + } +} + +func appendReadBillingDemoJoinUnits(units []readBillingDemoUnit, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, fallbackWidth float64, fallbackWidthSource string, useBuildProbe bool) ([]readBillingDemoUnit, bool) { + for childOrder, childIdx := range tree[idx].ChildrenIdx { + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { + continue + } + rows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + if !ok { + return nil, false + } + width, widthSource := explainRURowWidth(sctx, tree[childIdx].Origin, true) + if width <= 0 { + width = fallbackWidth + widthSource = fallbackWidthSource + } + side := readBillingDemoInputSideAll + if useBuildProbe { + switch tree[childIdx].Label { + case BuildSide: + side = readBillingDemoInputSideBuild + case ProbeSide: + side = readBillingDemoInputSideProbe + default: + if childOrder == 0 { + side = readBillingDemoInputSideBuild + } else { + side = readBillingDemoInputSideProbe + } + } + } else if childOrder == 0 { + side = readBillingDemoInputSideLeft + } else { + side = readBillingDemoInputSideRight + } + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeRows, side: side, value: float64(rows), rowWidth: width, widthSource: widthSource}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeRows, side: side, value: float64(rows) * width, rowWidth: width, widthSource: widthSource}, + ) + } + return units, true +} + +func readBillingDemoDirectLocalInputRowsAndWidth(sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, fallbackWidth float64) (int64, float64, bool) { + if idx < 0 || idx >= len(tree) || tree[idx] == nil { + return 0, fallbackWidth, true + } + var rows int64 + inputBytes := 0.0 + for _, childIdx := range tree[idx].ChildrenIdx { + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { + continue + } + childRows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + if !ok { + return 0, 0, false + } + childWidth, _ := explainRURowWidth(sctx, tree[childIdx].Origin, true) + rows += childRows + inputBytes += float64(childRows) * childWidth + } + if rows == 0 { + return 0, fallbackWidth, true + } + return rows, inputBytes / float64(rows), true +} + +func readBillingDemoPlanActRows(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, bool) { + if runtimeStats == nil || !runtimeStats.ExistsRootStats(planID) { + return 0, false + } + return runtimeStats.GetPlanActRows(planID), true +} + +func readBillingDemoCopUnits(sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, bool) { + copStats := readBillingDemoCopStats(runtimeStats, tree, idx, operator.opClass) + if copStats == nil { + return nil, false + } + scanDetail := copStats.GetScanDetail() + rowWidth, rowWidthSource := explainRURowWidth(sctx, op.Origin, false) + units := []readBillingDemoUnit{{ + unit: readBillingDemoUnitFixedEvents, + source: readBillingDemoInputSourceScanDetail, + side: readBillingDemoInputSideAll, + value: 1, + rowWidth: rowWidth, + widthSource: rowWidthSource, + }} + switch operator.opClass { + case readBillingDemoOpClassRangeScan: + scanInputRows, scanInputBytes, scanInputSource := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeysSize, copStats.GetActRows(), rowWidth) + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: scanInputSource, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: rowWidth, widthSource: rowWidthSource}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: scanInputSource, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: rowWidthSource}, + ) + default: + rows, inputRowWidth, inputRowWidthSource, ok := readBillingDemoDirectCopInputRowsAndWidth(sctx, runtimeStats, tree, idx, rowWidth, rowWidthSource, copStats) + if !ok { + return nil, false + } + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: float64(rows), rowWidth: inputRowWidth, widthSource: inputRowWidthSource}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: float64(rows) * inputRowWidth, rowWidth: inputRowWidth, widthSource: inputRowWidthSource}, + ) + } + return units, true +} + +func readBillingDemoRangeScanInput(totalKeys, processedKeysSize, actRows int64, rowWidth float64) (int64, float64, string) { + if totalKeys != 0 || processedKeysSize != 0 { + inputBytes := float64(processedKeysSize) + if inputBytes == 0 && totalKeys > 0 { + inputBytes = float64(totalKeys) * rowWidth + } + return totalKeys, inputBytes, readBillingDemoInputSourceScanDetail + } + // Mock-store based tests can return executor rows without legacy scan-detail + // key counters. Keep the source explicit instead of pretending actRows are + // scan-detail TotalKeys. + if actRows > 0 { + return actRows, float64(actRows) * rowWidth, readBillingDemoInputSourceRuntimeRows + } + return 0, 0, readBillingDemoInputSourceScanDetail +} + +func readBillingDemoDirectCopInputRowsAndWidth( + sctx base.PlanContext, + runtimeStats *execdetails.RuntimeStatsColl, + tree FlatPlanTree, + idx int, + fallbackWidth float64, + fallbackWidthSource string, + ownStats *execdetails.CopRuntimeStats, +) (int64, float64, string, bool) { + var rows int64 + inputBytes := 0.0 + inputRowWidth := fallbackWidth + inputRowWidthSource := fallbackWidthSource + hasCopChild := false + for _, childIdx := range tree[idx].ChildrenIdx { + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || tree[childIdx].IsRoot { + continue + } + hasCopChild = true + childStats := runtimeStats.GetCopStats(tree[childIdx].Origin.ID()) + if childStats == nil { + return 0, 0, "", false + } + childRows := childStats.GetActRows() + childWidth, childWidthSource := explainRURowWidth(sctx, tree[childIdx].Origin, false) + if childWidth <= 0 { + childWidth = fallbackWidth + childWidthSource = fallbackWidthSource + } + inputRowWidth = childWidth + inputRowWidthSource = childWidthSource + rows += childRows + inputBytes += float64(childRows) * childWidth + } + if !hasCopChild { + return ownStats.GetActRows(), fallbackWidth, fallbackWidthSource, true + } + if rows == 0 { + return 0, inputRowWidth, inputRowWidthSource, true + } + return rows, inputBytes / float64(rows), inputRowWidthSource, true +} + +func readBillingDemoCopStats(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, opClass string) *execdetails.CopRuntimeStats { + if runtimeStats == nil || idx < 0 || idx >= len(tree) || tree[idx] == nil || tree[idx].Origin == nil { + return nil + } + exact := runtimeStats.GetCopStats(tree[idx].Origin.ID()) + if readBillingDemoCopStatsUsable(exact, opClass) { + return exact + } + // distsql may attach scan detail to the last cop plan ID in a task, not + // necessarily to the scan node. Search the same cop subtree before failing. + if end := tree[idx].ChildrenEndIdx; end > idx { + if end >= len(tree) { + end = len(tree) - 1 + } + for i := end; i > idx; i-- { + if tree[i] == nil || tree[i].IsRoot || tree[i].Origin == nil || tree[i].StoreType != tree[idx].StoreType { + continue + } + if stats := runtimeStats.GetCopStats(tree[i].Origin.ID()); readBillingDemoCopStatsUsable(stats, opClass) { + return stats + } + } + } + for i := idx - 1; i >= 0; i-- { + if tree[i] == nil || tree[i].IsRoot || tree[i].Origin == nil || tree[i].StoreType != tree[idx].StoreType || tree[i].ChildrenEndIdx < idx { + continue + } + if stats := runtimeStats.GetCopStats(tree[i].Origin.ID()); readBillingDemoCopStatsUsable(stats, opClass) { + return stats + } + } + return exact +} + +func readBillingDemoCopStatsUsable(copStats *execdetails.CopRuntimeStats, opClass string) bool { + if copStats == nil { + return false + } + if opClass != readBillingDemoOpClassRangeScan { + return true + } + scanDetail := copStats.GetScanDetail() + return scanDetail.TotalKeys != 0 || scanDetail.ProcessedKeysSize != 0 +} + +func recordReadBillingDemoResult(result readBillingDemoResult) { + status := result.status + if status == "" { + status = readBillingDemoStatusUnknownInput + } + metrics.RecordReadBillingDemoStatement(status, readBillingDemoModelVersion) + for _, op := range result.operators { + opStatus := op.status + if opStatus == "" { + opStatus = status + } + reason := op.reason + if reason == "" { + reason = result.reason + } + if reason == "" { + reason = readBillingDemoReasonNone + } + metrics.RecordReadBillingDemoOperatorStatus(op.site, op.opClass, op.operatorKind, opStatus, reason, readBillingDemoModelVersion) + if status != readBillingDemoStatusSuccess { + continue + } + for _, unit := range op.units { + metrics.AddReadBillingDemoBaseUnits(op.site, op.opClass, op.operatorKind, unit.unit, unit.source, unit.side, readBillingDemoModelVersion, unit.value) + metrics.ObserveReadBillingDemoRowWidth(op.site, op.opClass, op.operatorKind, unit.widthSource, readBillingDemoModelVersion, unit.rowWidth) + } + } +} + +func explainRUError(status explainRUStatus) error { + return errors.NewNoStackErrorf("EXPLAIN ANALYZE FORMAT='RU' is not supported for this target: %s", status) +} + +func recordExplainRUStatus(status explainRUStatus) { + metrics.RecordExplainRUStatus(string(status)) +} + +func (e *Explain) recordExplainRUStatus(status explainRUStatus) { + if e == nil || e.ruStatusRecorded { + return + } + e.ruStatusRecorded = true + recordExplainRUStatus(status) +} + +// explainRUSelectGateStatus is the first-demo pre-execution safety gate. It +// accepts only SELECT keyword surfaces and set operations whose leaves can be +// checked before EXPLAIN ANALYZE can run the target statement. +func explainRUSelectGateStatus(stmt ast.StmtNode) explainRUStatus { + switch x := stmt.(type) { + case *ast.SelectStmt: + return explainRUValidateSelectNode(x) + case *ast.SetOprStmt: + if x == nil || x.SelectList == nil { + return explainRUStatusUnsupportedNonSelect + } + return explainRUValidateSetOprSelectList(x.SelectList) + default: + return explainRUStatusUnsupportedNonSelect + } +} + +func explainRUValidateSetOprSelectList(list *ast.SetOprSelectList) explainRUStatus { + if list == nil || len(list.Selects) == 0 { + return explainRUStatusUnsupportedNonSelect + } + for _, sel := range list.Selects { + switch x := sel.(type) { + case *ast.SelectStmt: + if status := explainRUValidateSelectNode(x); status != explainRUStatusSuccess { + return status + } + case *ast.SetOprSelectList: + if status := explainRUValidateSetOprSelectList(x); status != explainRUStatusSuccess { + return status + } + default: + // SetOprSelectList is documented as SELECT/TABLE/VALUES capable. Fail + // closed until non-SELECT leaves have an explicit attribution design. + return explainRUStatusUnsupportedNonSelect + } + } + visitor := &explainRUSideEffectVisitor{status: explainRUStatusSuccess} + list.Accept(visitor) + return visitor.status +} + +func explainRUValidateSelectNode(sel *ast.SelectStmt) explainRUStatus { + if sel == nil || sel.Kind != ast.SelectStmtKindSelect { + return explainRUStatusUnsupportedNonSelect + } + if sel.SelectIntoOpt != nil { + return explainRUStatusUnsupportedSideEffecting + } + if sel.LockInfo != nil && sel.LockInfo.LockType != ast.SelectLockNone { + return explainRUStatusUnsupportedLockingSelect + } + visitor := &explainRUSideEffectVisitor{status: explainRUStatusSuccess} + sel.Accept(visitor) + return visitor.status +} + +type explainRUSideEffectVisitor struct { + status explainRUStatus +} + +func (v *explainRUSideEffectVisitor) Enter(n ast.Node) (ast.Node, bool) { + if v.status != explainRUStatusSuccess { + return n, true + } + switch x := n.(type) { + case *ast.SelectStmt: + if x.Kind != ast.SelectStmtKindSelect { + v.status = explainRUStatusUnsupportedNonSelect + return n, true + } + if x.SelectIntoOpt != nil { + v.status = explainRUStatusUnsupportedSideEffecting + return n, true + } + if x.LockInfo != nil && x.LockInfo.LockType != ast.SelectLockNone { + v.status = explainRUStatusUnsupportedLockingSelect + return n, true + } + case *ast.VariableExpr: + // User-variable assignment is syntactically a SELECT expression but + // mutates session state, so it is outside the side-effect-free demo scope. + if x.Value != nil { + v.status = explainRUStatusUnsupportedSideEffecting + return n, true + } + case *ast.FuncCallExpr: + if explainRUFuncCallHasSideEffect(x) { + v.status = explainRUStatusUnsupportedSideEffecting + return n, true + } + } + return n, false +} + +func (v *explainRUSideEffectVisitor) Leave(n ast.Node) (ast.Node, bool) { + return n, v.status == explainRUStatusSuccess +} + +func explainRUFuncCallHasSideEffect(fn *ast.FuncCallExpr) bool { + if fn == nil { + return false + } + switch strings.ToLower(fn.FnName.L) { + case ast.GetLock, ast.ReleaseLock, ast.ReleaseAllLocks, ast.NextVal, ast.SetVal, ast.Sleep: + return true + case ast.LastInsertId: + return len(fn.Args) > 0 + default: + return false + } +} + +func (e *Explain) renderRUExplain() (err error) { + start := time.Now() + status := explainRUStatusError + defer func() { + metrics.ObserveExplainRURenderDuration(string(status), time.Since(start).Seconds()) + e.recordExplainRUStatus(status) + }() + + if !e.Analyze { + status = explainRUStatusUnsupportedNonAnalyze + return explainRUError(explainRUStatusUnsupportedNonAnalyze) + } + if gateStatus := explainRUSelectGateStatus(e.ExecStmt); gateStatus != explainRUStatusSuccess { + status = gateStatus + return explainRUError(gateStatus) + } + flat := FlattenPhysicalPlan(e.TargetPlan, true) + if flat == nil || len(flat.Main) == 0 || flat.InExplain { + return errors.NewNoStackError("EXPLAIN ANALYZE FORMAT='RU' cannot render an empty target plan") + } + runtimeStats := e.RuntimeStatsColl + if runtimeStats == nil && e.SCtx() != nil && e.SCtx().GetSessionVars() != nil { + runtimeStats = e.SCtx().GetSessionVars().StmtCtx.RuntimeStatsColl + } + // The snapshot belongs to the target statement execution. Returning this + // EXPLAIN result can add more result-chunk counters later, so render output + // and Demo Metrics are derived from this frozen input and the generated rows. + _, snapshotStatus := explainRUExtractComponentSnapshot(runtimeStats, e.TargetPlan.ID()) + metrics.RecordExplainRUComponentSnapshot(string(snapshotStatus)) + result := buildReadBillingDemoResult(e.SCtx(), e.TargetPlan, e.ExecStmt, nil) + if result.status != readBillingDemoStatusSuccess { + operator := "" + if len(result.operators) > 0 { + op := result.operators[0] + operator = " operator=" + op.site + "/" + op.opClass + "/" + op.operatorKind + } + status = explainRUStatusError + return errors.NewNoStackErrorf( + "EXPLAIN ANALYZE FORMAT='RU' cannot render a complete read billing model result: status=%s reason=%s%s", + result.status, + result.reason, + operator, + ) + } + rows := explainRUBuildReadBillingRows(result, snapshotStatus) + + e.Rows = make([][]string, 0, len(rows)) + for _, row := range rows { + e.Rows = append(e.Rows, row.toStrings()) + explainRUObserveRow(row) + } + status = explainRUStatusSuccess + return nil +} + +func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus explainRUComponentSnapshotStatus) []explainRURow { + rows := []explainRURow{{ + section: explainRUSectionSummary, + component: "total_preview_ru", + hasPreviewRU: true, + source: explainRUSourceSummaryTotal, + note: explainRUReadBillingSummaryNote(snapshotStatus), + }} + totalPreviewRU := 0.0 + for _, op := range result.operators { + if op.status != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + continue + } + weights, hasWeights := readBillingDemoResolveWeights(op.site, op.opClass, readBillingDemoWeightVersion) + for _, unit := range op.units { + row := explainRUReadBillingUnitRow(op, unit) + if hasWeights { + if weight, previewRU, ok := readBillingDemoUnitPreviewRU(unit, weights); ok { + row.weight = weight + row.hasWeight = true + row.previewRU = previewRU + row.hasPreviewRU = true + totalPreviewRU += previewRU + } + } else { + row.note = appendExplainRUNote(row.note, "missing_weight") + } + rows = append(rows, row) + } + } + rows[0].previewRU = totalPreviewRU + return rows +} + +func explainRUReadBillingSummaryNote(snapshotStatus explainRUComponentSnapshotStatus) string { + note := "weight_version=" + readBillingDemoWeightVersion + if snapshotStatus != explainRUComponentSnapshotOK { + note = appendExplainRUNote(note, "component_snapshot_"+string(snapshotStatus)) + } + return note +} + +func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBillingDemoUnit) explainRURow { + row := explainRURow{ + section: explainRUSectionPlan, + id: op.id, + component: op.operatorKind, + operatorClass: op.site + "/" + op.opClass, + rowWidth: unit.rowWidth, + hasRowWidth: unit.rowWidth > 0, + rowWidthSource: unit.widthSource, + unit: unit.unit, + source: unit.source, + note: "input_side=" + unit.side + ",weight_version=" + readBillingDemoWeightVersion, + } + if op.hasActRows { + row.actRows = op.actRows + row.hasActRows = true + row.outputRows = op.actRows + row.hasOutputRows = true + } + switch unit.unit { + case readBillingDemoUnitFixedEvents: + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitInputRows: + row.inputRows = int64(unit.value) + row.hasInputRows = true + row.workRows = int64(unit.value) + row.hasWorkRows = true + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitInputBytes: + row.workBytes = unit.value + row.hasWorkBytes = true + } + return row +} + +func appendExplainRUNote(note, extra string) string { + if note == "" { + return extra + } + if extra == "" { + return note + } + return note + "," + extra +} + +func explainRUExtractComponentSnapshot(runtimeStats *execdetails.RuntimeStatsColl, targetPlanID int) (*execdetails.RURuntimeStats, explainRUComponentSnapshotStatus) { + // GetRootStats creates an empty entry for a missing plan ID; check + // ExistsRootStats first so "missing snapshot" stays observable. + if runtimeStats == nil || !runtimeStats.ExistsRootStats(targetPlanID) { + return nil, explainRUComponentSnapshotMissing + } + _, groups := runtimeStats.GetRootStats(targetPlanID).MergeStats() + for _, group := range groups { + ruStats, ok := group.(*execdetails.RURuntimeStats) + if !ok { + continue + } + if ruStats.RUVersion != rmclient.RUVersionV2 { + return ruStats, explainRUComponentSnapshotNonV2 + } + if ruStats.Metrics == nil { + return ruStats, explainRUComponentSnapshotNilMetrics + } + if ruStats.Metrics.Bypass() { + return ruStats, explainRUComponentSnapshotBypassed + } + return ruStats, explainRUComponentSnapshotOK + } + return nil, explainRUComponentSnapshotMissing +} + +func explainRURowWidth(sctx base.PlanContext, p base.Plan, includedRoot bool) (float64, string) { + if includedRoot { + // Root reader helpers describe SQL-visible row width for included TiDB + // work. Scan helpers below describe TiKV cop-side scan input width. + switch x := p.(type) { + case *physicalop.PhysicalTableReader: + if width := x.GetAvgRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + case *physicalop.PhysicalIndexLookUpReader: + if width := x.GetAvgTableRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + case *physicalop.PhysicalIndexMergeReader: + if width := x.GetAvgTableRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + case *physicalop.PointGetPlan: + if width := x.GetAvgRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + case *physicalop.BatchPointGetPlan: + if width := x.GetAvgRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + } + } else { + switch x := p.(type) { + case *physicalop.PhysicalTableScan: + if width := x.GetScanRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + case *physicalop.PhysicalIndexScan: + if width := x.GetScanRowSize(); width > 0 { + return width, explainRUWidthSourceOperatorHelper + } + } + } + if sctx == nil { + sctx = p.SCtx() + } + if stats := p.StatsInfo(); sctx != nil && stats != nil && stats.HistColl != nil && p.Schema() != nil { + if width := cardinality.GetAvgRowSize(sctx, stats.HistColl, p.Schema().Columns, false, false); width > 0 { + return width, explainRUWidthSourcePlanStats + } + } + if p.Schema() != nil { + width := 0 + for _, col := range p.Schema().Columns { + width += chunk.EstimateTypeWidth(col.GetStaticType()) + } + if width > 0 { + return float64(width), explainRUWidthSourceSchemaTypeWidth + } + if len(p.Schema().Columns) > 0 { + return float64(8 * len(p.Schema().Columns)), explainRUWidthSourceSchemaFallback + } + } + return 8, explainRUWidthSourceSchemaFallback +} + +func (row explainRURow) toStrings() []string { + return []string{ + row.section, + row.id, + row.component, + row.operatorClass, + formatOptionalInt(row.actRows, row.hasActRows), + formatOptionalInt(row.inputRows, row.hasInputRows), + formatOptionalInt(row.outputRows, row.hasOutputRows), + formatOptionalFloat(row.rowWidth, row.hasRowWidth), + row.rowWidthSource, + formatOptionalInt(row.workRows, row.hasWorkRows), + formatOptionalFloat(row.workBytes, row.hasWorkBytes), + row.unit, + formatOptionalInt(row.count, row.hasCount), + formatOptionalFloat(row.weight, row.hasWeight), + formatOptionalFloat(row.previewRU, row.hasPreviewRU), + row.source, + row.note, + } +} + +func formatOptionalInt(v int64, ok bool) string { + if !ok { + return "" + } + return strconv.FormatInt(v, 10) +} + +func formatOptionalFloat(v float64, ok bool) string { + if !ok { + return "" + } + return strconv.FormatFloat(v, 'f', 6, 64) +} + +func explainRUObserveRow(row explainRURow) { + // Metrics are emitted from rendered rows so the Prometheus view matches the + // SQL output and avoids reading live counters after render-side accounting. + previewRU := -1.0 + if row.hasPreviewRU { + previewRU = row.previewRU + } + workRows := -1.0 + if row.hasWorkRows { + workRows = float64(row.workRows) + } + workBytes := -1.0 + if row.hasWorkBytes { + workBytes = row.workBytes + } + rowWidth := -1.0 + if row.section == explainRUSectionPlan && row.hasRowWidth { + rowWidth = row.rowWidth + } + component, operator := explainRUMetricComponentOperator(row) + metrics.ObserveExplainRURow(row.section, component, operator, row.source, row.rowWidthSource, previewRU, workRows, workBytes, rowWidth) +} + +func explainRUMetricComponentOperator(row explainRURow) (component, operator string) { + switch row.section { + case explainRUSectionPlan: + return "", row.component + default: + return row.component, "" + } +} + +func explainRUUnsupportedFormatError(format string) error { + return errors.Errorf("'explain format=%v' cannot work without 'analyze', please use 'explain analyze format=%v'", format, format) +} + +func isExplainRUFormat(format string) bool { + return strings.ToLower(format) == types.ExplainFormatRU +} diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index efc9dbbe2498b..7bfa7bdb051b3 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -5712,6 +5712,18 @@ func (b *PlanBuilder) buildTrace(trace *ast.TraceStmt) (base.Plan, error) { func (b *PlanBuilder) buildExplainPlan(targetPlan base.Plan, format string, explainBriefBinary string, analyze, explore bool, execStmt ast.StmtNode, runtimeStats *execdetails.RuntimeStatsColl, sqlDigest, replayerFile string) (base.Plan, error) { format = strings.ToLower(format) + if format == types.ExplainFormatRU && !analyze { + recordExplainRUStatus(explainRUStatusUnsupportedNonAnalyze) + return nil, explainRUUnsupportedFormatError(format) + } + if format == types.ExplainFormatRU && analyze { + // Secondary guard for callers that already have a target plan. The main + // SQL path below rejects before optimization or no-delay execution. + if status := explainRUSelectGateStatus(execStmt); status != explainRUStatusSuccess { + recordExplainRUStatus(status) + return nil, explainRUError(status) + } + } if format == types.ExplainFormatTrueCardCost && !analyze { return nil, errors.Errorf("'explain format=%v' cannot work without 'analyze', please use 'explain analyze format=%v'", format, format) } @@ -5748,6 +5760,12 @@ func (b *PlanBuilder) buildExplainFor(explainFor *ast.ExplainForStmt) (base.Plan targetPlan, ok := processInfo.Plan.(base.Plan) explainForFormat := strings.ToLower(explainFor.Format) + // Check RU before the missing-plan branch so idle and running connections + // return the same unsupported status. + if explainForFormat == types.ExplainFormatRU { + recordExplainRUStatus(explainRUStatusUnsupportedForConnection) + return nil, errors.Errorf("explain format '%s' for connection is not supported now: %s", explainForFormat, explainRUStatusUnsupportedForConnection) + } if !ok || targetPlan == nil { return &Explain{Format: explainForFormat}, nil } @@ -5801,6 +5819,23 @@ func (b *PlanBuilder) buildExplain(ctx context.Context, explain *ast.ExplainStmt } explain.Stmt = hintedStmt } + if isExplainRUFormat(explain.Format) { + // This is the primary FORMAT='RU' safety boundary. It runs after plan + // digest expansion but before target optimization/execution, which keeps + // DML no-delay paths and side-effecting SELECT forms from running first. + if !explain.Analyze { + recordExplainRUStatus(explainRUStatusUnsupportedNonAnalyze) + return nil, explainRUUnsupportedFormatError(strings.ToLower(explain.Format)) + } + if explain.Explore { + recordExplainRUStatus(explainRUStatusUnsupportedNonSelect) + return nil, explainRUError(explainRUStatusUnsupportedNonSelect) + } + if status := explainRUSelectGateStatus(explain.Stmt); status != explainRUStatusSuccess { + recordExplainRUStatus(status) + return nil, explainRUError(status) + } + } sctx, err := AsSctx(b.ctx) if err != nil { diff --git a/pkg/session/session.go b/pkg/session/session.go index a414b90cd3941..869a24dcaafea 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -2424,10 +2424,16 @@ func (s *session) ExecuteStmt(ctx context.Context, stmtNode ast.StmtNode) (sqlex return rs, err } -func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (sqlexec.RecordSet, error) { +func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (recordSet sqlexec.RecordSet, err error) { r, ctx := tracing.StartRegionEx(ctx, "session.ExecuteStmt") defer r.End() ctx = execdetails.ContextWithMissingExecDetailsInitialized(ctx) + readBillingDemoHandledByStmt := false + defer func() { + if err != nil && !readBillingDemoHandledByStmt { + s.recordReadBillingDemoEarlyError(stmtNode, err) + } + }() if err := s.PrepareTxnCtx(ctx, stmtNode); err != nil { return nil, err @@ -2572,7 +2578,6 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s } var stmt *executor.ExecStmt - var err error { // Transform abstract syntax tree to a physical plan(stored in executor.ExecStmt). @@ -2678,7 +2683,6 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s }() } - var recordSet sqlexec.RecordSet if stmt.PsStmt != nil { // point plan short path ctx, prevTraceID := resetStmtTraceID(ctx, s) @@ -2711,6 +2715,7 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (s s.setLastTxnInfoBeforeTxnEnd() s.txn.changeToInvalid() } else { + readBillingDemoHandledByStmt = true recordSet, err = runStmt(ctx, s, stmt) } @@ -2766,6 +2771,17 @@ func resolvePreparedStmt(stmt ast.StmtNode, vars *variable.SessionVars) (ast.Stm return prepareStmt.PreparedAst.Stmt, nil } +func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err error) { + if err == nil { + return + } + metricsStmt := stmtNode + if resolvedStmt, resolveErr := resolvePreparedStmt(stmtNode, s.sessionVars); resolveErr == nil && resolvedStmt != nil { + metricsStmt = resolvedStmt + } + plannercore.RecordReadBillingDemoForStatement(s, nil, metricsStmt, err) +} + func shouldBypass(ctx context.Context, stmtNode ast.StmtNode, sessVars *variable.SessionVars) bool { switch kv.GetInternalSourceType(ctx) { case kv.InternalTxnOthers: @@ -2992,6 +3008,7 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. origTxnCtx := sessVars.TxnCtx err = se.checkTxnAborted(s) if err != nil { + se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) return nil, err } if sessVars.TxnCtx.CouldRetry && !s.IsReadOnly(sessVars) { @@ -2999,6 +3016,7 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. // otherwise, the stmt won't be add into stmt history, and also don't need check. // About `stmt-count-limit`, see more in https://docs.pingcap.com/tidb/stable/tidb-configuration-file#stmt-count-limit if err := checkStmtLimit(ctx, se, false); err != nil { + se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) return nil, err } } diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index cbe8896a9c9cd..fda9d39bf8d60 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -768,6 +768,9 @@ const ( // TiDBEnableCollectExecutionInfo indicates that whether execution info is collected. TiDBEnableCollectExecutionInfo = "tidb_enable_collect_execution_info" + // TiDBEnableReadBillingDemo enables the first-version read billing demo base-unit metrics. + TiDBEnableReadBillingDemo = "tidb_enable_read_billing_demo" + // TiDBExecutorConcurrency is used for controlling the concurrency of all types of executors. TiDBExecutorConcurrency = "tidb_executor_concurrency" @@ -1594,6 +1597,7 @@ const ( DefTiDBFoundInPlanCache = false DefTiDBFoundInBinding = false DefTiDBEnableCollectExecutionInfo = true + DefTiDBEnableReadBillingDemo = false DefTiDBAllowAutoRandExplicitInsert = false DefTiDBEnableClusteredIndex = ClusteredIndexDefModeOn DefTiDBRedactLog = Off diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 718703e440cfd..a0816309e0619 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1231,6 +1231,9 @@ type SessionVars struct { // AllowProjectionPushDown enables pushdown projection on TiKV. AllowProjectionPushDown bool + // EnableReadBillingDemo indicates whether read billing demo metrics are emitted for SELECT reads. + EnableReadBillingDemo bool + // EnableStrictNotNullCheck enables strict not-null check for single-row insert in non-strict mode. EnableStrictNotNullCheck bool @@ -2491,6 +2494,7 @@ func NewSessionVars(hctx HookContext) *SessionVars { OptOrderingIdxSelRatio: vardef.DefTiDBOptOrderingIdxSelRatio, RegardNULLAsPoint: vardef.DefTiDBRegardNULLAsPoint, AllowProjectionPushDown: vardef.DefOptEnableProjectionPushDown, + EnableReadBillingDemo: vardef.DefTiDBEnableReadBillingDemo, SkipMissingPartitionStats: vardef.DefTiDBSkipMissingPartitionStats, IndexLookUpPushDownPolicy: vardef.DefTiDBIndexLookUpPushDownPolicy, OptPartialOrderedIndexForTopN: vardef.DefTiDBOptPartialOrderedIndexForTopN, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index cc4c97f600e67..843f91caeb6b1 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -609,6 +609,10 @@ var defaultSysVars = []*SysVar{ }, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return BoolToOnOff(config.GetGlobalConfig().Instance.EnableCollectExecutionInfo.Load()), nil }}, + {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableReadBillingDemo, Value: BoolToOnOff(vardef.DefTiDBEnableReadBillingDemo), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableReadBillingDemo = TiDBOptOn(val) + return nil + }}, {Scope: vardef.ScopeInstance, Name: vardef.PluginLoad, Value: "", ReadOnly: true, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return config.GetGlobalConfig().Instance.PluginLoad, nil }}, diff --git a/pkg/types/explain_format.go b/pkg/types/explain_format.go index 67aabe4b27287..95a8dd07385c5 100644 --- a/pkg/types/explain_format.go +++ b/pkg/types/explain_format.go @@ -41,6 +41,8 @@ var ( ExplainFormatPlanCache = "plan_cache" // ExplainFormatPlanTree displays the plan in a tree structure format ExplainFormatPlanTree = "plan_tree" + // ExplainFormatRU displays read billing model attribution for EXPLAIN ANALYZE. + ExplainFormatRU = "ru" // ExplainFormats stores the valid formats for explain statement, used by validator. ExplainFormats = []string{ @@ -57,5 +59,6 @@ var ( ExplainFormatCostTrace, ExplainFormatPlanCache, ExplainFormatPlanTree, + ExplainFormatRU, } ) diff --git a/pkg/util/execdetails/runtime_stats.go b/pkg/util/execdetails/runtime_stats.go index 7dbf875c4faab..7ff46a5401b2f 100644 --- a/pkg/util/execdetails/runtime_stats.go +++ b/pkg/util/execdetails/runtime_stats.go @@ -231,6 +231,14 @@ func (crs *CopRuntimeStats) GetTasks() int32 { return int32(crs.stats.procTimes.size) } +// GetScanDetail returns a copy of the coprocessor scan detail collected for this plan. +func (crs *CopRuntimeStats) GetScanDetail() util.ScanDetail { + if crs == nil { + return util.ScanDetail{} + } + return crs.scanDetail +} + var zeroTimeDetail = util.TimeDetail{} func (crs *CopRuntimeStats) String() string { From e644416eda7feb3450158f8e76a9e518047e564b Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 1 Jul 2026 03:40:00 +0800 Subject: [PATCH 02/25] test: cover read billing RU model --- pkg/executor/explain_test.go | 361 ++++++++++++++++++++++ pkg/executor/explainfor_test.go | 22 ++ pkg/metrics/metrics_internal_test.go | 102 +++++++ pkg/planner/core/common_plans_test.go | 424 ++++++++++++++++++++++++++ 4 files changed, 909 insertions(+) diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 8d7e04e436d6b..a8f273a5178e2 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -16,8 +16,11 @@ package executor_test import ( "bytes" + "context" "encoding/json" "fmt" + "os" + "path/filepath" "regexp" "strconv" "strings" @@ -25,9 +28,15 @@ import ( "time" "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/metrics" + "github.com/pingcap/tidb/pkg/parser/auth" plannercore "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/sqlexec" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -496,6 +505,7 @@ func TestExplainFormatInCtx(t *testing.T) { types.ExplainFormatTiDBJSON, types.ExplainFormatCostTrace, types.ExplainFormatPlanCache, + types.ExplainFormatRU, } tk.MustExec("select * from t") @@ -515,6 +525,357 @@ func TestExplainFormatInCtx(t *testing.T) { } } +func TestExplainAnalyzeFormatRUOutput(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + rows := tk.MustQuery("explain analyze format='ru' select 1").Rows() + require.NotEmpty(t, rows) + require.Equal(t, "summary", rows[0][0]) + require.Equal(t, "total_preview_ru", rows[0][2]) + require.Equal(t, "summary_total", rows[0][15]) + requireExplainRUPlanRow(t, rows) + + tk.MustExec("drop table if exists explain_ru_t") + tk.MustExec("create table explain_ru_t(a int primary key, b varchar(20))") + tk.MustExec("insert into explain_ru_t values (1, 'x'), (2, 'yy')") + rows = tk.MustQuery("explain analyze format='ru' select * from explain_ru_t where a > 0").Rows() + requireExplainRUPlanRow(t, rows) + requireExplainRUOperatorClass(t, rows, "tikv/kv_range_scan") + + rows = tk.MustQuery("explain analyze format='ru' select * from explain_ru_t where a = 1").Rows() + requireExplainRUWeightedOperatorClass(t, rows, "tikv/kv_point_lookup") +} + +func TestExplainAnalyzeFormatRUTiKVCopOperatorClasses(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_cop") + tk.MustExec("create table explain_ru_cop(a int primary key, b int, c varchar(20), key idx_b(b))") + tk.MustExec("insert into explain_ru_cop values (1, 10, 'a'), (2, 20, 'bb'), (3, 20, 'ccc'), (4, 30, 'dddd')") + + cases := []struct { + sql string + opClasses []string + }{ + { + sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) where b > 10", + opClasses: []string{ + "tikv/filter_eval", + "tikv/kv_range_scan", + }, + }, + { + sql: "explain analyze format='ru' select a from explain_ru_cop where b > 10", + opClasses: []string{ + "tikv/projection_eval", + "tikv/kv_range_scan", + }, + }, + { + sql: "explain analyze format='ru' select * from explain_ru_cop limit 2", + opClasses: []string{ + "tikv/row_limit", + "tikv/kv_range_scan", + }, + }, + { + sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) order by c limit 2", + opClasses: []string{ + "tikv/bounded_topn", + "tikv/kv_range_scan", + }, + }, + { + sql: "explain analyze format='ru' select /*+ agg_to_cop(), hash_agg() */ b, count(*) from explain_ru_cop group by b", + opClasses: []string{ + "tikv/agg_hash", + "tikv/kv_range_scan", + }, + }, + { + sql: "explain analyze format='ru' select /*+ agg_to_cop(), stream_agg() */ b, count(*) from explain_ru_cop group by b", + opClasses: []string{ + "tikv/agg_stream", + "tikv/kv_range_scan", + }, + }, + } + for _, tc := range cases { + rows := tk.MustQuery(tc.sql).Rows() + for _, opClass := range tc.opClasses { + requireExplainRUWeightedOperatorClass(t, rows, opClass) + } + } +} + +func requireExplainRUPlanRow(t *testing.T, rows [][]any) { + t.Helper() + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" { + continue + } + require.NotEmpty(t, row[1]) + require.NotEmpty(t, row[2]) + require.Contains(t, fmt.Sprint(row[3]), "/") + require.NotEmpty(t, row[4]) + require.NotEmpty(t, row[7]) + require.NotEmpty(t, row[8]) + require.NotEmpty(t, row[11]) + require.NotEmpty(t, row[12]) + require.NotEmpty(t, row[13]) + require.NotEmpty(t, row[14]) + require.NotEmpty(t, row[15]) + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v1") + return + } + require.Fail(t, "missing FORMAT='RU' plan row") +} + +func requireExplainRUOperatorClass(t *testing.T, rows [][]any, operatorClass string) { + t.Helper() + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" || row[3] != operatorClass { + continue + } + return + } + require.Failf(t, "missing FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) +} + +func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorClass string) { + t.Helper() + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" || row[3] != operatorClass { + continue + } + require.NotEmpty(t, row[13], "missing weight for %s row %v", operatorClass, row) + require.NotEmpty(t, row[14], "missing preview RU for %s row %v", operatorClass, row) + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v1") + return + } + require.Failf(t, "missing weighted FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) +} + +func TestExplainAnalyzeFormatRUPlanDigest(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + + originEnableStmtSummary := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_stmt_summary").Rows()[0][0]) + originStmtSummaryRefreshInterval := fmt.Sprint(tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Rows()[0][0]) + originStmtSummaryHistorySize := fmt.Sprint(tk.MustQuery("select @@global.tidb_stmt_summary_history_size").Rows()[0][0]) + defer tk.MustExec(fmt.Sprintf("set global tidb_enable_stmt_summary = %s", originEnableStmtSummary)) + defer tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_refresh_interval = %s", originStmtSummaryRefreshInterval)) + defer tk.MustExec(fmt.Sprintf("set global tidb_stmt_summary_history_size = %s", originStmtSummaryHistorySize)) + tk.MustExec("set global tidb_stmt_summary_history_size = 24") + tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + + tk.MustExec("drop table if exists explain_ru_digest") + tk.MustExec("create table explain_ru_digest(a int primary key, b int)") + tk.MustExec("insert into explain_ru_digest values (1, 10)") + tk.MustQuery("select b from explain_ru_digest where a = 1").Check(testkit.Rows("10")) + + digestRows := tk.MustQuery("select plan_digest from information_schema.statements_summary_history where digest_text like 'select `b` from `explain_ru_digest`%' and plan_digest != ''").Rows() + require.NotEmpty(t, digestRows) + planDigest := fmt.Sprint(digestRows[0][0]) + rows := tk.MustQuery(fmt.Sprintf("explain analyze format='ru' '%s'", planDigest)).Rows() + require.NotEmpty(t, rows) + require.Equal(t, "summary", rows[0][0]) + requireExplainRUPlanRow(t, rows) +} + +func TestExplainAnalyzeFormatRUUnsupportedTargetsBeforeExecution(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_dml") + tk.MustExec("create table explain_ru_dml(a int primary key)") + tk.MustExec("insert into explain_ru_dml values (1)") + + err := tk.ExecToErr("explain format='ru' select 1") + require.Error(t, err) + require.Contains(t, err.Error(), "cannot work without 'analyze'") + require.Contains(t, err.Error(), "format=ru") + + err = tk.ExecToErr("explain analyze format=ru select 1") + require.Error(t, err) + + err = tk.ExecToErr("explain analyze format='ru' insert into explain_ru_dml values (2)") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_non_select") + tk.MustQuery("select * from explain_ru_dml order by a").Check(testkit.Rows("1")) + + for _, sql := range []string{ + "explain analyze format='ru' update explain_ru_dml set a = 2 where a = 1", + "explain analyze format='ru' delete from explain_ru_dml where a = 1", + "explain analyze format='ru' replace into explain_ru_dml values (2)", + "explain analyze format='ru' alter table explain_ru_dml add column b int", + "explain analyze format='ru' import into explain_ru_dml from select * from explain_ru_dml", + } { + err = tk.ExecToErr(sql) + require.Error(t, err, sql) + require.Contains(t, err.Error(), "unsupported_non_select", sql) + tk.MustQuery("select * from explain_ru_dml order by a").Check(testkit.Rows("1")) + } + tk.MustQuery("show columns from explain_ru_dml").Check(testkit.Rows("a int(11) NO PRI ")) + + err = tk.ExecToErr("explain analyze format='ru' values (1)") + require.Error(t, err) + + err = tk.ExecToErr("explain analyze format='ru' table explain_ru_dml") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_non_select") + + tk.MustExec("set @explain_ru_var := 7") + err = tk.ExecToErr("explain analyze format='ru' select @explain_ru_var := 9") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + tk.MustQuery("select @explain_ru_var").Check(testkit.Rows("7")) + + tk.MustQuery("select last_insert_id(11)").Check(testkit.Rows("11")) + err = tk.ExecToErr("explain analyze format='ru' select last_insert_id(123)") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + tk.MustQuery("select last_insert_id()").Check(testkit.Rows("11")) + + outFile := filepath.Join(t.TempDir(), "explain_ru.csv") + err = tk.ExecToErr(fmt.Sprintf("explain analyze format='ru' select 1 into outfile %q", outFile)) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + _, statErr := os.Stat(outFile) + require.True(t, os.IsNotExist(statErr)) + + err = tk.ExecToErr("explain analyze format='ru' select * from explain_ru_dml for update skip locked") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_locking_select") + + err = tk.ExecToErr("explain analyze format='ru' select get_lock('explain_ru', 0)") + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_side_effecting_select") + tk.MustQuery("select is_free_lock('explain_ru')").Check(testkit.Rows("1")) +} + +func TestReadBillingDemoMetricsHook(t *testing.T) { + metrics.InitExplainRUMetrics() + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustQuery("select @@tidb_enable_read_billing_demo").Check(testkit.Rows("0")) + tk.MustExec("drop table if exists read_billing_demo") + tk.MustExec("create table read_billing_demo(a int primary key)") + tk.MustExec("insert into read_billing_demo values (1), (2)") + + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v1") + unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v1") + unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v1") + errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v1") + projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_act_rows", "all", "v1") + + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Equal(t, 0.0, readExecutorCounterValue(t, success)) + + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Equal(t, 1.0, readExecutorCounterValue(t, success)) + require.Equal(t, 1.0, readExecutorCounterValue(t, projectionFixedEvents)) + + beforeRestrictedSuccess := readExecutorCounterValue(t, success) + beforeRestrictedBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) + internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnOthers) + _, _, restrictedErr := tk.Session().GetRestrictedSQLExecutor().ExecRestrictedSQL(internalCtx, []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession}, "select 1 + 1") + require.NoError(t, restrictedErr) + require.Equal(t, beforeRestrictedSuccess, readExecutorCounterValue(t, success)) + require.Equal(t, beforeRestrictedBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("prepare read_billing_demo_stmt from 'select ? + 1'") + tk.MustExec("set @read_billing_demo_param := 2") + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustQuery("execute read_billing_demo_stmt using @read_billing_demo_param").Check(testkit.Rows("3")) + require.Equal(t, 2.0, readExecutorCounterValue(t, success)) + + beforeBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) + beforeBaseUnitsTotal := readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + beforeUnsupported := readExecutorCounterValue(t, unsupported) + tk.MustExec("insert into read_billing_demo values (3)") + require.Equal(t, beforeUnsupported+1, readExecutorCounterValue(t, unsupported)) + require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + beforeUnknownInput := readExecutorCounterValue(t, unknownInput) + beforeBaseUnits = readExecutorCounterValue(t, projectionFixedEvents) + beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + plannercore.RecordReadBillingDemoForStatement(tk.Session(), nil, nil, nil) + require.Equal(t, beforeUnknownInput+1, readExecutorCounterValue(t, unknownInput)) + require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("create table read_billing_compile_error(a int)") + tk.MustExec("prepare read_billing_compile_error_stmt from 'select * from read_billing_compile_error'") + tk.MustExec("drop table read_billing_compile_error") + tk.MustExec("set tidb_enable_read_billing_demo=on") + beforeError := readExecutorCounterValue(t, errorStatus) + beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + err := tk.ExecToErr("execute read_billing_compile_error_stmt") + require.Error(t, err) + require.Equal(t, beforeError+1, readExecutorCounterValue(t, errorStatus)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + beforeEarlyError := readExecutorCounterValue(t, errorStatus) + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = tk.ExecWithContext(canceledCtx, "select 1") + require.Error(t, err) + require.Equal(t, beforeEarlyError+1, readExecutorCounterValue(t, errorStatus)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + + tk.MustExec("drop table if exists read_billing_outer") + tk.MustExec("create table read_billing_outer(a int)") + tk.MustExec("insert into read_billing_outer values (0)") + tk.MustExec("set @@tidb_init_chunk_size=1") + rs, err := tk.Exec("select (select t.a from read_billing_demo t where t.a > o.a) from read_billing_outer o") + require.NoError(t, err) + err = rs.Next(context.TODO(), rs.NewChunk(nil)) + require.Error(t, err) + require.NoError(t, rs.Close()) + require.Equal(t, beforeEarlyError+2, readExecutorCounterValue(t, errorStatus)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) +} + +func readExecutorCounterValue(t *testing.T, counter prometheus.Counter) float64 { + t.Helper() + m := &dto.Metric{} + require.NoError(t, counter.Write(m)) + return m.GetCounter().GetValue() +} + +func readExecutorCounterVecTotal(t *testing.T, collector prometheus.Collector) float64 { + t.Helper() + ch := make(chan prometheus.Metric) + go func() { + collector.Collect(ch) + close(ch) + }() + var total float64 + for metric := range ch { + m := &dto.Metric{} + require.NoError(t, metric.Write(m)) + total += m.GetCounter().GetValue() + } + return total +} + func TestExplainImportFromSelect(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/pkg/executor/explainfor_test.go b/pkg/executor/explainfor_test.go index d2238ae879c60..1d25cdd26c045 100644 --- a/pkg/executor/explainfor_test.go +++ b/pkg/executor/explainfor_test.go @@ -336,6 +336,28 @@ func TestExplainDotForExplainPlan(t *testing.T) { require.Contains(t, err.Error(), "explain format 'dot' for connection is not supported now") } +func TestExplainRUForConnectionUnsupported(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + + connIDRows := tk.MustQuery("select connection_id()").Rows() + require.Len(t, connIDRows, 1) + connID := connIDRows[0][0].(string) + + tkProcess := tk.Session().ShowProcess() + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: []*sessmgr.ProcessInfo{tkProcess}}) + err := tk.ExecToErr(fmt.Sprintf("explain format='ru' for connection %s", connID)) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_for_connection") + + tk.MustQuery("select 1") + tkProcess = tk.Session().ShowProcess() + tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: []*sessmgr.ProcessInfo{tkProcess}}) + err = tk.ExecToErr(fmt.Sprintf("explain format='ru' for connection %s", connID)) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported_for_connection") +} + func TestExplainDotForQuery(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index a0fd8e1b66f3b..747f3e9eb05f5 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -95,6 +95,95 @@ func TestRUV2ExecutorCounterReturnsCachedKnownLabels(t *testing.T) { } } +func TestExplainRUMetrics(t *testing.T) { + InitExplainRUMetrics() + + RecordExplainRUStatus("success") + ObserveExplainRURenderDuration("success", 0.01) + RecordExplainRUComponentSnapshot("ok") + ObserveExplainRURow("plan", "", "projection", "read_billing_model", "plan_stats", 1.25, 3, 24, 8) + RecordReadBillingDemoStatement("success", "v1") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v1") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_act_rows", "all", "v1", 3) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "plan_stats", "v1", 8) + + require.Equal(t, 1.0, readCounterValue(t, ExplainRUStatementsCounter.WithLabelValues("success"))) + require.Equal(t, 1.0, readCounterValue(t, ExplainRUComponentSnapshotCounter.WithLabelValues("ok"))) + require.Equal(t, 1.25, readCounterValue(t, ExplainRUPreviewRUCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) + require.Equal(t, 3.0, readCounterValue(t, ExplainRUWorkRowsCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) + require.Equal(t, 24.0, readCounterValue(t, ExplainRUWorkBytesCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v1"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v1"))) + require.Equal(t, 3.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "input_rows", "runtime_act_rows", "all", "v1"))) + + registry := prometheus.NewRegistry() + require.NoError(t, registry.Register(ExplainRUPreviewRUCounter)) + require.NoError(t, registry.Register(ExplainRUWorkRowsCounter)) + require.NoError(t, registry.Register(ExplainRUWorkBytesCounter)) + require.NoError(t, registry.Register(ExplainRURowWidthHistogram)) + require.NoError(t, registry.Register(ExplainRUStatementsCounter)) + require.NoError(t, registry.Register(ExplainRURenderDurationHistogram)) + require.NoError(t, registry.Register(ExplainRUComponentSnapshotCounter)) + require.NoError(t, registry.Register(ReadBillingDemoStatementsCounter)) + require.NoError(t, registry.Register(ReadBillingDemoOperatorStatusCounter)) + require.NoError(t, registry.Register(ReadBillingDemoBaseUnitsCounter)) + require.NoError(t, registry.Register(ReadBillingDemoRowWidthHistogram)) + families, err := registry.Gather() + require.NoError(t, err) + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_preview_ru_total")) + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_work_rows_total")) + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_work_bytes_total")) + rowWidthFamily := findMetricFamily(families, "tidb_explain_ru_row_width_bytes") + require.NotNil(t, rowWidthFamily) + requireMetricFamilyHasLabels(t, rowWidthFamily, "component", "operator", "source") + require.True(t, metricHasLabelValue(rowWidthFamily.GetMetric()[0], "source", "plan_stats")) + statementFamily := findMetricFamily(families, "tidb_explain_ru_statements_total") + require.NotNil(t, statementFamily) + requireMetricFamilyHasLabels(t, statementFamily, "status") + require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_render_duration_seconds")) + componentSnapshotFamily := findMetricFamily(families, "tidb_explain_ru_component_snapshot_total") + require.NotNil(t, componentSnapshotFamily) + requireMetricFamilyHasLabels(t, componentSnapshotFamily, "component_snapshot_status") + readBillingStatementFamily := findMetricFamily(families, "tidb_read_billing_demo_statements_total") + require.NotNil(t, readBillingStatementFamily) + requireMetricFamilyHasLabels(t, readBillingStatementFamily, "status", "model_version") + readBillingOperatorFamily := findMetricFamily(families, "tidb_read_billing_demo_operator_status_total") + require.NotNil(t, readBillingOperatorFamily) + requireMetricFamilyHasLabels(t, readBillingOperatorFamily, "site", "op_class", "operator_kind", "status", "reason", "model_version") + readBillingBaseUnitFamily := findMetricFamily(families, "tidb_read_billing_demo_base_units_total") + require.NotNil(t, readBillingBaseUnitFamily) + requireMetricFamilyHasLabels(t, readBillingBaseUnitFamily, "site", "op_class", "operator_kind", "unit", "input_source", "input_side", "model_version") + readBillingRowWidthFamily := findMetricFamily(families, "tidb_read_billing_demo_row_width_bytes") + require.NotNil(t, readBillingRowWidthFamily) + requireMetricFamilyHasLabels(t, readBillingRowWidthFamily, "site", "op_class", "operator_kind", "row_width_source", "model_version") +} + +func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { + InitExplainRUMetrics() + + RecordExplainRUStatus("") + ObserveExplainRURenderDuration("", 0.01) + RecordExplainRUComponentSnapshot("") + ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", 0, -1, -1, -1) + ObserveExplainRURow("plan", "", "", "read_billing_model", "operator_helper", -1, -1, -1, 32) + RecordReadBillingDemoStatement("", "v1") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v1") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_act_rows", "all", "v1", 0) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "plan_stats", "v1", 0) + + require.Equal(t, 0, countCollectedMetrics(ExplainRUStatementsCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRURenderDurationHistogram)) + require.Equal(t, 0, countCollectedMetrics(ExplainRUComponentSnapshotCounter)) + require.Equal(t, 1, countCollectedMetrics(ExplainRUPreviewRUCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRUWorkRowsCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRUWorkBytesCounter)) + require.Equal(t, 0, countCollectedMetrics(ExplainRURowWidthHistogram)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoStatementsCounter)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoOperatorStatusCounter)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoBaseUnitsCounter)) + require.Equal(t, 0, countCollectedMetrics(ReadBillingDemoRowWidthHistogram)) +} + func TestStmtSummaryMetricLabels(t *testing.T) { InitStmtSummaryMetrics() require.Equal(t, 0, countCollectedMetrics(StmtSummaryWindowRecordCount)) @@ -218,3 +307,16 @@ func metricHasLabelValue(metric *dto.Metric, name string, value string) bool { } return false } + +func requireMetricFamilyHasLabels(t *testing.T, family *dto.MetricFamily, names ...string) { + t.Helper() + require.NotEmpty(t, family.GetMetric()) + labels := make(map[string]struct{}, len(family.GetMetric()[0].GetLabel())) + for _, label := range family.GetMetric()[0].GetLabel() { + labels[label.GetName()] = struct{}{} + } + for _, name := range names { + _, ok := labels[name] + require.Truef(t, ok, "metric %s missing label %s", family.GetName(), name) + } +} diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index cec90eef77648..7a72238f8186a 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -16,10 +16,23 @@ package core import ( "testing" + "time" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" + "github.com/pingcap/tidb/pkg/planner/property" + "github.com/pingcap/tidb/pkg/statistics" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" + tikvutil "github.com/tikv/client-go/v2/util" + rmclient "github.com/tikv/pd/client/resource_group/controller" ) func TestNewLineFieldsInfo(t *testing.T) { @@ -114,3 +127,414 @@ func TestNewLineFieldsInfo(t *testing.T) { require.Equal(t, c.expected, lineFieldsInfo) } } + +func TestExplainRUSelectGateStatus(t *testing.T) { + cases := []struct { + sql string + expected explainRUStatus + }{ + {"explain analyze format='ru' select 1", explainRUStatusSuccess}, + {"explain analyze format='ru' with cte as (select 1) select * from cte", explainRUStatusSuccess}, + {"explain analyze format='ru' select rand(), uuid()", explainRUStatusSuccess}, + {"explain analyze format='ru' select last_insert_id()", explainRUStatusSuccess}, + {"explain analyze format='ru' insert into t values (1)", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' table t", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' select 1 union table t", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' select 1 into outfile '/tmp/explain_ru.csv'", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select @a := 1", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select 1 union select @a := 2", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' with cte as (select get_lock('x', 0)) select * from cte", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select release_lock('x')", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select release_all_locks()", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select last_insert_id(1)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select nextval(seq)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select setval(seq, 1)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select sleep(1)", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' select * from t for update skip locked", explainRUStatusUnsupportedLockingSelect}, + {"explain analyze format='ru' select * from t for share skip locked", explainRUStatusUnsupportedLockingSelect}, + {"explain analyze format='ru' select 1 union select * from t for update", explainRUStatusUnsupportedLockingSelect}, + } + p := parser.New() + for _, tc := range cases { + stmt, err := p.ParseOneStmt(tc.sql, "", "") + require.NoError(t, err, tc.sql) + explain := stmt.(*ast.ExplainStmt) + require.Equal(t, tc.expected, explainRUSelectGateStatus(explain.Stmt), tc.sql) + } + require.Equal(t, explainRUStatusUnsupportedNonSelect, explainRUSelectGateStatus(&ast.SelectStmt{Kind: ast.SelectStmtKindValues})) + require.Equal(t, explainRUStatusUnsupportedNonSelect, explainRUValidateSetOprSelectList(&ast.SetOprSelectList{ + Selects: []ast.Node{&ast.SelectStmt{Kind: ast.SelectStmtKindValues}}, + })) +} + +func TestExplainRURowFormatting(t *testing.T) { + row := explainRURow{ + section: explainRUSectionPlan, + id: "Projection_1", + component: "projection", + operatorClass: "tidb/projection_eval", + actRows: 1, + hasActRows: true, + inputRows: 2, + hasInputRows: true, + outputRows: 1, + hasOutputRows: true, + rowWidth: 8, + hasRowWidth: true, + rowWidthSource: explainRUWidthSourcePlanStats, + workRows: 2, + hasWorkRows: true, + unit: readBillingDemoUnitInputRows, + count: 2, + hasCount: true, + weight: 0.25, + hasWeight: true, + previewRU: 6, + hasPreviewRU: true, + source: readBillingDemoInputSourceRuntimeRows, + note: "input_side=all,weight_version=v1", + } + require.Equal(t, []string{ + "plan", "Projection_1", "projection", "tidb/projection_eval", "1", "2", "1", "8.000000", "plan_stats", "2", "", "input_rows", "2", "0.250000", "6.000000", "runtime_act_rows", "input_side=all,weight_version=v1", + }, row.toStrings()) +} + +func TestExplainRUPlanFormulaUsesRowsAndModeledBytes(t *testing.T) { + tidbWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) + require.True(t, ok) + tikvWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) + require.True(t, ok) + require.NotEqual(t, tidbWeights, tikvWeights) + _, ok = readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion) + require.True(t, ok) + _, ok = readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion) + require.False(t, ok) + + weight, previewRU, ok := readBillingDemoUnitPreviewRU( + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, value: 4096}, + tidbWeights, + ) + require.True(t, ok) + require.Equal(t, tidbWeights.byte, weight) + require.Equal(t, 4096*tidbWeights.byte, previewRU) + + _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: "scan_total_keys", value: 4}, tidbWeights) + require.False(t, ok) + + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 5} + for _, tc := range []struct { + name string + site string + opClass string + op func() *FlatOperator + }{ + { + name: "range scan", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassRangeScan, + op: func() *FlatOperator { + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + return &FlatOperator{Origin: scan, IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "filter", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassFilter, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalSelection{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "projection", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassProjection, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalProjection{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "limit", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassLimit, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalLimit{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "topn", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassTopN, + op: func() *FlatOperator { + return &FlatOperator{Origin: physicalop.PhysicalTopN{}.Init(ctx, stats, 0), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "hash agg", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassHashAgg, + op: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema), IsRoot: false, StoreType: kv.TiKV} + }, + }, + { + name: "stream agg", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassStreamAgg, + op: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForStream(ctx, stats, 0, schema), IsRoot: false, StoreType: kv.TiKV} + }, + }, + } { + t.Run(tc.site+" "+tc.name, func(t *testing.T) { + requireReadBillingDemoClass(t, tc.op(), tc.site, tc.opClass, true, "") + }) + } + requireReadBillingDemoClass(t, &FlatOperator{ + Origin: physicalop.PhysicalIndexScan{}.Init(ctx, 0), + IsRoot: false, + StoreType: kv.TiFlash, + }, readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, false, readBillingDemoReasonUnsupportedTiFlash) + requireReadBillingDemoClass(t, &FlatOperator{ + Origin: physicalop.PhysicalExchangeReceiver{}.Init(ctx, stats), + IsRoot: true, + }, readBillingDemoSiteTiDB, readBillingDemoOpClassReaderReceive, false, readBillingDemoReasonUnsupportedMPP) + indexMerge := &physicalop.PhysicalIndexMergeReader{} + indexMerge.BasePhysicalPlan = physicalop.NewBasePhysicalPlan(ctx, "IndexMerge", indexMerge, 0) + requireReadBillingDemoClass(t, &FlatOperator{ + Origin: indexMerge, + IsRoot: true, + }, readBillingDemoSiteTiDB, readBillingDemoOpClassLookupReader, false, readBillingDemoReasonUnsupportedIndexMerge) +} + +func requireReadBillingDemoClass(t *testing.T, op *FlatOperator, site, opClass string, supported bool, reason string) { + t.Helper() + operator, ok, actualReason := readBillingDemoClassifyOperator(op) + require.Equal(t, supported, ok) + require.Equal(t, reason, actualReason) + require.Equal(t, site, operator.site) + require.Equal(t, opClass, operator.opClass) + if supported && readBillingDemoOperatorBillable(operator) { + _, hasWeights := readBillingDemoResolveWeights(operator.site, operator.opClass, readBillingDemoWeightVersion) + require.True(t, hasWeights, "missing read billing demo weights for %s/%s", operator.site, operator.opClass) + } +} + +func TestExplainRUComponentSnapshotStatusAndWeights(t *testing.T) { + require.Equal(t, explainRUComponentSnapshotMissing, extractExplainRUTestSnapshotStatus(nil)) + require.Equal(t, explainRUComponentSnapshotMissing, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{})) + require.Equal(t, explainRUComponentSnapshotNonV2, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV1, + Metrics: &execdetails.RUV2Metrics{}, + })) + require.Equal(t, explainRUComponentSnapshotNilMetrics, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV2, + })) + + bypassedMetrics := &execdetails.RUV2Metrics{} + bypassedMetrics.SetBypass(true) + require.Equal(t, explainRUComponentSnapshotBypassed, extractExplainRUTestSnapshotStatus(&execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV2, + Metrics: bypassedMetrics, + })) + + okStats := &execdetails.RURuntimeStats{ + RUVersion: rmclient.RUVersionV2, + Metrics: &execdetails.RUV2Metrics{}, + } + snapshot, status := extractExplainRUTestSnapshot(okStats) + require.Equal(t, explainRUComponentSnapshotOK, status) + require.Same(t, okStats, snapshot) +} + +func extractExplainRUTestSnapshotStatus(stats *execdetails.RURuntimeStats) explainRUComponentSnapshotStatus { + _, status := extractExplainRUTestSnapshot(stats) + return status +} + +func extractExplainRUTestSnapshot(stats *execdetails.RURuntimeStats) (*execdetails.RURuntimeStats, explainRUComponentSnapshotStatus) { + coll := execdetails.NewRuntimeStatsColl(nil) + if stats != nil && (stats.RUVersion != 0 || stats.Metrics != nil) { + coll.RegisterStats(1, stats) + } + return explainRUExtractComponentSnapshot(coll, 1) +} + +func TestReadBillingDemoDirectCopInputRowsAndWidthUsesChildRows(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 5} + parent := (&physicalop.PhysicalHashAgg{}).InitForHash(ctx, stats, 0, schema).(*physicalop.PhysicalHashAgg) + child := (&physicalop.PhysicalSelection{}).Init(ctx, stats, 0) + childInput := physicalop.PhysicalTableScan{}.Init(ctx, 0) + childInput.SetSchema(schema) + child.SetChildren(childInput) + tree := FlatPlanTree{ + {Origin: parent, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, + {Origin: child, IsRoot: false, StoreType: kv.TiKV}, + } + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordCopRows := func(planID int, rows uint64) { + iterations := uint64(1) + duration := uint64(1) + runtimeStats.RecordOneCopTask(planID, kv.TiKV, &tipb.ExecutorExecutionSummary{ + NumProducedRows: &rows, + NumIterations: &iterations, + TimeProcessedNs: &duration, + }) + } + recordCopRows(parent.ID(), 1) + recordCopRows(child.ID(), 5) + + proj := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + proj.SetSchema(schema) + operator, supported, reason := readBillingDemoClassifyOperator(&FlatOperator{ + Origin: proj, + IsRoot: false, + StoreType: kv.TiKV, + }) + require.True(t, supported) + require.Empty(t, reason) + require.Equal(t, readBillingDemoSiteTiKV, operator.site) + require.Equal(t, readBillingDemoOpClassProjection, operator.opClass) + + rows, width, widthSource, ok := readBillingDemoDirectCopInputRowsAndWidth( + ctx, + runtimeStats, + tree, + 0, + 8, + explainRUWidthSourceSchemaFallback, + runtimeStats.GetCopStats(parent.ID()), + ) + require.True(t, ok) + require.Equal(t, int64(5), rows) + require.Equal(t, float64(8), width) + require.Equal(t, explainRUWidthSourceSchemaTypeWidth, widthSource) +} + +func TestReadBillingDemoRangeScanKeepsFixedEventForEmptyInput(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + scan := physicalop.PhysicalTableScan{}.Init(ctx, 0) + scan.SetSchema(schema) + scan.StoreType = kv.TiKV + scan.TblColHists = &statistics.HistColl{Pseudo: true} + scan.TblCols = []*expression.Column{col} + tree := FlatPlanTree{ + {Origin: scan, IsRoot: false, StoreType: kv.TiKV}, + } + + buildUnits := func(scanDetail *tikvutil.ScanDetail, producedRows uint64) ([]readBillingDemoUnit, bool) { + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, scanDetail, tikvutil.TimeDetail{}, nil) + iterations := uint64(1) + duration := uint64(1) + runtimeStats.RecordOneCopTask(scan.ID(), kv.TiKV, &tipb.ExecutorExecutionSummary{ + NumProducedRows: &producedRows, + NumIterations: &iterations, + TimeProcessedNs: &duration, + }) + return readBillingDemoCopUnits( + ctx, + runtimeStats, + tree, + 0, + tree[0], + readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "tablescan"}, + ) + } + + units, ok := buildUnits(&tikvutil.ScanDetail{}, 0) + require.True(t, ok) + require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) + require.Equal(t, 0.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 0.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(units, "scan_total_keys", readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(units, "processed_key_size", readBillingDemoInputSideAll)) + + units, ok = buildUnits(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeysSize: 128}, 2) + require.True(t, ok) + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 128.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + + units, ok = buildUnits(&tikvutil.ScanDetail{}, 2) + require.True(t, ok) + require.Equal(t, 2.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeRows, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Greater(t, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll), 0.0) +} + +func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 10} + join := (&physicalop.PhysicalHashJoin{}).Init(ctx, stats, 0) + left := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + right := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + join.SetSchema(schema) + left.SetSchema(schema) + right.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: join, ChildrenIdx: []int{1, 2}, IsRoot: true}, + {Origin: left, IsRoot: true, Label: BuildSide}, + {Origin: right, IsRoot: true, Label: ProbeSide}, + } + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRootRows := func(planID int, rows int) { + runtimeStats.GetBasicRuntimeStats(planID, true).Record(time.Millisecond, rows) + } + recordRootRows(join.ID(), 6) + recordRootRows(left.ID(), 4) + recordRootRows(right.ID(), 6) + + units, ok := readBillingDemoRootUnits( + ctx, + runtimeStats, + tree, + 0, + tree[0], + readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashJoin, operatorKind: "hashjoin"}, + ) + require.True(t, ok) + require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideBuild)) + require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideProbe)) +} + +func readBillingDemoUnitValue(units []readBillingDemoUnit, unitName, side string) float64 { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return unit.value + } + } + return -1 +} + +func readBillingDemoUnitSource(units []readBillingDemoUnit, unitName, side string) string { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return unit.source + } + } + return "" +} + +func readBillingDemoUnitExists(units []readBillingDemoUnit, unitName, side string) bool { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return true + } + } + return false +} From e7b96897a4a54225b873b1aac7e2991e9ff9e903 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 2 Jul 2026 23:05:14 +0800 Subject: [PATCH 03/25] executor: expose read billing base units in statement summary --- pkg/executor/adapter.go | 7 +-- pkg/executor/explain_test.go | 10 +++++ pkg/infoschema/tables.go | 3 ++ pkg/planner/core/BUILD.bazel | 1 + pkg/planner/core/explain_ru.go | 36 +++++++++++++--- pkg/util/stmtsummary/BUILD.bazel | 2 +- pkg/util/stmtsummary/evicted.go | 1 + pkg/util/stmtsummary/reader.go | 12 ++++++ pkg/util/stmtsummary/statement_summary.go | 43 +++++++++++++++---- .../stmtsummary/statement_summary_test.go | 43 +++++++++++++++++++ pkg/util/stmtsummary/v2/column.go | 12 ++++++ pkg/util/stmtsummary/v2/column_test.go | 9 ++++ pkg/util/stmtsummary/v2/record.go | 16 +++++-- pkg/util/stmtsummary/v2/record_test.go | 7 +++ 14 files changed, 180 insertions(+), 22 deletions(-) diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index 7f9c4e631b317..eb6f25a72fa35 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -1701,11 +1701,11 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo // Lazy SELECT metrics are emitted when the record set is closed. If a // client abandons a result without closing/draining it, this first demo can // miss that statement instead of guessing an incomplete status. - plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) + readBillingDemoBaseUnits := plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) a.updateNetworkTrafficStatsAndMetrics() // `LowSlowQuery` and `SummaryStmt` must be called before recording `PrevStmt`. a.LogSlowQuery(txnTS, succ, hasMoreResults) - a.SummaryStmt(succ) + a.SummaryStmt(succ, readBillingDemoBaseUnits) a.observeStmtFinishedForTopProfiling() a.UpdatePlanCacheRuntimeInfo() if sessVars.StmtCtx.IsTiFlash.Load() { @@ -2192,7 +2192,7 @@ func (digest planDigestAlias) planDigestDumpTriggerCheck(config *traceevent.Dump } // SummaryStmt collects statements for information_schema.statements_summary -func (a *ExecStmt) SummaryStmt(succ bool) { +func (a *ExecStmt) SummaryStmt(succ bool, readBillingDemoBaseUnits stmtsummary.ReadBillingDemoBaseUnitSummary) { sessVars := a.Ctx.GetSessionVars() var userString string if sessVars.User != nil { @@ -2296,6 +2296,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) { stmtExecInfo.KeyspaceID = keyspaceID stmtExecInfo.RUDetail = ruDetail stmtExecInfo.TotalRUV2 = calculateStatementTotalRUV2(sessVars.RUV2Metrics, sessVars.RUV2Weights(), ruDetail) + stmtExecInfo.ReadBillingDemoBaseUnits = readBillingDemoBaseUnits stmtExecInfo.ResourceGroupName = sessVars.StmtCtx.ResourceGroupName stmtExecInfo.CPUUsages = sessVars.SQLCPUUsages.GetCPUUsages() stmtExecInfo.PlanCacheUnqualified = sessVars.StmtCtx.PlanCacheUnqualified() diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index a8f273a5178e2..e765aec7f9b9a 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -769,7 +769,14 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) tk.MustExec("use test") + + originEnableStmtSummary := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_stmt_summary").Rows()[0][0]) + defer tk.MustExec(fmt.Sprintf("set global tidb_enable_stmt_summary = %s", originEnableStmtSummary)) + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + tk.MustQuery("select @@tidb_enable_read_billing_demo").Check(testkit.Rows("0")) tk.MustExec("drop table if exists read_billing_demo") tk.MustExec("create table read_billing_demo(a int primary key)") @@ -788,6 +795,9 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) require.Equal(t, 1.0, readExecutorCounterValue(t, success)) require.Equal(t, 1.0, readExecutorCounterValue(t, projectionFixedEvents)) + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events > 0, sum_read_billing_demo_input_rows > 0, sum_read_billing_demo_input_bytes > 0 from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 1 1 1")) + tk.MustExec("set tidb_enable_read_billing_demo=on") beforeRestrictedSuccess := readExecutorCounterValue(t, success) beforeRestrictedBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index 3072d29e07c87..6d9b7293bb612 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -1429,6 +1429,9 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.AvgQueuedRcTimeStr, tp: mysql.TypeLonglong, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of waiting for available request-units"}, {name: stmtsummary.MaxRequestUnitV2Str, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Max request-unit v2 cost of these statements"}, {name: stmtsummary.AvgRequestUnitV2Str, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Average request-unit v2 cost of these statements"}, + {name: stmtsummary.SumReadBillingDemoFixedEventsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Total read billing demo fixed-event base units"}, + {name: stmtsummary.SumReadBillingDemoInputRowsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Total read billing demo input-row base units"}, + {name: stmtsummary.SumReadBillingDemoInputBytesStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Total read billing demo input-byte base units"}, {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, {name: stmtsummary.PlanCacheUnqualifiedStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag, comment: "The number of times that these statements are not supported by the plan cache"}, {name: stmtsummary.PlanCacheUnqualifiedLastReasonStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "The last reason why the statement is not supported by the plan cache"}, diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 21235f1d6403a..e7f2e928a2ea9 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -194,6 +194,7 @@ go_library( "//pkg/util/size", "//pkg/util/slice", "//pkg/util/sqlexec", + "//pkg/util/stmtsummary", "//pkg/util/stringutil", "//pkg/util/syncutil", "//pkg/util/texttree", diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 82d6402cad408..0512d637b82ff 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/plancodec" + "github.com/pingcap/tidb/pkg/util/stmtsummary" rmclient "github.com/tikv/pd/client/resource_group/controller" ) @@ -213,15 +214,16 @@ type explainRURow struct { } // RecordReadBillingDemoForStatement emits coefficient-free read billing demo -// metrics for a completed statement. It is intentionally independent from RU -// v2 billing/reporting and never calls resource-control reporters. -func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, stmt ast.StmtNode, execErr error) { +// metrics for a completed statement and returns the statement-level base-unit +// totals. It is intentionally independent from RU v2 billing/reporting and +// never calls resource-control reporters. +func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, stmt ast.StmtNode, execErr error) stmtsummary.ReadBillingDemoBaseUnitSummary { if sctx == nil || sctx.GetSessionVars() == nil || !sctx.GetSessionVars().EnableReadBillingDemo { - return + return stmtsummary.ReadBillingDemoBaseUnitSummary{} } // Restricted/internal SQL is not external workload calibration input. if sctx.GetSessionVars().InRestrictedSQL || sctx.GetSessionVars().StmtCtx.InRestrictedSQL { - return + return stmtsummary.ReadBillingDemoBaseUnitSummary{} } planCtx := readBillingDemoPlanContext(plan) if planCtx == nil { @@ -229,6 +231,7 @@ func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, } result := buildReadBillingDemoResult(planCtx, plan, stmt, execErr) recordReadBillingDemoResult(result) + return summarizeReadBillingDemoBaseUnits(result) } func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error) readBillingDemoResult { @@ -330,6 +333,29 @@ func readBillingDemoFailedOperator(status string, op readBillingDemoOperatorResu } } +func summarizeReadBillingDemoBaseUnits(result readBillingDemoResult) stmtsummary.ReadBillingDemoBaseUnitSummary { + if result.status != readBillingDemoStatusSuccess { + return stmtsummary.ReadBillingDemoBaseUnitSummary{} + } + var summary stmtsummary.ReadBillingDemoBaseUnitSummary + for _, op := range result.operators { + if op.status != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + continue + } + for _, unit := range op.units { + switch unit.unit { + case readBillingDemoUnitFixedEvents: + summary.SumReadBillingDemoFixedEvents += unit.value + case readBillingDemoUnitInputRows: + summary.SumReadBillingDemoInputRows += unit.value + case readBillingDemoUnitInputBytes: + summary.SumReadBillingDemoInputBytes += unit.value + } + } + } + return summary +} + func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) (string, readBillingDemoOperatorResult) { for i, op := range tree { if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { diff --git a/pkg/util/stmtsummary/BUILD.bazel b/pkg/util/stmtsummary/BUILD.bazel index f98a374797061..7c506280969f9 100644 --- a/pkg/util/stmtsummary/BUILD.bazel +++ b/pkg/util/stmtsummary/BUILD.bazel @@ -40,7 +40,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 28, + shard_count = 29, deps = [ "//pkg/meta/model", "//pkg/metrics", diff --git a/pkg/util/stmtsummary/evicted.go b/pkg/util/stmtsummary/evicted.go index fec9e5f9399a6..07d85c70ea296 100644 --- a/pkg/util/stmtsummary/evicted.go +++ b/pkg/util/stmtsummary/evicted.go @@ -390,6 +390,7 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem addTo.sumErrors += addWith.sumErrors addTo.StmtRUSummary.Merge(&addWith.StmtRUSummary) + addTo.ReadBillingDemoBaseUnitSummary.Merge(&addWith.ReadBillingDemoBaseUnitSummary) // resourceGroupName might not be inited because when it is a evicted item. addTo.resourceGroupName = addWith.resourceGroupName } diff --git a/pkg/util/stmtsummary/reader.go b/pkg/util/stmtsummary/reader.go index 90edc89cbbb33..cbbab4a47ebce 100644 --- a/pkg/util/stmtsummary/reader.go +++ b/pkg/util/stmtsummary/reader.go @@ -368,6 +368,9 @@ const ( MaxQueuedRcTimeStr = "MAX_QUEUED_RC_TIME" AvgRequestUnitV2Str = "AVG_REQUEST_UNIT_V2" MaxRequestUnitV2Str = "MAX_REQUEST_UNIT_V2" + SumReadBillingDemoFixedEventsStr = "SUM_READ_BILLING_DEMO_FIXED_EVENTS" + SumReadBillingDemoInputRowsStr = "SUM_READ_BILLING_DEMO_INPUT_ROWS" + SumReadBillingDemoInputBytesStr = "SUM_READ_BILLING_DEMO_INPUT_BYTES" ResourceGroupName = "RESOURCE_GROUP" SumUnpackedBytesSentTiKVTotalStr = "SUM_UNPACKED_BYTES_SENT_TIKV_TOTAL" SumUnpackedBytesReceivedTiKVTotalStr = "SUM_UNPACKED_BYTES_RECEIVED_TIKV_TOTAL" @@ -925,6 +928,15 @@ var columnValueFactoryMap = map[string]columnValueFactory{ MaxRequestUnitV2Str: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { return ssStats.MaxRUV2 }, + SumReadBillingDemoFixedEventsStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.SumReadBillingDemoFixedEvents + }, + SumReadBillingDemoInputRowsStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.SumReadBillingDemoInputRows + }, + SumReadBillingDemoInputBytesStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.SumReadBillingDemoInputBytes + }, ResourceGroupName: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { return ssStats.resourceGroupName }, diff --git a/pkg/util/stmtsummary/statement_summary.go b/pkg/util/stmtsummary/statement_summary.go index d81e1a72c2275..68b96c1d3e900 100644 --- a/pkg/util/stmtsummary/statement_summary.go +++ b/pkg/util/stmtsummary/statement_summary.go @@ -254,6 +254,7 @@ type stmtSummaryStats struct { // request-units resourceGroupName string StmtRUSummary + ReadBillingDemoBaseUnitSummary StmtNetworkTrafficSummary planCacheUnqualifiedCount int64 @@ -296,15 +297,16 @@ type StmtExecInfo struct { WriteSQLRespDuration time.Duration - ResultRows int64 - TiKVExecDetails *util.ExecDetails - Prepared bool - KeyspaceName string - KeyspaceID uint32 - ResourceGroupName string - RUDetail *util.RUDetails - TotalRUV2 float64 - CPUUsages ppcpuusage.CPUUsages + ResultRows int64 + TiKVExecDetails *util.ExecDetails + Prepared bool + KeyspaceName string + KeyspaceID uint32 + ResourceGroupName string + RUDetail *util.RUDetails + TotalRUV2 float64 + ReadBillingDemoBaseUnits ReadBillingDemoBaseUnitSummary + CPUUsages ppcpuusage.CPUUsages PlanCacheUnqualified string @@ -997,6 +999,7 @@ func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affect // request-units ssStats.StmtRUSummary.Add(sei.RUDetail, sei.TotalRUV2) + ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoBaseUnits) ssStats.storageKV = sei.StmtCtx.IsTiKV.Load() ssStats.storageMPP = sei.StmtCtx.IsTiFlash.Load() @@ -1141,6 +1144,28 @@ func (s *StmtRUSummary) Merge(other *StmtRUSummary) { } } +// ReadBillingDemoBaseUnitSummary is the read billing demo base-unit summary for each type of statements. +type ReadBillingDemoBaseUnitSummary struct { + SumReadBillingDemoFixedEvents float64 `json:"sum_read_billing_demo_fixed_events"` + SumReadBillingDemoInputRows float64 `json:"sum_read_billing_demo_input_rows"` + SumReadBillingDemoInputBytes float64 `json:"sum_read_billing_demo_input_bytes"` +} + +// Add adds one read billing demo base-unit sample to the summary. +func (s *ReadBillingDemoBaseUnitSummary) Add(other *ReadBillingDemoBaseUnitSummary) { + if other == nil { + return + } + s.SumReadBillingDemoFixedEvents += other.SumReadBillingDemoFixedEvents + s.SumReadBillingDemoInputRows += other.SumReadBillingDemoInputRows + s.SumReadBillingDemoInputBytes += other.SumReadBillingDemoInputBytes +} + +// Merge merges another read billing demo base-unit summary. +func (s *ReadBillingDemoBaseUnitSummary) Merge(other *ReadBillingDemoBaseUnitSummary) { + s.Add(other) +} + // StmtNetworkTrafficSummary is the network traffic summary for each type of statements. type StmtNetworkTrafficSummary struct { UnpackedBytesSentTiKVTotal int64 `json:"unpacked_bytes_send_tikv_total"` diff --git a/pkg/util/stmtsummary/statement_summary_test.go b/pkg/util/stmtsummary/statement_summary_test.go index bdf4fac921c78..08c8da6fe8848 100644 --- a/pkg/util/stmtsummary/statement_summary_test.go +++ b/pkg/util/stmtsummary/statement_summary_test.go @@ -1053,6 +1053,49 @@ func TestToDatum(t *testing.T) { match(t, datums[1], expectedEvictedDatum...) } +func TestReadBillingDemoBaseUnitsToDatum(t *testing.T) { + ssMap := newStmtSummaryByDigestMap() + now := time.Now().Unix() + // to disable expiration + ssMap.beginTimeForCurInterval = now + 60 + + stmtExecInfo1 := generateAnyExecInfo() + stmtExecInfo1.ReadBillingDemoBaseUnits = ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 2, + SumReadBillingDemoInputRows: 100, + SumReadBillingDemoInputBytes: 2048, + } + ssMap.AddStatement(stmtExecInfo1) + + stmtExecInfo2 := generateAnyExecInfo() + stmtExecInfo2.ReadBillingDemoBaseUnits = ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 3, + SumReadBillingDemoInputRows: 200, + SumReadBillingDemoInputBytes: 4096, + } + ssMap.AddStatement(stmtExecInfo2) + + key := &StmtDigestKey{} + key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "") + value, ok := ssMap.summaryMap.Get(key) + require.True(t, ok) + ssElement := value.(*stmtSummaryByDigest).history.Back().Value.(*stmtSummaryByDigestElement) + require.Equal(t, 5.0, ssElement.SumReadBillingDemoFixedEvents) + require.Equal(t, 300.0, ssElement.SumReadBillingDemoInputRows) + require.Equal(t, 6144.0, ssElement.SumReadBillingDemoInputBytes) + + cols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(SumReadBillingDemoFixedEventsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputRowsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputBytesStr)}, + } + reader := NewStmtSummaryReader(nil, true, cols, "", time.UTC) + reader.ssMap = ssMap + datums := reader.GetStmtSummaryCurrentRows() + require.Len(t, datums, 1) + match(t, datums[0], 5.0, 300.0, 6144.0) +} + // Test AddStatement and ToDatum parallel. func TestAddStatementParallel(t *testing.T) { ssMap := newStmtSummaryByDigestMap() diff --git a/pkg/util/stmtsummary/v2/column.go b/pkg/util/stmtsummary/v2/column.go index 68ec4e170a88e..2ada88544d364 100644 --- a/pkg/util/stmtsummary/v2/column.go +++ b/pkg/util/stmtsummary/v2/column.go @@ -144,6 +144,9 @@ const ( MaxQueuedRcTimeStr = "MAX_QUEUED_RC_TIME" AvgRequestUnitV2 = "AVG_REQUEST_UNIT_V2" MaxRequestUnitV2 = "MAX_REQUEST_UNIT_V2" + SumReadBillingDemoFixedEventsStr = "SUM_READ_BILLING_DEMO_FIXED_EVENTS" + SumReadBillingDemoInputRowsStr = "SUM_READ_BILLING_DEMO_INPUT_ROWS" + SumReadBillingDemoInputBytesStr = "SUM_READ_BILLING_DEMO_INPUT_BYTES" ResourceGroupName = "RESOURCE_GROUP" SumUnpackedBytesSentTiKVTotalStr = "SUM_UNPACKED_BYTES_SENT_TIKV_TOTAL" SumUnpackedBytesReceivedTiKVTotalStr = "SUM_UNPACKED_BYTES_RECEIVED_TIKV_TOTAL" @@ -523,6 +526,15 @@ var columnFactoryMap = map[string]columnFactory{ MaxRequestUnitV2: func(_ columnInfo, record *StmtRecord) any { return record.MaxRUV2 }, + SumReadBillingDemoFixedEventsStr: func(_ columnInfo, record *StmtRecord) any { + return record.SumReadBillingDemoFixedEvents + }, + SumReadBillingDemoInputRowsStr: func(_ columnInfo, record *StmtRecord) any { + return record.SumReadBillingDemoInputRows + }, + SumReadBillingDemoInputBytesStr: func(_ columnInfo, record *StmtRecord) any { + return record.SumReadBillingDemoInputBytes + }, ResourceGroupName: func(_ columnInfo, record *StmtRecord) any { return record.ResourceGroupName }, diff --git a/pkg/util/stmtsummary/v2/column_test.go b/pkg/util/stmtsummary/v2/column_test.go index dd07fd5e22077..6431beaabde3b 100644 --- a/pkg/util/stmtsummary/v2/column_test.go +++ b/pkg/util/stmtsummary/v2/column_test.go @@ -37,6 +37,9 @@ func TestColumn(t *testing.T) { {Name: ast.NewCIStr(ExecCountStr)}, {Name: ast.NewCIStr(SumLatencyStr)}, {Name: ast.NewCIStr(MaxLatencyStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoFixedEventsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputRowsStr)}, + {Name: ast.NewCIStr(SumReadBillingDemoInputBytesStr)}, {Name: ast.NewCIStr(AvgTidbCPUTimeStr)}, {Name: ast.NewCIStr(AvgTikvCPUTimeStr)}, } @@ -69,6 +72,12 @@ func TestColumn(t *testing.T) { require.Equal(t, int64(record.SumLatency), column) case MaxLatencyStr: require.Equal(t, int64(record.MaxLatency), column) + case SumReadBillingDemoFixedEventsStr: + require.Equal(t, record.SumReadBillingDemoFixedEvents, column) + case SumReadBillingDemoInputRowsStr: + require.Equal(t, record.SumReadBillingDemoInputRows, column) + case SumReadBillingDemoInputBytesStr: + require.Equal(t, record.SumReadBillingDemoInputBytes, column) case AvgTidbCPUTimeStr: require.Equal(t, int64(record.SumTidbCPU), column) case AvgTikvCPUTimeStr: diff --git a/pkg/util/stmtsummary/v2/record.go b/pkg/util/stmtsummary/v2/record.go index fdd9a2de1d471..2d2172425cae5 100644 --- a/pkg/util/stmtsummary/v2/record.go +++ b/pkg/util/stmtsummary/v2/record.go @@ -155,6 +155,7 @@ type StmtRecord struct { // request units(RU) ResourceGroupName string `json:"resource_group_name"` stmtsummary.StmtRUSummary + stmtsummary.ReadBillingDemoBaseUnitSummary PlanCacheUnqualifiedCount int64 `json:"plan_cache_unqualified_count"` PlanCacheUnqualifiedLastReason string `json:"plan_cache_unqualified_last_reason"` // the reason why this query is unqualified for the plan cache @@ -444,6 +445,7 @@ func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { r.StmtNetworkTrafficSummary.Add(&tikvExecDetails) // RU r.StmtRUSummary.Add(info.RUDetail, info.TotalRUV2) + r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoBaseUnits) r.StorageKV = info.StmtCtx.IsTiKV.Load() r.StorageMPP = info.StmtCtx.IsTiFlash.Load() @@ -608,6 +610,7 @@ func (r *StmtRecord) Merge(other *StmtRecord) { r.SumTikvCPU += other.SumTikvCPU r.SumErrors += other.SumErrors r.StmtRUSummary.Merge(&other.StmtRUSummary) + r.ReadBillingDemoBaseUnitSummary.Merge(&other.ReadBillingDemoBaseUnitSummary) } // Truncate SQL to maxSQLLength. @@ -712,10 +715,15 @@ func GenerateStmtExecInfo4Test(digest string) *stmtsummary.StmtExecInfo { ResourceGroupName: "rg1", RUDetail: util.NewRUDetailsWith(1.2, 3.4, 2*time.Millisecond), TotalRUV2: 12345, - TiKVExecDetails: &util.ExecDetails{}, - CPUUsages: ppcpuusage.CPUUsages{TidbCPUTime: time.Duration(20), TikvCPUTime: time.Duration(10000)}, - LazyInfo: &mockLazyInfo{}, - MemArbitration: 22222, + ReadBillingDemoBaseUnits: stmtsummary.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 2, + SumReadBillingDemoInputRows: 100, + SumReadBillingDemoInputBytes: 2048, + }, + TiKVExecDetails: &util.ExecDetails{}, + CPUUsages: ppcpuusage.CPUUsages{TidbCPUTime: time.Duration(20), TikvCPUTime: time.Duration(10000)}, + LazyInfo: &mockLazyInfo{}, + MemArbitration: 22222, } stmtExecInfo.StmtCtx.AddAffectedRows(10000) return stmtExecInfo diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index d35d3807a970f..7d0e0f55c1a6d 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -67,6 +67,9 @@ func TestStmtRecord(t *testing.T) { require.Equal(t, info.RUDetail.RUWaitDuration(), record1.SumRUWaitDuration) require.Equal(t, info.TotalRUV2, record1.MaxRUV2) require.Equal(t, info.TotalRUV2, record1.SumRUV2) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoFixedEvents, record1.SumReadBillingDemoFixedEvents) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputRows, record1.SumReadBillingDemoInputRows) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputBytes, record1.SumReadBillingDemoInputBytes) require.Equal(t, info.CPUUsages.TidbCPUTime, record1.SumTidbCPU) require.Equal(t, info.CPUUsages.TikvCPUTime, record1.SumTikvCPU) @@ -83,6 +86,9 @@ func TestStmtRecord(t *testing.T) { require.Equal(t, info.RUDetail.WRU()*2, record2.SumWRU) require.Equal(t, info.RUDetail.RUWaitDuration()*2, record2.SumRUWaitDuration) require.Equal(t, info.TotalRUV2*2, record2.SumRUV2) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoFixedEvents*2, record2.SumReadBillingDemoFixedEvents) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputRows*2, record2.SumReadBillingDemoInputRows) + require.Equal(t, info.ReadBillingDemoBaseUnits.SumReadBillingDemoInputBytes*2, record2.SumReadBillingDemoInputBytes) require.Equal(t, info.CPUUsages.TidbCPUTime*2, record2.SumTidbCPU) require.Equal(t, info.CPUUsages.TikvCPUTime*2, record2.SumTikvCPU) @@ -103,6 +109,7 @@ func TestStmtRecord(t *testing.T) { require.NoError(t, json.Unmarshal(b, &items)) require.Equal(t, map[string]any{"stmt_meta_a": "value_a"}, items["additional_fields"]) require.Equal(t, record2.Digest, items["digest"]) + require.Equal(t, record2.SumReadBillingDemoInputBytes, items["sum_read_billing_demo_input_bytes"]) b, err = marshalEvictedStmtRecord(record2) require.NoError(t, err) From 690b858f3525f36a73715331964ed7e11c370df4 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 2 Jul 2026 23:40:57 +0800 Subject: [PATCH 04/25] docs: design read billing calibration surface --- .../2026-07-01-read-billing-demo-ru-model.md | 711 ++++++++++++++++++ 1 file changed, 711 insertions(+) create mode 100644 docs/design/2026-07-01-read-billing-demo-ru-model.md diff --git a/docs/design/2026-07-01-read-billing-demo-ru-model.md b/docs/design/2026-07-01-read-billing-demo-ru-model.md new file mode 100644 index 0000000000000..87f5474687e93 --- /dev/null +++ b/docs/design/2026-07-01-read-billing-demo-ru-model.md @@ -0,0 +1,711 @@ +# Read Billing Demo RU 模型与校准统计面设计 + +- Author(s): TBD +- Discussion PR: TBD +- Tracking Issue: TBD + +## 目录 + +* [简介](#简介) +* [背景和目标](#背景和目标) +* [当前代码边界](#当前代码边界) +* [模型和支持范围](#模型和支持范围) +* [推荐统计面](#推荐统计面) +* [采集和生命周期](#采集和生命周期) +* [查询和权重校准方法](#查询和权重校准方法) +* [初始权重](#初始权重) +* [性能和容量控制](#性能和容量控制) +* [兼容性和迁移计划](#兼容性和迁移计划) +* [测试设计](#测试设计) +* [备选方案](#备选方案) +* [风险和未决问题](#风险和未决问题) + +## 简介 + +本文描述 read billing demo 的 SELECT 读请求 RU 模型，以及用于 workload 权重校准的 TiDB-native 统计面。 + +模型把读请求执行过程拆成有限数量的 operator class。每个 operator class 都按版本管理权重，权重作用在无系数的 base units 上： + +```text +preview_ru = + fixed_events * fixed_weight + + input_rows * row_weight + + input_bytes * byte_weight +``` + +第一版是校准 demo，不是最终生产 billing 语义。它和现有 RU v2 resource-control 上报保持隔离，并分成两个输出面： + +1. `EXPLAIN ANALYZE FORMAT='RU'`：单语句诊断输出，显示 base units、weight 和 preview RU。 +2. statement-summary-associated statistics tables：workload 级无系数统计面，用于离线重新套任意 weights，不依赖 Prometheus scrape，也不打印 per-SQL log。 + +Prometheus `tidb_read_billing_demo_*` metrics 可以继续作为 bounded observability 辅助，但不能作为 workload testers 的必需接口；校准契约以 SQL 查询面为准。 + +## 背景和目标 + +现有 RU v2 主要是 statement-level billing 逻辑，不适合回答两个问题： + +1. 一条 SELECT 语句里的 TiDB / TiKV 哪些 executor 消耗了 RU； +2. 在 workload 下，如何收集无系数、可重放的原始输入，用真实压测数据回归出更合理的 weights。 + +read billing demo 的目标是： + +1. 让单条语句可以通过 `EXPLAIN ANALYZE FORMAT='RU'` 看到 operator-level 的 base units、weight 和 preview RU； +2. 让 workload 可以通过 SQL 查询收集 `site/op_class/unit` 维度的 `fixed_events`、`input_rows`、`input_bytes` 等基础量； +3. 让 testers 可以在 workload 结束后直接替换 weights 并离线重算 RU，不必重跑 workload； +4. 让 unsupported、unknown input、statement error、aggregation overflow 等失败或降级状态有可查询位置； +5. 在 weights 还没有校准完成前，避免把 demo 结果混入现有 RU v2 billing 或 resource-control consumption 上报。 + +非目标： + +- 不把 per-SQL log 作为校准接口； +- 不要求 downstream testers 搭 Prometheus 或依赖 metrics retention； +- 不在第一版覆盖 TiFlash、MPP、IndexMerge 或复杂 storage path； +- 不把当前 `v1` weights 解释成生产校准值。 + +## 当前代码边界 + +当前实现已经有以下锚点： + +- `pkg/sessionctx/vardef/tidb_vars.go`、`pkg/sessionctx/variable/session.go`、`pkg/sessionctx/variable/sysvar.go` 定义 `tidb_enable_read_billing_demo`，默认 OFF。 +- `pkg/executor/adapter.go::FinishExecuteStmt` 在 statement 完成时调用 `plannercore.RecordReadBillingDemoForStatement`，随后 `SummaryStmt` 把返回的 read-billing summary 放进 `stmtsummary.StmtExecInfo`。 +- `pkg/session/session.go::recordReadBillingDemoEarlyError` 覆盖 compile / resolve / pre-completion error 等没有正常走到 `FinishExecuteStmt` 的错误出口。 +- `pkg/planner/core/explain_ru.go::buildReadBillingDemoResult` 生成 frozen result，包含 statement status、operator status 和 operator base units。 +- `pkg/planner/core/explain_ru.go::recordReadBillingDemoResult` 当前会写 bounded Prometheus metrics。 +- `pkg/planner/core/explain_ru.go::summarizeReadBillingDemoBaseUnits` 当前只把成功 statement 的 base units 汇总成三个 scalar totals。 +- `pkg/util/stmtsummary/statement_summary.go`、`pkg/util/stmtsummary/reader.go`、`pkg/infoschema/tables.go` 当前在 statement summary v1 路径暴露三列： + - `SUM_READ_BILLING_DEMO_FIXED_EVENTS` + - `SUM_READ_BILLING_DEMO_INPUT_ROWS` + - `SUM_READ_BILLING_DEMO_INPUT_BYTES` +- `pkg/util/stmtsummary/v2/record.go`、`pkg/util/stmtsummary/v2/column.go` 当前在 persistent statement summary v2 路径保持同样三列。 + +三列 totals 可以作为 convenience summary，但它们丢失了 `site/op_class/unit/input_source/input_side` 维度，无法支持 per-opclass weight calibration。因此本设计推荐新增维度化 SQL 查询面，并从同一个 statement summary 生命周期里聚合。 + +## 模型和支持范围 + +### 支持范围和安全 gate + +demo 只接受 side-effect-free SELECT： + +- 普通 `SELECT`； +- set operation，前提是所有叶子都是 side-effect-free SELECT； +- prepared statement / `EXECUTE`，前提是 resolved prepared target 是可支持的 side-effect-free SELECT。 + +以下语句或路径会被拒绝、跳过或 fail closed： + +- 非 SELECT； +- `SELECT ... INTO`； +- locking SELECT； +- 带副作用的函数，例如 lock 函数、sequence mutation、`LAST_INSERT_ID(expr)` 和 `SLEEP`； +- internal / restricted SQL； +- TiFlash、MPP exchange、IndexMerge 和未支持的 operator。 + +当出现 unsupported、unknown input 或 statement error 时，demo 必须记录 status/reason，但不能上报 partial base units 或 partial preview RU。 + +### Base-unit result + +每个 supported 且实际执行的 billable operator 至少产生一个： + +```text +fixed_events = 1 +``` + +大多数 operator 还会产生： + +- `input_rows`：该 operator 消耗的行数或 key 数； +- `input_bytes`：该 operator 消耗的 byte-shaped 输入。 + +每个 operator 结果都有以下身份字段： + +- `site`：`tidb`、`tikv`，statement-level status 用 `statement`； +- `op_class`：用于解析权重的有限 class； +- `operator_kind`：真实 plan/operator 名称，只用于观测和调参，不参与 weight lookup； +- `model_version`：当前统计模型版本，当前是 `v1`； +- `weight_version`：当前 weight table 版本，当前是 `v1`。 + +权重按下面的 key 解析： + +```text +site + op_class + weight_version +``` + +因此 TiDB 和 TiKV 即使共享类似的 opclass 名称，例如 `filter_eval`，也可以使用不同权重。 + +### TiKV Cop Executor 模型 + +TiKV 侧不是只按 scan 计费，而是把 cop executor 也纳入模型。第一版包括： + +| TiKV opclass | 覆盖的 operator 形态 | base-unit 输入 | +|---|---|---| +| `kv_range_scan` | table/index range scan | `input_rows = ScanDetail.TotalKeys`；`input_bytes = ScanDetail.ProcessedKeysSize` 优先，缺失时回退 `TotalKeys * row_width` | +| `kv_point_lookup` | point get / batch point get | point lookup work；当前 variable input 基于 runtime rows，fixed event 覆盖 operator setup | +| `filter_eval` | selection | 直接 cop child runtime rows 和 modeled row width | +| `projection_eval` | projection | 直接 cop child runtime rows 和 modeled row width | +| `row_limit` | limit | 直接 cop child runtime rows 和 modeled row width | +| `bounded_topn` | TopN | 直接 cop child runtime rows 和 modeled row width | +| `agg_hash` | hash aggregation | aggregation input rows 和 modeled input bytes | +| `agg_stream` | stream aggregation | aggregation input rows 和 modeled input bytes | + +range scan 的 byte 输入优先使用 `ProcessedKeysSize`，因为 scan 成本不仅和输出行数有关，还和 key/value bytes、MVCC 数据移动、row decode 有关。如果 runtime 只有 `TotalKeys` 而没有 processed size，则回退为： + +```text +input_bytes = TotalKeys * row_width +``` + +如果是 mock-store 或其他没有 scan detail 的路径，则回退为： + +```text +input_rows = actRows +input_bytes = actRows * row_width +input_source = runtime_act_rows +``` + +这个 fallback 会通过 `input_source` 明确标出，避免把 runtime rows 伪装成 scan detail。 + +### TiDB Root Executor 模型 + +TiDB 侧覆盖 root executor CPU、row movement、reader materialization、join 和 local overlay： + +| TiDB opclass | 覆盖的 operator 形态 | base-unit 输入 | +|---|---|---| +| `filter_eval` | root selection | local child rows 和 modeled input bytes | +| `projection_eval` | root projection | local child rows 和 modeled input bytes | +| `row_limit` | root limit | local child rows 和 modeled input bytes | +| `bounded_topn` | root TopN | local child rows 和 modeled input bytes | +| `full_ordering` | sort | local child rows 和 modeled input bytes | +| `window_eval` | window executor | local child rows 和 modeled input bytes | +| `agg_hash` | hash aggregation | aggregation input rows 和 modeled input bytes | +| `agg_stream` | stream aggregation | aggregation input rows 和 modeled input bytes | +| `join_hash` | hash join | build/probe 两侧 input rows 和 bytes | +| `join_merge` | merge join | left/right 两侧 input rows 和 bytes | +| `join_lookup` | index lookup join family | left/right 两侧 input rows 和 bytes | +| `reader_receive` | table/index reader receive | runtime rows 和 modeled received bytes | +| `lookup_reader` | index lookup reader orchestration | runtime rows 和 modeled bytes | +| `overlay_reader` | union scan / local overlay | runtime rows 和 modeled bytes | +| `metadata_reader` | memory / cluster metadata reads | runtime rows 和 modeled bytes | + +Join 的 side 会通过 `input_side` 保留下来： + +- hash join：`build`、`probe`； +- merge join / lookup join：`left`、`right`； +- 其他 operator：`all`。 + +## 推荐统计面 + +推荐新增 statement-summary-associated virtual tables，按 statement summary 的 digest/window/history 生命周期聚合 read billing demo 明细。表名前缀和现有 statement summary 保持一致： + +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS` + +### Base-unit table schema + +每一行表示一个 statement summary window 内、一个 digest/plan_digest 下的一组 base-unit key 聚合。 + +| Column | Type | 含义 | +|---|---|---| +| `SUMMARY_BEGIN_TIME` | timestamp | statement summary window begin | +| `SUMMARY_END_TIME` | timestamp | statement summary window end | +| `STMT_TYPE` | varchar | statement type | +| `SCHEMA_NAME` | varchar | schema | +| `DIGEST` | varchar | SQL digest | +| `DIGEST_TEXT` | blob | normalized SQL，沿用 statement summary 截断策略 | +| `PLAN_DIGEST` | varchar | plan digest；point get 可由 lazy plan digest 补齐 | +| `RESOURCE_GROUP` | varchar | resource group | +| `MODEL_VERSION` | varchar | read billing model version | +| `WEIGHT_VERSION` | varchar | 生成这些 base units 时匹配的 weight version | +| `SITE` | varchar | `tidb` / `tikv` | +| `OP_CLASS` | varchar | bounded opclass | +| `OPERATOR_KIND` | varchar | bounded plan/operator kind | +| `UNIT` | varchar | `fixed_events` / `input_rows` / `input_bytes` | +| `INPUT_SOURCE` | varchar | `scan_detail` / `runtime_act_rows` | +| `INPUT_SIDE` | varchar | `all` / `build` / `probe` / `left` / `right` | +| `ROW_WIDTH_SOURCE` | varchar | `plan_stats` / `schema_type_width` / `schema_fallback` / `operator_helper` | +| `VALUE` | double unsigned | 该 key 的 base-unit total | +| `SAMPLE_COUNT` | bigint unsigned | 汇入该 key 的 unit samples 数 | +| `ROW_WIDTH_SUM` | double unsigned | row-width sample sum，用于诊断 modeled bytes | +| `AVG_ROW_WIDTH` | double unsigned | `ROW_WIDTH_SUM / SAMPLE_COUNT` | + +`DIGEST_TEXT` 不是 key，只是查询便利字段。校准时推荐按 `DIGEST`、`PLAN_DIGEST`、`SITE`、`OP_CLASS`、`UNIT` 等机器字段聚合。 + +### Status table schema + +每一行表示一个 statement summary window 内、一个 digest/plan_digest 下的一组 statement/operator status 聚合。 + +| Column | Type | 含义 | +|---|---|---| +| `SUMMARY_BEGIN_TIME` | timestamp | statement summary window begin | +| `SUMMARY_END_TIME` | timestamp | statement summary window end | +| `STMT_TYPE` | varchar | statement type | +| `SCHEMA_NAME` | varchar | schema | +| `DIGEST` | varchar | SQL digest | +| `DIGEST_TEXT` | blob | normalized SQL | +| `PLAN_DIGEST` | varchar | plan digest | +| `RESOURCE_GROUP` | varchar | resource group | +| `MODEL_VERSION` | varchar | read billing model version | +| `WEIGHT_VERSION` | varchar | current weight version at accounting time | +| `SITE` | varchar | `statement` / `tidb` / `tikv` | +| `OP_CLASS` | varchar | statement or operator opclass | +| `OPERATOR_KIND` | varchar | statement or plan/operator kind | +| `STATUS` | varchar | `success` / `unsupported` / `unknown_input` / `error` / `ok` | +| `REASON` | varchar | bounded reason，例如 `unsupported_mpp`、`missing_scan_detail`、`statement_error` | +| `COUNT` | bigint unsigned | 该 status key 的出现次数 | + +失败 statement 只写 status table，不写 base-unit table。成功 statement 写一条 statement-level `success` status，并为每个 billable operator 写 operator-level `ok` status。 + +### 为什么不用 statement summary 单表 JSON 列 + +JSON/encoded column 的写入成本可以较低，但对校准不够直接： + +- testers 需要额外 JSON 解析才能 group by `site/op_class/unit`； +- SQL 里直接套 weights、按 digest/window 过滤和 join 会变复杂； +- 历史持久化中的 schema 演进、旧 JSON 兼容和 partial parse failure 更难测试； +- 失败 status 和 base units 混在一个 encoded blob 里，容易再次变成不可直接观测的状态。 + +窄表把 row explosion 放到查询阶段，而不是把复杂性塞进单列解析；同时它复用 statement summary 的 digest/window/LRU/history 控制。 + +## 采集和生命周期 + +### Statement completion path + +后续实现应把 `RecordReadBillingDemoForStatement` 的返回值从三个 totals 扩成结构化 snapshot，例如： + +```go +type ReadBillingDemoStatementStats struct { + ModelVersion string + WeightVersion string + Statuses []ReadBillingDemoStatusSample + BaseUnits []ReadBillingDemoBaseUnitSample + Totals ReadBillingDemoBaseUnitSummary +} +``` + +其中 `Totals` 从 `BaseUnits` 派生，用于继续填充现有三列；`Statuses` 和 `BaseUnits` 则进入新表所需的维度化聚合。 + +采集顺序： + +1. statement 执行结束或早期错误出口触发 read billing demo hook； +2. `buildReadBillingDemoResult` 生成 frozen result； +3. 如果 result status 是 success： + - 为 statement 写 `success` status； + - 为 billable operator 写 operator `ok` status； + - 把 operator units 展平成 base-unit samples； + - 从 base-unit samples 派生三列 totals； +4. 如果 result status 不是 success： + - 为 statement 或 failed operator 写 status/reason； + - 不写任何 base-unit samples； + - 三列 totals 保持 0； +5. 正常执行结束路径由 `ExecStmt.SummaryStmt` 把结构化 snapshot 挂到 `StmtExecInfo`； +6. statement summary v1/v2 通过同一个 `Add`/`Merge` 路径聚合到当前 window 和 history； +7. 早期错误路径不能假设一定有 `ExecStmt.SummaryStmt`。`session.recordReadBillingDemoEarlyError` 需要在调用 `RecordReadBillingDemoForStatement` 后，把 status-only snapshot 通过一个 read-billing 专用 helper 写入同一套 statement-summary-associated status 聚合，例如 `stmtsummaryv2.AddReadBillingDemoStatusOnly(...)`。该 helper 只记录 status table 所需字段，`PLAN_DIGEST` 可为空，不写 base units，不影响普通 statement summary latency/RU 等通用列。 + +### Statement summary v1 + +v1 in-memory path 位于 `pkg/util/stmtsummary/statement_summary.go`： + +- `stmtSummaryStats` 新增 bounded in-memory maps： + - `ReadBillingDemoBaseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg` + - `ReadBillingDemoStatusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg` +- `newStmtSummaryStats`、`stmtSummaryStats.add`、`ReadBillingDemo...Merge` 负责把 `StmtExecInfo` 的 samples 合并进去； +- status-only early error helper 使用同样的 key 构造和 window 选择逻辑，但只 upsert status map；如果 statement digest 不可用，则使用空 digest/空 plan digest 聚合到当前 window，并保留 bounded reason； +- `stmtSummaryReader` 新增面向 base-unit/status 表的 reader，遍历同一个 current/history/cumulative window 后展开 rows； +- `STATEMENTS_SUMMARY_READ_BILLING_DEMO_*` 表沿用 statement summary 的 auth 行为：无 `PROCESS` privilege 时只能看到自己的 statement 聚合。 + +### Statement summary v2 + +v2 persistent path 位于 `pkg/util/stmtsummary/v2`： + +- `StmtRecord` 新增 JSON-stable entry slices，而不是 `map[struct]...` 这类不能稳定 marshal/unmarshal 的字段： + - `ReadBillingDemoBaseUnitAggs []ReadBillingDemoBaseUnitAggEntry` + - `ReadBillingDemoStatusAggs []ReadBillingDemoStatusAggEntry` +- 每个 entry 直接携带 key 字段和 aggregate value；`StmtRecord.Add` / `StmtRecord.Merge` 可以用临时 canonical string key 或小规模线性 upsert 合并后再保持 entry slice 排序； +- `MemReader` 从当前 `StmtRecord` 展开新表 rows； +- `HistoryReader` 读取历史 JSON log 时，如果旧记录没有这些字段，则 entry slices 为 nil，新表返回 0 行，不影响旧 log 解析； +- evicted aggregate record 也合并 read billing entries。对校准而言，evicted row 只作为覆盖率/容量诊断，不应用来拟合精细 weights，因为 digest/plan 维度已经被合并。 + +### Current 和 history 表 + +当前表读取未持久化窗口，history 表读取内存当前窗口加已持久化历史，与现有 `STATEMENTS_SUMMARY` / `STATEMENTS_SUMMARY_HISTORY` 语义保持一致。 + +如果 user 需要完整 workload run，推荐： + +1. 开启 statement summary； +2. 确认 `tidb_enable_read_billing_demo = ON`； +3. 记录 workload 起止时间； +4. 从 `CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_*` 用 `SUMMARY_BEGIN_TIME` / `SUMMARY_END_TIME` 过滤； +5. 同时查 status 表确认失败和 overflow 比例。 + +## 查询和权重校准方法 + +### 开启采集 + +```sql +SET GLOBAL tidb_enable_read_billing_demo = ON; +``` + +建议只在校准 workload 的 session 或测试集群中开启。生产默认仍然是 OFF。 + +### 第一步：检查覆盖率和失败状态 + +先看 statement/operator status： + +```sql +SELECT + status, + reason, + SUM(count) AS count +FROM information_schema.cluster_statements_summary_history_read_billing_demo_status +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v1' +GROUP BY status, reason +ORDER BY count DESC; +``` + +如果 `unsupported`、`unknown_input`、`error` 或 `aggregation_overflow` 占比高，说明 base-unit 数据不能代表整个 workload，需要先收敛覆盖范围或降低维度压力。 + +进一步按 operator 看原因： + +```sql +SELECT + site, + op_class, + operator_kind, + status, + reason, + SUM(count) AS count +FROM information_schema.cluster_statements_summary_history_read_billing_demo_status +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v1' +GROUP BY site, op_class, operator_kind, status, reason +ORDER BY count DESC; +``` + +### 第二步：收集 base units + +按 `site/op_class/unit/source/side` 聚合 base units： + +```sql +SELECT + site, + op_class, + unit, + input_source, + input_side, + SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v1' +GROUP BY site, op_class, unit, input_source, input_side +ORDER BY site, op_class, unit, input_source, input_side; +``` + +这个查询是校准模型的主要输入，相当于得到无系数特征矩阵： + +```text +X[site, op_class, unit] = 当前时间窗口内观测到的 base-unit total +``` + +如果需要区分 plan shape，可以保留 `DIGEST` 和 `PLAN_DIGEST`： + +```sql +SELECT + digest, + plan_digest, + site, + op_class, + unit, + SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v1' +GROUP BY digest, plan_digest, site, op_class, unit; +``` + +### 第三步：检查 row-width evidence + +row width 是模型 factor，不是 runtime 真实 byte sample。拟合前需要检查它是否稳定： + +```sql +SELECT + site, + op_class, + row_width_source, + SUM(sample_count) AS samples, + SUM(row_width_sum) / NULLIF(SUM(sample_count), 0) AS avg_row_width +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND unit = 'input_bytes' + AND model_version = 'v1' +GROUP BY site, op_class, row_width_source +ORDER BY site, op_class, row_width_source; +``` + +如果 `schema_fallback` 占比高，或某些 opclass 的 row width 异常，拟合出来的 byte weight 可能是在补偿 row-width 估计误差，而不是真实 byte cost。 + +### 第四步：用任意 weights 重算 RU + +testers 可以把新 weights 放到 CTE、临时表或外部 notebook。SQL 侧的最小形态如下： + +```sql +WITH observed AS ( + SELECT site, op_class, unit, SUM(value) AS value + FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units + WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' + AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' + AND model_version = 'v1' + GROUP BY site, op_class, unit +), +weights(site, op_class, fixed_weight, row_weight, byte_weight) AS ( + SELECT 'tikv', 'kv_range_scan', 0.070, 0.000045, 0.000020 UNION ALL + SELECT 'tidb', 'projection_eval', 0.020, 0.000020, 0.000004 +) +SELECT + SUM(CASE observed.unit + WHEN 'fixed_events' THEN observed.value * weights.fixed_weight + WHEN 'input_rows' THEN observed.value * weights.row_weight + WHEN 'input_bytes' THEN observed.value * weights.byte_weight + ELSE 0 + END) AS recalculated_ru +FROM observed +JOIN weights + ON observed.site = weights.site + AND observed.op_class = weights.op_class; +``` + +因为统计面保留的是 base units，重算时只需要替换 `weights` 数据，不需要重新跑 workload。 + +### 第五步：拟合新 weights + +对每个校准窗口构造： + +```text +y_window = 观测目标成本 +X_window = 按 site/opclass/unit 聚合的 base-unit totals +``` + +`y_window` 可以选择： + +- workload 窗口内 TiDB/TiKV CPU time； +- 现有 RU consumption，用于兼容性对比； +- benchmark 手工标注成本； +- 混合目标，例如 CPU + IO-normalized scan cost。 + +推荐使用非负线性模型： + +```text +minimize || X * w - y ||^2 +subject to w >= 0 +``` + +拟合规则建议： + +1. 按 `site/op_class` 分别拟合 `fixed`、`row`、`byte` 三类系数。 +2. 保持权重非负。 +3. 使用 hold-out workload window 验证预测误差。 +4. 默认保留符合源码成本形态的单调关系，除非数据强烈反证，例如： + - `agg_hash.row >= agg_stream.row`； + - `bounded_topn.row >= row_limit.row`； + - `kv_range_scan.byte >= filter_eval.byte`； + - `join_hash.row >= join_merge.row`。 +5. 拟合后的新权重应该发布为新的 `weight_version`，不要静默改变 `v1` 的含义。 + +## 初始权重 + +当前 `v1` weights 是 demo calibration seed，不是 production-calibrated billing constants。它们来自源码成本形态和相对排序： + +- scan 同时受 row/key 数和 bytes 影响，比纯 expression eval 更重； +- hash aggregation 和 hash join 需要维护 hash map / hash table / aggregate state，row weight 更高； +- stream aggregation 和 merge join 依赖有序输入，通常低于 hash 版本； +- limit 是最轻的 row-shaped operator； +- TopN 需要 order expression 和 heap maintenance，比 limit 重； +- TiDB reader class 对 bytes 更敏感，因为它负责接收、materialize chunk 和协调 lookup； +- TiDB 与 TiKV 即使 opclass 名称相同，也运行不同代码路径，因此权重分开设置。 + +当前 `v1` 权重如下： + +| site | opclass | fixed_weight | row_weight | byte_weight | 初始设置原因 | +|---|---|---:|---:|---:|---| +| `tikv` | `kv_range_scan` | 0.070 | 0.000045 | 0.000020 | range iterator、MVCC/key-value movement、row decode 和 scan limiter 使 scan 同时对 row/key 和 bytes 敏感。 | +| `tikv` | `kv_point_lookup` | 0.045 | 0.000030 | 0.000012 | point get 有 per-key dispatch 和 decode，但没有 range iterator loop。 | +| `tikv` | `filter_eval` | 0.020 | 0.000040 | 0.000006 | RPN predicate eval 主要按 logical rows 消耗 CPU，byte movement 权重较低。 | +| `tikv` | `projection_eval` | 0.020 | 0.000030 | 0.000006 | projection 会执行表达式或 materialize column reference，row 成本低于 filter。 | +| `tikv` | `row_limit` | 0.010 | 0.000008 | 0.000002 | 主要是 pass-through counter / gate，是最轻的 executor。 | +| `tikv` | `bounded_topn` | 0.060 | 0.000075 | 0.000012 | order expression eval 加 bounded heap maintenance。 | +| `tikv` | `agg_hash` | 0.080 | 0.000100 | 0.000014 | group expression eval、hash group lookup/insert、aggregate state update。 | +| `tikv` | `agg_stream` | 0.060 | 0.000065 | 0.000010 | ordered group compare 加 state update，不构建 hash table。 | +| `tidb` | `filter_eval` | 0.020 | 0.000030 | 0.000005 | TiDB chunk 上的 expression eval，低于 scan 和 hash 类 operator。 | +| `tidb` | `projection_eval` | 0.020 | 0.000020 | 0.000004 | projection / materialization，per-row 成本较低。 | +| `tidb` | `row_limit` | 0.010 | 0.000006 | 0.000001 | 轻量 pass-through control。 | +| `tidb` | `bounded_topn` | 0.060 | 0.000060 | 0.000010 | heap / order expression work，比 full sort 范围小。 | +| `tidb` | `full_ordering` | 0.080 | 0.000070 | 0.000012 | full sort 需要处理完整输入。 | +| `tidb` | `window_eval` | 0.070 | 0.000070 | 0.000010 | partition / order / frame state maintenance。 | +| `tidb` | `agg_hash` | 0.070 | 0.000085 | 0.000012 | hash grouping 和 aggregate state update。 | +| `tidb` | `agg_stream` | 0.050 | 0.000055 | 0.000008 | ordered streaming aggregate，低于 hash agg。 | +| `tidb` | `join_hash` | 0.110 | 0.000115 | 0.000020 | build/probe hash table work，是 TiDB 侧最高 row cost 之一。 | +| `tidb` | `join_merge` | 0.090 | 0.000075 | 0.000012 | ordered comparison / merge，低于 hash join。 | +| `tidb` | `join_lookup` | 0.120 | 0.000120 | 0.000020 | lookup task orchestration 加 join-side work，固定成本和 row 成本都较高。 | +| `tidb` | `reader_receive` | 0.040 | 0.000025 | 0.000014 | network/result receive 和 chunk materialization，对 bytes 敏感。 | +| `tidb` | `lookup_reader` | 0.070 | 0.000045 | 0.000016 | index/table two-phase lookup reader coordination。 | +| `tidb` | `overlay_reader` | 0.050 | 0.000035 | 0.000012 | UnionScan/local overlay merge。 | +| `tidb` | `metadata_reader` | 0.020 | 0.000008 | 0.000002 | metadata / memory-table read，故意设得较轻。 | + +## 性能和容量控制 + +### 低开销原则 + +新统计面必须满足： + +- `tidb_enable_read_billing_demo` 默认 OFF，关闭时不构造 read billing result，也不写 statement summary maps； +- 开启后仍只在 statement end 聚合一次，不打印 per-SQL log； +- base-unit/status key 由 bounded enum-like 字段组成，不包含 SQL text、table name、index name、region、store address 或 plan id； +- `DIGEST_TEXT` 和 `PLAN_DIGEST` 复用 statement summary 已有字段，不进入 read-billing map key； +- 读表时才把 map 展开成多行，写入时保持 map 聚合，不为每次执行追加 unbounded slice； +- cluster 表沿用现有 cluster memtable fan-out，不引入新的后台 scrape。 + +### 单 digest/window map 上限 + +statement summary 已有 `max-stmt-count`、`refresh-interval`、`history-size` 和 LRU eviction，但这些只限制 digest/window 数量。read billing 还需要限制单 digest/window 内的维度 key 数。 + +建议第一版使用保守上限： + +```text +max_base_unit_keys_per_record = 256 +max_status_keys_per_record = 128 +``` + +当超过上限： + +- status table 增加 `status='unknown_input'`、`reason='aggregation_overflow'` 的 reserved row； +- 超出上限的新 base-unit key 不再写入 base-unit table，避免把不完整特征矩阵伪装成完整数据； +- 现有已聚合 key 继续累加； +- 三列 totals 仍只从已接收 base-unit samples 派生，并且用户必须通过 status 表判断该窗口是否可用于拟合。 + +overflow row 必须永远可见，不能被 status map 自身的上限挤掉。实现时预留两个不计入 `max_status_keys_per_record` 的 fixed buckets： + +```text +statement/statement/statement unknown_input aggregation_overflow +statement/statement/statement unknown_input status_aggregation_overflow +``` + +如果 base-unit map 超限，写 `aggregation_overflow` bucket。如果 status map 对普通 status key 超限，写 `status_aggregation_overflow` bucket。reserved buckets 只累加 count，不触发新的 overflow，也不写 base units。 + +这个策略牺牲超高维异常 workload 的完整性，但保证整体性能和内存不会被单个 digest 放大。 + +### 估算内存上界 + +粗略上界： + +```text +max_records = stmt-summary.max-stmt-count * (history-size 或 current window) +per_record_read_billing_keys <= 256 + 128 +``` + +实际 key 数通常远低于上限，因为第一版 opclass、unit、input_source 和 input_side 都是有限集合。典型 SELECT digest/window 只会产生数十个 key。 + +## 兼容性和迁移计划 + +1. 保留 `SUM_READ_BILLING_DEMO_FIXED_EVENTS`、`SUM_READ_BILLING_DEMO_INPUT_ROWS`、`SUM_READ_BILLING_DEMO_INPUT_BYTES` 三列作为 convenience totals。 +2. 文档和后续测试明确：三列不能用于 per-opclass calibration，因为它们丢失维度。 +3. 新 base-unit/status 表成为 workload calibration 的主接口。 +4. Prometheus `tidb_read_billing_demo_*` metrics 可继续保留为 optional observability，但不作为 testers 必需依赖。 +5. v2 persisted statement summary 的 JSON 结构新增 read-billing maps；旧日志没有这些字段时读出来为空，不需要 migration。 +6. 如果后续模型稳定并准备生产化，可以再决定是否移除 demo 三列或把 demo 表改名为非 demo 表；第一版不做破坏性删除。 + +## 测试设计 + +后续实现 loop 至少需要补以下测试。 + +### Planner / calculator tests + +- `RecordReadBillingDemoForStatement` 返回结构化 status/base-unit snapshot； +- success path 只为 billable operators 产生 base units； +- unsupported、unknown-input、error path 不产生 base units，但产生 status/reason； +- normal finish path 通过 `StmtExecInfo` 写 status/base units，early error path 通过 status-only helper 写 SQL-visible status； +- `aggregation_overflow` 和 `status_aggregation_overflow` reserved buckets 可见且不会写 partial unknown key； +- current three-column totals 从 base-unit samples 派生，与现有测试保持兼容。 + +### Statement summary v1 tests + +- `stmtSummaryStats.add` 能按 `site/op_class/unit/input_source/input_side/row_width_source` 聚合 value； +- status-only early error helper 能写入 current window 的 status table，且不写 base-unit rows； +- `Merge` / evicted aggregate 能合并 read billing maps； +- `STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS` current/history rows 与 digest/window 对齐； +- `STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS` 能看到 failed billing； +- base-unit/status map 超限时 reserved overflow buckets 不受 cap 影响； +- auth 行为与 statement summary 保持一致。 + +### Statement summary v2 tests + +- `StmtRecord.Add` / `Merge` 聚合 JSON-stable read billing entry slices； +- current reader 展开 base-unit/status rows； +- history reader 能读取新 JSON 字段； +- old history JSON 没有 read-billing 字段时返回 0 行且不报错； +- entry slice marshal/unmarshal 后 key 字段和值不丢失，排序保持 deterministic； +- evicted aggregate 包含 read billing maps，但测试中明确它不适合精细拟合。 + +### Information schema / cluster table tests + +- 新表注册、列顺序、cluster `INSTANCE` 列； +- digest/time predicate extractor 复用 statement summary extractor； +- `CLUSTER_*` 表能查询多个 TiDB instance 的 rows； +- 默认 OFF 时新表无 read-billing rows； +- 开启 demo 后可以用 SQL 直接按 `site/op_class/unit` 重算 RU。 + +### 性能验证 + +- targeted benchmark 比较 demo OFF、demo ON + scalar totals、demo ON + dimension maps； +- 验证正常 workload 下 statement end 额外开销不显著； +- 验证高维异常 workload 触发 overflow 后内存和写入开销保持 bounded。 + +## 备选方案 + +### 继续只用 Prometheus metrics + +不采用。metrics label shape 可以保存 `site/op_class/unit`，但 downstream testers 明确不希望依赖 Prometheus；metrics retention、scrape interval 和 workload 窗口对齐也会增加校准流程复杂度。 + +### 每条 SQL 打日志 + +不采用。per-SQL log 会显著影响 QPS，也会把校准流程变成日志采集和解析问题。 + +### 只扩展现有 statement summary 三列 + +不采用。三列 totals 丢失 `site/op_class/unit`，只能重算一个全局粗粒度模型，不能直接优化不同 opclass 的 weights。 + +### 在 statement summary 单表里放 JSON/encoded column + +暂不采用。它减少表数量，但把校准所需的 group-by 和 join 复杂度推给用户，也让 failed billing status 不够直接。可以作为后续低表面面积方案重新评估。 + +### 每个 operator kind 一个 weight + +不采用。这样会显著提高模型和统计面复杂度，也会让校准更脆弱。当前保留 `operator_kind` 用于观测，但 weight 绑定在有限 opclass 上。 + +### workload 统计面直接上报 preview RU + +不采用。preview RU 是加权结果，会遮住校准需要的 base-unit 原始证据。workload 表上报 base units；单语句调试通过 `EXPLAIN ANALYZE FORMAT='RU'` 看 preview RU。 + +## 风险和未决问题 + +### 风险 + +- 新表会增加 statement summary 内存占用；必须用单 digest/window key 上限和 overflow status 控制。 +- v2 persistent history 会扩大 statement summary JSON record；需要 benchmark 和 history-reader 兼容测试。 +- row-width factor 仍然是 modeled value，如果大量来自 `schema_fallback`，byte weight 可能补偿 row-width 误差。 +- evicted aggregate 会丢 digest/plan 细节，不适合作为精细校准输入；用户需要查询 status 和 evicted 表判断窗口质量。 +- 当前 point lookup miss 的 variable row cost 证据仍有限；没有 requested-key 证据时主要由 fixed event 覆盖。 + +### 未决问题 + +- 第一轮生产化拟合应该选什么目标信号：CPU、现有 RU、latency-normalized cost，还是混合目标？ +- point lookup 是否需要接入 requested key count，使 key miss 也能产生 variable row cost？ +- scan bytes 后续是否应该拆成 total key size、processed key size、value size 等多个 bounded units？ +- `aggregation_overflow` 的默认上限是否需要暴露为配置，还是作为 demo 常量即可？ +- TiFlash、MPP、IndexMerge 何时从 fail-closed 进入 supported opclasses？ From 421e3cedf9befd15fba8cf9eae1f008d777ad18c Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 3 Jul 2026 00:14:59 +0800 Subject: [PATCH 05/25] stmt-summary read billing demo calibration tables --- pkg/executor/adapter.go | 9 +- pkg/executor/builder.go | 8 + pkg/executor/explain_test.go | 6 + pkg/executor/stmtsummary.go | 59 +- pkg/infoschema/cluster.go | 38 +- pkg/infoschema/tables.go | 185 +++-- pkg/planner/core/explain_ru.go | 94 ++- pkg/planner/core/logical_plan_builder.go | 10 +- .../operator/logicalop/logical_mem_table.go | 8 + pkg/planner/core/pb_to_plan.go | 6 +- pkg/session/BUILD.bazel | 2 + pkg/session/session.go | 33 +- pkg/util/stmtsummary/BUILD.bazel | 4 +- pkg/util/stmtsummary/evicted.go | 6 + pkg/util/stmtsummary/read_billing.go | 632 ++++++++++++++++++ pkg/util/stmtsummary/read_billing_test.go | 87 +++ pkg/util/stmtsummary/reader.go | 242 ++++++- pkg/util/stmtsummary/statement_summary.go | 194 +++++- .../stmtsummary/statement_summary_test.go | 187 ++++++ pkg/util/stmtsummary/v2/BUILD.bazel | 2 +- pkg/util/stmtsummary/v2/reader.go | 300 ++++++++- pkg/util/stmtsummary/v2/reader_test.go | 165 +++++ pkg/util/stmtsummary/v2/record.go | 77 ++- pkg/util/stmtsummary/v2/record_test.go | 86 +++ pkg/util/stmtsummary/v2/stmtsummary.go | 54 ++ 25 files changed, 2385 insertions(+), 109 deletions(-) create mode 100644 pkg/util/stmtsummary/read_billing.go create mode 100644 pkg/util/stmtsummary/read_billing_test.go diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index eb6f25a72fa35..79c20aff81045 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -1701,11 +1701,11 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo // Lazy SELECT metrics are emitted when the record set is closed. If a // client abandons a result without closing/draining it, this first demo can // miss that statement instead of guessing an incomplete status. - readBillingDemoBaseUnits := plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) + readBillingDemoStats := plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) a.updateNetworkTrafficStatsAndMetrics() // `LowSlowQuery` and `SummaryStmt` must be called before recording `PrevStmt`. a.LogSlowQuery(txnTS, succ, hasMoreResults) - a.SummaryStmt(succ, readBillingDemoBaseUnits) + a.SummaryStmt(succ, readBillingDemoStats) a.observeStmtFinishedForTopProfiling() a.UpdatePlanCacheRuntimeInfo() if sessVars.StmtCtx.IsTiFlash.Load() { @@ -2192,7 +2192,7 @@ func (digest planDigestAlias) planDigestDumpTriggerCheck(config *traceevent.Dump } // SummaryStmt collects statements for information_schema.statements_summary -func (a *ExecStmt) SummaryStmt(succ bool, readBillingDemoBaseUnits stmtsummary.ReadBillingDemoBaseUnitSummary) { +func (a *ExecStmt) SummaryStmt(succ bool, readBillingDemoStats stmtsummary.ReadBillingDemoStatementStats) { sessVars := a.Ctx.GetSessionVars() var userString string if sessVars.User != nil { @@ -2296,7 +2296,8 @@ func (a *ExecStmt) SummaryStmt(succ bool, readBillingDemoBaseUnits stmtsummary.R stmtExecInfo.KeyspaceID = keyspaceID stmtExecInfo.RUDetail = ruDetail stmtExecInfo.TotalRUV2 = calculateStatementTotalRUV2(sessVars.RUV2Metrics, sessVars.RUV2Weights(), ruDetail) - stmtExecInfo.ReadBillingDemoBaseUnits = readBillingDemoBaseUnits + stmtExecInfo.ReadBillingDemoStats = readBillingDemoStats + stmtExecInfo.ReadBillingDemoBaseUnits = readBillingDemoStats.Totals stmtExecInfo.ResourceGroupName = sessVars.StmtCtx.ResourceGroupName stmtExecInfo.CPUUsages = sessVars.SQLCPUUsages.GetCPUUsages() stmtExecInfo.PlanCacheUnqualified = sessVars.StmtCtx.PlanCacheUnqualified() diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go index 7644929184311..8ffa1e7c5f473 100644 --- a/pkg/executor/builder.go +++ b/pkg/executor/builder.go @@ -2616,10 +2616,18 @@ func (b *executorBuilder) buildMemTable(v *physicalop.PhysicalMemTable) exec.Exe case strings.ToLower(infoschema.TableStatementsSummary), strings.ToLower(infoschema.TableStatementsSummaryHistory), strings.ToLower(infoschema.TableStatementsSummaryEvicted), + strings.ToLower(infoschema.TableStatementsSummaryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.TableStatementsSummaryReadBillingDemoStatus), + strings.ToLower(infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus), strings.ToLower(infoschema.TableTiDBStatementsStats), strings.ToLower(infoschema.ClusterTableStatementsSummary), strings.ToLower(infoschema.ClusterTableStatementsSummaryHistory), strings.ToLower(infoschema.ClusterTableStatementsSummaryEvicted), + strings.ToLower(infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits), + strings.ToLower(infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus), + strings.ToLower(infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus), strings.ToLower(infoschema.ClusterTableTiDBStatementsStats): var extractor *plannercore.StatementsSummaryExtractor if v.Extractor != nil { diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index e765aec7f9b9a..f5e848c09af5e 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -797,6 +797,9 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, 1.0, readExecutorCounterValue(t, projectionFixedEvents)) tk.MustExec("set tidb_enable_read_billing_demo=off") tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events > 0, sum_read_billing_demo_input_rows > 0, sum_read_billing_demo_input_bytes > 0 from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 1 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_act_rows all v1 v1 1 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) + tk.MustQuery(`select column_name from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and ordinal_position = 1`).Check(testkit.Rows("INSTANCE")) tk.MustExec("set tidb_enable_read_billing_demo=on") beforeRestrictedSuccess := readExecutorCounterValue(t, success) @@ -821,6 +824,7 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, beforeUnsupported+1, readExecutorCounterValue(t, unsupported)) require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + tk.MustQuery("select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_demo`%' and status = 'unsupported' group by status, reason").Check(testkit.Rows("unsupported unsupported_non_select 1")) beforeUnknownInput := readExecutorCounterValue(t, unknownInput) beforeBaseUnits = readExecutorCounterValue(t, projectionFixedEvents) @@ -861,6 +865,8 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.NoError(t, rs.Close()) require.Equal(t, beforeEarlyError+2, readExecutorCounterValue(t, errorStatus)) require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + tk.MustQuery(`select sum(count) from information_schema.statements_summary_read_billing_demo_status where site = 'statement' and status = 'error' and reason = 'statement_error'`).Check(testkit.Rows("3")) + tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units b join information_schema.statements_summary_read_billing_demo_status s on b.digest = s.digest where s.status in ('unsupported', 'error')`).Check(testkit.Rows("0")) } func readExecutorCounterValue(t *testing.T, counter prometheus.Counter) float64 { diff --git a/pkg/executor/stmtsummary.go b/pkg/executor/stmtsummary.go index 990a1808a66f3..12c2b7a71115e 100644 --- a/pkg/executor/stmtsummary.go +++ b/pkg/executor/stmtsummary.go @@ -159,7 +159,13 @@ func (e *stmtSummaryRetriever) initSummaryRowsReader(sctx sessionctx.Context) (* } var rows [][]types.Datum - if isCumulativeTable(e.table.Name.O) { + if kind, ok := readBillingDemoTableKind(e.table.Name.O); ok { + if isCurrentTable(e.table.Name.O) { + rows = reader.GetReadBillingDemoCurrentRows(kind) + } else if isHistoryTable(e.table.Name.O) { + rows = reader.GetReadBillingDemoHistoryRows(kind) + } + } else if isCumulativeTable(e.table.Name.O) { rows = reader.GetStmtSummaryCumulativeRows() } else if isCurrentTable(e.table.Name.O) { rows = reader.GetStmtSummaryCurrentRows() @@ -254,8 +260,14 @@ func (r *stmtSummaryRetrieverV2) initSummaryRowsReader(ctx context.Context, sctx return nil, err } - mem := stmtsummaryv2.NewMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges) - memRows := mem.Rows() + var memRows [][]types.Datum + var readBillingDemoKind stmtsummary.ReadBillingDemoTableKind + readBillingDemoKind, isReadBillingDemoTable := readBillingDemoTableKind(r.table.Name.O) + if isReadBillingDemoTable { + memRows = stmtsummaryv2.NewReadBillingDemoMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges, readBillingDemoKind).Rows() + } else { + memRows = stmtsummaryv2.NewMemReader(stmtSummary, columns, instanceAddr, tz, user, priv, digests, timeRanges).Rows() + } var rowsReader *rowsReader if isCurrentTable(r.table.Name.O) { @@ -264,7 +276,13 @@ func (r *stmtSummaryRetrieverV2) initSummaryRowsReader(ctx context.Context, sctx if isHistoryTable(r.table.Name.O) { // history table should return all rows including mem and disk concurrent := sctx.GetSessionVars().Concurrency.DistSQLScanConcurrency() - history, err := stmtsummaryv2.NewHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent) + var history rowsPuller + var err error + if isReadBillingDemoTable { + history, err = stmtsummaryv2.NewReadBillingDemoHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent, readBillingDemoKind) + } else { + history, err = stmtsummaryv2.NewHistoryReader(ctx, columns, instanceAddr, tz, user, priv, digests, timeRanges, concurrent) + } if err != nil { return nil, err } @@ -347,6 +365,10 @@ func isClusterTable(originalTableName string) bool { case infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory, infoschema.ClusterTableStatementsSummaryEvicted, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus, infoschema.ClusterTableTiDBStatementsStats: return true } @@ -367,7 +389,11 @@ func isCumulativeTable(originalTableName string) bool { func isCurrentTable(originalTableName string) bool { switch originalTableName { case infoschema.TableStatementsSummary, - infoschema.ClusterTableStatementsSummary: + infoschema.ClusterTableStatementsSummary, + infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus: return true } @@ -377,13 +403,34 @@ func isCurrentTable(originalTableName string) bool { func isHistoryTable(originalTableName string) bool { switch originalTableName { case infoschema.TableStatementsSummaryHistory, - infoschema.ClusterTableStatementsSummaryHistory: + infoschema.ClusterTableStatementsSummaryHistory, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: return true } return false } +func readBillingDemoTableKind(originalTableName string) (stmtsummary.ReadBillingDemoTableKind, bool) { + switch originalTableName { + case infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits: + return stmtsummary.ReadBillingDemoTableBaseUnits, true + case infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: + return stmtsummary.ReadBillingDemoTableStatus, true + default: + return 0, false + } +} + func isEvictedTable(originalTableName string) bool { switch originalTableName { case infoschema.TableStatementsSummaryEvicted, diff --git a/pkg/infoschema/cluster.go b/pkg/infoschema/cluster.go index 6ed27e538af87..9bc4934a9ccff 100644 --- a/pkg/infoschema/cluster.go +++ b/pkg/infoschema/cluster.go @@ -43,6 +43,14 @@ const ( ClusterTableStatementsSummaryHistory = "CLUSTER_STATEMENTS_SUMMARY_HISTORY" // ClusterTableStatementsSummaryEvicted is the string constant of cluster statement summary evict table. ClusterTableStatementsSummaryEvicted = "CLUSTER_STATEMENTS_SUMMARY_EVICTED" + // ClusterTableStatementsSummaryReadBillingDemoBaseUnits is the string constant of cluster read billing demo base-unit summary table. + ClusterTableStatementsSummaryReadBillingDemoBaseUnits = "CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS" + // ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits is the string constant of cluster read billing demo history base-unit summary table. + ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits = "CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS" + // ClusterTableStatementsSummaryReadBillingDemoStatus is the string constant of cluster read billing demo status summary table. + ClusterTableStatementsSummaryReadBillingDemoStatus = "CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS" + // ClusterTableStatementsSummaryHistoryReadBillingDemoStatus is the string constant of cluster read billing demo history status summary table. + ClusterTableStatementsSummaryHistoryReadBillingDemoStatus = "CLUSTER_STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS" // ClusterTableTiDBStatementsStats is the string constant of the cluster statement stats table. ClusterTableTiDBStatementsStats = "CLUSTER_TIDB_STATEMENTS_STATS" // ClusterTableTiDBTrx is the string constant of cluster transaction running table. @@ -63,19 +71,23 @@ const ( // memTableToAllTiDBClusterTables means add memory table to cluster table that will send cop request to all TiDB nodes. var memTableToAllTiDBClusterTables = map[string]string{ - TableSlowQuery: ClusterTableSlowLog, - TableProcesslist: ClusterTableProcesslist, - TableStatementsSummary: ClusterTableStatementsSummary, - TableStatementsSummaryHistory: ClusterTableStatementsSummaryHistory, - TableStatementsSummaryEvicted: ClusterTableStatementsSummaryEvicted, - TableTiDBStatementsStats: ClusterTableTiDBStatementsStats, - TableTiDBTrx: ClusterTableTiDBTrx, - TableDeadlocks: ClusterTableDeadlocks, - TableTrxSummary: ClusterTableTrxSummary, - TableMemoryUsage: ClusterTableMemoryUsage, - TableMemoryUsageOpsHistory: ClusterTableMemoryUsageOpsHistory, - TableTiDBIndexUsage: ClusterTableTiDBIndexUsage, - TableTiDBPlanCache: ClusterTableTiDBPlanCache, + TableSlowQuery: ClusterTableSlowLog, + TableProcesslist: ClusterTableProcesslist, + TableStatementsSummary: ClusterTableStatementsSummary, + TableStatementsSummaryHistory: ClusterTableStatementsSummaryHistory, + TableStatementsSummaryEvicted: ClusterTableStatementsSummaryEvicted, + TableStatementsSummaryReadBillingDemoBaseUnits: ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + TableStatementsSummaryHistoryReadBillingDemoBaseUnits: ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + TableStatementsSummaryReadBillingDemoStatus: ClusterTableStatementsSummaryReadBillingDemoStatus, + TableStatementsSummaryHistoryReadBillingDemoStatus: ClusterTableStatementsSummaryHistoryReadBillingDemoStatus, + TableTiDBStatementsStats: ClusterTableTiDBStatementsStats, + TableTiDBTrx: ClusterTableTiDBTrx, + TableDeadlocks: ClusterTableDeadlocks, + TableTrxSummary: ClusterTableTrxSummary, + TableMemoryUsage: ClusterTableMemoryUsage, + TableMemoryUsageOpsHistory: ClusterTableMemoryUsageOpsHistory, + TableTiDBIndexUsage: ClusterTableTiDBIndexUsage, + TableTiDBPlanCache: ClusterTableTiDBPlanCache, } var memTableToAllTiDBClusterTablesWithLowerCase = make(map[string]string) diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index 6d9b7293bb612..a620816e06703 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -170,6 +170,14 @@ const ( TableStatementsSummaryHistory = "STATEMENTS_SUMMARY_HISTORY" // TableStatementsSummaryEvicted is the string constant of statements summary evicted table. TableStatementsSummaryEvicted = "STATEMENTS_SUMMARY_EVICTED" + // TableStatementsSummaryReadBillingDemoBaseUnits is the string constant of read billing demo base-unit summary table. + TableStatementsSummaryReadBillingDemoBaseUnits = "STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS" + // TableStatementsSummaryHistoryReadBillingDemoBaseUnits is the string constant of read billing demo history base-unit summary table. + TableStatementsSummaryHistoryReadBillingDemoBaseUnits = "STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_BASE_UNITS" + // TableStatementsSummaryReadBillingDemoStatus is the string constant of read billing demo status summary table. + TableStatementsSummaryReadBillingDemoStatus = "STATEMENTS_SUMMARY_READ_BILLING_DEMO_STATUS" + // TableStatementsSummaryHistoryReadBillingDemoStatus is the string constant of read billing demo history status summary table. + TableStatementsSummaryHistoryReadBillingDemoStatus = "STATEMENTS_SUMMARY_HISTORY_READ_BILLING_DEMO_STATUS" // TableTiDBStatementsStats is the string constant of the TiDB statement stats table. TableTiDBStatementsStats = "TIDB_STATEMENTS_STATS" // TableStorageStats is a table that contains all tables disk usage @@ -319,41 +327,49 @@ var tableIDMap = map[string]int64{ TableTiFlashSegments: autoid.InformationSchemaDBID + 65, // Removed, see https://github.com/pingcap/tidb/issues/28890 //TablePlacementPolicy: autoid.InformationSchemaDBID + 66, - TableClientErrorsSummaryGlobal: autoid.InformationSchemaDBID + 67, - TableClientErrorsSummaryByUser: autoid.InformationSchemaDBID + 68, - TableClientErrorsSummaryByHost: autoid.InformationSchemaDBID + 69, - TableTiDBTrx: autoid.InformationSchemaDBID + 70, - ClusterTableTiDBTrx: autoid.InformationSchemaDBID + 71, - TableDeadlocks: autoid.InformationSchemaDBID + 72, - ClusterTableDeadlocks: autoid.InformationSchemaDBID + 73, - TableDataLockWaits: autoid.InformationSchemaDBID + 74, - TableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 75, - ClusterTableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 76, - TableAttributes: autoid.InformationSchemaDBID + 77, - TableTiDBHotRegionsHistory: autoid.InformationSchemaDBID + 78, - TablePlacementPolicies: autoid.InformationSchemaDBID + 79, - TableTrxSummary: autoid.InformationSchemaDBID + 80, - ClusterTableTrxSummary: autoid.InformationSchemaDBID + 81, - TableVariablesInfo: autoid.InformationSchemaDBID + 82, - TableUserAttributes: autoid.InformationSchemaDBID + 83, - TableMemoryUsage: autoid.InformationSchemaDBID + 84, - TableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 85, - ClusterTableMemoryUsage: autoid.InformationSchemaDBID + 86, - ClusterTableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 87, - TableResourceGroups: autoid.InformationSchemaDBID + 88, - TableRunawayWatches: autoid.InformationSchemaDBID + 89, - TableCheckConstraints: autoid.InformationSchemaDBID + 90, - TableTiDBCheckConstraints: autoid.InformationSchemaDBID + 91, - TableKeywords: autoid.InformationSchemaDBID + 92, - TableTiDBIndexUsage: autoid.InformationSchemaDBID + 93, - ClusterTableTiDBIndexUsage: autoid.InformationSchemaDBID + 94, - TableTiFlashIndexes: autoid.InformationSchemaDBID + 95, - TableTiDBPlanCache: autoid.InformationSchemaDBID + 96, - ClusterTableTiDBPlanCache: autoid.InformationSchemaDBID + 97, - TableTiDBStatementsStats: autoid.InformationSchemaDBID + 98, - ClusterTableTiDBStatementsStats: autoid.InformationSchemaDBID + 99, - TableKeyspaceMeta: autoid.InformationSchemaDBID + 100, - TableSchemataExtensions: autoid.InformationSchemaDBID + 101, + TableClientErrorsSummaryGlobal: autoid.InformationSchemaDBID + 67, + TableClientErrorsSummaryByUser: autoid.InformationSchemaDBID + 68, + TableClientErrorsSummaryByHost: autoid.InformationSchemaDBID + 69, + TableTiDBTrx: autoid.InformationSchemaDBID + 70, + ClusterTableTiDBTrx: autoid.InformationSchemaDBID + 71, + TableDeadlocks: autoid.InformationSchemaDBID + 72, + ClusterTableDeadlocks: autoid.InformationSchemaDBID + 73, + TableDataLockWaits: autoid.InformationSchemaDBID + 74, + TableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 75, + ClusterTableStatementsSummaryEvicted: autoid.InformationSchemaDBID + 76, + TableAttributes: autoid.InformationSchemaDBID + 77, + TableTiDBHotRegionsHistory: autoid.InformationSchemaDBID + 78, + TablePlacementPolicies: autoid.InformationSchemaDBID + 79, + TableTrxSummary: autoid.InformationSchemaDBID + 80, + ClusterTableTrxSummary: autoid.InformationSchemaDBID + 81, + TableVariablesInfo: autoid.InformationSchemaDBID + 82, + TableUserAttributes: autoid.InformationSchemaDBID + 83, + TableMemoryUsage: autoid.InformationSchemaDBID + 84, + TableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 85, + ClusterTableMemoryUsage: autoid.InformationSchemaDBID + 86, + ClusterTableMemoryUsageOpsHistory: autoid.InformationSchemaDBID + 87, + TableResourceGroups: autoid.InformationSchemaDBID + 88, + TableRunawayWatches: autoid.InformationSchemaDBID + 89, + TableCheckConstraints: autoid.InformationSchemaDBID + 90, + TableTiDBCheckConstraints: autoid.InformationSchemaDBID + 91, + TableKeywords: autoid.InformationSchemaDBID + 92, + TableTiDBIndexUsage: autoid.InformationSchemaDBID + 93, + ClusterTableTiDBIndexUsage: autoid.InformationSchemaDBID + 94, + TableTiFlashIndexes: autoid.InformationSchemaDBID + 95, + TableTiDBPlanCache: autoid.InformationSchemaDBID + 96, + ClusterTableTiDBPlanCache: autoid.InformationSchemaDBID + 97, + TableTiDBStatementsStats: autoid.InformationSchemaDBID + 98, + ClusterTableTiDBStatementsStats: autoid.InformationSchemaDBID + 99, + TableKeyspaceMeta: autoid.InformationSchemaDBID + 100, + TableSchemataExtensions: autoid.InformationSchemaDBID + 101, + TableStatementsSummaryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 102, + TableStatementsSummaryHistoryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 103, + ClusterTableStatementsSummaryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 104, + ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits: autoid.InformationSchemaDBID + 105, + TableStatementsSummaryReadBillingDemoStatus: autoid.InformationSchemaDBID + 106, + TableStatementsSummaryHistoryReadBillingDemoStatus: autoid.InformationSchemaDBID + 107, + ClusterTableStatementsSummaryReadBillingDemoStatus: autoid.InformationSchemaDBID + 108, + ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: autoid.InformationSchemaDBID + 109, } // columnInfo represents the basic column information of all kinds of INFORMATION_SCHEMA tables @@ -1447,6 +1463,49 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.StorageMPPStr, tp: mysql.TypeTiny, size: 1, flag: mysql.NotNullFlag, comment: "Whether the last statement read data from TiFlash"}, } +var tableStatementsSummaryReadBillingDemoBaseUnitCols = []columnInfo{ + {name: stmtsummary.SummaryBeginTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "Begin time of this summary"}, + {name: stmtsummary.SummaryEndTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "End time of this summary"}, + {name: stmtsummary.StmtTypeStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"}, + {name: stmtsummary.SchemaNameStr, tp: mysql.TypeVarchar, size: 64, comment: "Current schema"}, + {name: stmtsummary.DigestStr, tp: mysql.TypeVarchar, size: 64}, + {name: stmtsummary.DigestTextStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, flag: mysql.NotNullFlag, comment: "Normalized statement"}, + {name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"}, + {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, + {name: stmtsummary.ReadBillingDemoModelVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo model version"}, + {name: stmtsummary.ReadBillingDemoWeightVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo weight version"}, + {name: stmtsummary.ReadBillingDemoSiteStr, tp: mysql.TypeVarchar, size: 16, flag: mysql.NotNullFlag, comment: "Read billing execution site"}, + {name: stmtsummary.ReadBillingDemoOpClassStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Read billing operator class"}, + {name: stmtsummary.ReadBillingDemoOperatorKindStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Physical operator kind"}, + {name: stmtsummary.ReadBillingDemoUnitStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Coefficient-free base unit"}, + {name: stmtsummary.ReadBillingDemoInputSourceStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Runtime input evidence source"}, + {name: stmtsummary.ReadBillingDemoInputSideStr, tp: mysql.TypeVarchar, size: 16, flag: mysql.NotNullFlag, comment: "Operator input side"}, + {name: stmtsummary.ReadBillingDemoRowWidthSource, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Row-width evidence source"}, + {name: stmtsummary.ReadBillingDemoValueStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Aggregated base-unit value"}, + {name: stmtsummary.ReadBillingDemoSampleCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Number of unit samples"}, + {name: stmtsummary.ReadBillingDemoRowWidthSumStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Sum of row-width samples"}, + {name: stmtsummary.ReadBillingDemoAvgRowWidthStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average row-width sample"}, +} + +var tableStatementsSummaryReadBillingDemoStatusCols = []columnInfo{ + {name: stmtsummary.SummaryBeginTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "Begin time of this summary"}, + {name: stmtsummary.SummaryEndTimeStr, tp: mysql.TypeTimestamp, size: 26, flag: mysql.NotNullFlag, comment: "End time of this summary"}, + {name: stmtsummary.StmtTypeStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"}, + {name: stmtsummary.SchemaNameStr, tp: mysql.TypeVarchar, size: 64, comment: "Current schema"}, + {name: stmtsummary.DigestStr, tp: mysql.TypeVarchar, size: 64}, + {name: stmtsummary.DigestTextStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, flag: mysql.NotNullFlag, comment: "Normalized statement"}, + {name: stmtsummary.PlanDigestStr, tp: mysql.TypeVarchar, size: 64, comment: "Digest of its execution plan"}, + {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, + {name: stmtsummary.ReadBillingDemoModelVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo model version"}, + {name: stmtsummary.ReadBillingDemoWeightVersionStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing demo weight version"}, + {name: stmtsummary.ReadBillingDemoSiteStr, tp: mysql.TypeVarchar, size: 16, flag: mysql.NotNullFlag, comment: "Read billing execution site"}, + {name: stmtsummary.ReadBillingDemoOpClassStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Read billing operator class"}, + {name: stmtsummary.ReadBillingDemoOperatorKindStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Physical operator kind"}, + {name: stmtsummary.ReadBillingDemoStatusStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Read billing accounting status"}, + {name: stmtsummary.ReadBillingDemoReasonStr, tp: mysql.TypeVarchar, size: 128, flag: mysql.NotNullFlag, comment: "Read billing accounting status reason"}, + {name: stmtsummary.ReadBillingDemoCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Number of status samples"}, +} + var tableTiDBStatementsStatsCols = []columnInfo{ {name: stmtsummary.StmtTypeStr, tp: mysql.TypeVarchar, size: 64, flag: mysql.NotNullFlag, comment: "Statement type"}, {name: stmtsummary.SchemaNameStr, tp: mysql.TypeVarchar, size: 64, comment: "Current schema"}, @@ -2519,32 +2578,36 @@ var tableNameToColumns = map[string][]columnInfo{ TableStatementsSummary: tableStatementsSummaryCols, TableStatementsSummaryHistory: tableStatementsSummaryCols, TableStatementsSummaryEvicted: tableStatementsSummaryEvictedCols, - TableStorageStats: tableStorageStatsCols, - TableTiDBStatementsStats: tableTiDBStatementsStatsCols, - TableTiFlashTables: tableTableTiFlashTablesCols, - TableTiFlashSegments: tableTableTiFlashSegmentsCols, - TableTiFlashIndexes: tableTiFlashIndexesCols, - TableClientErrorsSummaryGlobal: tableClientErrorsSummaryGlobalCols, - TableClientErrorsSummaryByUser: tableClientErrorsSummaryByUserCols, - TableClientErrorsSummaryByHost: tableClientErrorsSummaryByHostCols, - TableTiDBTrx: tableTiDBTrxCols, - TableDeadlocks: tableDeadlocksCols, - TableDataLockWaits: tableDataLockWaitsCols, - TableAttributes: tableAttributesCols, - TablePlacementPolicies: tablePlacementPoliciesCols, - TableTrxSummary: tableTrxSummaryCols, - TableVariablesInfo: tableVariablesInfoCols, - TableUserAttributes: tableUserAttributesCols, - TableMemoryUsage: tableMemoryUsageCols, - TableMemoryUsageOpsHistory: tableMemoryUsageOpsHistoryCols, - TableResourceGroups: tableResourceGroupsCols, - TableRunawayWatches: tableRunawayWatchListCols, - TableCheckConstraints: tableCheckConstraintsCols, - TableTiDBCheckConstraints: tableTiDBCheckConstraintsCols, - TableKeywords: tableKeywords, - TableTiDBIndexUsage: tableTiDBIndexUsage, - TableTiDBPlanCache: tablePlanCache, - TableKeyspaceMeta: tableKeyspaceMetaCols, + TableStatementsSummaryReadBillingDemoBaseUnits: tableStatementsSummaryReadBillingDemoBaseUnitCols, + TableStatementsSummaryHistoryReadBillingDemoBaseUnits: tableStatementsSummaryReadBillingDemoBaseUnitCols, + TableStatementsSummaryReadBillingDemoStatus: tableStatementsSummaryReadBillingDemoStatusCols, + TableStatementsSummaryHistoryReadBillingDemoStatus: tableStatementsSummaryReadBillingDemoStatusCols, + TableStorageStats: tableStorageStatsCols, + TableTiDBStatementsStats: tableTiDBStatementsStatsCols, + TableTiFlashTables: tableTableTiFlashTablesCols, + TableTiFlashSegments: tableTableTiFlashSegmentsCols, + TableTiFlashIndexes: tableTiFlashIndexesCols, + TableClientErrorsSummaryGlobal: tableClientErrorsSummaryGlobalCols, + TableClientErrorsSummaryByUser: tableClientErrorsSummaryByUserCols, + TableClientErrorsSummaryByHost: tableClientErrorsSummaryByHostCols, + TableTiDBTrx: tableTiDBTrxCols, + TableDeadlocks: tableDeadlocksCols, + TableDataLockWaits: tableDataLockWaitsCols, + TableAttributes: tableAttributesCols, + TablePlacementPolicies: tablePlacementPoliciesCols, + TableTrxSummary: tableTrxSummaryCols, + TableVariablesInfo: tableVariablesInfoCols, + TableUserAttributes: tableUserAttributesCols, + TableMemoryUsage: tableMemoryUsageCols, + TableMemoryUsageOpsHistory: tableMemoryUsageOpsHistoryCols, + TableResourceGroups: tableResourceGroupsCols, + TableRunawayWatches: tableRunawayWatchListCols, + TableCheckConstraints: tableCheckConstraintsCols, + TableTiDBCheckConstraints: tableTiDBCheckConstraintsCols, + TableKeywords: tableKeywords, + TableTiDBIndexUsage: tableTiDBIndexUsage, + TableTiDBPlanCache: tablePlanCache, + TableKeyspaceMeta: tableKeyspaceMetaCols, } func createInfoSchemaTable(_ autoid.Allocators, _ func() (pools.Resource, error), meta *model.TableInfo) (table.Table, error) { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 0512d637b82ff..c77c8cbf7336d 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -214,16 +214,16 @@ type explainRURow struct { } // RecordReadBillingDemoForStatement emits coefficient-free read billing demo -// metrics for a completed statement and returns the statement-level base-unit -// totals. It is intentionally independent from RU v2 billing/reporting and -// never calls resource-control reporters. -func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, stmt ast.StmtNode, execErr error) stmtsummary.ReadBillingDemoBaseUnitSummary { +// metrics for a completed statement and returns the structured statement +// summary stats. It is intentionally independent from RU v2 billing/reporting +// and never calls resource-control reporters. +func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, stmt ast.StmtNode, execErr error) stmtsummary.ReadBillingDemoStatementStats { if sctx == nil || sctx.GetSessionVars() == nil || !sctx.GetSessionVars().EnableReadBillingDemo { - return stmtsummary.ReadBillingDemoBaseUnitSummary{} + return stmtsummary.ReadBillingDemoStatementStats{} } // Restricted/internal SQL is not external workload calibration input. if sctx.GetSessionVars().InRestrictedSQL || sctx.GetSessionVars().StmtCtx.InRestrictedSQL { - return stmtsummary.ReadBillingDemoBaseUnitSummary{} + return stmtsummary.ReadBillingDemoStatementStats{} } planCtx := readBillingDemoPlanContext(plan) if planCtx == nil { @@ -231,7 +231,7 @@ func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, } result := buildReadBillingDemoResult(planCtx, plan, stmt, execErr) recordReadBillingDemoResult(result) - return summarizeReadBillingDemoBaseUnits(result) + return buildReadBillingDemoStatementStats(result) } func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error) readBillingDemoResult { @@ -356,6 +356,86 @@ func summarizeReadBillingDemoBaseUnits(result readBillingDemoResult) stmtsummary return summary } +func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummary.ReadBillingDemoStatementStats { + stats := stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoWeightVersion, + } + status := result.status + if status == "" { + status = readBillingDemoStatusUnknownInput + } + reason := result.reason + if reason == "" { + reason = readBillingDemoReasonNone + } + stats.Statuses = append(stats.Statuses, stmtsummary.ReadBillingDemoStatusSample{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoWeightVersion, + Site: readBillingDemoSiteStatement, + OpClass: readBillingDemoOpClassStatement, + OperatorKind: readBillingDemoOperatorStatement, + Status: status, + Reason: reason, + }) + for _, op := range result.operators { + opStatus := op.status + if opStatus == "" { + opStatus = status + } + opReason := op.reason + if opReason == "" { + opReason = reason + } + if opReason == "" { + opReason = readBillingDemoReasonNone + } + if !(op.site == readBillingDemoSiteStatement && + op.opClass == readBillingDemoOpClassStatement && + op.operatorKind == readBillingDemoOperatorStatement && + opStatus == status && + opReason == reason) { + stats.Statuses = append(stats.Statuses, stmtsummary.ReadBillingDemoStatusSample{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoWeightVersion, + Site: op.site, + OpClass: op.opClass, + OperatorKind: op.operatorKind, + Status: opStatus, + Reason: opReason, + }) + } + if status != readBillingDemoStatusSuccess || opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + continue + } + for _, unit := range op.units { + sample := stmtsummary.ReadBillingDemoBaseUnitSample{ + ModelVersion: readBillingDemoModelVersion, + WeightVersion: readBillingDemoWeightVersion, + Site: op.site, + OpClass: op.opClass, + OperatorKind: op.operatorKind, + Unit: unit.unit, + InputSource: unit.source, + InputSide: unit.side, + RowWidthSource: unit.widthSource, + Value: unit.value, + RowWidth: unit.rowWidth, + } + stats.BaseUnits = append(stats.BaseUnits, sample) + switch unit.unit { + case readBillingDemoUnitFixedEvents: + stats.Totals.SumReadBillingDemoFixedEvents += unit.value + case readBillingDemoUnitInputRows: + stats.Totals.SumReadBillingDemoInputRows += unit.value + case readBillingDemoUnitInputBytes: + stats.Totals.SumReadBillingDemoInputBytes += unit.value + } + } + } + return stats +} + func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) (string, readBillingDemoOperatorResult) { for i, op := range tree { if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index d37b0d418f2ee..1c974681aabd3 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -5439,7 +5439,15 @@ func (b *PlanBuilder) buildMemTable(_ context.Context, dbName ast.CIStr, tableIn p.Extractor = &TableStorageStatsExtractor{} case infoschema.TableTiFlashTables, infoschema.TableTiFlashSegments, infoschema.TableTiFlashIndexes: p.Extractor = &TiFlashSystemTableExtractor{} - case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, infoschema.TableTiDBStatementsStats: + case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, infoschema.TableTiDBStatementsStats, + infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: p.Extractor = &StatementsSummaryExtractor{} case infoschema.TableTiKVRegionPeers: p.Extractor = &TikvRegionPeersExtractor{} diff --git a/pkg/planner/core/operator/logicalop/logical_mem_table.go b/pkg/planner/core/operator/logicalop/logical_mem_table.go index f9854098e301b..2aeb5260b0465 100644 --- a/pkg/planner/core/operator/logicalop/logical_mem_table.go +++ b/pkg/planner/core/operator/logicalop/logical_mem_table.go @@ -81,10 +81,18 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column) (bas case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, infoschema.TableTiDBStatementsStats, + infoschema.TableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.TableStatementsSummaryReadBillingDemoStatus, + infoschema.TableStatementsSummaryHistoryReadBillingDemoStatus, infoschema.TableSlowQuery, infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory, infoschema.ClusterTableTiDBStatementsStats, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus, infoschema.ClusterTableSlowLog, infoschema.TableTiDBTrx, infoschema.ClusterTableTiDBTrx, diff --git a/pkg/planner/core/pb_to_plan.go b/pkg/planner/core/pb_to_plan.go index e8588cc9fc047..659d5ac0d92f4 100644 --- a/pkg/planner/core/pb_to_plan.go +++ b/pkg/planner/core/pb_to_plan.go @@ -146,7 +146,11 @@ func (b *PBPlanBuilder) pbToTableScan(e *tipb.Executor) (base.PhysicalPlan, erro } } p.Extractor = extractor - case infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory: + case infoschema.ClusterTableStatementsSummary, infoschema.ClusterTableStatementsSummaryHistory, + infoschema.ClusterTableStatementsSummaryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoBaseUnits, + infoschema.ClusterTableStatementsSummaryReadBillingDemoStatus, + infoschema.ClusterTableStatementsSummaryHistoryReadBillingDemoStatus: p.Extractor = &StatementsSummaryExtractor{} case infoschema.ClusterTableTiDBIndexUsage: p.Extractor = NewInfoSchemaTiDBIndexUsageExtractor() diff --git a/pkg/session/BUILD.bazel b/pkg/session/BUILD.bazel index 3dbd454337571..f1013f3bff3bd 100644 --- a/pkg/session/BUILD.bazel +++ b/pkg/session/BUILD.bazel @@ -126,6 +126,8 @@ go_library( "//pkg/util/sli", "//pkg/util/sqlescape", "//pkg/util/sqlexec", + "//pkg/util/stmtsummary", + "//pkg/util/stmtsummary/v2:stmtsummary", "//pkg/util/syncutil", "//pkg/util/timeutil", "//pkg/util/topsql", diff --git a/pkg/session/session.go b/pkg/session/session.go index 869a24dcaafea..9b32c7da5f6dc 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -136,6 +136,8 @@ import ( "github.com/pingcap/tidb/pkg/util/sli" "github.com/pingcap/tidb/pkg/util/sqlescape" "github.com/pingcap/tidb/pkg/util/sqlexec" + "github.com/pingcap/tidb/pkg/util/stmtsummary" + stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2" "github.com/pingcap/tidb/pkg/util/syncutil" "github.com/pingcap/tidb/pkg/util/timeutil" "github.com/pingcap/tidb/pkg/util/topsql" @@ -2779,7 +2781,36 @@ func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err err if resolvedStmt, resolveErr := resolvePreparedStmt(stmtNode, s.sessionVars); resolveErr == nil && resolvedStmt != nil { metricsStmt = resolvedStmt } - plannercore.RecordReadBillingDemoForStatement(s, nil, metricsStmt, err) + readBillingDemoStats := plannercore.RecordReadBillingDemoForStatement(s, nil, metricsStmt, err) + if readBillingDemoStats.IsEmpty() || s.sessionVars == nil || s.sessionVars.StmtCtx == nil { + return + } + userString := "" + if s.sessionVars.User != nil { + userString = s.sessionVars.User.Username + } + stmtCtx := s.sessionVars.StmtCtx + if stmtCtx.StmtType == "" { + stmtCtx.StmtType = stmtctx.GetStmtLabel(context.Background(), metricsStmt) + } + normalizedSQL, digest := stmtCtx.SQLDigest() + digestString := "" + if digest != nil { + digestString = digest.String() + } + isInternalSQL := (s.sessionVars.InRestrictedSQL || len(userString) == 0) && !s.sessionVars.InExplainExplore + stmtsummaryv2.AddReadBillingDemoStatusOnly(&stmtsummary.StmtExecInfo{ + SchemaName: strings.ToLower(s.sessionVars.CurrentDB), + NormalizedSQL: normalizedSQL, + Digest: digestString, + User: userString, + StmtCtx: stmtCtx, + StartTime: s.sessionVars.StartTime, + IsInternal: isInternalSQL, + Succeed: false, + ResourceGroupName: stmtCtx.ResourceGroupName, + ReadBillingDemoStats: readBillingDemoStats, + }) } func shouldBypass(ctx context.Context, stmtNode ast.StmtNode, sessVars *variable.SessionVars) bool { diff --git a/pkg/util/stmtsummary/BUILD.bazel b/pkg/util/stmtsummary/BUILD.bazel index 7c506280969f9..03fee3b1f974c 100644 --- a/pkg/util/stmtsummary/BUILD.bazel +++ b/pkg/util/stmtsummary/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "stmtsummary", srcs = [ "evicted.go", + "read_billing.go", "reader.go", "statement_summary.go", ], @@ -36,11 +37,12 @@ go_test( srcs = [ "evicted_test.go", "main_test.go", + "read_billing_test.go", "statement_summary_test.go", ], embed = [":stmtsummary"], flaky = True, - shard_count = 29, + shard_count = 31, deps = [ "//pkg/meta/model", "//pkg/metrics", diff --git a/pkg/util/stmtsummary/evicted.go b/pkg/util/stmtsummary/evicted.go index 07d85c70ea296..b4f4d675b1c71 100644 --- a/pkg/util/stmtsummary/evicted.go +++ b/pkg/util/stmtsummary/evicted.go @@ -391,6 +391,12 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem addTo.StmtRUSummary.Merge(&addWith.StmtRUSummary) addTo.ReadBillingDemoBaseUnitSummary.Merge(&addWith.ReadBillingDemoBaseUnitSummary) + addTo.ReadBillingDemoBaseUnitAggs, addTo.ReadBillingDemoStatusAggs = MergeReadBillingDemoAggMaps( + addTo.ReadBillingDemoBaseUnitAggs, + addTo.ReadBillingDemoStatusAggs, + addWith.ReadBillingDemoBaseUnitAggs, + addWith.ReadBillingDemoStatusAggs, + ) // resourceGroupName might not be inited because when it is a evicted item. addTo.resourceGroupName = addWith.resourceGroupName } diff --git a/pkg/util/stmtsummary/read_billing.go b/pkg/util/stmtsummary/read_billing.go new file mode 100644 index 0000000000000..02123a147e0e5 --- /dev/null +++ b/pkg/util/stmtsummary/read_billing.go @@ -0,0 +1,632 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "cmp" + "slices" +) + +const ( + // MaxReadBillingDemoBaseUnitKeysPerRecord bounds read billing dimensions per + // statement summary digest/window record. + MaxReadBillingDemoBaseUnitKeysPerRecord = 256 + // MaxReadBillingDemoStatusKeysPerRecord bounds non-reserved read billing + // status dimensions per statement summary digest/window record. + MaxReadBillingDemoStatusKeysPerRecord = 128 +) + +// Read billing demo column names used by INFORMATION_SCHEMA table definitions +// and statement-summary readers. +const ( + ReadBillingDemoModelVersionStr = "MODEL_VERSION" + ReadBillingDemoWeightVersionStr = "WEIGHT_VERSION" + ReadBillingDemoSiteStr = "SITE" + ReadBillingDemoOpClassStr = "OP_CLASS" + ReadBillingDemoOperatorKindStr = "OPERATOR_KIND" + ReadBillingDemoUnitStr = "UNIT" + ReadBillingDemoInputSourceStr = "INPUT_SOURCE" + ReadBillingDemoInputSideStr = "INPUT_SIDE" + ReadBillingDemoRowWidthSource = "ROW_WIDTH_SOURCE" + ReadBillingDemoValueStr = "VALUE" + ReadBillingDemoSampleCountStr = "SAMPLE_COUNT" + ReadBillingDemoRowWidthSumStr = "ROW_WIDTH_SUM" + ReadBillingDemoAvgRowWidthStr = "AVG_ROW_WIDTH" + ReadBillingDemoStatusStr = "STATUS" + ReadBillingDemoReasonStr = "REASON" + ReadBillingDemoCountStr = "COUNT" +) + +const ( + readBillingDemoSiteStatement = "statement" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOperatorStatement = "statement" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoReasonAggregation = "aggregation_overflow" + readBillingDemoReasonStatusAggregation = "status_aggregation_overflow" +) + +// ReadBillingDemoTableKind identifies which read billing demo table is being read. +type ReadBillingDemoTableKind int + +const ( + // ReadBillingDemoTableBaseUnits expands coefficient-free base-unit aggregates. + ReadBillingDemoTableBaseUnits ReadBillingDemoTableKind = iota + // ReadBillingDemoTableStatus expands statement/operator status aggregates. + ReadBillingDemoTableStatus +) + +// ReadBillingDemoStatementStats is the structured read billing demo snapshot +// attached to one statement execution before statement summary aggregation. +type ReadBillingDemoStatementStats struct { + ModelVersion string + WeightVersion string + Statuses []ReadBillingDemoStatusSample + BaseUnits []ReadBillingDemoBaseUnitSample + Totals ReadBillingDemoBaseUnitSummary +} + +// IsEmpty returns whether the snapshot contains no read billing data. +func (s *ReadBillingDemoStatementStats) IsEmpty() bool { + if s == nil { + return true + } + return len(s.Statuses) == 0 && len(s.BaseUnits) == 0 && + s.Totals.SumReadBillingDemoFixedEvents == 0 && + s.Totals.SumReadBillingDemoInputRows == 0 && + s.Totals.SumReadBillingDemoInputBytes == 0 +} + +// ReadBillingDemoBaseUnitSample is a coefficient-free read billing unit sample. +type ReadBillingDemoBaseUnitSample struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + Unit string + InputSource string + InputSide string + RowWidthSource string + Value float64 + RowWidth float64 +} + +// ReadBillingDemoStatusSample is a statement/operator read billing status sample. +type ReadBillingDemoStatusSample struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + Status string + Reason string +} + +// ReadBillingDemoBaseUnitKey is the dimension key for statement-summary base-unit aggregation. +type ReadBillingDemoBaseUnitKey struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + Unit string + InputSource string + InputSide string + RowWidthSource string +} + +// ReadBillingDemoBaseUnitAgg is the aggregate for one base-unit key. +type ReadBillingDemoBaseUnitAgg struct { + Value float64 + SampleCount uint64 + RowWidthSum float64 +} + +// ReadBillingDemoBaseUnitAggEntry is a JSON-stable base-unit aggregate entry. +type ReadBillingDemoBaseUnitAggEntry struct { + ModelVersion string `json:"model_version"` + WeightVersion string `json:"weight_version"` + Site string `json:"site"` + OpClass string `json:"op_class"` + OperatorKind string `json:"operator_kind"` + Unit string `json:"unit"` + InputSource string `json:"input_source"` + InputSide string `json:"input_side"` + RowWidthSource string `json:"row_width_source"` + Value float64 `json:"value"` + SampleCount uint64 `json:"sample_count"` + RowWidthSum float64 `json:"row_width_sum"` +} + +// ReadBillingDemoStatusKey is the dimension key for statement-summary status aggregation. +type ReadBillingDemoStatusKey struct { + ModelVersion string + WeightVersion string + Site string + OpClass string + OperatorKind string + Status string + Reason string +} + +// ReadBillingDemoStatusAgg is the aggregate for one status key. +type ReadBillingDemoStatusAgg struct { + Count uint64 +} + +// ReadBillingDemoStatusAggEntry is a JSON-stable status aggregate entry. +type ReadBillingDemoStatusAggEntry struct { + ModelVersion string `json:"model_version"` + WeightVersion string `json:"weight_version"` + Site string `json:"site"` + OpClass string `json:"op_class"` + OperatorKind string `json:"operator_kind"` + Status string `json:"status"` + Reason string `json:"reason"` + Count uint64 `json:"count"` +} + +func makeReadBillingDemoBaseUnitKey(sample ReadBillingDemoBaseUnitSample) ReadBillingDemoBaseUnitKey { + return ReadBillingDemoBaseUnitKey{ + ModelVersion: sample.ModelVersion, + WeightVersion: sample.WeightVersion, + Site: sample.Site, + OpClass: sample.OpClass, + OperatorKind: sample.OperatorKind, + Unit: sample.Unit, + InputSource: sample.InputSource, + InputSide: sample.InputSide, + RowWidthSource: sample.RowWidthSource, + } +} + +func makeReadBillingDemoStatusKey(sample ReadBillingDemoStatusSample) ReadBillingDemoStatusKey { + return ReadBillingDemoStatusKey{ + ModelVersion: sample.ModelVersion, + WeightVersion: sample.WeightVersion, + Site: sample.Site, + OpClass: sample.OpClass, + OperatorKind: sample.OperatorKind, + Status: sample.Status, + Reason: sample.Reason, + } +} + +func (e ReadBillingDemoBaseUnitAggEntry) key() ReadBillingDemoBaseUnitKey { + return ReadBillingDemoBaseUnitKey{ + ModelVersion: e.ModelVersion, + WeightVersion: e.WeightVersion, + Site: e.Site, + OpClass: e.OpClass, + OperatorKind: e.OperatorKind, + Unit: e.Unit, + InputSource: e.InputSource, + InputSide: e.InputSide, + RowWidthSource: e.RowWidthSource, + } +} + +func (e ReadBillingDemoStatusAggEntry) key() ReadBillingDemoStatusKey { + return ReadBillingDemoStatusKey{ + ModelVersion: e.ModelVersion, + WeightVersion: e.WeightVersion, + Site: e.Site, + OpClass: e.OpClass, + OperatorKind: e.OperatorKind, + Status: e.Status, + Reason: e.Reason, + } +} + +func (a *ReadBillingDemoBaseUnitAgg) addSample(sample ReadBillingDemoBaseUnitSample) { + a.Value += sample.Value + a.SampleCount++ + a.RowWidthSum += sample.RowWidth +} + +func (a *ReadBillingDemoBaseUnitAgg) addEntry(entry ReadBillingDemoBaseUnitAggEntry) { + a.Value += entry.Value + a.SampleCount += entry.SampleCount + a.RowWidthSum += entry.RowWidthSum +} + +func (a *ReadBillingDemoStatusAgg) addSample() { + a.Count++ +} + +func (a *ReadBillingDemoStatusAgg) addEntry(entry ReadBillingDemoStatusAggEntry) { + a.Count += entry.Count +} + +func readBillingDemoBaseUnitEntry(key ReadBillingDemoBaseUnitKey, agg ReadBillingDemoBaseUnitAgg) ReadBillingDemoBaseUnitAggEntry { + return ReadBillingDemoBaseUnitAggEntry{ + ModelVersion: key.ModelVersion, + WeightVersion: key.WeightVersion, + Site: key.Site, + OpClass: key.OpClass, + OperatorKind: key.OperatorKind, + Unit: key.Unit, + InputSource: key.InputSource, + InputSide: key.InputSide, + RowWidthSource: key.RowWidthSource, + Value: agg.Value, + SampleCount: agg.SampleCount, + RowWidthSum: agg.RowWidthSum, + } +} + +func readBillingDemoStatusEntry(key ReadBillingDemoStatusKey, agg ReadBillingDemoStatusAgg) ReadBillingDemoStatusAggEntry { + return ReadBillingDemoStatusAggEntry{ + ModelVersion: key.ModelVersion, + WeightVersion: key.WeightVersion, + Site: key.Site, + OpClass: key.OpClass, + OperatorKind: key.OperatorKind, + Status: key.Status, + Reason: key.Reason, + Count: agg.Count, + } +} + +// ReadBillingDemoBaseUnitEntriesFromMap converts map aggregates into deterministic entries. +func ReadBillingDemoBaseUnitEntriesFromMap(aggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg) []ReadBillingDemoBaseUnitAggEntry { + entries := make([]ReadBillingDemoBaseUnitAggEntry, 0, len(aggs)) + for key, agg := range aggs { + entries = append(entries, readBillingDemoBaseUnitEntry(key, agg)) + } + sortReadBillingDemoBaseUnitEntries(entries) + return entries +} + +// ReadBillingDemoStatusEntriesFromMap converts map aggregates into deterministic entries. +func ReadBillingDemoStatusEntriesFromMap(aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) []ReadBillingDemoStatusAggEntry { + entries := make([]ReadBillingDemoStatusAggEntry, 0, len(aggs)) + for key, agg := range aggs { + entries = append(entries, readBillingDemoStatusEntry(key, agg)) + } + sortReadBillingDemoStatusEntries(entries) + return entries +} + +// AddReadBillingDemoStatementStatsToMaps merges one statement snapshot into v1 maps. +func AddReadBillingDemoStatementStatsToMaps( + baseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + statusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + stats *ReadBillingDemoStatementStats, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) { + if stats == nil { + return baseUnitAggs, statusAggs + } + for _, status := range stats.Statuses { + statusAggs, _ = addReadBillingDemoStatusSampleToMap(statusAggs, status, false) + } + for _, sample := range stats.BaseUnits { + var overflow bool + baseUnitAggs, overflow = addReadBillingDemoBaseUnitSampleToMap(baseUnitAggs, sample) + if overflow { + statusAggs = addReadBillingDemoReservedStatusToMap(statusAggs, stats.ModelVersion, stats.WeightVersion, readBillingDemoReasonAggregation) + } + } + return baseUnitAggs, statusAggs +} + +// MergeReadBillingDemoAggMaps merges source maps into destination maps with the same caps used on write. +func MergeReadBillingDemoAggMaps( + dstBase map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + dstStatus map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + srcBase map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + srcStatus map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) { + for key, agg := range srcStatus { + dstStatus, _ = addReadBillingDemoStatusEntryToMap(dstStatus, readBillingDemoStatusEntry(key, agg), false) + } + for key, agg := range srcBase { + var overflow bool + dstBase, overflow = addReadBillingDemoBaseUnitEntryToMap(dstBase, readBillingDemoBaseUnitEntry(key, agg)) + if overflow { + dstStatus = addReadBillingDemoReservedStatusToMap(dstStatus, key.ModelVersion, key.WeightVersion, readBillingDemoReasonAggregation) + } + } + return dstBase, dstStatus +} + +// AddReadBillingDemoStatementStatsToEntries merges one statement snapshot into v2 entries. +func AddReadBillingDemoStatementStatsToEntries( + baseUnitEntries []ReadBillingDemoBaseUnitAggEntry, + statusEntries []ReadBillingDemoStatusAggEntry, + stats *ReadBillingDemoStatementStats, +) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry) { + if stats == nil { + return baseUnitEntries, statusEntries + } + for _, status := range stats.Statuses { + statusEntries, _ = addReadBillingDemoStatusSampleToEntries(statusEntries, status, false) + } + for _, sample := range stats.BaseUnits { + var overflow bool + baseUnitEntries, overflow = addReadBillingDemoBaseUnitSampleToEntries(baseUnitEntries, sample) + if overflow { + statusEntries = addReadBillingDemoReservedStatusToEntries(statusEntries, stats.ModelVersion, stats.WeightVersion, readBillingDemoReasonAggregation) + } + } + return baseUnitEntries, statusEntries +} + +// MergeReadBillingDemoEntrySlices merges source entries into destination entries with caps. +func MergeReadBillingDemoEntrySlices( + dstBase []ReadBillingDemoBaseUnitAggEntry, + dstStatus []ReadBillingDemoStatusAggEntry, + srcBase []ReadBillingDemoBaseUnitAggEntry, + srcStatus []ReadBillingDemoStatusAggEntry, +) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry) { + for _, entry := range srcStatus { + dstStatus, _ = addReadBillingDemoStatusEntryToEntries(dstStatus, entry, false) + } + for _, entry := range srcBase { + var overflow bool + dstBase, overflow = addReadBillingDemoBaseUnitEntryToEntries(dstBase, entry) + if overflow { + dstStatus = addReadBillingDemoReservedStatusToEntries(dstStatus, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonAggregation) + } + } + return dstBase, dstStatus +} + +func addReadBillingDemoBaseUnitSampleToMap( + aggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + sample ReadBillingDemoBaseUnitSample, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, bool) { + key := makeReadBillingDemoBaseUnitKey(sample) + if aggs == nil { + aggs = make(map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg) + } + agg, exists := aggs[key] + if !exists && len(aggs) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return aggs, true + } + agg.addSample(sample) + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoBaseUnitEntryToMap( + aggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, + entry ReadBillingDemoBaseUnitAggEntry, +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, bool) { + key := entry.key() + if aggs == nil { + aggs = make(map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg) + } + agg, exists := aggs[key] + if !exists && len(aggs) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return aggs, true + } + agg.addEntry(entry) + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoStatusSampleToMap( + aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + sample ReadBillingDemoStatusSample, + reserved bool, +) (map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, bool) { + key := makeReadBillingDemoStatusKey(sample) + if aggs == nil { + aggs = make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) + } + agg, exists := aggs[key] + if !exists && !reserved && readBillingDemoNonReservedStatusKeyCount(aggs) >= MaxReadBillingDemoStatusKeysPerRecord { + aggs = addReadBillingDemoReservedStatusToMap(aggs, sample.ModelVersion, sample.WeightVersion, readBillingDemoReasonStatusAggregation) + return aggs, true + } + agg.addSample() + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoStatusEntryToMap( + aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + entry ReadBillingDemoStatusAggEntry, + reserved bool, +) (map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, bool) { + key := entry.key() + if aggs == nil { + aggs = make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) + } + agg, exists := aggs[key] + if !exists && !reserved && readBillingDemoNonReservedStatusKeyCount(aggs) >= MaxReadBillingDemoStatusKeysPerRecord { + aggs = addReadBillingDemoReservedStatusToMap(aggs, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonStatusAggregation) + return aggs, true + } + agg.addEntry(entry) + aggs[key] = agg + return aggs, false +} + +func addReadBillingDemoReservedStatusToMap( + aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, + modelVersion string, + weightVersion string, + reason string, +) map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg { + sample := readBillingDemoReservedStatusSample(modelVersion, weightVersion, reason) + aggs, _ = addReadBillingDemoStatusSampleToMap(aggs, sample, true) + return aggs +} + +func readBillingDemoNonReservedStatusKeyCount(aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) int { + count := 0 + for key := range aggs { + if key.Reason == readBillingDemoReasonAggregation || key.Reason == readBillingDemoReasonStatusAggregation { + continue + } + count++ + } + return count +} + +func addReadBillingDemoBaseUnitSampleToEntries( + entries []ReadBillingDemoBaseUnitAggEntry, + sample ReadBillingDemoBaseUnitSample, +) ([]ReadBillingDemoBaseUnitAggEntry, bool) { + key := makeReadBillingDemoBaseUnitKey(sample) + for i := range entries { + if entries[i].key() == key { + entries[i].Value += sample.Value + entries[i].SampleCount++ + entries[i].RowWidthSum += sample.RowWidth + return entries, false + } + } + if len(entries) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return entries, true + } + entries = append(entries, readBillingDemoBaseUnitEntry(key, ReadBillingDemoBaseUnitAgg{ + Value: sample.Value, + SampleCount: 1, + RowWidthSum: sample.RowWidth, + })) + sortReadBillingDemoBaseUnitEntries(entries) + return entries, false +} + +func addReadBillingDemoBaseUnitEntryToEntries( + entries []ReadBillingDemoBaseUnitAggEntry, + entry ReadBillingDemoBaseUnitAggEntry, +) ([]ReadBillingDemoBaseUnitAggEntry, bool) { + key := entry.key() + for i := range entries { + if entries[i].key() == key { + entries[i].Value += entry.Value + entries[i].SampleCount += entry.SampleCount + entries[i].RowWidthSum += entry.RowWidthSum + return entries, false + } + } + if len(entries) >= MaxReadBillingDemoBaseUnitKeysPerRecord { + return entries, true + } + entries = append(entries, entry) + sortReadBillingDemoBaseUnitEntries(entries) + return entries, false +} + +func addReadBillingDemoStatusSampleToEntries( + entries []ReadBillingDemoStatusAggEntry, + sample ReadBillingDemoStatusSample, + reserved bool, +) ([]ReadBillingDemoStatusAggEntry, bool) { + key := makeReadBillingDemoStatusKey(sample) + for i := range entries { + if entries[i].key() == key { + entries[i].Count++ + return entries, false + } + } + if !reserved && readBillingDemoNonReservedStatusEntryCount(entries) >= MaxReadBillingDemoStatusKeysPerRecord { + entries = addReadBillingDemoReservedStatusToEntries(entries, sample.ModelVersion, sample.WeightVersion, readBillingDemoReasonStatusAggregation) + return entries, true + } + entries = append(entries, readBillingDemoStatusEntry(key, ReadBillingDemoStatusAgg{Count: 1})) + sortReadBillingDemoStatusEntries(entries) + return entries, false +} + +func addReadBillingDemoStatusEntryToEntries( + entries []ReadBillingDemoStatusAggEntry, + entry ReadBillingDemoStatusAggEntry, + reserved bool, +) ([]ReadBillingDemoStatusAggEntry, bool) { + key := entry.key() + for i := range entries { + if entries[i].key() == key { + entries[i].Count += entry.Count + return entries, false + } + } + if !reserved && readBillingDemoNonReservedStatusEntryCount(entries) >= MaxReadBillingDemoStatusKeysPerRecord { + entries = addReadBillingDemoReservedStatusToEntries(entries, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonStatusAggregation) + return entries, true + } + entries = append(entries, entry) + sortReadBillingDemoStatusEntries(entries) + return entries, false +} + +func addReadBillingDemoReservedStatusToEntries( + entries []ReadBillingDemoStatusAggEntry, + modelVersion string, + weightVersion string, + reason string, +) []ReadBillingDemoStatusAggEntry { + sample := readBillingDemoReservedStatusSample(modelVersion, weightVersion, reason) + entries, _ = addReadBillingDemoStatusSampleToEntries(entries, sample, true) + return entries +} + +func readBillingDemoReservedStatusSample(modelVersion string, weightVersion string, reason string) ReadBillingDemoStatusSample { + return ReadBillingDemoStatusSample{ + ModelVersion: modelVersion, + WeightVersion: weightVersion, + Site: readBillingDemoSiteStatement, + OpClass: readBillingDemoOpClassStatement, + OperatorKind: readBillingDemoOperatorStatement, + Status: readBillingDemoStatusUnknownInput, + Reason: reason, + } +} + +func readBillingDemoNonReservedStatusEntryCount(entries []ReadBillingDemoStatusAggEntry) int { + count := 0 + for _, entry := range entries { + if entry.Reason == readBillingDemoReasonAggregation || entry.Reason == readBillingDemoReasonStatusAggregation { + continue + } + count++ + } + return count +} + +func sortReadBillingDemoBaseUnitEntries(entries []ReadBillingDemoBaseUnitAggEntry) { + slices.SortFunc(entries, func(i, j ReadBillingDemoBaseUnitAggEntry) int { + return cmp.Or( + cmp.Compare(i.ModelVersion, j.ModelVersion), + cmp.Compare(i.WeightVersion, j.WeightVersion), + cmp.Compare(i.Site, j.Site), + cmp.Compare(i.OpClass, j.OpClass), + cmp.Compare(i.OperatorKind, j.OperatorKind), + cmp.Compare(i.Unit, j.Unit), + cmp.Compare(i.InputSource, j.InputSource), + cmp.Compare(i.InputSide, j.InputSide), + cmp.Compare(i.RowWidthSource, j.RowWidthSource), + ) + }) +} + +func sortReadBillingDemoStatusEntries(entries []ReadBillingDemoStatusAggEntry) { + slices.SortFunc(entries, func(i, j ReadBillingDemoStatusAggEntry) int { + return cmp.Or( + cmp.Compare(i.ModelVersion, j.ModelVersion), + cmp.Compare(i.WeightVersion, j.WeightVersion), + cmp.Compare(i.Site, j.Site), + cmp.Compare(i.OpClass, j.OpClass), + cmp.Compare(i.OperatorKind, j.OperatorKind), + cmp.Compare(i.Status, j.Status), + cmp.Compare(i.Reason, j.Reason), + ) + }) +} diff --git a/pkg/util/stmtsummary/read_billing_test.go b/pkg/util/stmtsummary/read_billing_test.go new file mode 100644 index 0000000000000..fe7f493b0233c --- /dev/null +++ b/pkg/util/stmtsummary/read_billing_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stmtsummary + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadBillingDemoAggregationCaps(t *testing.T) { + stats := ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + } + for i := 0; i < MaxReadBillingDemoBaseUnitKeysPerRecord+2; i++ { + stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Unit: "input_rows", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: float64(i + 1), + RowWidth: 8, + }) + } + + baseAggs, statusAggs := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) + require.Len(t, baseAggs, MaxReadBillingDemoBaseUnitKeysPerRecord) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonAggregation).Count) + + baseEntries, statusEntries := AddReadBillingDemoStatementStatsToEntries(nil, nil, &stats) + require.Len(t, baseEntries, MaxReadBillingDemoBaseUnitKeysPerRecord) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonAggregation).Count) + + statusOnly := ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + } + for i := 0; i < MaxReadBillingDemoStatusKeysPerRecord+2; i++ { + statusOnly.Statuses = append(statusOnly.Statuses, ReadBillingDemoStatusSample{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Status: "unsupported", + Reason: "unsupported_operator", + }) + } + + _, statusAggs = AddReadBillingDemoStatementStatsToMaps(nil, nil, &statusOnly) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusKeyCount(statusAggs)) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonStatusAggregation).Count) + + _, statusEntries = AddReadBillingDemoStatementStatsToEntries(nil, nil, &statusOnly) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusEntryCount(statusEntries)) + require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonStatusAggregation).Count) +} + +func requireReadBillingDemoStatusReason(t *testing.T, entries []ReadBillingDemoStatusAggEntry, reason string) ReadBillingDemoStatusAggEntry { + t.Helper() + for _, entry := range entries { + if entry.Reason == reason { + return entry + } + } + require.Failf(t, "missing read billing status reason", "reason=%s entries=%v", reason, entries) + return ReadBillingDemoStatusAggEntry{} +} diff --git a/pkg/util/stmtsummary/reader.go b/pkg/util/stmtsummary/reader.go index cbbab4a47ebce..c0f5ca66b5f8d 100644 --- a/pkg/util/stmtsummary/reader.go +++ b/pkg/util/stmtsummary/reader.go @@ -15,7 +15,6 @@ package stmtsummary import ( - "fmt" "strings" "time" @@ -56,10 +55,9 @@ func NewStmtSummaryReader(user *auth.UserIdentity, hasProcessPriv bool, cols []* reader.columnValueFactories = make([]columnValueFactory, len(reader.columns)) for i, col := range reader.columns { factory, ok := columnValueFactoryMap[col.Name.O] - if !ok { - panic(fmt.Sprintf("should never happen, should register new column %v into columnValueFactoryMap", col.Name.O)) + if ok { + reader.columnValueFactories[i] = factory } - reader.columnValueFactories[i] = factory } return reader } @@ -135,6 +133,236 @@ func (ssr *stmtSummaryReader) GetStmtSummaryHistoryRows() [][]types.Datum { return rows } +// GetReadBillingDemoCurrentRows gets current-window read billing demo rows. +func (ssr *stmtSummaryReader) GetReadBillingDemoCurrentRows(kind ReadBillingDemoTableKind) [][]types.Datum { + ssMap := ssr.ssMap + ssMap.Lock() + values := ssMap.summaryMap.Values() + beginTime := ssMap.beginTimeForCurInterval + other := ssMap.other + ssMap.Unlock() + + rows := make([][]types.Datum, 0, len(values)) + for _, value := range values { + ssbd := value.(*stmtSummaryByDigest) + if ssr.checker != nil && !ssr.checker.isDigestValid(ssbd.digest) { + continue + } + rows = append(rows, ssr.getReadBillingDemoCurrentRows(ssbd, beginTime, kind)...) + } + if ssr.checker == nil { + rows = append(rows, ssr.getReadBillingDemoEvictedOtherRows(other, kind)...) + } + return rows +} + +// GetReadBillingDemoHistoryRows gets history-window read billing demo rows. +func (ssr *stmtSummaryReader) GetReadBillingDemoHistoryRows(kind ReadBillingDemoTableKind) [][]types.Datum { + ssMap := ssr.ssMap + ssMap.Lock() + values := ssMap.summaryMap.Values() + other := ssMap.other + ssMap.Unlock() + + historySize := ssMap.historySize() + rows := make([][]types.Datum, 0, len(values)*historySize) + for _, value := range values { + ssbd := value.(*stmtSummaryByDigest) + ssElements := ssbd.collectHistorySummaries(ssr.checker, historySize) + for _, ssElement := range ssElements { + rows = append(rows, ssr.getReadBillingDemoElementRows(ssElement, ssbd, kind)...) + } + } + if ssr.checker == nil { + rows = append(rows, ssr.getReadBillingDemoEvictedOtherHistoryRows(other, historySize, kind)...) + } + return rows +} + +func (ssr *stmtSummaryReader) getReadBillingDemoCurrentRows(ssbd *stmtSummaryByDigest, beginTimeForCurInterval int64, kind ReadBillingDemoTableKind) [][]types.Datum { + var ssElement *stmtSummaryByDigestElement + + ssbd.Lock() + if ssbd.initialized && ssbd.history.Len() > 0 { + ssElement = ssbd.history.Back().Value.(*stmtSummaryByDigestElement) + } + ssbd.Unlock() + + if ssElement == nil || ssElement.beginTime < beginTimeForCurInterval { + return nil + } + return ssr.getReadBillingDemoElementRows(ssElement, ssbd, kind) +} + +func (ssr *stmtSummaryReader) getReadBillingDemoEvictedOtherRows(ssbde *stmtSummaryByDigestEvicted, kind ReadBillingDemoTableKind) [][]types.Datum { + var seElement *stmtSummaryByDigestEvictedElement + + ssbde.Lock() + if ssbde.history.Len() > 0 { + seElement = ssbde.history.Back().Value.(*stmtSummaryByDigestEvictedElement) + } + ssbde.Unlock() + + if seElement == nil { + return nil + } + return ssr.getReadBillingDemoElementRows(seElement.otherSummary, new(stmtSummaryByDigest), kind) +} + +func (ssr *stmtSummaryReader) getReadBillingDemoEvictedOtherHistoryRows(ssbde *stmtSummaryByDigestEvicted, historySize int, kind ReadBillingDemoTableKind) [][]types.Datum { + ssbde.Lock() + seElements := ssbde.collectHistorySummaries(historySize) + ssbde.Unlock() + rows := make([][]types.Datum, 0, len(seElements)) + + ssbd := new(stmtSummaryByDigest) + for _, seElement := range seElements { + rows = append(rows, ssr.getReadBillingDemoElementRows(seElement.otherSummary, ssbd, kind)...) + } + return rows +} + +func (ssr *stmtSummaryReader) getReadBillingDemoElementRows(ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, kind ReadBillingDemoTableKind) [][]types.Datum { + ssElement.Lock() + defer ssElement.Unlock() + if !ssr.isAuthed(&ssElement.stmtSummaryStats) { + return nil + } + + switch kind { + case ReadBillingDemoTableBaseUnits: + entries := ReadBillingDemoBaseUnitEntriesFromMap(ssElement.ReadBillingDemoBaseUnitAggs) + rows := make([][]types.Datum, 0, len(entries)) + for _, entry := range entries { + rows = append(rows, ssr.readBillingDemoBaseUnitRow(ssElement, ssbd, entry)) + } + return rows + case ReadBillingDemoTableStatus: + entries := ReadBillingDemoStatusEntriesFromMap(ssElement.ReadBillingDemoStatusAggs) + rows := make([][]types.Datum, 0, len(entries)) + for _, entry := range entries { + rows = append(rows, ssr.readBillingDemoStatusRow(ssElement, ssbd, entry)) + } + return rows + default: + return nil + } +} + +func (ssr *stmtSummaryReader) readBillingDemoBaseUnitRow(ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoBaseUnitAggEntry) []types.Datum { + row := make([]types.Datum, len(ssr.columns)) + for i, col := range ssr.columns { + row[i] = types.NewDatum(ssr.readBillingDemoBaseUnitColumnValue(col.Name.O, ssElement, ssbd, entry)) + } + return row +} + +func (ssr *stmtSummaryReader) readBillingDemoStatusRow(ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoStatusAggEntry) []types.Datum { + row := make([]types.Datum, len(ssr.columns)) + for i, col := range ssr.columns { + row[i] = types.NewDatum(ssr.readBillingDemoStatusColumnValue(col.Name.O, ssElement, ssbd, entry)) + } + return row +} + +func (ssr *stmtSummaryReader) readBillingDemoCommonColumnValue(col string, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest) (any, bool) { + switch col { + case ClusterTableInstanceColumnNameStr: + return ssr.instanceAddr, true + case SummaryBeginTimeStr: + beginTime := time.Unix(ssElement.beginTime, 0) + if beginTime.Location() != ssr.tz { + beginTime = beginTime.In(ssr.tz) + } + return types.NewTime(types.FromGoTime(beginTime), mysql.TypeTimestamp, 0), true + case SummaryEndTimeStr: + endTime := time.Unix(ssElement.endTime, 0) + if endTime.Location() != ssr.tz { + endTime = endTime.In(ssr.tz) + } + return types.NewTime(types.FromGoTime(endTime), mysql.TypeTimestamp, 0), true + case StmtTypeStr: + return ssbd.stmtType, true + case SchemaNameStr: + return convertEmptyToNil(ssbd.schemaName), true + case DigestStr: + return convertEmptyToNil(ssbd.digest), true + case DigestTextStr: + return ssbd.normalizedSQL, true + case PlanDigestStr: + return convertEmptyToNil(ssbd.planDigest), true + case ResourceGroupName: + return ssElement.resourceGroupName, true + default: + return nil, false + } +} + +func (ssr *stmtSummaryReader) readBillingDemoBaseUnitColumnValue(col string, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoBaseUnitAggEntry) any { + if value, ok := ssr.readBillingDemoCommonColumnValue(col, ssElement, ssbd); ok { + return value + } + switch col { + case ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case ReadBillingDemoSiteStr: + return entry.Site + case ReadBillingDemoOpClassStr: + return entry.OpClass + case ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case ReadBillingDemoUnitStr: + return entry.Unit + case ReadBillingDemoInputSourceStr: + return entry.InputSource + case ReadBillingDemoInputSideStr: + return entry.InputSide + case ReadBillingDemoRowWidthSource: + return entry.RowWidthSource + case ReadBillingDemoValueStr: + return entry.Value + case ReadBillingDemoSampleCountStr: + return entry.SampleCount + case ReadBillingDemoRowWidthSumStr: + return entry.RowWidthSum + case ReadBillingDemoAvgRowWidthStr: + if entry.SampleCount == 0 { + return float64(0) + } + return entry.RowWidthSum / float64(entry.SampleCount) + default: + return nil + } +} + +func (ssr *stmtSummaryReader) readBillingDemoStatusColumnValue(col string, ssElement *stmtSummaryByDigestElement, ssbd *stmtSummaryByDigest, entry ReadBillingDemoStatusAggEntry) any { + if value, ok := ssr.readBillingDemoCommonColumnValue(col, ssElement, ssbd); ok { + return value + } + switch col { + case ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case ReadBillingDemoSiteStr: + return entry.Site + case ReadBillingDemoOpClassStr: + return entry.OpClass + case ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case ReadBillingDemoStatusStr: + return entry.Status + case ReadBillingDemoReasonStr: + return entry.Reason + case ReadBillingDemoCountStr: + return entry.Count + default: + return nil + } +} + func (ssr *stmtSummaryReader) SetChecker(checker *stmtSummaryChecker) { ssr.checker = checker } @@ -153,6 +381,9 @@ func (ssr *stmtSummaryReader) getStmtByDigestCumulativeRow(ssbd *stmtSummaryByDi if !ssr.isAuthed(&ssbd.cumulative) { return nil } + if ssbd.cumulative.isReadBillingDemoStatusOnly() { + return nil + } datums := make([]types.Datum, len(ssr.columnValueFactories)) for i, factory := range ssr.columnValueFactories { @@ -184,6 +415,9 @@ func (ssr *stmtSummaryReader) getStmtByDigestElementRow(ssElement *stmtSummaryBy if !ssr.isAuthed(&ssElement.stmtSummaryStats) { return nil } + if ssElement.stmtSummaryStats.isReadBillingDemoStatusOnly() { + return nil + } datums := make([]types.Datum, len(ssr.columnValueFactories)) for i, factory := range ssr.columnValueFactories { diff --git a/pkg/util/stmtsummary/statement_summary.go b/pkg/util/stmtsummary/statement_summary.go index 68b96c1d3e900..c6d2f21055ed1 100644 --- a/pkg/util/stmtsummary/statement_summary.go +++ b/pkg/util/stmtsummary/statement_summary.go @@ -255,6 +255,8 @@ type stmtSummaryStats struct { resourceGroupName string StmtRUSummary ReadBillingDemoBaseUnitSummary + ReadBillingDemoBaseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg + ReadBillingDemoStatusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg StmtNetworkTrafficSummary planCacheUnqualifiedCount int64 @@ -306,6 +308,7 @@ type StmtExecInfo struct { RUDetail *util.RUDetails TotalRUV2 float64 ReadBillingDemoBaseUnits ReadBillingDemoBaseUnitSummary + ReadBillingDemoStats ReadBillingDemoStatementStats CPUUsages ppcpuusage.CPUUsages PlanCacheUnqualified string @@ -434,6 +437,73 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { ssMap.updateMetricsLocked() } +// AddReadBillingDemoStatusOnly records read billing demo statuses that happen +// before the normal statement finish path can build a full StmtExecInfo. +func (ssMap *stmtSummaryByDigestMap) AddReadBillingDemoStatusOnly(sei *StmtExecInfo) { + if sei == nil || sei.ReadBillingDemoStats.IsEmpty() { + return + } + now := time.Now().Unix() + failpoint.Inject("mockTimeForStatementsSummary", func(val failpoint.Value) { + if unixTimeStr, ok := val.(string); ok { + unixTime, err := strconv.ParseInt(unixTimeStr, 10, 64) + if err != nil { + panic(err.Error()) + } + now = unixTime + } + }) + + intervalSeconds := ssMap.refreshInterval() + historySize := 0 + if ssMap.historyEnabled() { + historySize = ssMap.historySize() + } + + key := StmtDigestKeyPool.Get().(*StmtDigestKey) + + ssMap.Lock() + defer ssMap.Unlock() + + userForKey := "" + if ssMap.optGroupByUser.Load() { + userForKey = sei.User + } + key.Init(sei.SchemaName, sei.Digest, sei.PrevSQLDigest, sei.PlanDigest, sei.ResourceGroupName, userForKey) + + if !ssMap.Enabled() { + StmtDigestKeyPool.Put(key) + return + } + if sei.IsInternal && !ssMap.EnabledInternal() { + StmtDigestKeyPool.Put(key) + return + } + + if ssMap.beginTimeForCurInterval+intervalSeconds <= now { + ssMap.beginTimeForCurInterval = now / intervalSeconds * intervalSeconds + ssMap.currentWindowEvictedCount = 0 + } + + beginTime := ssMap.beginTimeForCurInterval + value, exist := ssMap.summaryMap.Get(key) + var summary *stmtSummaryByDigest + if !exist { + summary = new(stmtSummaryByDigest) + ssMap.summaryMap.Put(key, summary) + } else { + summary = value.(*stmtSummaryByDigest) + } + summary.isInternal = summary.isInternal && sei.IsInternal + if summary != nil { + summary.addReadBillingDemoStatusOnly(sei, beginTime, intervalSeconds, historySize) + } + if exist { + StmtDigestKeyPool.Put(key) + } + ssMap.updateMetricsLocked() +} + // Clear removes all statement summaries. func (ssMap *stmtSummaryByDigestMap) Clear() { ssMap.Lock() @@ -694,6 +764,59 @@ func (ssbd *stmtSummaryByDigest) add(sei *StmtExecInfo, beginTime int64, interva } } +func (ssbd *stmtSummaryByDigest) initReadBillingDemoStatusOnly(sei *StmtExecInfo) { + ssbd.cumulative = *newReadBillingDemoStatusOnlyStats(sei) + ssbd.schemaName = sei.SchemaName + ssbd.digest = sei.Digest + ssbd.planDigest = sei.PlanDigest + if sei.StmtCtx != nil { + ssbd.stmtType = sei.StmtCtx.StmtType + } + ssbd.normalizedSQL = formatSQL(sei.NormalizedSQL) + ssbd.history = list.New() + ssbd.initialized = true +} + +func (ssbd *stmtSummaryByDigest) addReadBillingDemoStatusOnly(sei *StmtExecInfo, beginTime int64, intervalSeconds int64, historySize int) { + ssElement, isElementNew := func() (*stmtSummaryByDigestElement, bool) { + ssbd.Lock() + defer ssbd.Unlock() + + initializedNow := false + if !ssbd.initialized { + ssbd.initReadBillingDemoStatusOnly(sei) + initializedNow = true + } + if !initializedNow { + ssbd.cumulative.addReadBillingDemoStatementStats(sei.User, &sei.ReadBillingDemoStats) + } + + var ssElement *stmtSummaryByDigestElement + isElementNew := true + if ssbd.history.Len() > 0 { + lastElement := ssbd.history.Back().Value.(*stmtSummaryByDigestElement) + if lastElement.beginTime >= beginTime { + ssElement = lastElement + isElementNew = false + } else { + lastElement.onExpire(intervalSeconds) + } + } + if isElementNew { + ssElement = newReadBillingDemoStatusOnlyElement(sei, beginTime) + ssbd.history.PushBack(ssElement) + } + for ssbd.history.Len() > historySize && ssbd.history.Len() > 1 { + ssbd.history.Remove(ssbd.history.Front()) + } + return ssElement, isElementNew + }() + + if !isElementNew { + ssElement.addReadBillingDemoStatusOnly(sei) + } +} + // collectHistorySummaries puts at most `historySize` summaries to an array. func (ssbd *stmtSummaryByDigest) collectHistorySummaries(checker *stmtSummaryChecker, historySize int) []*stmtSummaryByDigestElement { ssbd.Lock() @@ -756,6 +879,24 @@ func newStmtSummaryStats(sei *StmtExecInfo) *stmtSummaryStats { } } +func newReadBillingDemoStatusOnlyStats(sei *StmtExecInfo) *stmtSummaryStats { + startTime := sei.StartTime + if startTime.IsZero() { + startTime = time.Now() + } + stats := &stmtSummaryStats{ + backoffTypes: make(map[string]int), + authUsers: make(map[string]struct{}), + minLatency: time.Duration(math.MaxInt64), + minResultRows: math.MaxInt64, + firstSeen: startTime, + lastSeen: startTime, + resourceGroupName: sei.ResourceGroupName, + } + stats.addReadBillingDemoStatementStats(sei.User, &sei.ReadBillingDemoStats) + return stats +} + func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalSeconds int64, warningCount int, affectedRows uint64) *stmtSummaryByDigestElement { ssElement := &stmtSummaryByDigestElement{ beginTime: beginTime, @@ -765,6 +906,13 @@ func newStmtSummaryByDigestElement(sei *StmtExecInfo, beginTime int64, intervalS return ssElement } +func newReadBillingDemoStatusOnlyElement(sei *StmtExecInfo, beginTime int64) *stmtSummaryByDigestElement { + return &stmtSummaryByDigestElement{ + beginTime: beginTime, + stmtSummaryStats: *newReadBillingDemoStatusOnlyStats(sei), + } +} + // onExpire is called when this element expires to history. func (ssElement *stmtSummaryByDigestElement) onExpire(intervalSeconds int64) { ssElement.Lock() @@ -783,6 +931,34 @@ func (ssElement *stmtSummaryByDigestElement) onExpire(intervalSeconds int64) { } } +func (ssStats *stmtSummaryStats) addReadBillingDemoStatementStats(user string, stats *ReadBillingDemoStatementStats) { + if stats == nil { + return + } + if len(user) > 0 { + if ssStats.authUsers == nil { + ssStats.authUsers = make(map[string]struct{}) + } + ssStats.authUsers[user] = struct{}{} + } + ssStats.ReadBillingDemoBaseUnitSummary.Add(&stats.Totals) + ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs = AddReadBillingDemoStatementStatsToMaps( + ssStats.ReadBillingDemoBaseUnitAggs, + ssStats.ReadBillingDemoStatusAggs, + stats, + ) +} + +func (ssStats *stmtSummaryStats) isReadBillingDemoStatusOnly() bool { + if ssStats == nil || ssStats.execCount != 0 { + return false + } + return len(ssStats.ReadBillingDemoBaseUnitAggs) > 0 || len(ssStats.ReadBillingDemoStatusAggs) > 0 || + ssStats.SumReadBillingDemoFixedEvents != 0 || + ssStats.SumReadBillingDemoInputRows != 0 || + ssStats.SumReadBillingDemoInputBytes != 0 +} + func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affectedRows uint64) { // add user to auth users set if len(sei.User) > 0 { @@ -999,7 +1175,16 @@ func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affect // request-units ssStats.StmtRUSummary.Add(sei.RUDetail, sei.TotalRUV2) - ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoBaseUnits) + if !sei.ReadBillingDemoStats.IsEmpty() { + ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoStats.Totals) + ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs = AddReadBillingDemoStatementStatsToMaps( + ssStats.ReadBillingDemoBaseUnitAggs, + ssStats.ReadBillingDemoStatusAggs, + &sei.ReadBillingDemoStats, + ) + } else { + ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoBaseUnits) + } ssStats.storageKV = sei.StmtCtx.IsTiKV.Load() ssStats.storageMPP = sei.StmtCtx.IsTiFlash.Load() @@ -1014,6 +1199,13 @@ func (ssElement *stmtSummaryByDigestElement) add(sei *StmtExecInfo, intervalSeco ssElement.stmtSummaryStats.add(sei, warningCount, affectedRows) } +func (ssElement *stmtSummaryByDigestElement) addReadBillingDemoStatusOnly(sei *StmtExecInfo) { + ssElement.Lock() + defer ssElement.Unlock() + + ssElement.stmtSummaryStats.addReadBillingDemoStatementStats(sei.User, &sei.ReadBillingDemoStats) +} + // Truncate SQL to maxSQLLength. func formatSQL(sql string) string { maxSQLLength := StmtSummaryByDigestMap.maxSQLLength() diff --git a/pkg/util/stmtsummary/statement_summary_test.go b/pkg/util/stmtsummary/statement_summary_test.go index 08c8da6fe8848..9d3f1e88d2404 100644 --- a/pkg/util/stmtsummary/statement_summary_test.go +++ b/pkg/util/stmtsummary/statement_summary_test.go @@ -1096,6 +1096,193 @@ func TestReadBillingDemoBaseUnitsToDatum(t *testing.T) { match(t, datums[0], 5.0, 300.0, 6144.0) } +func TestReadBillingDemoStructuredRowsToDatum(t *testing.T) { + ssMap := newStmtSummaryByDigestMap() + now := time.Now().Unix() + // to disable expiration + ssMap.beginTimeForCurInterval = now + 60 + + stmtExecInfo := generateAnyExecInfo() + stmtExecInfo.ReadBillingDemoStats = ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []ReadBillingDemoStatusSample{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "success", + Reason: "none", + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Status: "ok", + Reason: "none", + }, + }, + BaseUnits: []ReadBillingDemoBaseUnitSample{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 2, + RowWidth: 16, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 3, + RowWidth: 24, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "join_hash", + OperatorKind: "hashjoin", + Unit: "input_rows", + InputSource: "runtime_act_rows", + InputSide: "build", + RowWidthSource: "schema_type_width", + Value: 100, + RowWidth: 32, + }, + }, + Totals: ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 5, + SumReadBillingDemoInputRows: 100, + }, + } + ssMap.AddStatement(stmtExecInfo) + + statusOnlyInfo := generateAnyExecInfo() + statusOnlyInfo.Digest = "status_digest" + statusOnlyInfo.NormalizedSQL = "status_only_sql" + statusOnlyInfo.ReadBillingDemoStats = ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + } + ssMap.AddReadBillingDemoStatusOnly(statusOnlyInfo) + + unknownInputInfo := generateAnyExecInfo() + unknownInputInfo.Digest = "unknown_input_digest" + unknownInputInfo.NormalizedSQL = "unknown_input_sql" + unknownInputInfo.ReadBillingDemoStats = ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "unknown_input", + Reason: "missing_runtime_stats", + }}, + } + ssMap.AddReadBillingDemoStatusOnly(unknownInputInfo) + + baseUnitCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOperatorKindStr)}, + {Name: ast.NewCIStr(ReadBillingDemoUnitStr)}, + {Name: ast.NewCIStr(ReadBillingDemoInputSourceStr)}, + {Name: ast.NewCIStr(ReadBillingDemoInputSideStr)}, + {Name: ast.NewCIStr(ReadBillingDemoRowWidthSource)}, + {Name: ast.NewCIStr(ReadBillingDemoValueStr)}, + {Name: ast.NewCIStr(ReadBillingDemoSampleCountStr)}, + {Name: ast.NewCIStr(ReadBillingDemoRowWidthSumStr)}, + {Name: ast.NewCIStr(ReadBillingDemoAvgRowWidthStr)}, + } + reader := NewStmtSummaryReader(nil, true, baseUnitCols, "", time.UTC) + reader.ssMap = ssMap + baseRows := reader.GetReadBillingDemoCurrentRows(ReadBillingDemoTableBaseUnits) + require.Len(t, baseRows, 2) + expectedBaseRows := map[string]string{ + "tidb/join_hash/hashjoin/input_rows/runtime_act_rows/build/schema_type_width": "100 1 32 32", + "tidb/projection_eval/projection/fixed_events/runtime_act_rows/all/operator_helper": "5 2 40 20", + } + require.Equal(t, expectedBaseRows, readBillingDemoRowsByKey(baseRows, 7)) + require.Equal(t, expectedBaseRows, readBillingDemoRowsByKey(reader.GetReadBillingDemoHistoryRows(ReadBillingDemoTableBaseUnits), 7)) + + statusCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestTextStr)}, + {Name: ast.NewCIStr(ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(ReadBillingDemoOperatorKindStr)}, + {Name: ast.NewCIStr(ReadBillingDemoStatusStr)}, + {Name: ast.NewCIStr(ReadBillingDemoReasonStr)}, + {Name: ast.NewCIStr(ReadBillingDemoCountStr)}, + } + statusReader := NewStmtSummaryReader(nil, true, statusCols, "", time.UTC) + statusReader.ssMap = ssMap + statusRows := statusReader.GetReadBillingDemoCurrentRows(ReadBillingDemoTableStatus) + require.Len(t, statusRows, 4) + expectedStatusRows := map[string]string{ + "normalized_sql/statement/statement/statement/success/none": "1", + "normalized_sql/tidb/projection_eval/projection/ok/none": "1", + "status_only_sql/statement/statement/statement/error/statement_error": "1", + "unknown_input_sql/statement/statement/statement/unknown_input/missing_runtime_stats": "1", + } + require.Equal(t, expectedStatusRows, readBillingDemoRowsByKey(statusRows, 6)) + require.Equal(t, expectedStatusRows, readBillingDemoRowsByKey(statusReader.GetReadBillingDemoHistoryRows(ReadBillingDemoTableStatus), 6)) + + normalCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(ExecCountStr)}, + } + normalReader := NewStmtSummaryReader(nil, true, normalCols, "", time.UTC) + normalReader.ssMap = ssMap + normalRows := normalReader.GetStmtSummaryCurrentRows() + require.Len(t, normalRows, 1) + match(t, normalRows[0], "digest", 1) +} + +func readBillingDemoRowsByKey(rows [][]types.Datum, keyColumns int) map[string]string { + result := make(map[string]string, len(rows)) + for _, row := range rows { + keyParts := make([]string, 0, keyColumns) + for i := 0; i < keyColumns; i++ { + keyParts = append(keyParts, fmt.Sprintf("%v", row[i].GetValue())) + } + valueParts := make([]string, 0, len(row)-keyColumns) + for i := keyColumns; i < len(row); i++ { + valueParts = append(valueParts, fmt.Sprintf("%v", row[i].GetValue())) + } + result[strings.Join(keyParts, "/")] = strings.Join(valueParts, " ") + } + return result +} + // Test AddStatement and ToDatum parallel. func TestAddStatementParallel(t *testing.T) { ssMap := newStmtSummaryByDigestMap() diff --git a/pkg/util/stmtsummary/v2/BUILD.bazel b/pkg/util/stmtsummary/v2/BUILD.bazel index 2ad5781b7b0b9..92e00fe7bec61 100644 --- a/pkg/util/stmtsummary/v2/BUILD.bazel +++ b/pkg/util/stmtsummary/v2/BUILD.bazel @@ -49,7 +49,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 18, + shard_count = 21, deps = [ "//pkg/config", "//pkg/meta/model", diff --git a/pkg/util/stmtsummary/v2/reader.go b/pkg/util/stmtsummary/v2/reader.go index 589fc4aa0a59f..44fa1205b6d38 100644 --- a/pkg/util/stmtsummary/v2/reader.go +++ b/pkg/util/stmtsummary/v2/reader.go @@ -31,10 +31,12 @@ import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/auth" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/set" + stmtsummarybase "github.com/pingcap/tidb/pkg/util/stmtsummary" "go.uber.org/zap" ) @@ -63,6 +65,17 @@ type MemReader struct { checker *stmtChecker } +// ReadBillingDemoMemReader expands read billing demo aggregates from the +// current in-memory statement summary window. +type ReadBillingDemoMemReader struct { + s *StmtSummary + columns []*model.ColumnInfo + instanceAddr string + timeLocation *time.Location + kind stmtsummarybase.ReadBillingDemoTableKind + checker *stmtChecker +} + // NewMemReader creates a MemReader from StmtSummary and other necessary parameters. func NewMemReader(s *StmtSummary, columns []*model.ColumnInfo, @@ -87,6 +100,31 @@ func NewMemReader(s *StmtSummary, } } +// NewReadBillingDemoMemReader creates a reader for current read billing demo rows. +func NewReadBillingDemoMemReader(s *StmtSummary, + columns []*model.ColumnInfo, + instanceAddr string, + timeLocation *time.Location, + user *auth.UserIdentity, + hasProcessPriv bool, + digests set.StringSet, + timeRanges []*StmtTimeRange, + kind stmtsummarybase.ReadBillingDemoTableKind) *ReadBillingDemoMemReader { + return &ReadBillingDemoMemReader{ + s: s, + columns: columns, + instanceAddr: instanceAddr, + timeLocation: timeLocation, + kind: kind, + checker: &stmtChecker{ + user: user, + hasProcessPriv: hasProcessPriv, + digests: digests, + timeRanges: timeRanges, + }, + } +} + // Rows returns rows converted from the current window's data maintained // in memory by StmtSummary. All evicted data will be aggregated into a // single row appended at the end. @@ -116,6 +154,9 @@ func (r *MemReader) Rows() [][]types.Datum { if !r.checker.hasPrivilege(record.AuthUsers) { return } + if record.StmtRecord.isReadBillingDemoStatusOnly() { + return + } record.Begin = w.begin.Unix() record.End = end row := make([]types.Datum, len(r.columnFactories)) @@ -147,6 +188,53 @@ func (r *MemReader) Rows() [][]types.Datum { return rows } +// Rows returns expanded read billing demo rows from the current window. +func (r *ReadBillingDemoMemReader) Rows() [][]types.Datum { + if r.s == nil { + return nil + } + end := timeNow().Unix() + r.s.windowLock.Lock() + w := r.s.window + if !r.checker.isTimeValid(w.begin.Unix(), end) { + r.s.windowLock.Unlock() + return nil + } + values := w.lru.Values() + evicted := w.evicted + r.s.windowLock.Unlock() + rows := make([][]types.Datum, 0, len(values)) + for _, v := range values { + record := v.(*lockedStmtRecord) + if !r.checker.isDigestValid(record.Digest) { + continue + } + func() { + record.Lock() + defer record.Unlock() + if !r.checker.hasPrivilege(record.AuthUsers) { + return + } + record.Begin = w.begin.Unix() + record.End = end + rows = append(rows, readBillingDemoRowsFromRecord(r, r.columns, record.StmtRecord, r.kind)...) + }() + } + if r.checker.digests == nil { + func() { + evicted.Lock() + defer evicted.Unlock() + if !r.checker.hasPrivilege(evicted.other.AuthUsers) { + return + } + evicted.other.Begin = w.begin.Unix() + evicted.other.End = end + rows = append(rows, readBillingDemoRowsFromRecord(r, r.columns, evicted.other, r.kind)...) + }() + } + return rows +} + // getInstanceAddr implements columnInfo. func (r *MemReader) getInstanceAddr() string { return r.instanceAddr @@ -157,6 +245,16 @@ func (r *MemReader) getTimeLocation() *time.Location { return r.timeLocation } +// getInstanceAddr implements columnInfo. +func (r *ReadBillingDemoMemReader) getInstanceAddr() string { + return r.instanceAddr +} + +// getInstanceAddr implements columnInfo. +func (r *ReadBillingDemoMemReader) getTimeLocation() *time.Location { + return r.timeLocation +} + // HistoryReader is used to read data that has been persisted to files. type HistoryReader struct { ctx context.Context @@ -166,9 +264,12 @@ type HistoryReader struct { instanceAddr string timeLocation *time.Location + columns []*model.ColumnInfo columnFactories []columnFactory checker *stmtChecker files *stmtFiles + readBillingDemo bool + readBillingKind stmtsummarybase.ReadBillingDemoTableKind concurrent int rowsCh <-chan [][]types.Datum @@ -188,6 +289,22 @@ func NewHistoryReader( digests set.StringSet, timeRanges []*StmtTimeRange, concurrent int, +) (*HistoryReader, error) { + return newHistoryReader(ctx, columns, instanceAddr, timeLocation, user, hasProcessPriv, digests, timeRanges, concurrent, false, 0) +} + +func newHistoryReader( + ctx context.Context, + columns []*model.ColumnInfo, + instanceAddr string, + timeLocation *time.Location, + user *auth.UserIdentity, + hasProcessPriv bool, + digests set.StringSet, + timeRanges []*StmtTimeRange, + concurrent int, + readBillingDemo bool, + readBillingKind stmtsummarybase.ReadBillingDemoTableKind, ) (*HistoryReader, error) { files, err := newStmtFiles(ctx, timeRanges) if err != nil { @@ -201,23 +318,30 @@ func NewHistoryReader( errCh := make(chan error, concurrent) ctx, cancel := context.WithCancel(ctx) + var columnFactories []columnFactory + if !readBillingDemo { + columnFactories = makeColumnFactories(columns) + } r := &HistoryReader{ ctx: ctx, cancel: cancel, instanceAddr: instanceAddr, timeLocation: timeLocation, - columnFactories: makeColumnFactories(columns), + columns: columns, + columnFactories: columnFactories, checker: &stmtChecker{ user: user, hasProcessPriv: hasProcessPriv, digests: digests, timeRanges: timeRanges, }, - files: files, - concurrent: concurrent, - rowsCh: rowsCh, - errCh: errCh, + files: files, + readBillingDemo: readBillingDemo, + readBillingKind: readBillingKind, + concurrent: concurrent, + rowsCh: rowsCh, + errCh: errCh, } r.wg.Add(1) @@ -228,6 +352,22 @@ func NewHistoryReader( return r, nil } +// NewReadBillingDemoHistoryReader creates a history reader for persisted read billing demo rows. +func NewReadBillingDemoHistoryReader( + ctx context.Context, + columns []*model.ColumnInfo, + instanceAddr string, + timeLocation *time.Location, + user *auth.UserIdentity, + hasProcessPriv bool, + digests set.StringSet, + timeRanges []*StmtTimeRange, + concurrent int, + kind stmtsummarybase.ReadBillingDemoTableKind, +) (*HistoryReader, error) { + return newHistoryReader(ctx, columns, instanceAddr, timeLocation, user, hasProcessPriv, digests, timeRanges, concurrent, true, kind) +} + // Rows returns rows converted from records in files. Reading and parsing // works asynchronously. If (nil, nil) is returned, it means that the // reading has been completed. @@ -311,7 +451,10 @@ func (r *HistoryReader) scheduleTasks( instanceAddr: r.instanceAddr, timeLocation: r.timeLocation, checker: r.checker, + columns: r.columns, columnFactories: r.columnFactories, + readBillingDemo: r.readBillingDemo, + readBillingKind: r.readBillingKind, } concurrent := r.concurrent @@ -443,6 +586,139 @@ func (c *stmtChecker) needStop(curBegin int64) bool { return stop } +func readBillingDemoRowsFromRecord(info columnInfo, columns []*model.ColumnInfo, record *StmtRecord, kind stmtsummarybase.ReadBillingDemoTableKind) [][]types.Datum { + switch kind { + case stmtsummarybase.ReadBillingDemoTableBaseUnits: + rows := make([][]types.Datum, 0, len(record.ReadBillingDemoBaseUnitAggs)) + for _, entry := range record.ReadBillingDemoBaseUnitAggs { + rows = append(rows, readBillingDemoBaseUnitRow(info, columns, record, entry)) + } + return rows + case stmtsummarybase.ReadBillingDemoTableStatus: + rows := make([][]types.Datum, 0, len(record.ReadBillingDemoStatusAggs)) + for _, entry := range record.ReadBillingDemoStatusAggs { + rows = append(rows, readBillingDemoStatusRow(info, columns, record, entry)) + } + return rows + default: + return nil + } +} + +func readBillingDemoBaseUnitRow(info columnInfo, columns []*model.ColumnInfo, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoBaseUnitAggEntry) []types.Datum { + row := make([]types.Datum, len(columns)) + for i, col := range columns { + row[i] = types.NewDatum(readBillingDemoBaseUnitColumnValue(info, col.Name.O, record, entry)) + } + return row +} + +func readBillingDemoStatusRow(info columnInfo, columns []*model.ColumnInfo, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoStatusAggEntry) []types.Datum { + row := make([]types.Datum, len(columns)) + for i, col := range columns { + row[i] = types.NewDatum(readBillingDemoStatusColumnValue(info, col.Name.O, record, entry)) + } + return row +} + +func readBillingDemoCommonColumnValue(info columnInfo, col string, record *StmtRecord) (any, bool) { + switch col { + case ClusterTableInstanceColumnNameStr: + return info.getInstanceAddr(), true + case SummaryBeginTimeStr: + beginTime := time.Unix(record.Begin, 0) + if beginTime.Location() != info.getTimeLocation() { + beginTime = beginTime.In(info.getTimeLocation()) + } + return types.NewTime(types.FromGoTime(beginTime), mysql.TypeTimestamp, 0), true + case SummaryEndTimeStr: + endTime := time.Unix(record.End, 0) + if endTime.Location() != info.getTimeLocation() { + endTime = endTime.In(info.getTimeLocation()) + } + return types.NewTime(types.FromGoTime(endTime), mysql.TypeTimestamp, 0), true + case StmtTypeStr: + return record.StmtType, true + case SchemaNameStr: + return convertEmptyToNil(record.SchemaName), true + case DigestStr: + return convertEmptyToNil(record.Digest), true + case DigestTextStr: + return record.NormalizedSQL, true + case PlanDigestStr: + return convertEmptyToNil(record.PlanDigest), true + case ResourceGroupName: + return record.ResourceGroupName, true + default: + return nil, false + } +} + +func readBillingDemoBaseUnitColumnValue(info columnInfo, col string, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoBaseUnitAggEntry) any { + if value, ok := readBillingDemoCommonColumnValue(info, col, record); ok { + return value + } + switch col { + case stmtsummarybase.ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case stmtsummarybase.ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case stmtsummarybase.ReadBillingDemoSiteStr: + return entry.Site + case stmtsummarybase.ReadBillingDemoOpClassStr: + return entry.OpClass + case stmtsummarybase.ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case stmtsummarybase.ReadBillingDemoUnitStr: + return entry.Unit + case stmtsummarybase.ReadBillingDemoInputSourceStr: + return entry.InputSource + case stmtsummarybase.ReadBillingDemoInputSideStr: + return entry.InputSide + case stmtsummarybase.ReadBillingDemoRowWidthSource: + return entry.RowWidthSource + case stmtsummarybase.ReadBillingDemoValueStr: + return entry.Value + case stmtsummarybase.ReadBillingDemoSampleCountStr: + return entry.SampleCount + case stmtsummarybase.ReadBillingDemoRowWidthSumStr: + return entry.RowWidthSum + case stmtsummarybase.ReadBillingDemoAvgRowWidthStr: + if entry.SampleCount == 0 { + return float64(0) + } + return entry.RowWidthSum / float64(entry.SampleCount) + default: + return nil + } +} + +func readBillingDemoStatusColumnValue(info columnInfo, col string, record *StmtRecord, entry stmtsummarybase.ReadBillingDemoStatusAggEntry) any { + if value, ok := readBillingDemoCommonColumnValue(info, col, record); ok { + return value + } + switch col { + case stmtsummarybase.ReadBillingDemoModelVersionStr: + return entry.ModelVersion + case stmtsummarybase.ReadBillingDemoWeightVersionStr: + return entry.WeightVersion + case stmtsummarybase.ReadBillingDemoSiteStr: + return entry.Site + case stmtsummarybase.ReadBillingDemoOpClassStr: + return entry.OpClass + case stmtsummarybase.ReadBillingDemoOperatorKindStr: + return entry.OperatorKind + case stmtsummarybase.ReadBillingDemoStatusStr: + return entry.Status + case stmtsummarybase.ReadBillingDemoReasonStr: + return entry.Reason + case stmtsummarybase.ReadBillingDemoCountStr: + return entry.Count + default: + return nil + } +} + type stmtTinyRecord struct { Begin int64 `json:"begin"` End int64 `json:"end"` @@ -730,7 +1006,10 @@ type stmtParseWorker struct { instanceAddr string timeLocation *time.Location checker *stmtChecker + columns []*model.ColumnInfo columnFactories []columnFactory + readBillingDemo bool + readBillingKind stmtsummarybase.ReadBillingDemoTableKind } func (w *stmtParseWorker) run( @@ -779,8 +1058,12 @@ func (w *stmtParseWorker) handleLines( continue } - row := w.buildRow(record) - rows = append(rows, row) + if w.readBillingDemo { + rows = append(rows, readBillingDemoRowsFromRecord(w, w.columns, record, w.readBillingKind)...) + } else { + row := w.buildRow(record) + rows = append(rows, row) + } } if len(rows) > 0 { @@ -823,6 +1106,9 @@ func (w *stmtParseWorker) matchConds(record *StmtRecord) bool { if !w.checker.hasPrivilege(record.AuthUsers) { return false } + if !w.readBillingDemo && record.isReadBillingDemoStatusOnly() { + return false + } return true } diff --git a/pkg/util/stmtsummary/v2/reader_test.go b/pkg/util/stmtsummary/v2/reader_test.go index 6f24b1c3422fe..12d75d64adc6f 100644 --- a/pkg/util/stmtsummary/v2/reader_test.go +++ b/pkg/util/stmtsummary/v2/reader_test.go @@ -19,6 +19,7 @@ import ( "context" "fmt" "os" + "strings" "testing" "time" @@ -28,6 +29,7 @@ import ( "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/set" + stmtsummarybase "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/stretchr/testify/require" ) @@ -253,6 +255,98 @@ func TestMemReader(t *testing.T) { require.Len(t, evicted, 3) // begin, end, count } +func TestReadBillingDemoMemReader(t *testing.T) { + timeLocation, err := time.LoadLocation("Asia/Shanghai") + require.NoError(t, err) + + ss := NewStmtSummary4Test(3) + defer ss.Close() + + info := GenerateStmtExecInfo4Test("digest_read_billing") + info.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "success", + Reason: "none", + }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 2, + RowWidth: 16, + }}, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 2, + }, + } + ss.Add(info) + + statusOnly := GenerateStmtExecInfo4Test("digest_read_billing_error") + statusOnly.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + } + ss.AddReadBillingDemoStatusOnly(statusOnly) + + baseUnitCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoUnitStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoValueStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSampleCountStr)}, + } + baseRows := NewReadBillingDemoMemReader(ss, baseUnitCols, "", timeLocation, nil, false, nil, nil, stmtsummarybase.ReadBillingDemoTableBaseUnits).Rows() + require.Len(t, baseRows, 1) + require.Equal(t, map[string]string{ + "digest_read_billing/tidb/projection_eval/fixed_events": "2 1", + }, readBillingDemoRowsByKey(baseRows, 4)) + + statusCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoStatusStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoReasonStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoCountStr)}, + } + statusRows := NewReadBillingDemoMemReader(ss, statusCols, "", timeLocation, nil, false, nil, nil, stmtsummarybase.ReadBillingDemoTableStatus).Rows() + require.Len(t, statusRows, 2) + require.Equal(t, map[string]string{ + "digest_read_billing/success/none": "1", + "digest_read_billing_error/error/statement_error": "1", + }, readBillingDemoRowsByKey(statusRows, 3)) + + normalReader := NewMemReader(ss, []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(ExecCountStr)}, + }, "", timeLocation, nil, false, nil, nil) + normalRows := normalReader.Rows() + require.Len(t, normalRows, 1) + require.Equal(t, "digest_read_billing", normalRows[0][0].GetString()) + require.Equal(t, int64(1), normalRows[0][1].GetInt64()) +} + func TestHistoryReader(t *testing.T) { filename1 := "tidb-statements-2022-12-27T16-21-20.245.log" filename2 := "tidb-statements.log" @@ -409,6 +503,61 @@ func TestHistoryReader(t *testing.T) { }() } +func TestReadBillingDemoHistoryReader(t *testing.T) { + filename := "tidb-statements.log" + file, err := os.Create(filename) + require.NoError(t, err) + defer func() { + require.NoError(t, os.Remove(filename)) + }() + _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_base_unit_aggs":[{"model_version":"v1","weight_version":"v1","site":"tidb","op_class":"projection_eval","operator_kind":"projection","unit":"fixed_events","input_source":"runtime_act_rows","input_side":"all","row_width_source":"operator_helper","value":2,"sample_count":1,"row_width_sum":16}],"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"success","reason":"none","count":1}]}` + "\n") + require.NoError(t, err) + _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing_error","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"error","reason":"statement_error","count":1}]}` + "\n") + require.NoError(t, err) + require.NoError(t, file.Close()) + + timeLocation, err := time.LoadLocation("Asia/Shanghai") + require.NoError(t, err) + baseUnitCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSiteStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoUnitStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoValueStr)}, + } + baseReader, err := NewReadBillingDemoHistoryReader(context.Background(), baseUnitCols, "", timeLocation, nil, false, nil, nil, 2, stmtsummarybase.ReadBillingDemoTableBaseUnits) + require.NoError(t, err) + baseRows := readAllRows(t, baseReader) + require.NoError(t, baseReader.Close()) + require.Equal(t, map[string]string{ + "digest_read_billing/tidb/projection_eval/fixed_events": "2", + }, readBillingDemoRowsByKey(baseRows, 4)) + + statusCols := []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoStatusStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoReasonStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoCountStr)}, + } + statusReader, err := NewReadBillingDemoHistoryReader(context.Background(), statusCols, "", timeLocation, nil, false, nil, nil, 2, stmtsummarybase.ReadBillingDemoTableStatus) + require.NoError(t, err) + statusRows := readAllRows(t, statusReader) + require.NoError(t, statusReader.Close()) + require.Equal(t, map[string]string{ + "digest_read_billing/success/none": "1", + "digest_read_billing_error/error/statement_error": "1", + }, readBillingDemoRowsByKey(statusRows, 3)) + + normalReader, err := NewHistoryReader(context.Background(), []*model.ColumnInfo{ + {Name: ast.NewCIStr(DigestStr)}, + {Name: ast.NewCIStr(ExecCountStr)}, + }, "", timeLocation, nil, false, nil, nil, 2) + require.NoError(t, err) + normalRows := readAllRows(t, normalReader) + require.NoError(t, normalReader.Close()) + require.Empty(t, normalRows) +} + func TestHistoryReaderInvalidLine(t *testing.T) { filename := "tidb-statements.log" @@ -458,3 +607,19 @@ func readAllRows(t *testing.T, reader *HistoryReader) [][]types.Datum { } return results } + +func readBillingDemoRowsByKey(rows [][]types.Datum, keyColumns int) map[string]string { + result := make(map[string]string, len(rows)) + for _, row := range rows { + keyParts := make([]string, 0, keyColumns) + for i := 0; i < keyColumns; i++ { + keyParts = append(keyParts, fmt.Sprintf("%v", row[i].GetValue())) + } + valueParts := make([]string, 0, len(row)-keyColumns) + for i := keyColumns; i < len(row); i++ { + valueParts = append(valueParts, fmt.Sprintf("%v", row[i].GetValue())) + } + result[strings.Join(keyParts, "/")] = strings.Join(valueParts, " ") + } + return result +} diff --git a/pkg/util/stmtsummary/v2/record.go b/pkg/util/stmtsummary/v2/record.go index 2d2172425cae5..2f525228053ac 100644 --- a/pkg/util/stmtsummary/v2/record.go +++ b/pkg/util/stmtsummary/v2/record.go @@ -156,6 +156,8 @@ type StmtRecord struct { ResourceGroupName string `json:"resource_group_name"` stmtsummary.StmtRUSummary stmtsummary.ReadBillingDemoBaseUnitSummary + ReadBillingDemoBaseUnitAggs []stmtsummary.ReadBillingDemoBaseUnitAggEntry `json:"read_billing_demo_base_unit_aggs,omitempty"` + ReadBillingDemoStatusAggs []stmtsummary.ReadBillingDemoStatusAggEntry `json:"read_billing_demo_status_aggs,omitempty"` PlanCacheUnqualifiedCount int64 `json:"plan_cache_unqualified_count"` PlanCacheUnqualifiedLastReason string `json:"plan_cache_unqualified_last_reason"` // the reason why this query is unqualified for the plan cache @@ -238,6 +240,34 @@ func NewStmtRecord(info *stmtsummary.StmtExecInfo) *StmtRecord { } } +// NewReadBillingDemoStatusOnlyRecord creates a minimal record for status-only +// read billing demo rows produced before the normal statement finish path. +func NewReadBillingDemoStatusOnlyRecord(info *stmtsummary.StmtExecInfo) *StmtRecord { + startTime := info.StartTime + if startTime.IsZero() { + startTime = time.Now() + } + record := &StmtRecord{ + SchemaName: info.SchemaName, + Digest: info.Digest, + PlanDigest: info.PlanDigest, + NormalizedSQL: info.NormalizedSQL, + IsInternal: info.IsInternal, + AuthUsers: make(map[string]struct{}), + BackoffTypes: make(map[string]int), + MinLatency: time.Duration(math.MaxInt64), + MinResultRows: math.MaxInt64, + FirstSeen: startTime, + LastSeen: startTime, + ResourceGroupName: info.ResourceGroupName, + } + if info.StmtCtx != nil { + record.StmtType = info.StmtCtx.StmtType + } + record.AddReadBillingDemoStatusOnly(info) + return record +} + // Add adds the statistics of StmtExecInfo to StmtRecord. func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { r.IsInternal = r.IsInternal && info.IsInternal @@ -445,12 +475,51 @@ func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { r.StmtNetworkTrafficSummary.Add(&tikvExecDetails) // RU r.StmtRUSummary.Add(info.RUDetail, info.TotalRUV2) - r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoBaseUnits) + if !info.ReadBillingDemoStats.IsEmpty() { + r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoStats.Totals) + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs = stmtsummary.AddReadBillingDemoStatementStatsToEntries( + r.ReadBillingDemoBaseUnitAggs, + r.ReadBillingDemoStatusAggs, + &info.ReadBillingDemoStats, + ) + } else { + r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoBaseUnits) + } r.StorageKV = info.StmtCtx.IsTiKV.Load() r.StorageMPP = info.StmtCtx.IsTiFlash.Load() } +// AddReadBillingDemoStatusOnly adds only read billing demo status/base-unit +// aggregates without changing ordinary statement summary counters. +func (r *StmtRecord) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo) { + if info == nil || info.ReadBillingDemoStats.IsEmpty() { + return + } + if len(info.User) > 0 { + if r.AuthUsers == nil { + r.AuthUsers = make(map[string]struct{}) + } + r.AuthUsers[info.User] = struct{}{} + } + r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoStats.Totals) + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs = stmtsummary.AddReadBillingDemoStatementStatsToEntries( + r.ReadBillingDemoBaseUnitAggs, + r.ReadBillingDemoStatusAggs, + &info.ReadBillingDemoStats, + ) +} + +func (r *StmtRecord) isReadBillingDemoStatusOnly() bool { + if r == nil || r.ExecCount != 0 { + return false + } + return len(r.ReadBillingDemoBaseUnitAggs) > 0 || len(r.ReadBillingDemoStatusAggs) > 0 || + r.SumReadBillingDemoFixedEvents != 0 || + r.SumReadBillingDemoInputRows != 0 || + r.SumReadBillingDemoInputBytes != 0 +} + // Merge merges the statistics of another StmtRecord to this StmtRecord. func (r *StmtRecord) Merge(other *StmtRecord) { // User @@ -611,6 +680,12 @@ func (r *StmtRecord) Merge(other *StmtRecord) { r.SumErrors += other.SumErrors r.StmtRUSummary.Merge(&other.StmtRUSummary) r.ReadBillingDemoBaseUnitSummary.Merge(&other.ReadBillingDemoBaseUnitSummary) + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs = stmtsummary.MergeReadBillingDemoEntrySlices( + r.ReadBillingDemoBaseUnitAggs, + r.ReadBillingDemoStatusAggs, + other.ReadBillingDemoBaseUnitAggs, + other.ReadBillingDemoStatusAggs, + ) } // Truncate SQL to maxSQLLength. diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index 7d0e0f55c1a6d..5d80620f14633 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/pingcap/tidb/pkg/config" + stmtsummarybase "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/stretchr/testify/require" ) @@ -119,3 +120,88 @@ func TestStmtRecord(t *testing.T) { require.Equal(t, true, items["evicted"]) require.Equal(t, record2.Digest, items["digest"]) } + +func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { + info := GenerateStmtExecInfo4Test("digest_read_billing") + info.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "success", + Reason: "none", + }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 2, + RowWidth: 16, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 4, + RowWidth: 24, + }, + }, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 6, + }, + } + + record := NewStmtRecord(info) + record.Add(info) + require.Equal(t, int64(1), record.ExecCount) + require.Equal(t, 6.0, record.SumReadBillingDemoFixedEvents) + require.Len(t, record.ReadBillingDemoBaseUnitAggs, 1) + require.Equal(t, 6.0, record.ReadBillingDemoBaseUnitAggs[0].Value) + require.Equal(t, uint64(2), record.ReadBillingDemoBaseUnitAggs[0].SampleCount) + require.Equal(t, 40.0, record.ReadBillingDemoBaseUnitAggs[0].RowWidthSum) + require.Len(t, record.ReadBillingDemoStatusAggs, 1) + + b, err := marshalStmtRecord(record) + require.NoError(t, err) + var restored StmtRecord + require.NoError(t, json.Unmarshal(b, &restored)) + require.Equal(t, record.ReadBillingDemoBaseUnitAggs, restored.ReadBillingDemoBaseUnitAggs) + require.Equal(t, record.ReadBillingDemoStatusAggs, restored.ReadBillingDemoStatusAggs) + + statusOnly := GenerateStmtExecInfo4Test("digest_read_billing_error") + statusOnly.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + } + statusOnlyRecord := NewReadBillingDemoStatusOnlyRecord(statusOnly) + require.Zero(t, statusOnlyRecord.ExecCount) + require.True(t, statusOnlyRecord.isReadBillingDemoStatusOnly()) + require.Len(t, statusOnlyRecord.ReadBillingDemoStatusAggs, 1) + require.Contains(t, statusOnlyRecord.AuthUsers, "user") +} diff --git a/pkg/util/stmtsummary/v2/stmtsummary.go b/pkg/util/stmtsummary/v2/stmtsummary.go index 2eab73de1ca2f..6e2f0914b07e5 100644 --- a/pkg/util/stmtsummary/v2/stmtsummary.go +++ b/pkg/util/stmtsummary/v2/stmtsummary.go @@ -348,6 +348,44 @@ func (s *StmtSummary) Add(info *stmtsummary.StmtExecInfo) { } } +// AddReadBillingDemoStatusOnly adds read billing demo status-only data to the +// current statistics window without changing ordinary statement counters. +func (s *StmtSummary) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo) { + if s.closed.Load() || info == nil || info.ReadBillingDemoStats.IsEmpty() { + return + } + + k := stmtsummary.StmtDigestKeyPool.Get().(*stmtsummary.StmtDigestKey) + + s.windowLock.Lock() + if s.closed.Load() { + s.windowLock.Unlock() + stmtsummary.StmtDigestKeyPool.Put(k) + return + } + userForKey := "" + if s.optGroupByUser.Load() { + userForKey = info.User + } + k.Init(info.SchemaName, info.Digest, info.PrevSQLDigest, info.PlanDigest, info.ResourceGroupName, userForKey) + var record *lockedStmtRecord + v, exist := s.window.lru.Get(k) + if exist { + record = v.(*lockedStmtRecord) + } else { + record = &lockedStmtRecord{StmtRecord: NewReadBillingDemoStatusOnlyRecord(info)} + s.window.lru.Put(k, record) + } + s.windowLock.Unlock() + + if exist { + record.Lock() + record.AddReadBillingDemoStatusOnly(info) + record.Unlock() + stmtsummary.StmtDigestKeyPool.Put(k) + } +} + // Evicted returns the number of statements evicted for the current // time window. The returned type is one row consisting of three // columns: [BEGIN_TIME, END_TIME, EVICTED_COUNT]. @@ -743,6 +781,22 @@ func Add(stmtExecInfo *stmtsummary.StmtExecInfo) { } } +// AddReadBillingDemoStatusOnly records read billing demo early-error statuses. +func AddReadBillingDemoStatusOnly(stmtExecInfo *stmtsummary.StmtExecInfo) { + if stmtExecInfo == nil || stmtExecInfo.ReadBillingDemoStats.IsEmpty() { + return + } + if config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent { + if GlobalStmtSummary == nil || !GlobalStmtSummary.Enabled() || + (stmtExecInfo.IsInternal && !GlobalStmtSummary.EnableInternalQuery()) { + return + } + GlobalStmtSummary.AddReadBillingDemoStatusOnly(stmtExecInfo) + return + } + stmtsummary.StmtSummaryByDigestMap.AddReadBillingDemoStatusOnly(stmtExecInfo) +} + // Enabled wraps GlobalStmtSummary.Enabled and stmtsummary.StmtSummaryByDigestMap.Enabled. func Enabled() bool { if config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent { From 2a68ce56656e84b3f4731dc3c6a9b6a75b6deaa1 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 3 Jul 2026 00:27:34 +0800 Subject: [PATCH 06/25] stmtsummary: tighten read billing aggregation bounds --- pkg/util/stmtsummary/evicted.go | 9 ++- pkg/util/stmtsummary/read_billing.go | 79 ++++++++++++++++--- pkg/util/stmtsummary/read_billing_test.go | 16 +++- pkg/util/stmtsummary/statement_summary.go | 18 +++-- .../stmtsummary/statement_summary_test.go | 16 ++++ pkg/util/stmtsummary/v2/BUILD.bazel | 2 +- pkg/util/stmtsummary/v2/logger.go | 2 +- pkg/util/stmtsummary/v2/reader_test.go | 16 ++++ pkg/util/stmtsummary/v2/record.go | 39 +++++++-- pkg/util/stmtsummary/v2/record_test.go | 18 +++++ pkg/util/stmtsummary/v2/stmtsummary.go | 8 +- pkg/util/stmtsummary/v2/stmtsummary_test.go | 46 +++++++++++ 12 files changed, 236 insertions(+), 33 deletions(-) diff --git a/pkg/util/stmtsummary/evicted.go b/pkg/util/stmtsummary/evicted.go index b4f4d675b1c71..4c77021ac2529 100644 --- a/pkg/util/stmtsummary/evicted.go +++ b/pkg/util/stmtsummary/evicted.go @@ -390,13 +390,18 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem addTo.sumErrors += addWith.sumErrors addTo.StmtRUSummary.Merge(&addWith.StmtRUSummary) - addTo.ReadBillingDemoBaseUnitSummary.Merge(&addWith.ReadBillingDemoBaseUnitSummary) - addTo.ReadBillingDemoBaseUnitAggs, addTo.ReadBillingDemoStatusAggs = MergeReadBillingDemoAggMaps( + var acceptedSummary ReadBillingDemoBaseUnitSummary + addTo.ReadBillingDemoBaseUnitAggs, addTo.ReadBillingDemoStatusAggs, acceptedSummary = MergeReadBillingDemoAggMaps( addTo.ReadBillingDemoBaseUnitAggs, addTo.ReadBillingDemoStatusAggs, addWith.ReadBillingDemoBaseUnitAggs, addWith.ReadBillingDemoStatusAggs, ) + if len(addWith.ReadBillingDemoBaseUnitAggs) == 0 && len(addWith.ReadBillingDemoStatusAggs) == 0 { + addTo.ReadBillingDemoBaseUnitSummary.Merge(&addWith.ReadBillingDemoBaseUnitSummary) + } else { + addTo.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) + } // resourceGroupName might not be inited because when it is a evicted item. addTo.resourceGroupName = addWith.resourceGroupName } diff --git a/pkg/util/stmtsummary/read_billing.go b/pkg/util/stmtsummary/read_billing.go index 02123a147e0e5..c6b93a9805f0c 100644 --- a/pkg/util/stmtsummary/read_billing.go +++ b/pkg/util/stmtsummary/read_billing.go @@ -89,6 +89,31 @@ func (s *ReadBillingDemoStatementStats) IsEmpty() bool { s.Totals.SumReadBillingDemoInputBytes == 0 } +// StatusOnly returns the status portion of a read billing snapshot. +func (s ReadBillingDemoStatementStats) StatusOnly() ReadBillingDemoStatementStats { + return ReadBillingDemoStatementStats{ + ModelVersion: s.ModelVersion, + WeightVersion: s.WeightVersion, + Statuses: append([]ReadBillingDemoStatusSample(nil), s.Statuses...), + } +} + +// ReadBillingDemoStatusOnlyExecInfo returns a shallow copy that is safe for the +// status-only statement summary path. +func ReadBillingDemoStatusOnlyExecInfo(sei *StmtExecInfo) (*StmtExecInfo, bool) { + if sei == nil { + return nil, false + } + statusOnly := sei.ReadBillingDemoStats.StatusOnly() + if statusOnly.IsEmpty() { + return nil, false + } + copied := *sei + copied.ReadBillingDemoStats = statusOnly + copied.ReadBillingDemoBaseUnits = ReadBillingDemoBaseUnitSummary{} + return &copied, true +} + // ReadBillingDemoBaseUnitSample is a coefficient-free read billing unit sample. type ReadBillingDemoBaseUnitSample struct { ModelVersion string @@ -251,6 +276,28 @@ func (a *ReadBillingDemoStatusAgg) addEntry(entry ReadBillingDemoStatusAggEntry) a.Count += entry.Count } +func (s *ReadBillingDemoBaseUnitSummary) addSample(sample ReadBillingDemoBaseUnitSample) { + switch sample.Unit { + case "fixed_events": + s.SumReadBillingDemoFixedEvents += sample.Value + case "input_rows": + s.SumReadBillingDemoInputRows += sample.Value + case "input_bytes": + s.SumReadBillingDemoInputBytes += sample.Value + } +} + +func (s *ReadBillingDemoBaseUnitSummary) addEntry(entry ReadBillingDemoBaseUnitAggEntry) { + switch entry.Unit { + case "fixed_events": + s.SumReadBillingDemoFixedEvents += entry.Value + case "input_rows": + s.SumReadBillingDemoInputRows += entry.Value + case "input_bytes": + s.SumReadBillingDemoInputBytes += entry.Value + } +} + func readBillingDemoBaseUnitEntry(key ReadBillingDemoBaseUnitKey, agg ReadBillingDemoBaseUnitAgg) ReadBillingDemoBaseUnitAggEntry { return ReadBillingDemoBaseUnitAggEntry{ ModelVersion: key.ModelVersion, @@ -306,9 +353,10 @@ func AddReadBillingDemoStatementStatsToMaps( baseUnitAggs map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, statusAggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, stats *ReadBillingDemoStatementStats, -) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) { +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary if stats == nil { - return baseUnitAggs, statusAggs + return baseUnitAggs, statusAggs, acceptedSummary } for _, status := range stats.Statuses { statusAggs, _ = addReadBillingDemoStatusSampleToMap(statusAggs, status, false) @@ -318,9 +366,11 @@ func AddReadBillingDemoStatementStatsToMaps( baseUnitAggs, overflow = addReadBillingDemoBaseUnitSampleToMap(baseUnitAggs, sample) if overflow { statusAggs = addReadBillingDemoReservedStatusToMap(statusAggs, stats.ModelVersion, stats.WeightVersion, readBillingDemoReasonAggregation) + continue } + acceptedSummary.addSample(sample) } - return baseUnitAggs, statusAggs + return baseUnitAggs, statusAggs, acceptedSummary } // MergeReadBillingDemoAggMaps merges source maps into destination maps with the same caps used on write. @@ -329,7 +379,8 @@ func MergeReadBillingDemoAggMaps( dstStatus map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, srcBase map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, srcStatus map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, -) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) { +) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary for key, agg := range srcStatus { dstStatus, _ = addReadBillingDemoStatusEntryToMap(dstStatus, readBillingDemoStatusEntry(key, agg), false) } @@ -338,9 +389,11 @@ func MergeReadBillingDemoAggMaps( dstBase, overflow = addReadBillingDemoBaseUnitEntryToMap(dstBase, readBillingDemoBaseUnitEntry(key, agg)) if overflow { dstStatus = addReadBillingDemoReservedStatusToMap(dstStatus, key.ModelVersion, key.WeightVersion, readBillingDemoReasonAggregation) + continue } + acceptedSummary.addEntry(readBillingDemoBaseUnitEntry(key, agg)) } - return dstBase, dstStatus + return dstBase, dstStatus, acceptedSummary } // AddReadBillingDemoStatementStatsToEntries merges one statement snapshot into v2 entries. @@ -348,9 +401,10 @@ func AddReadBillingDemoStatementStatsToEntries( baseUnitEntries []ReadBillingDemoBaseUnitAggEntry, statusEntries []ReadBillingDemoStatusAggEntry, stats *ReadBillingDemoStatementStats, -) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry) { +) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary if stats == nil { - return baseUnitEntries, statusEntries + return baseUnitEntries, statusEntries, acceptedSummary } for _, status := range stats.Statuses { statusEntries, _ = addReadBillingDemoStatusSampleToEntries(statusEntries, status, false) @@ -360,9 +414,11 @@ func AddReadBillingDemoStatementStatsToEntries( baseUnitEntries, overflow = addReadBillingDemoBaseUnitSampleToEntries(baseUnitEntries, sample) if overflow { statusEntries = addReadBillingDemoReservedStatusToEntries(statusEntries, stats.ModelVersion, stats.WeightVersion, readBillingDemoReasonAggregation) + continue } + acceptedSummary.addSample(sample) } - return baseUnitEntries, statusEntries + return baseUnitEntries, statusEntries, acceptedSummary } // MergeReadBillingDemoEntrySlices merges source entries into destination entries with caps. @@ -371,7 +427,8 @@ func MergeReadBillingDemoEntrySlices( dstStatus []ReadBillingDemoStatusAggEntry, srcBase []ReadBillingDemoBaseUnitAggEntry, srcStatus []ReadBillingDemoStatusAggEntry, -) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry) { +) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry, ReadBillingDemoBaseUnitSummary) { + var acceptedSummary ReadBillingDemoBaseUnitSummary for _, entry := range srcStatus { dstStatus, _ = addReadBillingDemoStatusEntryToEntries(dstStatus, entry, false) } @@ -380,9 +437,11 @@ func MergeReadBillingDemoEntrySlices( dstBase, overflow = addReadBillingDemoBaseUnitEntryToEntries(dstBase, entry) if overflow { dstStatus = addReadBillingDemoReservedStatusToEntries(dstStatus, entry.ModelVersion, entry.WeightVersion, readBillingDemoReasonAggregation) + continue } + acceptedSummary.addEntry(entry) } - return dstBase, dstStatus + return dstBase, dstStatus, acceptedSummary } func addReadBillingDemoBaseUnitSampleToMap( diff --git a/pkg/util/stmtsummary/read_billing_test.go b/pkg/util/stmtsummary/read_billing_test.go index fe7f493b0233c..86ec8b1f77b9e 100644 --- a/pkg/util/stmtsummary/read_billing_test.go +++ b/pkg/util/stmtsummary/read_billing_test.go @@ -42,13 +42,15 @@ func TestReadBillingDemoAggregationCaps(t *testing.T) { }) } - baseAggs, statusAggs := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) + baseAggs, statusAggs, acceptedSummary := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) require.Len(t, baseAggs, MaxReadBillingDemoBaseUnitKeysPerRecord) require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonAggregation).Count) + require.Equal(t, float64(MaxReadBillingDemoBaseUnitKeysPerRecord*(MaxReadBillingDemoBaseUnitKeysPerRecord+1)/2), acceptedSummary.SumReadBillingDemoInputRows) - baseEntries, statusEntries := AddReadBillingDemoStatementStatsToEntries(nil, nil, &stats) + baseEntries, statusEntries, acceptedSummary := AddReadBillingDemoStatementStatsToEntries(nil, nil, &stats) require.Len(t, baseEntries, MaxReadBillingDemoBaseUnitKeysPerRecord) require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonAggregation).Count) + require.Equal(t, float64(MaxReadBillingDemoBaseUnitKeysPerRecord*(MaxReadBillingDemoBaseUnitKeysPerRecord+1)/2), acceptedSummary.SumReadBillingDemoInputRows) statusOnly := ReadBillingDemoStatementStats{ ModelVersion: "v1", @@ -66,13 +68,19 @@ func TestReadBillingDemoAggregationCaps(t *testing.T) { }) } - _, statusAggs = AddReadBillingDemoStatementStatsToMaps(nil, nil, &statusOnly) + _, statusAggs, acceptedSummary = AddReadBillingDemoStatementStatsToMaps(nil, nil, &statusOnly) require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusKeyCount(statusAggs)) require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonStatusAggregation).Count) + require.Zero(t, acceptedSummary.SumReadBillingDemoFixedEvents) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputRows) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputBytes) - _, statusEntries = AddReadBillingDemoStatementStatsToEntries(nil, nil, &statusOnly) + _, statusEntries, acceptedSummary = AddReadBillingDemoStatementStatsToEntries(nil, nil, &statusOnly) require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusEntryCount(statusEntries)) require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonStatusAggregation).Count) + require.Zero(t, acceptedSummary.SumReadBillingDemoFixedEvents) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputRows) + require.Zero(t, acceptedSummary.SumReadBillingDemoInputBytes) } func requireReadBillingDemoStatusReason(t *testing.T, entries []ReadBillingDemoStatusAggEntry, reason string) ReadBillingDemoStatusAggEntry { diff --git a/pkg/util/stmtsummary/statement_summary.go b/pkg/util/stmtsummary/statement_summary.go index c6d2f21055ed1..b74fc046b5184 100644 --- a/pkg/util/stmtsummary/statement_summary.go +++ b/pkg/util/stmtsummary/statement_summary.go @@ -440,7 +440,9 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) { // AddReadBillingDemoStatusOnly records read billing demo statuses that happen // before the normal statement finish path can build a full StmtExecInfo. func (ssMap *stmtSummaryByDigestMap) AddReadBillingDemoStatusOnly(sei *StmtExecInfo) { - if sei == nil || sei.ReadBillingDemoStats.IsEmpty() { + var ok bool + sei, ok = ReadBillingDemoStatusOnlyExecInfo(sei) + if !ok { return } now := time.Now().Unix() @@ -880,6 +882,10 @@ func newStmtSummaryStats(sei *StmtExecInfo) *stmtSummaryStats { } func newReadBillingDemoStatusOnlyStats(sei *StmtExecInfo) *stmtSummaryStats { + sei, ok := ReadBillingDemoStatusOnlyExecInfo(sei) + if !ok { + return &stmtSummaryStats{} + } startTime := sei.StartTime if startTime.IsZero() { startTime = time.Now() @@ -941,12 +947,13 @@ func (ssStats *stmtSummaryStats) addReadBillingDemoStatementStats(user string, s } ssStats.authUsers[user] = struct{}{} } - ssStats.ReadBillingDemoBaseUnitSummary.Add(&stats.Totals) - ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs = AddReadBillingDemoStatementStatsToMaps( + var acceptedSummary ReadBillingDemoBaseUnitSummary + ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs, acceptedSummary = AddReadBillingDemoStatementStatsToMaps( ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs, stats, ) + ssStats.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) } func (ssStats *stmtSummaryStats) isReadBillingDemoStatusOnly() bool { @@ -1176,12 +1183,13 @@ func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affect // request-units ssStats.StmtRUSummary.Add(sei.RUDetail, sei.TotalRUV2) if !sei.ReadBillingDemoStats.IsEmpty() { - ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoStats.Totals) - ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs = AddReadBillingDemoStatementStatsToMaps( + var acceptedSummary ReadBillingDemoBaseUnitSummary + ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs, acceptedSummary = AddReadBillingDemoStatementStatsToMaps( ssStats.ReadBillingDemoBaseUnitAggs, ssStats.ReadBillingDemoStatusAggs, &sei.ReadBillingDemoStats, ) + ssStats.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) } else { ssStats.ReadBillingDemoBaseUnitSummary.Add(&sei.ReadBillingDemoBaseUnits) } diff --git a/pkg/util/stmtsummary/statement_summary_test.go b/pkg/util/stmtsummary/statement_summary_test.go index 9d3f1e88d2404..5b15bd02f1568 100644 --- a/pkg/util/stmtsummary/statement_summary_test.go +++ b/pkg/util/stmtsummary/statement_summary_test.go @@ -1189,6 +1189,22 @@ func TestReadBillingDemoStructuredRowsToDatum(t *testing.T) { Status: "error", Reason: "statement_error", }}, + BaseUnits: []ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 999, + RowWidth: 999, + }}, + Totals: ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 999, + }, } ssMap.AddReadBillingDemoStatusOnly(statusOnlyInfo) diff --git a/pkg/util/stmtsummary/v2/BUILD.bazel b/pkg/util/stmtsummary/v2/BUILD.bazel index 92e00fe7bec61..7fdf4dcaa3dae 100644 --- a/pkg/util/stmtsummary/v2/BUILD.bazel +++ b/pkg/util/stmtsummary/v2/BUILD.bazel @@ -49,7 +49,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 21, + shard_count = 22, deps = [ "//pkg/config", "//pkg/meta/model", diff --git a/pkg/util/stmtsummary/v2/logger.go b/pkg/util/stmtsummary/v2/logger.go index 028b20ed11fc9..c37956f9bd5ab 100644 --- a/pkg/util/stmtsummary/v2/logger.go +++ b/pkg/util/stmtsummary/v2/logger.go @@ -61,7 +61,7 @@ func (s *stmtLogStorage) persist(w *stmtWindow, end time.Time) { r.Unlock() } w.evicted.Lock() - if w.evicted.otherForPersist.ExecCount > 0 { + if w.evicted.otherForPersist.ExecCount > 0 || w.evicted.otherForPersist.hasReadBillingDemoData() { w.evicted.otherForPersist.Begin = begin w.evicted.otherForPersist.End = end.Unix() s.log(w.evicted.otherForPersist) diff --git a/pkg/util/stmtsummary/v2/reader_test.go b/pkg/util/stmtsummary/v2/reader_test.go index 12d75d64adc6f..e4aa0c4c18431 100644 --- a/pkg/util/stmtsummary/v2/reader_test.go +++ b/pkg/util/stmtsummary/v2/reader_test.go @@ -307,6 +307,22 @@ func TestReadBillingDemoMemReader(t *testing.T) { Status: "error", Reason: "statement_error", }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 999, + RowWidth: 999, + }}, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 999, + }, } ss.AddReadBillingDemoStatusOnly(statusOnly) diff --git a/pkg/util/stmtsummary/v2/record.go b/pkg/util/stmtsummary/v2/record.go index 2f525228053ac..531b2bb5e5152 100644 --- a/pkg/util/stmtsummary/v2/record.go +++ b/pkg/util/stmtsummary/v2/record.go @@ -243,6 +243,16 @@ func NewStmtRecord(info *stmtsummary.StmtExecInfo) *StmtRecord { // NewReadBillingDemoStatusOnlyRecord creates a minimal record for status-only // read billing demo rows produced before the normal statement finish path. func NewReadBillingDemoStatusOnlyRecord(info *stmtsummary.StmtExecInfo) *StmtRecord { + var ok bool + info, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(info) + if !ok { + return &StmtRecord{ + AuthUsers: make(map[string]struct{}), + BackoffTypes: make(map[string]int), + MinLatency: time.Duration(math.MaxInt64), + MinResultRows: math.MaxInt64, + } + } startTime := info.StartTime if startTime.IsZero() { startTime = time.Now() @@ -476,12 +486,13 @@ func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { // RU r.StmtRUSummary.Add(info.RUDetail, info.TotalRUV2) if !info.ReadBillingDemoStats.IsEmpty() { - r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoStats.Totals) - r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs = stmtsummary.AddReadBillingDemoStatementStatsToEntries( + var acceptedSummary stmtsummary.ReadBillingDemoBaseUnitSummary + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, acceptedSummary = stmtsummary.AddReadBillingDemoStatementStatsToEntries( r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, &info.ReadBillingDemoStats, ) + r.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) } else { r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoBaseUnits) } @@ -493,7 +504,9 @@ func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { // AddReadBillingDemoStatusOnly adds only read billing demo status/base-unit // aggregates without changing ordinary statement summary counters. func (r *StmtRecord) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo) { - if info == nil || info.ReadBillingDemoStats.IsEmpty() { + var ok bool + info, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(info) + if !ok { return } if len(info.User) > 0 { @@ -502,16 +515,21 @@ func (r *StmtRecord) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo } r.AuthUsers[info.User] = struct{}{} } - r.ReadBillingDemoBaseUnitSummary.Add(&info.ReadBillingDemoStats.Totals) - r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs = stmtsummary.AddReadBillingDemoStatementStatsToEntries( + var acceptedSummary stmtsummary.ReadBillingDemoBaseUnitSummary + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, acceptedSummary = stmtsummary.AddReadBillingDemoStatementStatsToEntries( r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, &info.ReadBillingDemoStats, ) + r.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) } func (r *StmtRecord) isReadBillingDemoStatusOnly() bool { - if r == nil || r.ExecCount != 0 { + return r != nil && r.ExecCount == 0 && r.hasReadBillingDemoData() +} + +func (r *StmtRecord) hasReadBillingDemoData() bool { + if r == nil { return false } return len(r.ReadBillingDemoBaseUnitAggs) > 0 || len(r.ReadBillingDemoStatusAggs) > 0 || @@ -679,13 +697,18 @@ func (r *StmtRecord) Merge(other *StmtRecord) { r.SumTikvCPU += other.SumTikvCPU r.SumErrors += other.SumErrors r.StmtRUSummary.Merge(&other.StmtRUSummary) - r.ReadBillingDemoBaseUnitSummary.Merge(&other.ReadBillingDemoBaseUnitSummary) - r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs = stmtsummary.MergeReadBillingDemoEntrySlices( + var acceptedSummary stmtsummary.ReadBillingDemoBaseUnitSummary + r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, acceptedSummary = stmtsummary.MergeReadBillingDemoEntrySlices( r.ReadBillingDemoBaseUnitAggs, r.ReadBillingDemoStatusAggs, other.ReadBillingDemoBaseUnitAggs, other.ReadBillingDemoStatusAggs, ) + if len(other.ReadBillingDemoBaseUnitAggs) == 0 && len(other.ReadBillingDemoStatusAggs) == 0 { + r.ReadBillingDemoBaseUnitSummary.Merge(&other.ReadBillingDemoBaseUnitSummary) + } else { + r.ReadBillingDemoBaseUnitSummary.Add(&acceptedSummary) + } } // Truncate SQL to maxSQLLength. diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index 5d80620f14633..7af6fd673759f 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -198,10 +198,28 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { Status: "error", Reason: "statement_error", }}, + BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "fixed_events", + InputSource: "runtime_act_rows", + InputSide: "all", + RowWidthSource: "operator_helper", + Value: 999, + RowWidth: 999, + }}, + Totals: stmtsummarybase.ReadBillingDemoBaseUnitSummary{ + SumReadBillingDemoFixedEvents: 999, + }, } statusOnlyRecord := NewReadBillingDemoStatusOnlyRecord(statusOnly) require.Zero(t, statusOnlyRecord.ExecCount) require.True(t, statusOnlyRecord.isReadBillingDemoStatusOnly()) require.Len(t, statusOnlyRecord.ReadBillingDemoStatusAggs, 1) + require.Empty(t, statusOnlyRecord.ReadBillingDemoBaseUnitAggs) + require.Zero(t, statusOnlyRecord.SumReadBillingDemoFixedEvents) require.Contains(t, statusOnlyRecord.AuthUsers, "user") } diff --git a/pkg/util/stmtsummary/v2/stmtsummary.go b/pkg/util/stmtsummary/v2/stmtsummary.go index 6e2f0914b07e5..b68e44c866d39 100644 --- a/pkg/util/stmtsummary/v2/stmtsummary.go +++ b/pkg/util/stmtsummary/v2/stmtsummary.go @@ -351,7 +351,9 @@ func (s *StmtSummary) Add(info *stmtsummary.StmtExecInfo) { // AddReadBillingDemoStatusOnly adds read billing demo status-only data to the // current statistics window without changing ordinary statement counters. func (s *StmtSummary) AddReadBillingDemoStatusOnly(info *stmtsummary.StmtExecInfo) { - if s.closed.Load() || info == nil || info.ReadBillingDemoStats.IsEmpty() { + var ok bool + info, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(info) + if s.closed.Load() || !ok { return } @@ -783,7 +785,9 @@ func Add(stmtExecInfo *stmtsummary.StmtExecInfo) { // AddReadBillingDemoStatusOnly records read billing demo early-error statuses. func AddReadBillingDemoStatusOnly(stmtExecInfo *stmtsummary.StmtExecInfo) { - if stmtExecInfo == nil || stmtExecInfo.ReadBillingDemoStats.IsEmpty() { + var ok bool + stmtExecInfo, ok = stmtsummary.ReadBillingDemoStatusOnlyExecInfo(stmtExecInfo) + if !ok { return } if config.GetGlobalConfig().Instance.StmtSummaryEnablePersistent { diff --git a/pkg/util/stmtsummary/v2/stmtsummary_test.go b/pkg/util/stmtsummary/v2/stmtsummary_test.go index c395045e7b387..9e455aeb16120 100644 --- a/pkg/util/stmtsummary/v2/stmtsummary_test.go +++ b/pkg/util/stmtsummary/v2/stmtsummary_test.go @@ -195,6 +195,52 @@ func TestStmtSummaryPersistEvictedDoesNotPersistLoggedRecordsAsAggregate(t *test require.Equal(t, int64(4), totalExecCount) } +func TestStmtSummaryPersistStatusOnlyEvictedAggregate(t *testing.T) { + var logBuf bytes.Buffer + storage := &stmtLogStorage{ + logger: zap.New(zapcore.NewCore(&stmtLogEncoder{}, zapcore.AddSync(&logBuf), zapcore.InfoLevel)), + } + statusOnlyInfo := func(digest string) *stmtsummary.StmtExecInfo { + info := GenerateStmtExecInfo4Test(digest) + info.ReadBillingDemoStats = stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v1", + WeightVersion: "v1", + Statuses: []stmtsummary.ReadBillingDemoStatusSample{{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "error", + Reason: "statement_error", + }}, + } + return info + } + + ss := NewStmtSummary4Test(1) + ss.storage = storage + ss.AddReadBillingDemoStatusOnly(statusOnlyInfo("digest_status_1")) + ss.AddReadBillingDemoStatusOnly(statusOnlyInfo("digest_status_2")) // evicts digest_status_1 + ss.Close() + + type loggedRecord struct { + Digest string `json:"digest"` + StatusAggs []stmtsummary.ReadBillingDemoStatusAggEntry `json:"read_billing_demo_status_aggs"` + } + var recordsWithStatus []loggedRecord + for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { + var record loggedRecord + require.NoError(t, json.Unmarshal([]byte(line), &record)) + if len(record.StatusAggs) > 0 { + recordsWithStatus = append(recordsWithStatus, record) + } + } + require.Len(t, recordsWithStatus, 2) + require.Contains(t, []string{recordsWithStatus[0].Digest, recordsWithStatus[1].Digest}, "") + require.Contains(t, []string{recordsWithStatus[0].Digest, recordsWithStatus[1].Digest}, "digest_status_2") +} + func TestStmtSummaryGroupByUser(t *testing.T) { ss := NewStmtSummary4Test(100) defer ss.Close() From 98c8fdfeb512cd3d8a75cc7ab8c523184ac57494 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 3 Jul 2026 00:37:51 +0800 Subject: [PATCH 07/25] stmtsummary: preserve read billing overflow status on merge --- pkg/util/stmtsummary/BUILD.bazel | 2 +- pkg/util/stmtsummary/read_billing.go | 13 ++++-- pkg/util/stmtsummary/read_billing_test.go | 46 +++++++++++++++++++ pkg/util/stmtsummary/v2/BUILD.bazel | 2 +- pkg/util/stmtsummary/v2/record_test.go | 56 +++++++++++++++++++++++ 5 files changed, 113 insertions(+), 6 deletions(-) diff --git a/pkg/util/stmtsummary/BUILD.bazel b/pkg/util/stmtsummary/BUILD.bazel index 03fee3b1f974c..dc6bf9ad49e60 100644 --- a/pkg/util/stmtsummary/BUILD.bazel +++ b/pkg/util/stmtsummary/BUILD.bazel @@ -42,7 +42,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 31, + shard_count = 32, deps = [ "//pkg/meta/model", "//pkg/metrics", diff --git a/pkg/util/stmtsummary/read_billing.go b/pkg/util/stmtsummary/read_billing.go index c6b93a9805f0c..c03c4022f6f62 100644 --- a/pkg/util/stmtsummary/read_billing.go +++ b/pkg/util/stmtsummary/read_billing.go @@ -382,7 +382,8 @@ func MergeReadBillingDemoAggMaps( ) (map[ReadBillingDemoBaseUnitKey]ReadBillingDemoBaseUnitAgg, map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg, ReadBillingDemoBaseUnitSummary) { var acceptedSummary ReadBillingDemoBaseUnitSummary for key, agg := range srcStatus { - dstStatus, _ = addReadBillingDemoStatusEntryToMap(dstStatus, readBillingDemoStatusEntry(key, agg), false) + entry := readBillingDemoStatusEntry(key, agg) + dstStatus, _ = addReadBillingDemoStatusEntryToMap(dstStatus, entry, readBillingDemoStatusReasonReserved(entry.Reason)) } for key, agg := range srcBase { var overflow bool @@ -430,7 +431,7 @@ func MergeReadBillingDemoEntrySlices( ) ([]ReadBillingDemoBaseUnitAggEntry, []ReadBillingDemoStatusAggEntry, ReadBillingDemoBaseUnitSummary) { var acceptedSummary ReadBillingDemoBaseUnitSummary for _, entry := range srcStatus { - dstStatus, _ = addReadBillingDemoStatusEntryToEntries(dstStatus, entry, false) + dstStatus, _ = addReadBillingDemoStatusEntryToEntries(dstStatus, entry, readBillingDemoStatusReasonReserved(entry.Reason)) } for _, entry := range srcBase { var overflow bool @@ -530,7 +531,7 @@ func addReadBillingDemoReservedStatusToMap( func readBillingDemoNonReservedStatusKeyCount(aggs map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) int { count := 0 for key := range aggs { - if key.Reason == readBillingDemoReasonAggregation || key.Reason == readBillingDemoReasonStatusAggregation { + if readBillingDemoStatusReasonReserved(key.Reason) { continue } count++ @@ -652,7 +653,7 @@ func readBillingDemoReservedStatusSample(modelVersion string, weightVersion stri func readBillingDemoNonReservedStatusEntryCount(entries []ReadBillingDemoStatusAggEntry) int { count := 0 for _, entry := range entries { - if entry.Reason == readBillingDemoReasonAggregation || entry.Reason == readBillingDemoReasonStatusAggregation { + if readBillingDemoStatusReasonReserved(entry.Reason) { continue } count++ @@ -660,6 +661,10 @@ func readBillingDemoNonReservedStatusEntryCount(entries []ReadBillingDemoStatusA return count } +func readBillingDemoStatusReasonReserved(reason string) bool { + return reason == readBillingDemoReasonAggregation || reason == readBillingDemoReasonStatusAggregation +} + func sortReadBillingDemoBaseUnitEntries(entries []ReadBillingDemoBaseUnitAggEntry) { slices.SortFunc(entries, func(i, j ReadBillingDemoBaseUnitAggEntry) int { return cmp.Or( diff --git a/pkg/util/stmtsummary/read_billing_test.go b/pkg/util/stmtsummary/read_billing_test.go index 86ec8b1f77b9e..58804863f45d9 100644 --- a/pkg/util/stmtsummary/read_billing_test.go +++ b/pkg/util/stmtsummary/read_billing_test.go @@ -83,6 +83,52 @@ func TestReadBillingDemoAggregationCaps(t *testing.T) { require.Zero(t, acceptedSummary.SumReadBillingDemoInputBytes) } +func TestReadBillingDemoReservedStatusMergeBypassesStatusCap(t *testing.T) { + fullStatusAggs := make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) + fullStatusEntries := make([]ReadBillingDemoStatusAggEntry, 0, MaxReadBillingDemoStatusKeysPerRecord) + for i := 0; i < MaxReadBillingDemoStatusKeysPerRecord; i++ { + key := ReadBillingDemoStatusKey{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Status: "unsupported", + Reason: "unsupported_operator", + } + fullStatusAggs[key] = ReadBillingDemoStatusAgg{Count: 1} + fullStatusEntries = append(fullStatusEntries, readBillingDemoStatusEntry(key, ReadBillingDemoStatusAgg{Count: 1})) + } + + srcStatusAggs := map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg{ + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonAggregation)): { + Count: 7, + }, + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonStatusAggregation)): { + Count: 11, + }, + } + _, mergedStatusAggs, _ := MergeReadBillingDemoAggMaps(nil, fullStatusAggs, nil, srcStatusAggs) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusKeyCount(mergedStatusAggs)) + require.Equal(t, uint64(7), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(mergedStatusAggs), readBillingDemoReasonAggregation).Count) + require.Equal(t, uint64(11), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(mergedStatusAggs), readBillingDemoReasonStatusAggregation).Count) + + srcStatusEntries := []ReadBillingDemoStatusAggEntry{ + readBillingDemoStatusEntry( + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonAggregation)), + ReadBillingDemoStatusAgg{Count: 7}, + ), + readBillingDemoStatusEntry( + makeReadBillingDemoStatusKey(readBillingDemoReservedStatusSample("v1", "v1", readBillingDemoReasonStatusAggregation)), + ReadBillingDemoStatusAgg{Count: 11}, + ), + } + _, mergedStatusEntries, _ := MergeReadBillingDemoEntrySlices(nil, fullStatusEntries, nil, srcStatusEntries) + require.Equal(t, MaxReadBillingDemoStatusKeysPerRecord, readBillingDemoNonReservedStatusEntryCount(mergedStatusEntries)) + require.Equal(t, uint64(7), requireReadBillingDemoStatusReason(t, mergedStatusEntries, readBillingDemoReasonAggregation).Count) + require.Equal(t, uint64(11), requireReadBillingDemoStatusReason(t, mergedStatusEntries, readBillingDemoReasonStatusAggregation).Count) +} + func requireReadBillingDemoStatusReason(t *testing.T, entries []ReadBillingDemoStatusAggEntry, reason string) ReadBillingDemoStatusAggEntry { t.Helper() for _, entry := range entries { diff --git a/pkg/util/stmtsummary/v2/BUILD.bazel b/pkg/util/stmtsummary/v2/BUILD.bazel index 7fdf4dcaa3dae..a8220186c06e5 100644 --- a/pkg/util/stmtsummary/v2/BUILD.bazel +++ b/pkg/util/stmtsummary/v2/BUILD.bazel @@ -49,7 +49,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 22, + shard_count = 23, deps = [ "//pkg/config", "//pkg/meta/model", diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index 7af6fd673759f..260004a292a11 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -16,6 +16,7 @@ package stmtsummary import ( "encoding/json" + "fmt" "testing" "github.com/pingcap/tidb/pkg/config" @@ -223,3 +224,58 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { require.Zero(t, statusOnlyRecord.SumReadBillingDemoFixedEvents) require.Contains(t, statusOnlyRecord.AuthUsers, "user") } + +func TestStmtRecordMergeReadBillingDemoReservedStatusBypassesStatusCap(t *testing.T) { + record := NewStmtRecord(GenerateStmtExecInfo4Test("digest_read_billing_dst")) + for i := 0; i < stmtsummarybase.MaxReadBillingDemoStatusKeysPerRecord; i++ { + record.ReadBillingDemoStatusAggs = append(record.ReadBillingDemoStatusAggs, stmtsummarybase.ReadBillingDemoStatusAggEntry{ + ModelVersion: "v1", + WeightVersion: "v1", + Site: "tidb", + OpClass: fmt.Sprintf("op_%03d", i), + OperatorKind: "projection", + Status: "unsupported", + Reason: "unsupported_operator", + Count: 1, + }) + } + + other := NewStmtRecord(GenerateStmtExecInfo4Test("digest_read_billing_src")) + other.ReadBillingDemoStatusAggs = []stmtsummarybase.ReadBillingDemoStatusAggEntry{ + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "unknown_input", + Reason: "aggregation_overflow", + Count: 7, + }, + { + ModelVersion: "v1", + WeightVersion: "v1", + Site: "statement", + OpClass: "statement", + OperatorKind: "statement", + Status: "unknown_input", + Reason: "status_aggregation_overflow", + Count: 11, + }, + } + + record.Merge(other) + require.Equal(t, uint64(7), requireReadBillingDemoStatusReason(t, record.ReadBillingDemoStatusAggs, "aggregation_overflow").Count) + require.Equal(t, uint64(11), requireReadBillingDemoStatusReason(t, record.ReadBillingDemoStatusAggs, "status_aggregation_overflow").Count) +} + +func requireReadBillingDemoStatusReason(t *testing.T, entries []stmtsummarybase.ReadBillingDemoStatusAggEntry, reason string) stmtsummarybase.ReadBillingDemoStatusAggEntry { + t.Helper() + for _, entry := range entries { + if entry.Reason == reason { + return entry + } + } + require.Failf(t, "missing read billing status reason", "reason=%s entries=%v", reason, entries) + return stmtsummarybase.ReadBillingDemoStatusAggEntry{} +} From ce53f6dd10f13b981fb397d96287c1cb44418a02 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 8 Jul 2026 01:04:27 +0800 Subject: [PATCH 08/25] docs: design TiDB write billing units --- ...7-08-write-billing-units-weights-design.md | 672 ++++++++++++++++++ 1 file changed, 672 insertions(+) create mode 100644 docs/design/2026-07-08-write-billing-units-weights-design.md diff --git a/docs/design/2026-07-08-write-billing-units-weights-design.md b/docs/design/2026-07-08-write-billing-units-weights-design.md new file mode 100644 index 0000000000000..53e5f0a5f45ac --- /dev/null +++ b/docs/design/2026-07-08-write-billing-units-weights-design.md @@ -0,0 +1,672 @@ +# TiDB 写入计费 Unit 和初始权重设计 + +- Author(s): Codex +- Discussion PR: N/A +- Tracking Issue: N/A + +## 目录 + +* [简介](#简介) +* [目标和非目标](#目标和非目标) +* [当前代码边界](#当前代码边界) +* [模型边界](#模型边界) +* [Taxonomy](#taxonomy) +* [Base Units](#base-units) +* [覆盖语句](#覆盖语句) +* [初始权重](#初始权重) +* [采集和暴露面](#采集和暴露面) +* [失败和降级状态](#失败和降级状态) +* [避免重复计费](#避免重复计费) +* [实现期 instrumentation 需求](#实现期-instrumentation-需求) +* [测试设计](#测试设计) +* [风险和未决问题](#风险和未决问题) + +## 简介 + +本文描述 TiDB 侧写入工作的 billing demo 设计，重点回答写入 executor 应该如何拆分 `unit` 和 `weight`。这是设计文档，不包含产品代码实现。 + +设计沿用 read billing demo 的基本方向：先暴露无系数 base units，再通过 `site/op_class/unit/weight_version` 离线套权重。不要只输出最终加权 RU，因为 workload 校准需要保留足够维度来重算不同权重。 + +写入侧 preview RU 推荐使用同一种可扩展公式形态： + +```text +preview_ru = sum(unit_value * weight(site, op_class, unit, weight_version)) +``` + +其中 `unit` 是有界枚举，不把表名、索引名、SQL 文本、region、store 或 plan id 放进权重 key。`operator_kind` 可以保留用于诊断，但不参与权重解析。 + +## 目标和非目标 + +目标： + +1. 覆盖 TiDB 侧主要 DML 写入 executor：`INSERT`、`INSERT ... SELECT`、`INSERT ... ON DUPLICATE KEY UPDATE`、`REPLACE`、`UPDATE`、`DELETE`、batch DML，以及 `LOAD DATA` 的 TiDB 解析和提交 worker 路径。 +2. 明确 `site`、`op_class`、`unit`、`operator_kind` 的 bounded taxonomy。 +3. 为每个 opclass 定义 base units、初始权重或相对权重顺序。 +4. 区分哪些 unit 可以从现有 executor / statement context / runtime stats 得到，哪些需要新增 instrumentation。 +5. 明确 `INSERT ... SELECT`、`UPDATE`、`DELETE` 等混合读写语句如何避免重复计算读路径和写路径。 +6. 明确 TiDB 写准备、表达式求值、table/index mutation、transaction/2PC orchestration、TiKV storage write cost 的边界。 +7. 让 failed / unsupported / unknown accounting 有 SQL 可查询 status。 + +非目标： + +- 不改变当前 RU v2 resource-control 生产上报。 +- 不把 storage-side TiKV/TiFlash write charging 纳入本设计的第一版权重；TiKV write keys / write bytes 只作为边界和未来校准参考。 +- 不把 commit / 2PC latency 当作 TiDB executor CPU 的替代输入。 +- 不用 `affectedRows` 直接代表物理写入行数。 +- 不要求第一版支持所有 DDL backfill、temporary index merge、FK cascade 的完整成本；这些路径必须能给出 status。 + +## 当前代码边界 + +写入入口和关键路径如下： + +- `pkg/executor/insert_common.go` + - `InsertValues` 保存 INSERT / REPLACE / LOAD DATA 共享状态。 + - `insertRows` 处理 `INSERT/REPLACE ... VALUES` 和 `SET`。 + - `insertRowsFromSelect` 处理 `INSERT/REPLACE ... SELECT`。 + - `evalRow`、`fastEvalRow`、`getRow`、`fillRow` 完成表达式求值、cast、default、generated column、auto id 和 rowid 填充。 + - `batchCheckAndInsert` 处理 `IGNORE` / `LOAD DATA IGNORE` / `LOAD DATA REPLACE` 的 duplicate 检查。 + - `addRecordWithAutoIDHint` 调用 table 层 `AddRecord`，并更新 SQL 语义 row counters。 +- `pkg/executor/insert.go` + - `InsertExec.exec` 分派普通 insert、insert ignore 和 ODKU。 + - `batchUpdateDupRows` 构造 duplicate check keys、prefetch，并对冲突行走 update path。 + - `doDupRowUpdate` 复用 `updateRecord`。 +- `pkg/executor/replace.go` + - `ReplaceExec.exec` 先构造 duplicate check keys，再删除冲突旧行并插入新行。 + - `replaceRow`、`removeIndexRow` 负责 handle / unique-key 冲突处理。 +- `pkg/executor/update.go` 和 `pkg/executor/write.go` + - `UpdateExec.updateRows` 从 child executor 读取待更新行。 + - `composeNewRow` / `fastComposeNewRow` 执行 assignment 表达式和 cast。 + - `updateRecord` 比较 old/new row，处理 no-op、handle changed、generated/on-update-now、FK、`RemoveRecord + AddRecord` 或 `UpdateRecord`。 +- `pkg/executor/delete.go` + - `deleteSingleTableByChunk` / `deleteMultiTablesByChunk` 从 child executor 读取待删行。 + - `removeRow` 调用 table 层 `RemoveRecord` 并处理 FK。 +- `pkg/executor/load_data.go` + - `LoadDataExec.Next` 进入 local / remote load。 + - `LoadDataWorker.load` 创建 reader、encode worker 和 commit worker pipeline。 + - `parserData2TableData` 做文件行解析、列映射、user variable、`SET` 表达式、cast/default/generated column。 + - `commitWorker.checkAndInsertOneBatch` 复用 insert / duplicate-check 写入路径。 +- `pkg/table/tables/tables.go` 和 `pkg/table/tables/index.go` + - `TableCommon.AddRecord` 构造 record key/value、写 memBuffer、创建索引。 + - `TableCommon.UpdateRecord` 重建受影响索引、编码新 row、写 memBuffer。 + - `TableCommon.RemoveRecord` delete record key 和 index keys。 + - `index.create` / `index.Delete` 生成 index key/value，做 unique check、temp index double-write、assertion 和 memBuffer mutation。 +- `pkg/sessionctx/stmtctx/stmtctx.go` + - `RecordRows`、`CopiedRows`、`UpdatedRows`、`DeletedRows`、`TouchedRows`、`AffectedRows` 是 MySQL 兼容语义计数，不是物理写入成本。 +- `pkg/executor/adapter.go` 和 `pkg/util/execdetails/ruv2_metrics.go` + - 当前 RU v2 从 `CommitDetails.WriteKeys` / `WriteSize` 汇总写入相关指标，但显式事务、batch DML 和 pipelined DML 的归因边界不同于 executor-local base units。 + +## 模型边界 + +写 billing demo 第一版按 `site` 分成三层： + +| Site | 含义 | 第一版动作 | +|---|---|---| +| `statement` | statement-level status | 只写 status | +| `tidb` | TiDB executor、row prepare、table/index mutation 构造、memBuffer mutation、LOAD DATA decode | 计入 base units | +| `tikv` | storage read/write、lock、2PC、raft/write bytes | 第一版不直接计入 TiDB 写 base units；可作为独立 storage billing 或 status/reference | + +TiDB 写计费只覆盖 TiDB 本地工作。下面这些工作不应在 TiDB write opclass 中计入： + +- `INSERT ... SELECT`、`UPDATE`、`DELETE` child plan 的 scan/filter/join/read executor 成本。它们应由 read billing demo 继续计入。 +- duplicate / FK check 过程中实际落到 TiKV 的 `txn.Get`、`txn.BatchGet`、snapshot iterator 的 storage read 成本。 +- pessimistic lock acquisition、`txn.LockKeys`、prewrite、commit、resolve lock、region grouping、local latch、raftstore write。 +- TiKV / TiFlash storage engine write amplification。 + +TiDB 写侧可以记录 duplicate probe keys、mutation keys、mutation bytes 等无系数输入，但这些 unit 的权重必须解释为 TiDB 本地 key construction / encoding / memBuffer work，而不是 storage RPC 或 2PC 成本。 + +## Taxonomy + +### Site + +第一版使用： + +- `statement` +- `tidb` + +保留未来扩展： + +- `tikv` +- `tiflash` + +### Op Class + +第一版推荐的 TiDB 写 opclasses： + +| Op class | 主要覆盖 | 代码锚点 | +|---|---|---| +| `write_row_prepare` | VALUES/SELECT/LOAD DATA 输入行到 table row 的表达式求值、cast、default、generated column、auto id/rowid/on-update-now | `InsertValues.evalRow`、`fastEvalRow`、`getRow`、`fillRow`、`UpdateExec.composeNewRow`、`parserData2TableData` | +| `write_duplicate_probe` | 构造 handle / unique index probe keys，批量 duplicate 检查的 TiDB 本地循环和 key handling | `getKeysNeedCheck`、`prefetchUniqueIndices`、`batchCheckAndInsert`、`batchUpdateDupRows`、`ReplaceExec.replaceRow` | +| `write_row_compare` | UPDATE / ODKU old/new row 比较、changed/no-op 判断、handle changed 判断 | `updateRecord` | +| `write_row_mutation` | record key/value encode，record key set/delete，row-level memBuffer mutation | `TableCommon.AddRecord`、`UpdateRecord`、`RemoveRecord` | +| `write_index_mutation` | index key/value encode，index set/delete，unique-index memBuffer checks，MV index expansion，temp-index double write | `TableCommon.addIndices`、`rebuildUpdateRecordIndices`、`removeRowIndices`、`index.create`、`index.Delete` | +| `write_foreign_key` | FK check/cascade 的 TiDB 本地 row/key preparation | `FKCheckExec`、`FKCascadeExec`、`checkRows`、`onRemoveRowForFK` | +| `write_txn_orchestrate` | batch DML statement commit boundary、`NewTxnInStmt`、`txn.MayFlush` 调用等 TiDB 事务编排事件 | `doBatchInsert`、`doBatchDelete`、`txn.MayFlush` callers | +| `load_data_decode` | LOAD DATA 文件行解析、列映射、user vars、SET 表达式进入 row prepare 前的 TiDB 本地处理 | `LoadDataWorker.load`、`readOneBatchRows`、`parserData2TableData` | + +`operator_kind` 是 bounded diagnostic 维度，建议使用具体执行路径而不是表名或索引名： + +- `insert_values` +- `insert_select` +- `insert_on_duplicate` +- `replace` +- `update` +- `delete` +- `load_data` +- `batch_insert` +- `batch_delete` +- `table_add_record` +- `table_update_record` +- `table_remove_record` +- `index_create` +- `index_delete` +- `foreign_key_check` +- `foreign_key_cascade` +- `txn_may_flush` +- `batch_dml_stmt_commit` + +### Unit + +第一版推荐 unit 枚举： + +| Unit | 含义 | +|---|---| +| `fixed_events` | opclass 发生次数；用于保留固定开销 | +| `input_rows` | opclass 消耗的逻辑输入行数 | +| `input_cells` | 被表达式、cast、default、compare 处理的列值数量；可近似 row * column | +| `input_bytes` | TiDB 本地处理的 row/file/value byte-shaped 输入 | +| `probe_keys` | duplicate / FK / unchanged-lock 等准备或检查的 key 数 | +| `probe_key_bytes` | probe keys 的 key bytes | +| `mutation_keys` | TiDB 写入 memBuffer 的 record/index set/delete key 数 | +| `mutation_bytes` | TiDB 构造并写入 memBuffer 的 key/value bytes；delete 只计 key bytes | +| `changed_rows` | 发生真实 row mutation 的逻辑行数 | +| `unchanged_rows` | UPDATE/ODKU/REPLACE 中比较后没有真实 row mutation 的行数 | +| `conflict_rows` | duplicate probe 找到冲突并进入 ignore/update/replace 分支的行数 | +| `batch_events` | batch DML / LOAD DATA commit worker / statement transaction boundary 事件 | + +这些 units 都是 coefficient-free。下游校准可以选择只使用其中一部分；TiDB 仍应保留所有可验证维度。 + +## Base Units + +### `write_row_prepare` + +覆盖： + +- INSERT VALUES / SET 的表达式求值和 cast。 +- INSERT SELECT 从 child row 到 table row 的 cast/default/generated/auto id。 +- UPDATE assignment 表达式、cast、generated/on-update-now。 +- LOAD DATA 行解析后的列映射、SET 表达式、cast/default/generated。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每次 executor 或 worker batch 发生 | 是 | +| `input_rows` | `InsertValues.rowCount` delta、`StmtCtx.RecordRows`、UPDATE child rows、LOAD DATA batch count | 部分需要 | +| `input_cells` | insert/update 当前已有 `rowsColMultiply` 口径，LOAD DATA 可用 parsed columns | 需要保存为维度化样本 | +| `input_bytes` | `types.EstimatedMemUsage`、LOAD DATA raw bytes、row datum estimated size | 需要 | + +`StmtCtx.AffectedRows` 不可用于这里，因为 ODKU、REPLACE、CLIENT_FOUND_ROWS 和 FK trigger 会改变 MySQL 语义。 + +### `write_duplicate_probe` + +覆盖： + +- `getKeysNeedCheck` 为每个 row 生成 handle key 和 unique index keys。 +- `prefetchUniqueIndices` / `prefetchConflictedOldRows` 的 TiDB key collection。 +- `batchCheckAndInsert`、ODKU、REPLACE 中逐 key duplicate handling。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 duplicate-check batch | 是 | +| `input_rows` | 被检查的 row 数 | `toBeCheckedRows` 长度可得 | +| `probe_keys` | handle key + unique keys + temp-index fallback keys | 是 | +| `probe_key_bytes` | probe key len sum | 是 | +| `conflict_rows` | duplicate path 找到冲突的 row 数 | 是 | + +注意：实际 `txn.Get` / `BatchGet` 的 storage read cost 不在本 opclass 权重里。若未来 TiKV read/write billing 覆盖这些请求，应通过 storage site 或独立 opclass 处理。 + +### `write_row_compare` + +覆盖： + +- UPDATE 和 ODKU 中 old/new row 比较。 +- no-op update / unchanged replace 判断。 +- handle changed 判断。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 update/ODKU compare path | 是 | +| `input_rows` | 进入 `updateRecord` 的 row 数 | 是 | +| `input_cells` | `len(t.Cols())` 或 writable columns | 是 | +| `changed_rows` | `updateRecord` changed=true | 是 | +| `unchanged_rows` | `updateRecord` changed=false 或 identical replace | 是 | + +`unchanged_rows` 保留为 base unit，是因为 no-op UPDATE 仍有 TiDB 比较和 lock-key preparation 成本，但不能被当成 row mutation。 + +### `write_row_mutation` + +覆盖： + +- `AddRecord` record key/value encode 和 memBuffer set。 +- `UpdateRecord` record value re-encode 和 memBuffer set。 +- `RemoveRecord` record key delete。 +- handle changed UPDATE 中的 remove + add 两段 mutation。 +- REPLACE 中删除旧 row 后插入新 row。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | `AddRecord` / `UpdateRecord` / `RemoveRecord` 调用 | 是 | +| `input_rows` | row mutation 数，remove + add 分开计 | 是 | +| `input_cells` | encoded / checked writable columns | 是 | +| `mutation_keys` | record key set/delete 数 | 是 | +| `mutation_bytes` | record key bytes + encoded row value bytes；delete 只计 key bytes | 是 | + +`CommitDetails.WriteKeys` / `WriteSize` 不适合作为第一版来源：它们发生在 commit/2PC 边界，显式事务会偏向 `COMMIT` 语句，pipelined DML 还存在 flush 归并口径风险。 + +### `write_index_mutation` + +覆盖： + +- AddRecord 创建 writable secondary indexes。 +- UpdateRecord 删除旧 index、创建新 index，跳过 untouched index 的优化。 +- RemoveRecord 删除 deletable indexes。 +- unique index 的 local/memBuffer 检查和 assertion 设置。 +- MV index expansion。 +- DDL 状态下的 temp index / double-write。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每次进入 index mutation path | 是 | +| `input_rows` | 参与 index mutation 的 row 数 | 是 | +| `input_cells` | index columns + affected columns + partial-index condition columns | 是 | +| `mutation_keys` | 实际 memBuffer set/delete 的 index key 数；temp-index double-write 分开计 | 是 | +| `mutation_bytes` | index key bytes + index value bytes；delete 只计 key bytes | 是 | +| `probe_keys` | unique index existence check keys | 是 | +| `probe_key_bytes` | unique index check key bytes | 是 | + +不要把 `index_name` 放入维度。索引数量和索引 key bytes 已经反映写放大；`operator_kind=index_create/index_delete` 足够诊断路径。 + +### `write_foreign_key` + +覆盖： + +- FK check rows/key preparation。 +- FK cascade 调用前后的 TiDB 本地 row preparation。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 FK checker/cascade executor | 是 | +| `input_rows` | 被检查或 cascade 的 rows | 部分可从 FK runtime stats 推导 | +| `probe_keys` | FK check keys | 是 | +| `probe_key_bytes` | FK key bytes | 是 | + +第一版可先对 simple FK check 计 base units；复杂 cascade 若无法完整归因，应写 status `unsupported` / `unsupported_fk_cascade`，不能发 partial units。 + +### `write_txn_orchestrate` + +覆盖： + +- batch insert/delete 每批 `StmtCommit` + `NewTxnInStmt`。 +- `txn.MayFlush` 调用事件。 +- LOAD DATA commit worker batch 事件。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 事务编排事件 | 是 | +| `batch_events` | batch commit / worker task 数 | 是 | +| `input_rows` | batch 中 rows | 可从 batch count 得到 | + +本 opclass 只表示 TiDB 事务编排固定成本，不包含 prewrite/commit/lock 的 storage/2PC 成本。 + +### `load_data_decode` + +覆盖： + +- 文件行读取和 parser 输出。 +- column list、user variables、`SET` 表达式。 +- restrictive / non-restrictive load 的 warning/error handling。 + +Base units： + +| Unit | 来源 | 需要新增 instrumentation | +|---|---|---| +| `fixed_events` | 每个 load stream / batch | 是 | +| `input_rows` | parsed rows | 可从 controller/encoder batch count 得到 | +| `input_cells` | parsed fields + SET expressions | 是 | +| `input_bytes` | raw file bytes 或 parser consumed bytes | 是 | + +后续插入阶段继续复用 `write_row_prepare`、`write_duplicate_probe`、`write_row_mutation`、`write_index_mutation`。 + +## 覆盖语句 + +### INSERT VALUES / SET + +推荐发出： + +- `write_row_prepare` +- `write_row_mutation` +- `write_index_mutation` +- `write_txn_orchestrate`，仅 batch DML、pipelined flush 或显式需要 statement transaction boundary 时 + +普通 INSERT 的 duplicate check 如果走 lazy check，只在 `write_index_mutation` 中记录 unique index key construction / memBuffer flags；不要把 future prewrite constraint check 算进 TiDB executor。 + +### INSERT ... SELECT + +读路径由 read billing 计费： + +- child SELECT 的 scan/filter/join/aggregation 继续使用 read billing units。 +- write billing 从 `insertRowsFromSelect` 从 child chunk 取出的 rows 开始，只计 row prepare 和 table/index mutation。 +- `input_source` 建议记录为 `child_output_rows`，避免把 child executor 的 input rows 再计一次。 + +### INSERT IGNORE + +推荐发出： + +- `write_row_prepare` +- `write_duplicate_probe` +- 对未冲突 rows 发出 `write_row_mutation` / `write_index_mutation` +- 对冲突 rows 发出 `conflict_rows`，但不发 mutation units + +### INSERT ... ON DUPLICATE KEY UPDATE + +推荐发出： + +- `write_row_prepare`，对 candidate insert rows +- `write_duplicate_probe` +- 对非冲突 rows 发出 insert mutation units +- 对冲突 rows 发出 `write_row_compare` +- 对 changed duplicate rows 发出 update mutation units +- 对 unchanged duplicate rows 发出 `unchanged_rows`，不发 row/index mutation units + +ODKU 的 `AffectedRows += 2` 是 MySQL 语义，不能作为 mutation row 数。 + +### REPLACE + +REPLACE 的 physical effect 应拆成 duplicate probe、delete mutation、insert mutation： + +- candidate row 发出 `write_row_prepare` +- duplicate path 发出 `write_duplicate_probe` +- 每个真正被删除的旧 row 发出 delete-flavored `write_row_mutation` / `write_index_mutation` +- 每个真正插入的新 row 发出 insert-flavored `write_row_mutation` / `write_index_mutation` +- identical row 不发 mutation units，只发 `unchanged_rows` + +这和 `AffectedRows` 的 1/2 语义分离。 + +### UPDATE + +读路径由 read billing 计费： + +- child executor 找 matched rows 的 scan/join/filter 成本属于 read side。 +- write billing 从 `UpdateExec.updateRows` 读取到 matched row 后开始。 + +推荐发出： + +- `write_row_prepare`，assignment/cast/generated/on-update-now +- `write_row_compare` +- handle unchanged 且 changed：`UpdateRecord` 作为 update-flavored row/index mutation +- handle changed：`RemoveRecord + AddRecord` 拆成 delete + insert mutation +- no-op update：只发 compare units 和 `unchanged_rows` + +### DELETE + +读路径由 read billing 计费： + +- child executor 找待删 rows 的成本属于 read side。 +- write billing 从 `DeleteExec.deleteOneRow` / `removeRow` 开始。 + +推荐发出： + +- `write_row_mutation`，record key delete +- `write_index_mutation`，deletable index delete +- batch delete 额外发 `write_txn_orchestrate` + +Multi-table DELETE 应按每个实际删除的 table row 发 mutation units，但不把 table id 纳入维度。 + +### Batch DML + +`tidb_batch_insert` / `tidb_batch_delete` 会在单条 SQL 内多次 `StmtCommit` 并 `NewTxnInStmt`。base units 应仍归属于原始 digest/window： + +- row/index units 累加到原 statement。 +- `write_txn_orchestrate.batch_events` 记录 batch boundary 次数。 +- 不新增 batch number 维度，避免高基数。 + +### LOAD DATA + +LOAD DATA 首版建议覆盖两个阶段： + +1. `load_data_decode`：文件 decode、列映射、user variables、SET expression。 +2. insert-like write path：按 `ERROR` / `IGNORE` / `REPLACE` 复用 INSERT / duplicate / REPLACE 的 units。 + +远端对象存储读取、网络下载和 decompression worker 是否属于 TiDB billing 需要单独决定。若没有可靠 byte evidence，先发 status `unknown_input` / `missing_load_data_bytes`，不要估算。 + +## 初始权重 + +以下权重只用于 demo preview 和离线试算，不代表生产校准结果。数值选择原则： + +- `fixed_events` 只覆盖小额固定成本； +- `input_cells` 捕捉表达式、cast、比较的 per-cell CPU； +- `probe_keys` / `probe_key_bytes` 只覆盖 TiDB key construction 和 local handling，不覆盖 TiKV read RPC； +- `mutation_keys` / `mutation_bytes` 覆盖 TiDB row/index encode 和 memBuffer set/delete，不覆盖 storage write； +- index mutation 通常比 row mutation 更贵，因为它包含 index value/key generation、partial/MV/unique 处理； +- duplicate probe、row compare、LOAD DATA decode 根据源码中的额外逻辑给予更高 per-row/per-key 权重。 + +推荐 seed weights，单位是 preview RU per unit： + +| Site | Op class | `fixed_events` | `input_rows` | `input_cells` | `input_bytes` | `probe_keys` | `probe_key_bytes` | `mutation_keys` | `mutation_bytes` | `changed_rows` | `unchanged_rows` | `conflict_rows` | `batch_events` | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `tidb` | `write_row_prepare` | 0.030 | 0.000020 | 0.000006 | 0.000001 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| `tidb` | `write_duplicate_probe` | 0.040 | 0.000015 | 0 | 0 | 0.000025 | 0.000001 | 0 | 0 | 0 | 0 | 0.000040 | 0 | +| `tidb` | `write_row_compare` | 0.030 | 0.000015 | 0.000005 | 0.000001 | 0 | 0 | 0 | 0 | 0.000010 | 0.000006 | 0 | 0 | +| `tidb` | `write_row_mutation` | 0.050 | 0.000020 | 0.000004 | 0.000001 | 0 | 0 | 0.000035 | 0.000004 | 0.000010 | 0 | 0 | 0 | +| `tidb` | `write_index_mutation` | 0.060 | 0.000015 | 0.000006 | 0.000001 | 0.000020 | 0.000001 | 0.000050 | 0.000005 | 0 | 0 | 0 | 0 | +| `tidb` | `write_foreign_key` | 0.050 | 0.000020 | 0.000004 | 0.000001 | 0.000030 | 0.000001 | 0 | 0 | 0 | 0 | 0 | 0 | +| `tidb` | `write_txn_orchestrate` | 0.020 | 0.000003 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.010 | +| `tidb` | `load_data_decode` | 0.060 | 0.000025 | 0.000007 | 0.000002 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + +相对顺序： + +```text +write_index_mutation + > write_row_mutation + > write_duplicate_probe ~= load_data_decode + > write_row_prepare ~= write_row_compare + > write_txn_orchestrate +``` + +这个顺序来自当前源码工作量： + +- `index.create` / `index.Delete` 需要 index value/key generation、MV expansion、unique/temp-index 处理和 memBuffer flags。 +- `AddRecord` / `UpdateRecord` / `RemoveRecord` 需要 row encoding、constraint check、assertion 和 record key mutation。 +- duplicate probe 在 `getKeysNeedCheck`、`prefetchUniqueIndices`、`batchCheckAndInsert` 中构造多组 keys 并循环处理冲突。 +- row prepare 和 row compare 主要是 expression/cast/default/generated/compare 的 per-row/per-cell CPU。 +- transaction orchestration 的本地固定开销存在，但 storage/2PC 成本不在本 opclass。 + +## 采集和暴露面 + +推荐新增与 read billing demo 平行的写侧开关和 SQL 面： + +- `tidb_enable_write_billing_demo`，默认 OFF。 +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_BASE_UNITS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_WRITE_BILLING_DEMO_STATUS` +- `INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY_WRITE_BILLING_DEMO_STATUS` + +Base-unit table columns 应沿用 read billing demo 的主要维度，并把 read-only 的 `ROW_WIDTH_SOURCE` 泛化为 `WIDTH_SOURCE` 或保留兼容命名： + +| Column | 含义 | +|---|---| +| statement summary window / digest / plan digest / resource group columns | 沿用 statement summary | +| `MODEL_VERSION` | write billing model version | +| `WEIGHT_VERSION` | 生成样本时使用的 weight version | +| `SITE` | `tidb` | +| `OP_CLASS` | bounded opclass | +| `OPERATOR_KIND` | bounded diagnostic kind | +| `UNIT` | bounded unit | +| `INPUT_SOURCE` | `executor_counter` / `stmtctx_counter` / `runtime_stats` / `mem_buffer_delta` / `encoded_bytes` / `load_data_parser` | +| `INPUT_SIDE` | `all`；必要时可用 `old` / `new` / `delete` / `insert`，但不要放 table/index name | +| `VALUE` | unit total | +| `SAMPLE_COUNT` | sample count | + +Status table 应和 read billing demo 一致，支持 statement-level 和 operator-level status。 + +Prometheus 可以新增 bounded observability metrics： + +- `tidb_write_billing_demo_statements_total` +- `tidb_write_billing_demo_operator_status_total` +- `tidb_write_billing_demo_base_units_total` + +但 workload 校准契约应以 SQL 查询面为准，不能要求 testers 依赖 Prometheus retention。 + +## 失败和降级状态 + +推荐 status： + +| Status | 含义 | +|---|---| +| `success` | statement-level accounting 完整 | +| `ok` | operator-level accounting 完整 | +| `unsupported` | 语句或路径明确不支持 | +| `unknown_input` | 必要输入缺失，不能安全估算 | +| `error` | statement 执行失败 | + +推荐 reason： + +| Reason | 触发场景 | +|---|---| +| `statement_error` | executor 返回错误 | +| `missing_plan` | 没有可归因 plan | +| `missing_stmtctx` | statement context 不可用 | +| `missing_runtime_stats` | 必要 runtime stats 不可用 | +| `missing_mutation_stats` | table/index mutation 没有 instrumentation | +| `missing_encoded_bytes` | bytes unit 需要的 encoded length 不可用 | +| `missing_load_data_bytes` | LOAD DATA 无可靠 input bytes | +| `unsupported_statement` | 非目标语句 | +| `unsupported_fk_cascade` | FK cascade 无法完整归因 | +| `unsupported_temporary_table` | temporary table 口径不同且未支持 | +| `unsupported_temp_index_merge` | DDL temp-index merge / double-write 无法完整归因 | +| `unsupported_internal_sql` | restricted/internal SQL | +| `aggregation_overflow` | base-unit key cap overflow | +| `status_aggregation_overflow` | status key cap overflow | + +失败或 unsupported statement 不写 base units。若某个必需 opclass 缺失输入，应 fail closed：写 status，跳过该 statement 的所有 base units，避免 partial units 被误用于校准。 + +## 避免重复计费 + +1. 读写混合语句： + - child SELECT / scan / join / filter 由 read billing 负责； + - write billing 只从 DML executor 消费 child rows 后开始； + - `input_source=child_output_rows` 可标明边界。 +2. duplicate / FK storage probes： + - TiDB write side 只计 key construction 和本地 loop； + - `txn.Get` / `BatchGet` / snapshot scan 成本属于 storage/read path； + - 不把 probe RPC latency 或 processed keys 乘入 TiDB write weights。 +3. row/index mutation 与 commit details： + - `mutation_keys` / `mutation_bytes` 是 TiDB 构造并放入 memBuffer 的 key/value； + - `CommitDetails.WriteKeys` / `WriteSize` 是 2PC mutation 口径，显式事务可能归到 COMMIT； + - 两者不能同时作为同一层权重输入。 +4. transaction orchestration： + - `write_txn_orchestrate` 只计本地 fixed/batch event； + - prewrite/commit/lock/resolve-lock/local-latch 不属于本 opclass。 +5. affected rows： + - `AffectedRows` 只用于 SQL 兼容展示； + - physical mutation rows 必须从 `AddRecord` / `UpdateRecord` / `RemoveRecord` 或新增 counters 得到。 + +## 实现期 Instrumentation 需求 + +可直接复用或改造的现有输入： + +| 输入 | 当前来源 | 限制 | +|---|---|---| +| statement type / digest / plan digest / resource group | statement summary | 可复用 | +| `RecordRows` / `CopiedRows` / `UpdatedRows` / `DeletedRows` / `TouchedRows` | `StatementContext` | SQL 语义，不够物理 | +| `rowsColMultiply` | `InsertValues.rowsColMultiply`、`UpdateExec.rowsColMultiplyForPreparedRow`、delete row/column count | 当前只进 RU v2 scalar counter，需要结构化保存 | +| insert/update runtime stats | `InsertRuntimeStat`、`updateRuntimeStats` | time/RPC 诊断，不是 base-unit source | +| commit write keys/size | `CommitDetails` / RU v2 metrics | storage/2PC 边界，不适合作为 TiDB executor-local source | + +需要新增： + +1. Statement-scoped `WriteBillingDemoStats`，与 read billing stats 类似，包含 status samples 和 base-unit samples。 +2. Table/index mutation instrumentation： + - record set/delete key count； + - record key bytes； + - encoded row value bytes； + - index set/delete key count； + - index key/value bytes； + - temp-index double-write count。 +3. Duplicate probe instrumentation： + - per row handle key count； + - unique key count； + - probe key bytes； + - conflict rows； + - temp-index fallback probes。 +4. Row prepare / compare instrumentation： + - input rows； + - input cells； + - estimated input bytes； + - changed / unchanged row counts。 +5. LOAD DATA instrumentation： + - parser consumed bytes； + - parsed rows / fields； + - SET expression cells； + - restrictive/non-restrictive skipped rows。 +6. Statement summary v1/v2 aggregation maps and JSON-stable v2 entry slices，保留 overflow status。 + +## 测试设计 + +后续实现应至少覆盖： + +1. INSERT VALUES： + - 无索引、普通二级索引、unique index、common handle、auto id。 + - 验证 row prepare、row mutation、index mutation units。 +2. INSERT SELECT： + - 验证 read side 和 write side 不重复计算 child input rows。 +3. INSERT IGNORE： + - 无冲突、handle 冲突、unique 冲突。 + - 冲突 rows 只产生 duplicate probe / conflict units，不产生 mutation units。 +4. ODKU： + - 无冲突插入、冲突更新、冲突 no-op。 + - `AffectedRows` 与 mutation rows 分离。 +5. REPLACE： + - identical row、replace one row、多个 unique key 触发多次 remove。 +6. UPDATE： + - no-op、普通 update、handle changed update、multi-table update。 +7. DELETE： + - single-table、multi-table、batch delete。 +8. LOAD DATA： + - ERROR / IGNORE / REPLACE 三种 duplicate mode； + - local 和 remote reader 的 input bytes 缺失时 status 行。 +9. FK： + - simple FK check； + - unsupported cascade status-only。 +10. Statement summary： + - v1 current/history、v2 persistent current/history、cluster table； + - base-unit aggregation overflow 和 status aggregation overflow； + - error / unsupported / unknown-input 不写 base units。 +11. RU v2 隔离： + - 开启 write billing demo 不改变 resource-control RU 上报； + - `CommitDetails.WriteKeys/WriteSize` 不被误当成 TiDB write base units。 + +## 风险和未决问题 + +1. `mutation_bytes` 的最佳口径需要实现时固定：建议用 TiDB 实际构造的 key bytes + value bytes，而不是 memBuffer memory footprint。 +2. 显式事务中一条 DML 产生 mutation，另一条 COMMIT 提交。若未来要 storage write billing，需要单独解决语句归因；本文只解决 TiDB executor-local write preparation。 +3. Pipelined DML 当前 commit details 可能无法完整反映 flushed keys/size。不要用该口径验证 TiDB write units。 +4. Temporary table、temp index merge、DDL backfill 期间的 double-write 口径复杂，第一版可以 status-only 或以 bounded units 记录 double-write count。 +5. FK cascade 可能递归触发多条 row mutation。第一版应先保证不漏报 status，再决定是否把 cascade 作为独立 nested write opclass。 +6. `LOAD DATA` 的文件读取、解压、对象存储网络是否属于 TiDB write billing 需要产品层确认；本设计只覆盖 parser/decode 和 insert-like write path。 From 81173506c0a606e5694ca2940e02cce45ddf8f1f Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 8 Jul 2026 01:14:50 +0800 Subject: [PATCH 09/25] docs: refine load data write billing design --- .../2026-07-08-write-billing-units-weights-design.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/design/2026-07-08-write-billing-units-weights-design.md b/docs/design/2026-07-08-write-billing-units-weights-design.md index 53e5f0a5f45ac..d5df0f6b026d7 100644 --- a/docs/design/2026-07-08-write-billing-units-weights-design.md +++ b/docs/design/2026-07-08-write-billing-units-weights-design.md @@ -83,8 +83,8 @@ preview_ru = sum(unit_value * weight(site, op_class, unit, weight_version)) - `pkg/executor/load_data.go` - `LoadDataExec.Next` 进入 local / remote load。 - `LoadDataWorker.load` 创建 reader、encode worker 和 commit worker pipeline。 - - `parserData2TableData` 做文件行解析、列映射、user variable、`SET` 表达式、cast/default/generated column。 - - `commitWorker.checkAndInsertOneBatch` 复用 insert / duplicate-check 写入路径。 + - `encodeWorker.readOneBatchRows` / `parserData2TableData` 做 parser 输出读取、列映射、user variable、`SET` 表达式、cast/default/generated column。 + - `commitWorker.checkAndInsertOneBatch` 按 worker task 复用 insert / duplicate-check 写入路径；不要把每个 task 等同为单独的 statement commit。 - `pkg/table/tables/tables.go` 和 `pkg/table/tables/index.go` - `TableCommon.AddRecord` 构造 record key/value、写 memBuffer、创建索引。 - `TableCommon.UpdateRecord` 重建受影响索引、编码新 row、写 memBuffer。 @@ -141,7 +141,7 @@ TiDB 写侧可以记录 duplicate probe keys、mutation keys、mutation bytes | `write_index_mutation` | index key/value encode，index set/delete，unique-index memBuffer checks，MV index expansion，temp-index double write | `TableCommon.addIndices`、`rebuildUpdateRecordIndices`、`removeRowIndices`、`index.create`、`index.Delete` | | `write_foreign_key` | FK check/cascade 的 TiDB 本地 row/key preparation | `FKCheckExec`、`FKCascadeExec`、`checkRows`、`onRemoveRowForFK` | | `write_txn_orchestrate` | batch DML statement commit boundary、`NewTxnInStmt`、`txn.MayFlush` 调用等 TiDB 事务编排事件 | `doBatchInsert`、`doBatchDelete`、`txn.MayFlush` callers | -| `load_data_decode` | LOAD DATA 文件行解析、列映射、user vars、SET 表达式进入 row prepare 前的 TiDB 本地处理 | `LoadDataWorker.load`、`readOneBatchRows`、`parserData2TableData` | +| `load_data_decode` | LOAD DATA 文件行解析、列映射、user vars、SET 表达式进入 row prepare 前的 TiDB 本地处理 | `LoadDataWorker.load`、`encodeWorker.readOneBatchRows`、`parserData2TableData` | `operator_kind` 是 bounded diagnostic 维度，建议使用具体执行路径而不是表名或索引名： @@ -325,7 +325,7 @@ Base units： | Unit | 来源 | 需要新增 instrumentation | |---|---|---| | `fixed_events` | 事务编排事件 | 是 | -| `batch_events` | batch commit / worker task 数 | 是 | +| `batch_events` | batch DML commit boundary / LOAD DATA worker task 数 | 是 | | `input_rows` | batch 中 rows | 可从 batch count 得到 | 本 opclass 只表示 TiDB 事务编排固定成本，不包含 prewrite/commit/lock 的 storage/2PC 成本。 From 612f0cf9b33e917f4aec740f70155f479e50c054 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 8 Jul 2026 19:15:57 +0800 Subject: [PATCH 10/25] docs: iterate read billing actual bytes design From 97791cc6af751df0f734e3d2d52317024f3a41c6 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 8 Jul 2026 20:04:36 +0800 Subject: [PATCH 11/25] executor: use actual bytes for read billing --- .../2026-07-01-read-billing-demo-ru-model.md | 10 + pkg/executor/explain_test.go | 134 +++--- pkg/executor/internal/exec/BUILD.bazel | 8 +- pkg/executor/internal/exec/executor.go | 83 +++- pkg/executor/internal/exec/executor_test.go | 79 +++- pkg/metrics/metrics_internal_test.go | 28 +- pkg/planner/core/common_plans_test.go | 131 +++--- pkg/planner/core/explain_ru.go | 421 +++++++----------- pkg/util/chunk/chunk.go | 35 ++ pkg/util/chunk/chunk_test.go | 26 ++ pkg/util/execdetails/execdetails_test.go | 17 + pkg/util/execdetails/runtime_stats.go | 39 ++ 12 files changed, 584 insertions(+), 427 deletions(-) diff --git a/docs/design/2026-07-01-read-billing-demo-ru-model.md b/docs/design/2026-07-01-read-billing-demo-ru-model.md index 87f5474687e93..bac1fdeeaf31f 100644 --- a/docs/design/2026-07-01-read-billing-demo-ru-model.md +++ b/docs/design/2026-07-01-read-billing-demo-ru-model.md @@ -40,6 +40,16 @@ preview_ru = Prometheus `tidb_read_billing_demo_*` metrics 可以继续作为 bounded observability 辅助，但不能作为 workload testers 的必需接口；校准契约以 SQL 查询面为准。 +### v2 实现说明 + +当前实现的 `model_version = 'v2'` 已经把 `input_bytes` 从 v1 的 planner/schema row-width 估算切换为执行期 byte evidence： + +- TiDB root operator 使用 runtime chunk 的 logical live bytes，`input_source = runtime_chunk_bytes`，`row_width_source = runtime_chunk_avg`。 +- TiKV range scan 使用 scan detail 中的 `processed_key_size / processed_keys * total_keys`，`row_width_source = scan_detail_processed_key_avg`。 +- 缺少 byte evidence 的路径以 `unknown_input` fail closed，不产出 partial billable base units。 + +本文后续保留的 `runtime_act_rows`、`plan_stats`、`schema_type_width`、`schema_fallback`、`operator_helper` 等描述只适用于 v1 历史估算模型或背景说明，不再代表 v2 的 `input_bytes` 构造规则。 + ## 背景和目标 现有 RU v2 主要是 statement-level billing 逻辑，不适合回答两个问题： diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index f5e848c09af5e..2b4155cd6992f 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser/auth" plannercore "github.com/pingcap/tidb/pkg/planner/core" + "github.com/pingcap/tidb/pkg/session" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/sqlexec" @@ -540,9 +541,11 @@ func TestExplainAnalyzeFormatRUOutput(t *testing.T) { tk.MustExec("drop table if exists explain_ru_t") tk.MustExec("create table explain_ru_t(a int primary key, b varchar(20))") tk.MustExec("insert into explain_ru_t values (1, 'x'), (2, 'yy')") - rows = tk.MustQuery("explain analyze format='ru' select * from explain_ru_t where a > 0").Rows() - requireExplainRUPlanRow(t, rows) - requireExplainRUOperatorClass(t, rows, "tikv/kv_range_scan") + _, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' select * from explain_ru_t where a > 0") + require.Error(t, err) + require.Contains(t, err.Error(), "status=unknown_input") + require.Contains(t, err.Error(), "reason=missing_scan_detail") + require.Contains(t, err.Error(), "operator=tikv/kv_range_scan") rows = tk.MustQuery("explain analyze format='ru' select * from explain_ru_t where a = 1").Rows() requireExplainRUWeightedOperatorClass(t, rows, "tikv/kv_point_lookup") @@ -557,60 +560,84 @@ func TestExplainAnalyzeFormatRUTiKVCopOperatorClasses(t *testing.T) { tk.MustExec("insert into explain_ru_cop values (1, 10, 'a'), (2, 20, 'bb'), (3, 20, 'ccc'), (4, 30, 'dddd')") cases := []struct { - sql string - opClasses []string + sql string + nonScanOpClass string }{ { - sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) where b > 10", - opClasses: []string{ - "tikv/filter_eval", - "tikv/kv_range_scan", - }, + sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) where b > 10", + nonScanOpClass: "tikv/filter_eval", }, { - sql: "explain analyze format='ru' select a from explain_ru_cop where b > 10", - opClasses: []string{ - "tikv/projection_eval", - "tikv/kv_range_scan", - }, + sql: "explain analyze format='ru' select a from explain_ru_cop where b > 10", + nonScanOpClass: "tikv/projection_eval", }, { - sql: "explain analyze format='ru' select * from explain_ru_cop limit 2", - opClasses: []string{ - "tikv/row_limit", - "tikv/kv_range_scan", - }, + sql: "explain analyze format='ru' select * from explain_ru_cop limit 2", + nonScanOpClass: "tikv/row_limit", }, { - sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) order by c limit 2", - opClasses: []string{ - "tikv/bounded_topn", - "tikv/kv_range_scan", - }, + sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) order by c limit 2", + nonScanOpClass: "tikv/bounded_topn", }, { - sql: "explain analyze format='ru' select /*+ agg_to_cop(), hash_agg() */ b, count(*) from explain_ru_cop group by b", - opClasses: []string{ - "tikv/agg_hash", - "tikv/kv_range_scan", - }, + sql: "explain analyze format='ru' select /*+ agg_to_cop(), hash_agg() */ b, count(*) from explain_ru_cop group by b", + nonScanOpClass: "tikv/agg_hash", }, { - sql: "explain analyze format='ru' select /*+ agg_to_cop(), stream_agg() */ b, count(*) from explain_ru_cop group by b", - opClasses: []string{ - "tikv/agg_stream", - "tikv/kv_range_scan", - }, + sql: "explain analyze format='ru' select /*+ agg_to_cop(), stream_agg() */ b, count(*) from explain_ru_cop group by b", + nonScanOpClass: "tikv/agg_stream", }, } for _, tc := range cases { - rows := tk.MustQuery(tc.sql).Rows() - for _, opClass := range tc.opClasses { - requireExplainRUWeightedOperatorClass(t, rows, opClass) + rows, err := queryExplainRURowsOrErr(t, tk, tc.sql) + if err != nil { + require.Contains(t, err.Error(), "status=unknown_input", tc.sql) + require.Contains(t, err.Error(), "reason=missing_runtime_bytes", tc.sql) + require.Contains(t, err.Error(), "operator="+tc.nonScanOpClass, tc.sql) + continue } + requireExplainRUWeightedOperatorClass(t, rows, "tikv/kv_range_scan") + requireNoExplainRUOperatorClass(t, rows, tc.nonScanOpClass) } } +func queryExplainRURowsOrErr(t *testing.T, tk *testkit.TestKit, sql string) ([][]any, error) { + t.Helper() + rs, err := tk.Exec(sql) + if err != nil { + if rs != nil { + require.NoError(t, rs.Close()) + } + return nil, err + } + fields := rs.Fields() + chunkRows, err := session.GetRows4Test(context.Background(), tk.Session(), rs) + closeErr := rs.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + anyRows := make([][]any, len(chunkRows)) + for i, row := range chunkRows { + anyRows[i] = make([]any, row.Len()) + for j := range row.Len() { + if row.IsNull(j) { + anyRows[i][j] = "" + continue + } + datum := row.GetDatum(j, &fields[j].Column.FieldType) + value, err := datum.ToString() + if err != nil { + return nil, err + } + anyRows[i][j] = value + } + } + return anyRows, nil +} + func requireExplainRUPlanRow(t *testing.T, rows [][]any) { t.Helper() for _, row := range rows { @@ -622,7 +649,6 @@ func requireExplainRUPlanRow(t *testing.T, rows [][]any) { require.NotEmpty(t, row[2]) require.Contains(t, fmt.Sprint(row[3]), "/") require.NotEmpty(t, row[4]) - require.NotEmpty(t, row[7]) require.NotEmpty(t, row[8]) require.NotEmpty(t, row[11]) require.NotEmpty(t, row[12]) @@ -635,16 +661,14 @@ func requireExplainRUPlanRow(t *testing.T, rows [][]any) { require.Fail(t, "missing FORMAT='RU' plan row") } -func requireExplainRUOperatorClass(t *testing.T, rows [][]any, operatorClass string) { +func requireNoExplainRUOperatorClass(t *testing.T, rows [][]any, operatorClass string) { t.Helper() for _, row := range rows { require.Len(t, row, 17) - if row[0] != "plan" || row[3] != operatorClass { - continue + if row[0] == "plan" && row[3] == operatorClass { + require.Failf(t, "unexpected FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) } - return } - require.Failf(t, "missing FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) } func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorClass string) { @@ -782,11 +806,11 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustExec("create table read_billing_demo(a int primary key)") tk.MustExec("insert into read_billing_demo values (1), (2)") - success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v1") - unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v1") - unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v1") - errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v1") - projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_act_rows", "all", "v1") + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2") + unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v2") + unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v2") + errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v2") + projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v2") tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) require.Equal(t, 0.0, readExecutorCounterValue(t, success)) @@ -796,8 +820,8 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, 1.0, readExecutorCounterValue(t, success)) require.Equal(t, 1.0, readExecutorCounterValue(t, projectionFixedEvents)) tk.MustExec("set tidb_enable_read_billing_demo=off") - tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events > 0, sum_read_billing_demo_input_rows > 0, sum_read_billing_demo_input_bytes > 0 from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 1 1 1")) - tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_act_rows all v1 v1 1 1 1")) + tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events > 0, sum_read_billing_demo_input_rows > 0, sum_read_billing_demo_input_bytes = 0 from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 1 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width = 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_chunk_bytes all v2 v1 1 1 1")) tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) tk.MustQuery(`select column_name from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and ordinal_position = 1`).Check(testkit.Rows("INSTANCE")) tk.MustExec("set tidb_enable_read_billing_demo=on") @@ -834,6 +858,14 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + beforeUnknownInput = readExecutorCounterValue(t, unknownInput) + beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + tk.MustQuery("select * from read_billing_demo where a > 0").Sort().Check(testkit.Rows("1", "2", "3")) + require.Equal(t, beforeUnknownInput+1, readExecutorCounterValue(t, unknownInput)) + require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) + tk.MustQuery(`select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?' and site = 'statement' group by status, reason`).Check(testkit.Rows("unknown_input missing_scan_detail 1")) + tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?'`).Check(testkit.Rows("0")) + tk.MustExec("set tidb_enable_read_billing_demo=off") tk.MustExec("create table read_billing_compile_error(a int)") tk.MustExec("prepare read_billing_compile_error_stmt from 'select * from read_billing_compile_error'") diff --git a/pkg/executor/internal/exec/BUILD.bazel b/pkg/executor/internal/exec/BUILD.bazel index c258a46e1eb71..f3976f9b54f89 100644 --- a/pkg/executor/internal/exec/BUILD.bazel +++ b/pkg/executor/internal/exec/BUILD.bazel @@ -42,16 +42,22 @@ go_test( ], embed = [":exec"], flaky = True, - shard_count = 8, + shard_count = 9, deps = [ "//pkg/domain", + "//pkg/expression", "//pkg/kv", "//pkg/parser/ast", + "//pkg/parser/mysql", "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", "//pkg/statistics", "//pkg/statistics/handle/usage/indexusage", "//pkg/testkit", + "//pkg/types", "//pkg/types/parser_driver", + "//pkg/util/chunk", + "//pkg/util/execdetails", "//pkg/util/logutil", "//pkg/util/mock", "@com_github_pingcap_tipb//go-tipb", diff --git a/pkg/executor/internal/exec/executor.go b/pkg/executor/internal/exec/executor.go index d0a1efe910bb2..637f8e643b907 100644 --- a/pkg/executor/internal/exec/executor.go +++ b/pkg/executor/internal/exec/executor.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" @@ -41,8 +40,10 @@ import ( // nextIOAcc accumulates input volume for a single Executor.Next() invocation. type nextIOAcc struct { - inRows int64 - inCells int64 + inRows int64 + inCells int64 + inBytes int64 + trackBytes bool } func (a *nextIOAcc) reset() { @@ -51,6 +52,8 @@ func (a *nextIOAcc) reset() { } stdatomic.StoreInt64(&a.inRows, 0) stdatomic.StoreInt64(&a.inCells, 0) + stdatomic.StoreInt64(&a.inBytes, 0) + a.trackBytes = false } func calcCellCount(rows, cols int) int64 { @@ -60,13 +63,16 @@ func calcCellCount(rows, cols int) int64 { return int64(rows) * int64(cols) } -// addInput adds rows and cells (rows*cols) into the accumulator. -func (a *nextIOAcc) addInput(rows, cols int) { +// addInput adds rows, cells (rows*cols), and optional bytes into the accumulator. +func (a *nextIOAcc) addInput(rows, cols int, bytes int64) { if a == nil || rows <= 0 { return } stdatomic.AddInt64(&a.inRows, int64(rows)) stdatomic.AddInt64(&a.inCells, calcCellCount(rows, cols)) + if bytes > 0 { + stdatomic.AddInt64(&a.inBytes, bytes) + } } type nextIOAccKeyType struct{} @@ -392,10 +398,12 @@ type executorStats struct { normalizedSQL string normalizedPlan string inRestrictedSQL bool + collectRuntimeBytes bool } // newExecutorStats creates a new `executorStats` -func newExecutorStats(stmtCtx *stmtctx.StatementContext, id int) executorStats { +func newExecutorStats(vars *variable.SessionVars, id int) executorStats { + stmtCtx := vars.StmtCtx normalizedSQL, sqlDigest := stmtCtx.SQLDigest() normalizedPlan, planDigest := stmtCtx.GetPlanDigest() e := executorStats{ @@ -405,6 +413,7 @@ func newExecutorStats(stmtCtx *stmtctx.StatementContext, id int) executorStats { normalizedPlan: normalizedPlan, planDigest: planDigest, inRestrictedSQL: stmtCtx.InRestrictedSQL, + collectRuntimeBytes: needReadBillingRuntimeBytes(vars), } if stmtCtx.RuntimeStatsColl != nil { @@ -416,6 +425,17 @@ func newExecutorStats(stmtCtx *stmtctx.StatementContext, id int) executorStats { return e } +func needReadBillingRuntimeBytes(vars *variable.SessionVars) bool { + if vars == nil || vars.StmtCtx == nil { + return false + } + if vars.EnableReadBillingDemo { + return true + } + stmtCtx := vars.StmtCtx + return stmtCtx.InExplainStmt && stmtCtx.InExplainAnalyzeStmt && stmtCtx.ExplainFormat == types.ExplainFormatRU +} + // RuntimeStats returns the runtime stats of an executor. func (e *executorStats) RuntimeStats() *execdetails.BasicRuntimeStats { return e.runtimeStats @@ -464,7 +484,7 @@ func NewBaseExecutorV2(vars *variable.SessionVars, schema *expression.Schema, id executorMeta := newExecutorMeta(schema, id, children...) e := BaseExecutorV2{ executorMeta: executorMeta, - executorStats: newExecutorStats(vars.StmtCtx, id), + executorStats: newExecutorStats(vars, id), executorChunkAllocator: newExecutorChunkAllocator(vars, executorMeta.RetFieldTypes()), executorKillerHandler: newExecutorKillerHandler(&vars.SQLKiller), } @@ -512,6 +532,10 @@ func (e *BaseExecutorV2) ruv2NextCache() *ruv2NextCacheState { return &e.ruv2CacheState } +func (e *BaseExecutorV2) needReadBillingRuntimeBytes() bool { + return e.collectRuntimeBytes && e.runtimeStats != nil +} + // BuildNewBaseExecutorV2 builds a new `BaseExecutorV2` based on the configuration of the current base executor. // It's used to build a new sub-executor from an existing executor. For example, the `IndexLookUpExecutor` will use // this function to build `TableReaderExecutor` @@ -630,9 +654,10 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { err = util.GetRecoverError(r) } }() - if e.RuntimeStats() != nil { + runtimeStats := e.RuntimeStats() + if runtimeStats != nil { start := time.Now() - defer func() { e.RuntimeStats().Record(time.Since(start), req.NumRows()) }() + defer func() { runtimeStats.Record(time.Since(start), req.NumRows()) }() } if err := e.HandleSQLKillerSignal(); err != nil { @@ -640,11 +665,12 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { } var ( - regionName string - info ruv2ExecutorMetric - trackRUV2 bool - ruv2Metrics *execdetails.RUV2Metrics - recorder execdetails.ExecutorMetricRecorder + regionName string + info ruv2ExecutorMetric + trackRUV2 bool + trackRuntimeBytes bool + ruv2Metrics *execdetails.RUV2Metrics + recorder execdetails.ExecutorMetricRecorder ) if provider, ok := e.(ruv2CacheProvider); ok { cache := provider.ruv2NextCache() @@ -670,16 +696,19 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { // A tracked-type executor whose statement is bypassed leaves ruv2Metrics nil // and must skip the per-child IO accumulator setup as well as the late update. trackRUV2 = ruv2Metrics != nil + if provider, ok := e.(interface{ needReadBillingRuntimeBytes() bool }); ok { + trackRuntimeBytes = provider.needReadBillingRuntimeBytes() + } r, ctx := tracing.StartRegionEx(ctx, regionName) defer r.End() parentAcc, _ := ctx.Value(nextIOAccKey).(*nextIOAcc) childCount := 0 - if trackRUV2 || parentAcc != nil { + if trackRUV2 || trackRuntimeBytes || parentAcc != nil { childCount = len(e.AllChildren()) } - needLocalAcc := needNextIOAcc(trackRUV2, parentAcc, childCount) + needLocalAcc := needNextIOAcc(trackRUV2 || trackRuntimeBytes, parentAcc, childCount) var myAcc *nextIOAcc if needLocalAcc { // Keep descendant IO local to this executor before optionally bubbling the @@ -687,6 +716,7 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { // local accumulator, and BaseExecutorV2-backed executors can reuse one // across Next() calls. myAcc = getReusableNextIOAcc(e) + myAcc.trackBytes = trackRuntimeBytes ctx = context.WithValue(ctx, nextIOAccKey, myAcc) } @@ -699,8 +729,22 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { outRows := req.NumRows() outCols := req.NumCols() + var outBytes int64 + if trackRuntimeBytes || (parentAcc != nil && parentAcc.trackBytes) { + outBytes = req.LogicalLiveBytes() + } if parentAcc != nil { - parentAcc.addInput(outRows, outCols) + parentAcc.addInput(outRows, outCols, outBytes) + } + + var inRows, inCells, inBytes int64 + if myAcc != nil { + inRows = stdatomic.LoadInt64(&myAcc.inRows) + inCells = stdatomic.LoadInt64(&myAcc.inCells) + inBytes = stdatomic.LoadInt64(&myAcc.inBytes) + } + if trackRuntimeBytes && runtimeStats != nil { + runtimeStats.RecordBytes(inBytes, outBytes) } if !trackRUV2 { @@ -708,11 +752,6 @@ func Next(ctx context.Context, e Executor, req *chunk.Chunk) (err error) { return e.HandleSQLKillerSignal() } - var inRows, inCells int64 - if myAcc != nil { - inRows = stdatomic.LoadInt64(&myAcc.inRows) - inCells = stdatomic.LoadInt64(&myAcc.inCells) - } outCells := calcCellCount(outRows, outCols) addRUV2ExecutorMetricCached(ruv2Metrics, info, recorder, inRows, int64(outRows), inCells, outCells) // recheck whether the session/query is killed during the Next() diff --git a/pkg/executor/internal/exec/executor_test.go b/pkg/executor/internal/exec/executor_test.go index 9d67c38a468f8..4944ddb0c14b9 100644 --- a/pkg/executor/internal/exec/executor_test.go +++ b/pkg/executor/internal/exec/executor_test.go @@ -15,8 +15,15 @@ package exec import ( + "context" "testing" + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx/variable" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/mock" "github.com/stretchr/testify/require" ) @@ -36,26 +43,28 @@ func TestNextIOAccAddInputCountsRowsWithZeroCols(t *testing.T) { t.Run("add input counts rows with zero cols", func(t *testing.T) { acc := &nextIOAcc{} - acc.addInput(3, 0) + acc.addInput(3, 0, 0) require.Equal(t, int64(3), acc.inRows) require.Equal(t, int64(0), acc.inCells) + require.Equal(t, int64(0), acc.inBytes) }) t.Run("base executor reuses local accumulator state", func(t *testing.T) { exec := newMockNextIOAccExecutor() first := getReusableNextIOAcc(exec) - first.addInput(4, 2) + first.addInput(4, 2, 16) second := getReusableNextIOAcc(exec) require.Same(t, first, second) require.Equal(t, int64(0), second.inRows) require.Equal(t, int64(0), second.inCells) + require.Equal(t, int64(0), second.inBytes) allocs := testing.AllocsPerRun(1000, func() { acc := getReusableNextIOAcc(exec) - acc.addInput(1, 1) + acc.addInput(1, 1, 1) }) require.Less(t, allocs, 1.) }) @@ -70,6 +79,70 @@ func TestNextIOAccAddInputCountsRowsWithZeroCols(t *testing.T) { }) } +type mockRuntimeBytesExecutor struct { + BaseExecutorV2 + produce func(*chunk.Chunk) + returned bool +} + +func newMockRuntimeBytesExecutor(vars *variable.SessionVars, schema *expression.Schema, id int, produce func(*chunk.Chunk), children ...Executor) *mockRuntimeBytesExecutor { + return &mockRuntimeBytesExecutor{ + BaseExecutorV2: NewBaseExecutorV2(vars, schema, id, children...), + produce: produce, + } +} + +func (e *mockRuntimeBytesExecutor) Next(ctx context.Context, req *chunk.Chunk) error { + req.Reset() + if e.returned { + return nil + } + e.returned = true + for _, child := range e.AllChildren() { + childReq := NewFirstChunk(child) + if err := Next(ctx, child, childReq); err != nil { + return err + } + } + if e.produce != nil { + e.produce(req) + } + return nil +} + +func TestNextRecordsRuntimeBytesForReadBilling(t *testing.T) { + ctx := mock.NewContext() + vars := ctx.GetSessionVars() + vars.EnableReadBillingDemo = true + vars.StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + schema := expression.NewSchema( + &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)}, + &expression.Column{RetType: types.NewFieldType(mysql.TypeVarchar)}, + ) + + child := newMockRuntimeBytesExecutor(vars, schema, 1, func(req *chunk.Chunk) { + req.AppendInt64(0, 1) + req.AppendString(1, "abc") + req.AppendInt64(0, 2) + req.AppendString(1, "d") + }) + parent := newMockRuntimeBytesExecutor(vars, schema, 2, func(req *chunk.Chunk) { + req.AppendInt64(0, 3) + req.AppendString(1, "ef") + }, child) + + req := NewFirstChunk(parent) + require.NoError(t, Next(context.Background(), parent, req)) + + childStats := vars.StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(1, false) + parentStats := vars.StmtCtx.RuntimeStatsColl.GetBasicRuntimeStats(2, false) + require.True(t, childStats.HasBytes()) + require.True(t, parentStats.HasBytes()) + require.Equal(t, int64(46), childStats.GetOutputBytes()) + require.Equal(t, childStats.GetOutputBytes(), parentStats.GetInputBytes()) + require.Equal(t, int64(28), parentStats.GetOutputBytes()) +} + func TestRUV2ExecutorMetricByTypeIncludesConcreteExecutorTypes(t *testing.T) { cases := map[string]ruv2ExecutorMetric{ "*aggregate.HashAggExec": {level: 2, label: "HashAggExec", useCells: false}, diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index 747f3e9eb05f5..535809eedc10e 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -101,20 +101,20 @@ func TestExplainRUMetrics(t *testing.T) { RecordExplainRUStatus("success") ObserveExplainRURenderDuration("success", 0.01) RecordExplainRUComponentSnapshot("ok") - ObserveExplainRURow("plan", "", "projection", "read_billing_model", "plan_stats", 1.25, 3, 24, 8) - RecordReadBillingDemoStatement("success", "v1") - RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v1") - AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_act_rows", "all", "v1", 3) - ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "plan_stats", "v1", 8) + ObserveExplainRURow("plan", "", "projection", "read_billing_model", "runtime_chunk_avg", 1.25, 3, 24, 8) + RecordReadBillingDemoStatement("success", "v2") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 3) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", 8) require.Equal(t, 1.0, readCounterValue(t, ExplainRUStatementsCounter.WithLabelValues("success"))) require.Equal(t, 1.0, readCounterValue(t, ExplainRUComponentSnapshotCounter.WithLabelValues("ok"))) require.Equal(t, 1.25, readCounterValue(t, ExplainRUPreviewRUCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) require.Equal(t, 3.0, readCounterValue(t, ExplainRUWorkRowsCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) require.Equal(t, 24.0, readCounterValue(t, ExplainRUWorkBytesCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) - require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v1"))) - require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v1"))) - require.Equal(t, 3.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "input_rows", "runtime_act_rows", "all", "v1"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v2"))) + require.Equal(t, 3.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2"))) registry := prometheus.NewRegistry() require.NoError(t, registry.Register(ExplainRUPreviewRUCounter)) @@ -136,7 +136,7 @@ func TestExplainRUMetrics(t *testing.T) { rowWidthFamily := findMetricFamily(families, "tidb_explain_ru_row_width_bytes") require.NotNil(t, rowWidthFamily) requireMetricFamilyHasLabels(t, rowWidthFamily, "component", "operator", "source") - require.True(t, metricHasLabelValue(rowWidthFamily.GetMetric()[0], "source", "plan_stats")) + require.True(t, metricHasLabelValue(rowWidthFamily.GetMetric()[0], "source", "runtime_chunk_avg")) statementFamily := findMetricFamily(families, "tidb_explain_ru_statements_total") require.NotNil(t, statementFamily) requireMetricFamilyHasLabels(t, statementFamily, "status") @@ -165,11 +165,11 @@ func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { ObserveExplainRURenderDuration("", 0.01) RecordExplainRUComponentSnapshot("") ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", 0, -1, -1, -1) - ObserveExplainRURow("plan", "", "", "read_billing_model", "operator_helper", -1, -1, -1, 32) - RecordReadBillingDemoStatement("", "v1") - RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v1") - AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_act_rows", "all", "v1", 0) - ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "plan_stats", "v1", 0) + ObserveExplainRURow("plan", "", "", "read_billing_model", "runtime_chunk_avg", -1, -1, -1, 32) + RecordReadBillingDemoStatement("", "v2") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v2") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 0) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", 0) require.Equal(t, 0, countCollectedMetrics(ExplainRUStatementsCounter)) require.Equal(t, 0, countCollectedMetrics(ExplainRURenderDurationHistogram)) diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index 7a72238f8186a..1c73f78ddb043 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -25,11 +25,9 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/mock" - "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" tikvutil "github.com/tikv/client-go/v2/util" rmclient "github.com/tikv/pd/client/resource_group/controller" @@ -181,7 +179,7 @@ func TestExplainRURowFormatting(t *testing.T) { hasOutputRows: true, rowWidth: 8, hasRowWidth: true, - rowWidthSource: explainRUWidthSourcePlanStats, + rowWidthSource: explainRUWidthSourceRuntimeChunkAvg, workRows: 2, hasWorkRows: true, unit: readBillingDemoUnitInputRows, @@ -191,15 +189,15 @@ func TestExplainRURowFormatting(t *testing.T) { hasWeight: true, previewRU: 6, hasPreviewRU: true, - source: readBillingDemoInputSourceRuntimeRows, + source: readBillingDemoInputSourceRuntimeChunkBytes, note: "input_side=all,weight_version=v1", } require.Equal(t, []string{ - "plan", "Projection_1", "projection", "tidb/projection_eval", "1", "2", "1", "8.000000", "plan_stats", "2", "", "input_rows", "2", "0.250000", "6.000000", "runtime_act_rows", "input_side=all,weight_version=v1", + "plan", "Projection_1", "projection", "tidb/projection_eval", "1", "2", "1", "8.000000", "runtime_chunk_avg", "2", "", "input_rows", "2", "0.250000", "6.000000", "runtime_chunk_bytes", "input_side=all,weight_version=v1", }, row.toStrings()) } -func TestExplainRUPlanFormulaUsesRowsAndModeledBytes(t *testing.T) { +func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { tidbWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) require.True(t, ok) tikvWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) @@ -363,36 +361,19 @@ func extractExplainRUTestSnapshot(stats *execdetails.RURuntimeStats) (*execdetai return explainRUExtractComponentSnapshot(coll, 1) } -func TestReadBillingDemoDirectCopInputRowsAndWidthUsesChildRows(t *testing.T) { +func TestReadBillingDemoNonScanCopWithoutBytesFailsClosed(t *testing.T) { ctx := mock.NewContext() col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) stats := &property.StatsInfo{RowCount: 5} - parent := (&physicalop.PhysicalHashAgg{}).InitForHash(ctx, stats, 0, schema).(*physicalop.PhysicalHashAgg) - child := (&physicalop.PhysicalSelection{}).Init(ctx, stats, 0) - childInput := physicalop.PhysicalTableScan{}.Init(ctx, 0) - childInput.SetSchema(schema) - child.SetChildren(childInput) + proj := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + proj.SetSchema(schema) tree := FlatPlanTree{ - {Origin: parent, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, - {Origin: child, IsRoot: false, StoreType: kv.TiKV}, + {Origin: proj, IsRoot: false, StoreType: kv.TiKV}, } runtimeStats := execdetails.NewRuntimeStatsColl(nil) - recordCopRows := func(planID int, rows uint64) { - iterations := uint64(1) - duration := uint64(1) - runtimeStats.RecordOneCopTask(planID, kv.TiKV, &tipb.ExecutorExecutionSummary{ - NumProducedRows: &rows, - NumIterations: &iterations, - TimeProcessedNs: &duration, - }) - } - recordCopRows(parent.ID(), 1) - recordCopRows(child.ID(), 5) - - proj := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) - proj.SetSchema(schema) + runtimeStats.RecordCopStats(proj.ID(), kv.TiKV, &tikvutil.ScanDetail{}, tikvutil.TimeDetail{}, nil) operator, supported, reason := readBillingDemoClassifyOperator(&FlatOperator{ Origin: proj, IsRoot: false, @@ -403,74 +384,65 @@ func TestReadBillingDemoDirectCopInputRowsAndWidthUsesChildRows(t *testing.T) { require.Equal(t, readBillingDemoSiteTiKV, operator.site) require.Equal(t, readBillingDemoOpClassProjection, operator.opClass) - rows, width, widthSource, ok := readBillingDemoDirectCopInputRowsAndWidth( - ctx, - runtimeStats, - tree, - 0, - 8, - explainRUWidthSourceSchemaFallback, - runtimeStats.GetCopStats(parent.ID()), - ) - require.True(t, ok) - require.Equal(t, int64(5), rows) - require.Equal(t, float64(8), width) - require.Equal(t, explainRUWidthSourceSchemaTypeWidth, widthSource) + units, actualReason, ok := readBillingDemoCopUnits(runtimeStats, tree, 0, operator) + require.False(t, ok) + require.Nil(t, units) + require.Equal(t, readBillingDemoReasonMissingRuntimeBytes, actualReason) } -func TestReadBillingDemoRangeScanKeepsFixedEventForEmptyInput(t *testing.T) { +func TestReadBillingDemoRangeScanUsesProcessedKeyAverage(t *testing.T) { ctx := mock.NewContext() col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) scan := physicalop.PhysicalTableScan{}.Init(ctx, 0) scan.SetSchema(schema) scan.StoreType = kv.TiKV - scan.TblColHists = &statistics.HistColl{Pseudo: true} scan.TblCols = []*expression.Column{col} tree := FlatPlanTree{ {Origin: scan, IsRoot: false, StoreType: kv.TiKV}, } - buildUnits := func(scanDetail *tikvutil.ScanDetail, producedRows uint64) ([]readBillingDemoUnit, bool) { + rows, bytes, ok := readBillingDemoRangeScanInput(10, 5, 100) + require.True(t, ok) + require.Equal(t, int64(10), rows) + require.Equal(t, 200.0, bytes) + for _, tc := range []struct { + totalKeys int64 + processedKeys int64 + processedKeysSize int64 + }{ + {0, 5, 100}, + {10, 0, 100}, + {10, 5, 0}, + } { + _, _, ok = readBillingDemoRangeScanInput(tc.totalKeys, tc.processedKeys, tc.processedKeysSize) + require.False(t, ok) + } + + buildUnits := func(scanDetail *tikvutil.ScanDetail) ([]readBillingDemoUnit, string, bool) { runtimeStats := execdetails.NewRuntimeStatsColl(nil) runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, scanDetail, tikvutil.TimeDetail{}, nil) - iterations := uint64(1) - duration := uint64(1) - runtimeStats.RecordOneCopTask(scan.ID(), kv.TiKV, &tipb.ExecutorExecutionSummary{ - NumProducedRows: &producedRows, - NumIterations: &iterations, - TimeProcessedNs: &duration, - }) return readBillingDemoCopUnits( - ctx, runtimeStats, tree, 0, - tree[0], readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "tablescan"}, ) } - units, ok := buildUnits(&tikvutil.ScanDetail{}, 0) + units, actualReason, ok := buildUnits(&tikvutil.ScanDetail{TotalKeys: 10, ProcessedKeys: 5, ProcessedKeysSize: 100}) require.True(t, ok) + require.Empty(t, actualReason) require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) - require.Equal(t, 0.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) - require.Equal(t, 0.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) - require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) - require.False(t, readBillingDemoUnitExists(units, "scan_total_keys", readBillingDemoInputSideAll)) - require.False(t, readBillingDemoUnitExists(units, "processed_key_size", readBillingDemoInputSideAll)) - - units, ok = buildUnits(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeysSize: 128}, 2) - require.True(t, ok) - require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) - require.Equal(t, 128.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 10.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 200.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceScanDetailProcessedAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) - units, ok = buildUnits(&tikvutil.ScanDetail{}, 2) - require.True(t, ok) - require.Equal(t, 2.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) - require.Equal(t, readBillingDemoInputSourceRuntimeRows, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) - require.Greater(t, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll), 0.0) + units, actualReason, ok = buildUnits(&tikvutil.ScanDetail{}) + require.False(t, ok) + require.Nil(t, units) + require.Equal(t, readBillingDemoReasonMissingScanDetail, actualReason) } func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { @@ -492,14 +464,15 @@ func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { runtimeStats := execdetails.NewRuntimeStatsColl(nil) recordRootRows := func(planID int, rows int) { - runtimeStats.GetBasicRuntimeStats(planID, true).Record(time.Millisecond, rows) + stats := runtimeStats.GetBasicRuntimeStats(planID, true) + stats.Record(time.Millisecond, rows) + stats.RecordBytes(0, int64(rows*10)) } recordRootRows(join.ID(), 6) recordRootRows(left.ID(), 4) recordRootRows(right.ID(), 6) - units, ok := readBillingDemoRootUnits( - ctx, + units, reason, ok := readBillingDemoRootUnits( runtimeStats, tree, 0, @@ -507,9 +480,14 @@ func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassHashJoin, operatorKind: "hashjoin"}, ) require.True(t, ok) + require.Empty(t, reason) require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideBuild)) require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideProbe)) + require.Equal(t, 40.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideBuild)) + require.Equal(t, 60.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideProbe)) + require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideBuild)) + require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideProbe)) } func readBillingDemoUnitValue(units []readBillingDemoUnit, unitName, side string) float64 { @@ -530,6 +508,15 @@ func readBillingDemoUnitSource(units []readBillingDemoUnit, unitName, side strin return "" } +func readBillingDemoUnitWidthSource(units []readBillingDemoUnit, unitName, side string) string { + for _, unit := range units { + if unit.unit == unitName && unit.side == side { + return unit.widthSource + } + } + return "" +} + func readBillingDemoUnitExists(units []readBillingDemoUnit, unitName, side string) bool { for _, unit := range units { if unit.unit == unitName && unit.side == side { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index c77c8cbf7336d..e6bb1264ec574 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -23,12 +23,9 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser/ast" - "github.com/pingcap/tidb/pkg/planner/cardinality" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/plancodec" "github.com/pingcap/tidb/pkg/util/stmtsummary" @@ -53,64 +50,65 @@ const ( explainRUSectionSummary = "summary" explainRUSectionPlan = "plan" explainRUSourceSummaryTotal = "summary_total" - explainRUWidthSourceOperatorHelper = "operator_helper" - explainRUWidthSourcePlanStats = "plan_stats" - explainRUWidthSourceSchemaTypeWidth = "schema_type_width" - explainRUWidthSourceSchemaFallback = "schema_fallback" - - readBillingDemoModelVersion = "v1" - readBillingDemoWeightVersion = "v1" - readBillingDemoStatusSuccess = "success" - readBillingDemoStatusUnsupported = "unsupported" - readBillingDemoStatusUnknownInput = "unknown_input" - readBillingDemoStatusError = "error" - readBillingDemoStatusOperatorOK = "ok" - readBillingDemoReasonNone = "none" - readBillingDemoReasonStatementError = "statement_error" - readBillingDemoReasonMissingPlan = "missing_plan" - readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" - readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" - readBillingDemoReasonMissingScanDetail = "missing_scan_detail" - readBillingDemoReasonUnsupportedOperator = "unsupported_operator" - readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" - readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" - readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" - readBillingDemoReasonUnsupportedLock = "unsupported_lock" - readBillingDemoReasonNonBillable = "non_billable" - readBillingDemoSiteStatement = "statement" - readBillingDemoSiteTiDB = "tidb" - readBillingDemoSiteTiKV = "tikv" - readBillingDemoOpClassStatement = "statement" - readBillingDemoOpClassFilter = "filter_eval" - readBillingDemoOpClassProjection = "projection_eval" - readBillingDemoOpClassLimit = "row_limit" - readBillingDemoOpClassTopN = "bounded_topn" - readBillingDemoOpClassSort = "full_ordering" - readBillingDemoOpClassWindow = "window_eval" - readBillingDemoOpClassHashAgg = "agg_hash" - readBillingDemoOpClassStreamAgg = "agg_stream" - readBillingDemoOpClassHashJoin = "join_hash" - readBillingDemoOpClassMergeJoin = "join_merge" - readBillingDemoOpClassLookupJoin = "join_lookup" - readBillingDemoOpClassReaderReceive = "reader_receive" - readBillingDemoOpClassLookupReader = "lookup_reader" - readBillingDemoOpClassOverlayReader = "overlay_reader" - readBillingDemoOpClassMetadataReader = "metadata_reader" - readBillingDemoOpClassPointLookup = "kv_point_lookup" - readBillingDemoOpClassRangeScan = "kv_range_scan" - readBillingDemoOpClassWrapper = "wrapper" - readBillingDemoOpClassSynthetic = "synthetic_source" - readBillingDemoOperatorStatement = "statement" - readBillingDemoUnitFixedEvents = "fixed_events" - readBillingDemoUnitInputRows = "input_rows" - readBillingDemoUnitInputBytes = "input_bytes" - readBillingDemoInputSourceRuntimeRows = "runtime_act_rows" - readBillingDemoInputSourceScanDetail = "scan_detail" - readBillingDemoInputSideAll = "all" - readBillingDemoInputSideBuild = "build" - readBillingDemoInputSideProbe = "probe" - readBillingDemoInputSideLeft = "left" - readBillingDemoInputSideRight = "right" + + explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" + explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" + explainRUWidthSourceNotApplicable = "not_applicable" + readBillingDemoModelVersion = "v2" + readBillingDemoWeightVersion = "v1" + readBillingDemoStatusSuccess = "success" + readBillingDemoStatusUnsupported = "unsupported" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoStatusError = "error" + readBillingDemoStatusOperatorOK = "ok" + readBillingDemoReasonNone = "none" + readBillingDemoReasonStatementError = "statement_error" + readBillingDemoReasonMissingPlan = "missing_plan" + readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" + readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" + readBillingDemoReasonMissingRuntimeBytes = "missing_runtime_bytes" + readBillingDemoReasonMissingInputBytes = "missing_input_bytes" + readBillingDemoReasonMissingScanDetail = "missing_scan_detail" + readBillingDemoReasonUnsupportedOperator = "unsupported_operator" + readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" + readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" + readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" + readBillingDemoReasonUnsupportedLock = "unsupported_lock" + readBillingDemoReasonNonBillable = "non_billable" + readBillingDemoSiteStatement = "statement" + readBillingDemoSiteTiDB = "tidb" + readBillingDemoSiteTiKV = "tikv" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOpClassFilter = "filter_eval" + readBillingDemoOpClassProjection = "projection_eval" + readBillingDemoOpClassLimit = "row_limit" + readBillingDemoOpClassTopN = "bounded_topn" + readBillingDemoOpClassSort = "full_ordering" + readBillingDemoOpClassWindow = "window_eval" + readBillingDemoOpClassHashAgg = "agg_hash" + readBillingDemoOpClassStreamAgg = "agg_stream" + readBillingDemoOpClassHashJoin = "join_hash" + readBillingDemoOpClassMergeJoin = "join_merge" + readBillingDemoOpClassLookupJoin = "join_lookup" + readBillingDemoOpClassReaderReceive = "reader_receive" + readBillingDemoOpClassLookupReader = "lookup_reader" + readBillingDemoOpClassOverlayReader = "overlay_reader" + readBillingDemoOpClassMetadataReader = "metadata_reader" + readBillingDemoOpClassPointLookup = "kv_point_lookup" + readBillingDemoOpClassRangeScan = "kv_range_scan" + readBillingDemoOpClassWrapper = "wrapper" + readBillingDemoOpClassSynthetic = "synthetic_source" + readBillingDemoOperatorStatement = "statement" + readBillingDemoUnitFixedEvents = "fixed_events" + readBillingDemoUnitInputRows = "input_rows" + readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" + readBillingDemoInputSourceScanDetail = "scan_detail" + readBillingDemoInputSideAll = "all" + readBillingDemoInputSideBuild = "build" + readBillingDemoInputSideProbe = "probe" + readBillingDemoInputSideLeft = "left" + readBillingDemoInputSideRight = "right" ) type explainRUComponentSnapshotStatus string @@ -456,17 +454,22 @@ func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanCont continue } var units []readBillingDemoUnit + var missingReason string var ok bool if op.IsRoot { - units, ok = readBillingDemoRootUnits(sctx, runtimeStats, tree, i, op, operator) + units, missingReason, ok = readBillingDemoRootUnits(runtimeStats, tree, i, op, operator) } else { - units, ok = readBillingDemoCopUnits(sctx, runtimeStats, tree, i, op, operator) + units, missingReason, ok = readBillingDemoCopUnits(runtimeStats, tree, i, operator) } if !ok { - if op.IsRoot { - return readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingRuntimeRows) + if missingReason == "" { + if op.IsRoot { + missingReason = readBillingDemoReasonMissingRuntimeBytes + } else { + missingReason = readBillingDemoReasonMissingScanDetail + } } - return readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingScanDetail) + return readBillingDemoStatusUnknownInput, operator.withReason(missingReason) } operator.status = readBillingDemoStatusOperatorOK operator.reason = readBillingDemoReasonNone @@ -576,43 +579,54 @@ func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorR } } -func readBillingDemoRootUnits(sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, bool) { - outputRows, ok := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) - if !ok { - return nil, false +func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { + if _, _, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, op.Origin.ID()); !ok { + if _, rowsOK := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()); !rowsOK { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + return nil, readBillingDemoReasonMissingRuntimeBytes, false } - rowWidth, rowWidthSource := explainRURowWidth(sctx, op.Origin, true) - units := []readBillingDemoUnit{{ - unit: readBillingDemoUnitFixedEvents, - source: readBillingDemoInputSourceRuntimeRows, - side: readBillingDemoInputSideAll, - value: 1, - rowWidth: rowWidth, - widthSource: rowWidthSource, - }} + units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChunkBytes)} switch operator.opClass { case readBillingDemoOpClassHashJoin: - return appendReadBillingDemoJoinUnits(units, sctx, runtimeStats, tree, idx, rowWidth, rowWidthSource, true) + return appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, true) case readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: - return appendReadBillingDemoJoinUnits(units, sctx, runtimeStats, tree, idx, rowWidth, rowWidthSource, false) + return appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, false) default: - inputRows, inputRowWidth, ok := readBillingDemoDirectLocalInputRowsAndWidth(sctx, runtimeStats, tree, idx, rowWidth) + inputRows, inputBytes, reason, ok := readBillingDemoDirectLocalInputRowsAndBytes(runtimeStats, tree, idx, operator.opClass) if !ok { - return nil, false - } - if inputRows == 0 && (len(tree[idx].ChildrenIdx) == 0 || readBillingDemoUseOutputRowsAsInput(operator.opClass)) { - inputRows = outputRows - inputRowWidth = rowWidth + return nil, reason, false } - inputBytes := float64(inputRows) * inputRowWidth - units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: float64(inputRows), rowWidth: inputRowWidth, widthSource: rowWidthSource}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: inputBytes, rowWidth: inputRowWidth, widthSource: rowWidthSource}, - ) - return units, true + units = append(units, readBillingDemoRuntimeChunkInputUnits(inputRows, inputBytes, readBillingDemoInputSideAll)...) + return units, "", true + } +} + +func readBillingDemoFixedEventUnit(inputSource string) readBillingDemoUnit { + return readBillingDemoUnit{ + unit: readBillingDemoUnitFixedEvents, + source: inputSource, + side: readBillingDemoInputSideAll, + value: 1, + widthSource: explainRUWidthSourceNotApplicable, + } +} + +func readBillingDemoRuntimeChunkInputUnits(rows, bytes int64, side string) []readBillingDemoUnit { + rowWidth := readBillingDemoAverageRowWidth(rows, float64(bytes)) + return []readBillingDemoUnit{ + {unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChunkBytes, side: side, value: float64(rows), rowWidth: rowWidth, widthSource: explainRUWidthSourceRuntimeChunkAvg}, + {unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeChunkBytes, side: side, value: float64(bytes), rowWidth: rowWidth, widthSource: explainRUWidthSourceRuntimeChunkAvg}, } } +func readBillingDemoAverageRowWidth(rows int64, bytes float64) float64 { + if rows <= 0 || bytes <= 0 { + return 0 + } + return bytes / float64(rows) +} + func readBillingDemoUseOutputRowsAsInput(opClass string) bool { switch opClass { case readBillingDemoOpClassReaderReceive, readBillingDemoOpClassLookupReader, readBillingDemoOpClassMetadataReader, readBillingDemoOpClassPointLookup: @@ -622,19 +636,14 @@ func readBillingDemoUseOutputRowsAsInput(opClass string) bool { } } -func appendReadBillingDemoJoinUnits(units []readBillingDemoUnit, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, fallbackWidth float64, fallbackWidthSource string, useBuildProbe bool) ([]readBillingDemoUnit, bool) { +func appendReadBillingDemoJoinUnits(units []readBillingDemoUnit, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, useBuildProbe bool) ([]readBillingDemoUnit, string, bool) { for childOrder, childIdx := range tree[idx].ChildrenIdx { if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { continue } - rows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + rows, bytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, tree[childIdx].Origin.ID()) if !ok { - return nil, false - } - width, widthSource := explainRURowWidth(sctx, tree[childIdx].Origin, true) - if width <= 0 { - width = fallbackWidth - widthSource = fallbackWidthSource + return nil, readBillingDemoReasonMissingInputBytes, false } side := readBillingDemoInputSideAll if useBuildProbe { @@ -655,36 +664,35 @@ func appendReadBillingDemoJoinUnits(units []readBillingDemoUnit, sctx base.PlanC } else { side = readBillingDemoInputSideRight } - units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeRows, side: side, value: float64(rows), rowWidth: width, widthSource: widthSource}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeRows, side: side, value: float64(rows) * width, rowWidth: width, widthSource: widthSource}, - ) + units = append(units, readBillingDemoRuntimeChunkInputUnits(rows, bytes, side)...) } - return units, true + return units, "", true } -func readBillingDemoDirectLocalInputRowsAndWidth(sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, fallbackWidth float64) (int64, float64, bool) { +func readBillingDemoDirectLocalInputRowsAndBytes(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, opClass string) (int64, int64, string, bool) { if idx < 0 || idx >= len(tree) || tree[idx] == nil { - return 0, fallbackWidth, true + return 0, 0, "", true } - var rows int64 - inputBytes := 0.0 + if len(tree[idx].ChildrenIdx) == 0 || readBillingDemoUseOutputRowsAsInput(opClass) { + rows, bytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, tree[idx].Origin.ID()) + if !ok { + return 0, 0, readBillingDemoReasonMissingRuntimeBytes, false + } + return rows, bytes, "", true + } + var rows, inputBytes int64 for _, childIdx := range tree[idx].ChildrenIdx { if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { continue } - childRows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + childRows, childBytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, tree[childIdx].Origin.ID()) if !ok { - return 0, 0, false + return 0, 0, readBillingDemoReasonMissingInputBytes, false } - childWidth, _ := explainRURowWidth(sctx, tree[childIdx].Origin, true) rows += childRows - inputBytes += float64(childRows) * childWidth + inputBytes += childBytes } - if rows == 0 { - return 0, fallbackWidth, true - } - return rows, inputBytes / float64(rows), true + return rows, inputBytes, "", true } func readBillingDemoPlanActRows(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, bool) { @@ -694,99 +702,45 @@ func readBillingDemoPlanActRows(runtimeStats *execdetails.RuntimeStatsColl, plan return runtimeStats.GetPlanActRows(planID), true } -func readBillingDemoCopUnits(sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, bool) { - copStats := readBillingDemoCopStats(runtimeStats, tree, idx, operator.opClass) - if copStats == nil { - return nil, false +func readBillingDemoRootOutputRowsAndBytes(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, int64, bool) { + if runtimeStats == nil || !runtimeStats.ExistsRootStats(planID) { + return 0, 0, false } - scanDetail := copStats.GetScanDetail() - rowWidth, rowWidthSource := explainRURowWidth(sctx, op.Origin, false) - units := []readBillingDemoUnit{{ - unit: readBillingDemoUnitFixedEvents, - source: readBillingDemoInputSourceScanDetail, - side: readBillingDemoInputSideAll, - value: 1, - rowWidth: rowWidth, - widthSource: rowWidthSource, - }} - switch operator.opClass { - case readBillingDemoOpClassRangeScan: - scanInputRows, scanInputBytes, scanInputSource := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeysSize, copStats.GetActRows(), rowWidth) - units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: scanInputSource, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: rowWidth, widthSource: rowWidthSource}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: scanInputSource, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: rowWidthSource}, - ) - default: - rows, inputRowWidth, inputRowWidthSource, ok := readBillingDemoDirectCopInputRowsAndWidth(sctx, runtimeStats, tree, idx, rowWidth, rowWidthSource, copStats) - if !ok { - return nil, false - } - units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: float64(rows), rowWidth: inputRowWidth, widthSource: inputRowWidthSource}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeRows, side: readBillingDemoInputSideAll, value: float64(rows) * inputRowWidth, rowWidth: inputRowWidth, widthSource: inputRowWidthSource}, - ) + basic := runtimeStats.GetBasicRuntimeStats(planID, false) + if basic == nil || !basic.HasBytes() { + return 0, 0, false } - return units, true + return basic.GetActRows(), basic.GetOutputBytes(), true } -func readBillingDemoRangeScanInput(totalKeys, processedKeysSize, actRows int64, rowWidth float64) (int64, float64, string) { - if totalKeys != 0 || processedKeysSize != 0 { - inputBytes := float64(processedKeysSize) - if inputBytes == 0 && totalKeys > 0 { - inputBytes = float64(totalKeys) * rowWidth - } - return totalKeys, inputBytes, readBillingDemoInputSourceScanDetail +func readBillingDemoCopUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { + if operator.opClass != readBillingDemoOpClassRangeScan { + return nil, readBillingDemoReasonMissingRuntimeBytes, false } - // Mock-store based tests can return executor rows without legacy scan-detail - // key counters. Keep the source explicit instead of pretending actRows are - // scan-detail TotalKeys. - if actRows > 0 { - return actRows, float64(actRows) * rowWidth, readBillingDemoInputSourceRuntimeRows + copStats := readBillingDemoCopStats(runtimeStats, tree, idx, operator.opClass) + if copStats == nil { + return nil, readBillingDemoReasonMissingScanDetail, false } - return 0, 0, readBillingDemoInputSourceScanDetail + scanDetail := copStats.GetScanDetail() + scanInputRows, scanInputBytes, ok := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeys, scanDetail.ProcessedKeysSize) + if !ok { + return nil, readBillingDemoReasonMissingScanDetail, false + } + rowWidth := readBillingDemoAverageRowWidth(scanInputRows, scanInputBytes) + units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceScanDetail)} + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + ) + return units, "", true } -func readBillingDemoDirectCopInputRowsAndWidth( - sctx base.PlanContext, - runtimeStats *execdetails.RuntimeStatsColl, - tree FlatPlanTree, - idx int, - fallbackWidth float64, - fallbackWidthSource string, - ownStats *execdetails.CopRuntimeStats, -) (int64, float64, string, bool) { - var rows int64 - inputBytes := 0.0 - inputRowWidth := fallbackWidth - inputRowWidthSource := fallbackWidthSource - hasCopChild := false - for _, childIdx := range tree[idx].ChildrenIdx { - if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || tree[childIdx].IsRoot { - continue - } - hasCopChild = true - childStats := runtimeStats.GetCopStats(tree[childIdx].Origin.ID()) - if childStats == nil { - return 0, 0, "", false - } - childRows := childStats.GetActRows() - childWidth, childWidthSource := explainRURowWidth(sctx, tree[childIdx].Origin, false) - if childWidth <= 0 { - childWidth = fallbackWidth - childWidthSource = fallbackWidthSource - } - inputRowWidth = childWidth - inputRowWidthSource = childWidthSource - rows += childRows - inputBytes += float64(childRows) * childWidth - } - if !hasCopChild { - return ownStats.GetActRows(), fallbackWidth, fallbackWidthSource, true - } - if rows == 0 { - return 0, inputRowWidth, inputRowWidthSource, true +func readBillingDemoRangeScanInput(totalKeys, processedKeys, processedKeysSize int64) (int64, float64, bool) { + if totalKeys <= 0 || processedKeys <= 0 || processedKeysSize <= 0 { + return 0, 0, false } - return rows, inputBytes / float64(rows), inputRowWidthSource, true + inputBytes := float64(processedKeysSize) / float64(processedKeys) * float64(totalKeys) + return totalKeys, inputBytes, true } func readBillingDemoCopStats(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, opClass string) *execdetails.CopRuntimeStats { @@ -831,7 +785,7 @@ func readBillingDemoCopStatsUsable(copStats *execdetails.CopRuntimeStats, opClas return true } scanDetail := copStats.GetScanDetail() - return scanDetail.TotalKeys != 0 || scanDetail.ProcessedKeysSize != 0 + return scanDetail.TotalKeys > 0 && scanDetail.ProcessedKeys > 0 && scanDetail.ProcessedKeysSize > 0 } func recordReadBillingDemoResult(result readBillingDemoResult) { @@ -1162,67 +1116,6 @@ func explainRUExtractComponentSnapshot(runtimeStats *execdetails.RuntimeStatsCol return nil, explainRUComponentSnapshotMissing } -func explainRURowWidth(sctx base.PlanContext, p base.Plan, includedRoot bool) (float64, string) { - if includedRoot { - // Root reader helpers describe SQL-visible row width for included TiDB - // work. Scan helpers below describe TiKV cop-side scan input width. - switch x := p.(type) { - case *physicalop.PhysicalTableReader: - if width := x.GetAvgRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - case *physicalop.PhysicalIndexLookUpReader: - if width := x.GetAvgTableRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - case *physicalop.PhysicalIndexMergeReader: - if width := x.GetAvgTableRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - case *physicalop.PointGetPlan: - if width := x.GetAvgRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - case *physicalop.BatchPointGetPlan: - if width := x.GetAvgRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - } - } else { - switch x := p.(type) { - case *physicalop.PhysicalTableScan: - if width := x.GetScanRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - case *physicalop.PhysicalIndexScan: - if width := x.GetScanRowSize(); width > 0 { - return width, explainRUWidthSourceOperatorHelper - } - } - } - if sctx == nil { - sctx = p.SCtx() - } - if stats := p.StatsInfo(); sctx != nil && stats != nil && stats.HistColl != nil && p.Schema() != nil { - if width := cardinality.GetAvgRowSize(sctx, stats.HistColl, p.Schema().Columns, false, false); width > 0 { - return width, explainRUWidthSourcePlanStats - } - } - if p.Schema() != nil { - width := 0 - for _, col := range p.Schema().Columns { - width += chunk.EstimateTypeWidth(col.GetStaticType()) - } - if width > 0 { - return float64(width), explainRUWidthSourceSchemaTypeWidth - } - if len(p.Schema().Columns) > 0 { - return float64(8 * len(p.Schema().Columns)), explainRUWidthSourceSchemaFallback - } - } - return 8, explainRUWidthSourceSchemaFallback -} - func (row explainRURow) toStrings() []string { return []string{ row.section, diff --git a/pkg/util/chunk/chunk.go b/pkg/util/chunk/chunk.go index 4065cc9285408..19ffd8e805e19 100644 --- a/pkg/util/chunk/chunk.go +++ b/pkg/util/chunk/chunk.go @@ -192,6 +192,41 @@ func (c *Chunk) MemoryUsage() (sum int64) { return } +// LogicalLiveBytes returns bytes for the logical live rows in this chunk. +// It counts payload bytes plus the logical null bitmap and offset arrays, not +// retained allocation capacity. +func (c *Chunk) LogicalLiveBytes() (sum int64) { + if c == nil { + return 0 + } + rows := c.NumRows() + if rows <= 0 || len(c.columns) == 0 { + return 0 + } + for _, col := range c.columns { + sum += col.logicalLiveBytes(c.sel, rows) + } + return sum +} + +func (c *Column) logicalLiveBytes(sel []int, rows int) int64 { + if c == nil || rows <= 0 { + return 0 + } + sum := int64((rows + 7) >> 3) + if c.IsFixed() { + return sum + int64(rows*len(c.elemBuf)) + } + sum += int64((rows + 1) * 8) + if sel == nil { + return sum + int64(c.offsets[rows]-c.offsets[0]) + } + for _, rowIdx := range sel { + sum += int64(c.offsets[rowIdx+1] - c.offsets[rowIdx]) + } + return sum +} + // RequiredRows returns how many rows is considered full. func (c *Chunk) RequiredRows() int { return c.requiredRows diff --git a/pkg/util/chunk/chunk_test.go b/pkg/util/chunk/chunk_test.go index 14194699ee811..395fadf91d110 100644 --- a/pkg/util/chunk/chunk_test.go +++ b/pkg/util/chunk/chunk_test.go @@ -610,6 +610,32 @@ func TestChunkMemoryUsage(t *testing.T) { require.Equal(t, int64(expectedUsage), chk.MemoryUsage()) } +func TestChunkLogicalLiveBytes(t *testing.T) { + fieldTypes := []*types.FieldType{ + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeVarchar), + } + chk := NewChunkWithCapacity(fieldTypes, 8) + chk.AppendInt64(0, 1) + chk.AppendString(1, "abc") + chk.AppendNull(0) + chk.AppendString(1, "d") + + require.Equal(t, int64(46), chk.LogicalLiveBytes()) + require.Greater(t, chk.MemoryUsage(), chk.LogicalLiveBytes()) + + chk.SetSel([]int{1}) + require.Equal(t, int64(27), chk.LogicalLiveBytes()) + + chk.SetSel(nil) + chk.MakeRef(0, 1) + require.Equal(t, int64(34), chk.LogicalLiveBytes()) + + virtualRows := NewChunkWithCapacity(nil, 8) + virtualRows.SetNumVirtualRows(3) + require.Equal(t, int64(0), virtualRows.LogicalLiveBytes()) +} + func TestSwapColumn(t *testing.T) { fieldTypes := make([]*types.FieldType, 0, 2) fieldTypes = append(fieldTypes, types.NewFieldType(mysql.TypeFloat)) diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index 2369dda4cb37e..984380d5216d2 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -1001,6 +1001,23 @@ func TestRootRuntimeStats(t *testing.T) { require.Equal(t, expect, stats.String()) } +func TestBasicRuntimeStatsBytes(t *testing.T) { + stats := &BasicRuntimeStats{} + require.False(t, stats.HasBytes()) + + stats.RecordBytes(0, 0) + require.True(t, stats.HasBytes()) + require.Equal(t, int64(0), stats.GetInputBytes()) + require.Equal(t, int64(0), stats.GetOutputBytes()) + + other := &BasicRuntimeStats{} + other.RecordBytes(11, 22) + stats.Merge(other) + require.True(t, stats.HasBytes()) + require.Equal(t, int64(11), stats.GetInputBytes()) + require.Equal(t, int64(22), stats.GetOutputBytes()) +} + func TestFormatDurationForExplain(t *testing.T) { cases := []struct { t string diff --git a/pkg/util/execdetails/runtime_stats.go b/pkg/util/execdetails/runtime_stats.go index 7ff46a5401b2f..b109429b5b6f1 100644 --- a/pkg/util/execdetails/runtime_stats.go +++ b/pkg/util/execdetails/runtime_stats.go @@ -331,6 +331,13 @@ type BasicRuntimeStats struct { close atomic.Int64 // executor return row count. rows atomic.Int64 + // executor input bytes counted from child output chunks. + inputBytes atomic.Int64 + // executor output bytes counted from returned chunks. + outputBytes atomic.Int64 + // runtime byte accounting records, used to distinguish zero bytes from + // missing byte evidence. + byteRecords atomic.Int64 } // GetActRows return total rows of BasicRuntimeStats. @@ -338,6 +345,21 @@ func (e *BasicRuntimeStats) GetActRows() int64 { return e.rows.Load() } +// GetInputBytes returns total input bytes of BasicRuntimeStats. +func (e *BasicRuntimeStats) GetInputBytes() int64 { + return e.inputBytes.Load() +} + +// GetOutputBytes returns total output bytes of BasicRuntimeStats. +func (e *BasicRuntimeStats) GetOutputBytes() int64 { + return e.outputBytes.Load() +} + +// HasBytes returns whether runtime byte accounting has recorded evidence. +func (e *BasicRuntimeStats) HasBytes() bool { + return e != nil && e.byteRecords.Load() > 0 +} + // Clone implements the RuntimeStats interface. // BasicRuntimeStats shouldn't implement Clone interface because all executors with the same executor_id // should share the same BasicRuntimeStats, duplicated BasicRuntimeStats are easy to cause mistakes. @@ -356,6 +378,9 @@ func (e *BasicRuntimeStats) Merge(rs RuntimeStats) { e.open.Add(tmp.open.Load()) e.close.Add(tmp.close.Load()) e.rows.Add(tmp.rows.Load()) + e.inputBytes.Add(tmp.inputBytes.Load()) + e.outputBytes.Add(tmp.outputBytes.Load()) + e.byteRecords.Add(tmp.byteRecords.Load()) } // Tp implements the RuntimeStats interface. @@ -410,6 +435,20 @@ func (e *BasicRuntimeStats) Record(d time.Duration, rowNum int) { e.rows.Add(int64(rowNum)) } +// RecordBytes records executor input and output bytes for one Next call. +func (e *BasicRuntimeStats) RecordBytes(inputBytes, outputBytes int64) { + if e == nil { + return + } + e.byteRecords.Add(1) + if inputBytes > 0 { + e.inputBytes.Add(inputBytes) + } + if outputBytes > 0 { + e.outputBytes.Add(outputBytes) + } +} + // RecordOpen records executor's open time. func (e *BasicRuntimeStats) RecordOpen(d time.Duration) { e.consume.Add(int64(d)) From fff344eecfbb627d983b64349f96d8388bd3690f Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 9 Jul 2026 19:58:05 +0800 Subject: [PATCH 12/25] planner: add write preview RU for DML --- pkg/executor/explain_test.go | 97 +++++++++- pkg/metrics/metrics.go | 1 + pkg/metrics/ru_v2.go | 10 + pkg/planner/core/BUILD.bazel | 1 + pkg/planner/core/common_plans_test.go | 115 ++++++++++++ pkg/planner/core/explain_ru.go | 223 ++++++++++++++++++++++- pkg/planner/core/planbuilder.go | 4 +- pkg/util/execdetails/execdetails_test.go | 15 +- pkg/util/execdetails/ruv2_metrics.go | 28 +++ 9 files changed, 469 insertions(+), 25 deletions(-) diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 2b4155cd6992f..3a05de01482df 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -686,6 +686,51 @@ func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorC require.Failf(t, "missing weighted FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) } +func requireExplainRUWriteRows(t *testing.T, rows [][]any, operatorKind string) { + t.Helper() + require.NotEmpty(t, rows) + require.Len(t, rows[0], 17) + require.Equal(t, "summary", rows[0][0]) + require.Equal(t, "total_preview_ru", rows[0][2]) + require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v1") + summaryPreviewRU, err := strconv.ParseFloat(fmt.Sprint(rows[0][14]), 64) + require.NoError(t, err) + + var writeKeysRows, writeByteRows int + var planPreviewRU float64 + for _, row := range rows { + require.Len(t, row, 17) + if row[0] != "plan" || row[2] != operatorKind || row[3] != "tikv/kv_write" { + continue + } + previewRU, err := strconv.ParseFloat(fmt.Sprint(row[14]), 64) + require.NoError(t, err) + planPreviewRU += previewRU + switch row[11] { + case "write_keys": + writeKeysRows++ + require.NotEmpty(t, row[12]) + require.NotEmpty(t, row[13]) + require.NotEmpty(t, row[14]) + require.Equal(t, "commit_detail", row[15]) + case "write_byte": + writeByteRows++ + require.NotEmpty(t, row[10]) + require.NotEmpty(t, row[13]) + require.NotEmpty(t, row[14]) + require.Equal(t, "commit_detail", row[15]) + case "prewrite_region_num", "tikv_write_rpc_count": + require.NotEmpty(t, row[12]) + require.Equal(t, "0.000000", row[13]) + require.Equal(t, "0.000000", row[14]) + require.Contains(t, fmt.Sprint(row[16]), "diagnostic_only=true") + } + } + require.Equal(t, 1, writeKeysRows, rows) + require.Equal(t, 1, writeByteRows, rows) + require.InEpsilon(t, planPreviewRU, summaryPreviewRU, 0.000001) +} + func TestExplainAnalyzeFormatRUPlanDigest(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -717,6 +762,34 @@ func TestExplainAnalyzeFormatRUPlanDigest(t *testing.T) { requireExplainRUPlanRow(t, rows) } +func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_write") + tk.MustExec("create table explain_ru_write(a int primary key, b int, c varchar(20), key idx_b(b))") + + rows, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write values (1, 10, 'aa')") + require.NoError(t, err) + requireExplainRUWriteRows(t, rows, "insert") + tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 10 aa")) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'bbb' where a = 1") + require.NoError(t, err) + requireExplainRUWriteRows(t, rows, "update") + tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 10 bbb")) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set b = 11 where a = 1") + require.NoError(t, err) + requireExplainRUWriteRows(t, rows, "update") + tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 11 bbb")) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' delete from explain_ru_write where a = 1") + require.NoError(t, err) + requireExplainRUWriteRows(t, rows, "delete") + tk.MustQuery("select count(*) from explain_ru_write").Check(testkit.Rows("0")) +} + func TestExplainAnalyzeFormatRUUnsupportedTargetsBeforeExecution(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -733,14 +806,7 @@ func TestExplainAnalyzeFormatRUUnsupportedTargetsBeforeExecution(t *testing.T) { err = tk.ExecToErr("explain analyze format=ru select 1") require.Error(t, err) - err = tk.ExecToErr("explain analyze format='ru' insert into explain_ru_dml values (2)") - require.Error(t, err) - require.Contains(t, err.Error(), "unsupported_non_select") - tk.MustQuery("select * from explain_ru_dml order by a").Check(testkit.Rows("1")) - for _, sql := range []string{ - "explain analyze format='ru' update explain_ru_dml set a = 2 where a = 1", - "explain analyze format='ru' delete from explain_ru_dml where a = 1", "explain analyze format='ru' replace into explain_ru_dml values (2)", "explain analyze format='ru' alter table explain_ru_dml add column b int", "explain analyze format='ru' import into explain_ru_dml from select * from explain_ru_dml", @@ -843,12 +909,21 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { beforeBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) beforeBaseUnitsTotal := readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) + writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "insert", "write_keys", "commit_detail", "all", "v2") + writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "insert", "write_byte", "commit_detail", "all", "v2") + beforeWriteKeys := readExecutorCounterValue(t, writeKeys) + beforeWriteByte := readExecutorCounterValue(t, writeByte) + beforeWriteSuccess := readExecutorCounterValue(t, success) beforeUnsupported := readExecutorCounterValue(t, unsupported) tk.MustExec("insert into read_billing_demo values (3)") - require.Equal(t, beforeUnsupported+1, readExecutorCounterValue(t, unsupported)) + require.Equal(t, beforeUnsupported, readExecutorCounterValue(t, unsupported)) + require.Equal(t, beforeWriteSuccess+1, readExecutorCounterValue(t, success)) require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) - require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) - tk.MustQuery("select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_demo`%' and status = 'unsupported' group by status, reason").Check(testkit.Rows("unsupported unsupported_non_select 1")) + require.Greater(t, readExecutorCounterValue(t, writeKeys), beforeWriteKeys) + require.Greater(t, readExecutorCounterValue(t, writeByte), beforeWriteByte) + require.Greater(t, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter), beforeBaseUnitsTotal) + tk.MustQuery("select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_demo`%' and site = 'statement' group by status, reason").Check(testkit.Rows("success none 1")) + tk.MustQuery("select unit, input_source, input_side, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_demo`%' and site = 'tikv' and op_class = 'kv_write' and operator_kind = 'insert' and unit in ('write_byte', 'write_keys') order by unit").Check(testkit.Rows("write_byte commit_detail all 1", "write_keys commit_detail all 1")) beforeUnknownInput := readExecutorCounterValue(t, unknownInput) beforeBaseUnits = readExecutorCounterValue(t, projectionFixedEvents) @@ -888,7 +963,9 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustExec("drop table if exists read_billing_outer") tk.MustExec("create table read_billing_outer(a int)") + tk.MustExec("set tidb_enable_read_billing_demo=off") tk.MustExec("insert into read_billing_outer values (0)") + tk.MustExec("set tidb_enable_read_billing_demo=on") tk.MustExec("set @@tidb_init_chunk_size=1") rs, err := tk.Exec("select (select t.a from read_billing_demo t where t.a > o.a) from read_billing_outer o") require.NoError(t, err) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 19031cc494802..dac36c01a43e2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -353,6 +353,7 @@ func RegisterMetrics() { prometheus.MustRegister(RUV2ResourceManagerWriteCnt) prometheus.MustRegister(RUV2WriteKeys) prometheus.MustRegister(RUV2WriteSize) + prometheus.MustRegister(RUV2PrewriteRegionNum) prometheus.MustRegister(RUV2SessionParserTotal) prometheus.MustRegister(RUV2TxnCnt) prometheus.MustRegister(RUV2TiKVKVEngineCacheMiss) diff --git a/pkg/metrics/ru_v2.go b/pkg/metrics/ru_v2.go index 0712db292beba..ed6bf13a14ccc 100644 --- a/pkg/metrics/ru_v2.go +++ b/pkg/metrics/ru_v2.go @@ -32,6 +32,7 @@ var ( RUV2ResourceManagerWriteCnt prometheus.Counter RUV2WriteKeys prometheus.Counter RUV2WriteSize prometheus.Counter + RUV2PrewriteRegionNum prometheus.Counter RUV2SessionParserTotal prometheus.Counter RUV2TxnCnt prometheus.Counter @@ -183,6 +184,15 @@ func InitRUV2Metrics() { }, ) + RUV2PrewriteRegionNum = metricscommon.NewCounter( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "ruv2", + Name: "prewrite_region_num", + Help: "Diagnostic counter of commit prewrite region fanout for RU v2.", + }, + ) + RUV2SessionParserTotal = metricscommon.NewCounter( prometheus.CounterOpts{ Namespace: "tidb", diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index e7f2e928a2ea9..6732335bba5a2 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -209,6 +209,7 @@ go_library( "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//util", "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap", diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index 1c73f78ddb043..1f8da270d47d7 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -165,6 +165,28 @@ func TestExplainRUSelectGateStatus(t *testing.T) { })) } +func TestExplainRUTargetGateStatus(t *testing.T) { + cases := []struct { + sql string + expected explainRUStatus + }{ + {"explain analyze format='ru' select 1", explainRUStatusSuccess}, + {"explain analyze format='ru' insert into t values (1)", explainRUStatusSuccess}, + {"explain analyze format='ru' update t set a = 2 where a = 1", explainRUStatusSuccess}, + {"explain analyze format='ru' delete from t where a = 1", explainRUStatusSuccess}, + {"explain analyze format='ru' replace into t values (1)", explainRUStatusUnsupportedNonSelect}, + {"explain analyze format='ru' select @a := 1", explainRUStatusUnsupportedSideEffecting}, + {"explain analyze format='ru' table t", explainRUStatusUnsupportedNonSelect}, + } + p := parser.New() + for _, tc := range cases { + stmt, err := p.ParseOneStmt(tc.sql, "", "") + require.NoError(t, err, tc.sql) + explain := stmt.(*ast.ExplainStmt) + require.Equal(t, tc.expected, explainRUTargetGateStatus(explain.Stmt), tc.sql) + } +} + func TestExplainRURowFormatting(t *testing.T) { row := explainRURow{ section: explainRUSectionPlan, @@ -219,6 +241,28 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: "scan_total_keys", value: 4}, tidbWeights) require.False(t, ok) + writeWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion) + require.True(t, ok) + require.InEpsilon(t, readBillingDemoWriteKeyWeight, writeWeights.writeKey, 0.000001) + require.Equal(t, readBillingDemoWriteByteWeight, writeWeights.writeByte) + require.Zero(t, writeWeights.region) + require.Zero(t, writeWeights.writeRPC) + + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitWriteKeys, value: 3}, writeWeights) + require.True(t, ok) + require.Equal(t, writeWeights.writeKey, weight) + require.Equal(t, 3*writeWeights.writeKey, previewRU) + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitWriteByte, value: 4096}, writeWeights) + require.True(t, ok) + require.Equal(t, writeWeights.writeByte, weight) + require.Equal(t, 4096*writeWeights.writeByte, previewRU) + for _, diagnosticUnit := range []string{readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount} { + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: diagnosticUnit, value: 99}, writeWeights) + require.True(t, ok) + require.Zero(t, weight) + require.Zero(t, previewRU) + } + ctx := mock.NewContext() col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) @@ -348,6 +392,77 @@ func TestExplainRUComponentSnapshotStatusAndWeights(t *testing.T) { require.Same(t, okStats, snapshot) } +func TestReadBillingDemoWriteDMLResult(t *testing.T) { + ruv2Metrics := execdetails.NewRUV2Metrics() + ruv2Metrics.AddResourceManagerWriteCnt(7) + commitDetail := &tikvutil.CommitDetails{ + WriteKeys: 3, + WriteSize: 66, + PrewriteRegionNum: 2, + } + result := buildWriteBillingDemoResultFromDetails("insert", commitDetail, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + require.Equal(t, readBillingDemoReasonNone, result.reason) + require.Len(t, result.operators, 1) + require.Equal(t, readBillingDemoStatusOperatorOK, result.operators[0].status) + + units := make(map[string]readBillingDemoUnit) + for _, unit := range result.operators[0].units { + units[unit.unit] = unit + } + require.Equal(t, 3.0, units[readBillingDemoUnitWriteKeys].value) + require.Equal(t, 66.0, units[readBillingDemoUnitWriteByte].value) + require.Equal(t, 2.0, units[readBillingDemoUnitPrewriteRegionNum].value) + require.Equal(t, 7.0, units[readBillingDemoUnitTiKVWriteRPCCount].value) + + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.InEpsilon(t, + 3*readBillingDemoWriteKeyWeight+66*readBillingDemoWriteByteWeight, + rows[0].previewRU, + 0.000001, + ) + var diagnosticRows int + for _, row := range rows[1:] { + if row.unit == readBillingDemoUnitPrewriteRegionNum || row.unit == readBillingDemoUnitTiKVWriteRPCCount { + diagnosticRows++ + require.True(t, row.hasWeight) + require.Zero(t, row.weight) + require.True(t, row.hasPreviewRU) + require.Zero(t, row.previewRU) + require.Contains(t, row.note, "diagnostic_only=true") + } + } + require.Equal(t, 2, diagnosticRows) + + stats := buildReadBillingDemoStatementStats(result) + require.Len(t, stats.BaseUnits, 4) + + partialResult := buildWriteBillingDemoResultFromDetails("delete", &tikvutil.CommitDetails{WriteKeys: 1, WriteSize: 2}, nil) + require.Equal(t, readBillingDemoStatusSuccess, partialResult.status) + require.Len(t, partialResult.operators, 3) + rows = explainRUBuildReadBillingRows(partialResult, explainRUComponentSnapshotMissing) + require.Contains(t, rows[0].note, "partial_missing_prewrite_region_num") + require.Contains(t, rows[0].note, "partial_missing_write_rpc_count") + + missingResult := buildWriteBillingDemoResultFromDetails("update", nil, ruv2Metrics) + require.Equal(t, readBillingDemoStatusUnknownInput, missingResult.status) + require.Equal(t, readBillingDemoReasonMissingCommitDetail, missingResult.reason) + + missingWriteKeys := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{WriteSize: 2}, ruv2Metrics) + require.Equal(t, readBillingDemoStatusUnknownInput, missingWriteKeys.status) + require.Equal(t, readBillingDemoReasonMissingWriteKeys, missingWriteKeys.reason) + + missingWriteByte := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{WriteKeys: 1}, ruv2Metrics) + require.Equal(t, readBillingDemoStatusUnknownInput, missingWriteByte.status) + require.Equal(t, readBillingDemoReasonMissingWriteByte, missingWriteByte.reason) + + zeroResult := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{}, ruv2Metrics) + require.Equal(t, readBillingDemoStatusSuccess, zeroResult.status) + require.Len(t, zeroResult.operators, 1) + require.Equal(t, readBillingDemoReasonZeroMutation, zeroResult.operators[0].reason) + require.Empty(t, zeroResult.operators[0].units) +} + func extractExplainRUTestSnapshotStatus(stats *execdetails.RURuntimeStats) explainRUComponentSnapshotStatus { _, status := extractExplainRUTestSnapshot(stats) return status diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index e6bb1264ec574..720f787d45e53 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -17,6 +17,7 @@ package core import ( "strconv" "strings" + "sync/atomic" "time" "github.com/pingcap/errors" @@ -29,6 +30,7 @@ import ( "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/plancodec" "github.com/pingcap/tidb/pkg/util/stmtsummary" + tikvutil "github.com/tikv/client-go/v2/util" rmclient "github.com/tikv/pd/client/resource_group/controller" ) @@ -61,6 +63,7 @@ const ( readBillingDemoStatusUnknownInput = "unknown_input" readBillingDemoStatusError = "error" readBillingDemoStatusOperatorOK = "ok" + readBillingDemoStatusPartial = "partial" readBillingDemoReasonNone = "none" readBillingDemoReasonStatementError = "statement_error" readBillingDemoReasonMissingPlan = "missing_plan" @@ -75,6 +78,12 @@ const ( readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" readBillingDemoReasonUnsupportedLock = "unsupported_lock" readBillingDemoReasonNonBillable = "non_billable" + readBillingDemoReasonMissingCommitDetail = "missing_commit_detail" + readBillingDemoReasonMissingWriteKeys = "missing_write_keys" + readBillingDemoReasonMissingWriteByte = "missing_write_byte" + readBillingDemoReasonZeroMutation = "zero_mutation" + readBillingDemoReasonMissingPrewriteRegion = "missing_prewrite_region_num" + readBillingDemoReasonMissingWriteRPCCount = "missing_write_rpc_count" readBillingDemoSiteStatement = "statement" readBillingDemoSiteTiDB = "tidb" readBillingDemoSiteTiKV = "tikv" @@ -96,19 +105,36 @@ const ( readBillingDemoOpClassMetadataReader = "metadata_reader" readBillingDemoOpClassPointLookup = "kv_point_lookup" readBillingDemoOpClassRangeScan = "kv_range_scan" + readBillingDemoOpClassKVWrite = "kv_write" readBillingDemoOpClassWrapper = "wrapper" readBillingDemoOpClassSynthetic = "synthetic_source" readBillingDemoOperatorStatement = "statement" readBillingDemoUnitFixedEvents = "fixed_events" readBillingDemoUnitInputRows = "input_rows" readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoUnitWriteKeys = "write_keys" + readBillingDemoUnitWriteByte = "write_byte" + readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" + readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" readBillingDemoInputSourceScanDetail = "scan_detail" + readBillingDemoInputSourceCommitDetail = "commit_detail" + readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" readBillingDemoInputSideAll = "all" readBillingDemoInputSideBuild = "build" readBillingDemoInputSideProbe = "probe" readBillingDemoInputSideLeft = "left" readBillingDemoInputSideRight = "right" + + // The first write-side preview formula intentionally has no fixed/RPC/region + // terms: write keys use the current scaled RUv2 write-key coefficient, and + // write bytes use the existing TiKV range-scan byte coefficient as a + // calibration seed because RUv2 only shadows commit WriteSize today. + readBillingDemoWriteRUScale = 2.01 + readBillingDemoRUV2WriteKeysWeight = 0.330760861554226 + readBillingDemoWriteKeyWeight = readBillingDemoWriteRUScale * readBillingDemoRUV2WriteKeysWeight + readBillingDemoWriteByteWeight = 0.000020 + readBillingDemoDiagnosticZeroWeight = 0.0 ) type explainRUComponentSnapshotStatus string @@ -144,6 +170,10 @@ type readBillingDemoOperatorWeights struct { fixedEvent float64 row float64 byte float64 + writeKey float64 + writeByte float64 + writeRPC float64 + region float64 } type readBillingDemoWeightKey struct { @@ -154,6 +184,7 @@ type readBillingDemoWeightKey struct { var readBillingDemoWeights = map[readBillingDemoWeightKey]readBillingDemoOperatorWeights{ {readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000020}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion}: {writeKey: readBillingDemoWriteKeyWeight, writeByte: readBillingDemoWriteByteWeight, writeRPC: readBillingDemoDiagnosticZeroWeight, region: readBillingDemoDiagnosticZeroWeight}, {readBillingDemoSiteTiKV, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000040, byte: 0.000006}, {readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000006}, {readBillingDemoSiteTiKV, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000008, byte: 0.000002}, @@ -227,21 +258,24 @@ func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, if planCtx == nil { planCtx = sctx.GetPlanCtx() } - result := buildReadBillingDemoResult(planCtx, plan, stmt, execErr) + result := buildReadBillingDemoResult(planCtx, plan, stmt, execErr, nil) recordReadBillingDemoResult(result) return buildReadBillingDemoStatementStats(result) } -func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error) readBillingDemoResult { +func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { if execErr != nil { return readBillingDemoFailure(readBillingDemoStatusError, readBillingDemoReasonStatementError) } if plan == nil { return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingPlan) } - if gateStatus := explainRUSelectGateStatus(stmt); gateStatus != explainRUStatusSuccess { + if gateStatus := explainRUTargetGateStatus(stmt); gateStatus != explainRUStatusSuccess { return readBillingDemoFailure(readBillingDemoStatusUnsupported, string(gateStatus)) } + if operatorKind, ok := explainRUWriteDMLKind(stmt); ok { + return buildWriteBillingDemoResult(sctx, operatorKind, ruv2Metrics) + } if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil { return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingRuntimeStats) } @@ -272,6 +306,110 @@ func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast. return result } +func buildWriteBillingDemoResult(sctx base.PlanContext, operatorKind string, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + var commitDetail *tikvutil.CommitDetails + if sctx != nil && sctx.GetSessionVars() != nil { + if ruv2Metrics == nil { + ruv2Metrics = sctx.GetSessionVars().RUV2Metrics + } + if sctx.GetSessionVars().StmtCtx != nil { + commitDetail = sctx.GetSessionVars().StmtCtx.GetExecDetails().CommitDetail + } + } + return buildWriteBillingDemoResultFromDetails(operatorKind, commitDetail, ruv2Metrics) +} + +func buildWriteBillingDemoResultFromDetails(operatorKind string, commitDetail *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + operator := readBillingDemoOperatorResult{ + id: "commit_txn", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassKVWrite, + operatorKind: operatorKind, + } + if commitDetail == nil { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingCommitDetail)) + } + + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + + writeKeys := int64(commitDetail.WriteKeys) + writeBytes := int64(commitDetail.WriteSize) + if writeKeys == 0 && writeBytes == 0 { + operator.reason = readBillingDemoReasonZeroMutation + result.operators = append(result.operators, operator) + return result + } + if writeKeys == 0 { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingWriteKeys)) + } + if writeBytes == 0 { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingWriteByte)) + } + operator.units = append(operator.units, + readBillingDemoUnit{ + unit: readBillingDemoUnitWriteKeys, + source: readBillingDemoInputSourceCommitDetail, + side: readBillingDemoInputSideAll, + value: float64(writeKeys), + widthSource: explainRUWidthSourceNotApplicable, + }, + readBillingDemoUnit{ + unit: readBillingDemoUnitWriteByte, + source: readBillingDemoInputSourceCommitDetail, + side: readBillingDemoInputSideAll, + value: float64(writeBytes), + widthSource: explainRUWidthSourceNotApplicable, + }, + ) + prewriteRegionNum := int64(atomic.LoadInt32(&commitDetail.PrewriteRegionNum)) + if prewriteRegionNum > 0 { + operator.units = append(operator.units, readBillingDemoUnit{ + unit: readBillingDemoUnitPrewriteRegionNum, + source: readBillingDemoInputSourceCommitDetail, + side: readBillingDemoInputSideAll, + value: float64(prewriteRegionNum), + widthSource: explainRUWidthSourceNotApplicable, + }) + } + writeRPCCount := int64(0) + if ruv2Metrics != nil && !ruv2Metrics.Bypass() { + writeRPCCount = ruv2Metrics.ResourceManagerWriteCnt() + } + if writeRPCCount > 0 { + operator.units = append(operator.units, readBillingDemoUnit{ + unit: readBillingDemoUnitTiKVWriteRPCCount, + source: readBillingDemoInputSourceRUV2Metrics, + side: readBillingDemoInputSideAll, + value: float64(writeRPCCount), + widthSource: explainRUWidthSourceNotApplicable, + }) + } + result.operators = append(result.operators, operator) + if prewriteRegionNum == 0 { + result.operators = append(result.operators, readBillingDemoWriteDiagnosticStatus(operatorKind, readBillingDemoReasonMissingPrewriteRegion)) + } + if writeRPCCount == 0 { + result.operators = append(result.operators, readBillingDemoWriteDiagnosticStatus(operatorKind, readBillingDemoReasonMissingWriteRPCCount)) + } + return result +} + +func readBillingDemoWriteDiagnosticStatus(operatorKind, reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "commit_txn", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassKVWrite, + operatorKind: operatorKind, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + func readBillingDemoPlanContext(plan base.Plan) base.PlanContext { if plan == nil { return nil @@ -292,6 +430,14 @@ func readBillingDemoUnitWeight(weights readBillingDemoOperatorWeights, unit stri return weights.row, true case readBillingDemoUnitInputBytes: return weights.byte, true + case readBillingDemoUnitWriteKeys: + return weights.writeKey, true + case readBillingDemoUnitWriteByte: + return weights.writeByte, true + case readBillingDemoUnitPrewriteRegionNum: + return weights.region, true + case readBillingDemoUnitTiKVWriteRPCCount: + return weights.writeRPC, true default: return 0, false } @@ -833,6 +979,38 @@ func (e *Explain) recordExplainRUStatus(status explainRUStatus) { recordExplainRUStatus(status) } +// explainRUTargetGateStatus is the pre-execution safety gate for FORMAT='RU'. +// SELECT keeps the side-effect-free checks from the read-side demo; write DML +// is limited to statements whose commit details expose foreground write volume. +func explainRUTargetGateStatus(stmt ast.StmtNode) explainRUStatus { + if _, ok := explainRUWriteDMLKind(stmt); ok { + return explainRUStatusSuccess + } + return explainRUSelectGateStatus(stmt) +} + +func explainRUWriteDMLKind(stmt ast.StmtNode) (string, bool) { + switch x := stmt.(type) { + case *ast.InsertStmt: + if x == nil || x.IsReplace { + return "", false + } + return "insert", true + case *ast.UpdateStmt: + if x == nil { + return "", false + } + return "update", true + case *ast.DeleteStmt: + if x == nil { + return "", false + } + return "delete", true + default: + return "", false + } +} + // explainRUSelectGateStatus is the first-demo pre-execution safety gate. It // accepts only SELECT keyword surfaces and set operations whose leaves can be // checked before EXPLAIN ANALYZE can run the target statement. @@ -958,7 +1136,7 @@ func (e *Explain) renderRUExplain() (err error) { status = explainRUStatusUnsupportedNonAnalyze return explainRUError(explainRUStatusUnsupportedNonAnalyze) } - if gateStatus := explainRUSelectGateStatus(e.ExecStmt); gateStatus != explainRUStatusSuccess { + if gateStatus := explainRUTargetGateStatus(e.ExecStmt); gateStatus != explainRUStatusSuccess { status = gateStatus return explainRUError(gateStatus) } @@ -973,9 +1151,13 @@ func (e *Explain) renderRUExplain() (err error) { // The snapshot belongs to the target statement execution. Returning this // EXPLAIN result can add more result-chunk counters later, so render output // and Demo Metrics are derived from this frozen input and the generated rows. - _, snapshotStatus := explainRUExtractComponentSnapshot(runtimeStats, e.TargetPlan.ID()) + snapshot, snapshotStatus := explainRUExtractComponentSnapshot(runtimeStats, e.TargetPlan.ID()) metrics.RecordExplainRUComponentSnapshot(string(snapshotStatus)) - result := buildReadBillingDemoResult(e.SCtx(), e.TargetPlan, e.ExecStmt, nil) + var snapshotRUV2Metrics *execdetails.RUV2Metrics + if snapshot != nil { + snapshotRUV2Metrics = snapshot.Metrics + } + result := buildReadBillingDemoResult(e.SCtx(), e.TargetPlan, e.ExecStmt, nil, snapshotRUV2Metrics) if result.status != readBillingDemoStatusSuccess { operator := "" if len(result.operators) > 0 { @@ -984,7 +1166,7 @@ func (e *Explain) renderRUExplain() (err error) { } status = explainRUStatusError return errors.NewNoStackErrorf( - "EXPLAIN ANALYZE FORMAT='RU' cannot render a complete read billing model result: status=%s reason=%s%s", + "EXPLAIN ANALYZE FORMAT='RU' cannot render a complete preview RU model result: status=%s reason=%s%s", result.status, result.reason, operator, @@ -1007,7 +1189,7 @@ func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus component: "total_preview_ru", hasPreviewRU: true, source: explainRUSourceSummaryTotal, - note: explainRUReadBillingSummaryNote(snapshotStatus), + note: explainRUReadBillingSummaryNote(snapshotStatus, result), }} totalPreviewRU := 0.0 for _, op := range result.operators { @@ -1035,11 +1217,16 @@ func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus return rows } -func explainRUReadBillingSummaryNote(snapshotStatus explainRUComponentSnapshotStatus) string { +func explainRUReadBillingSummaryNote(snapshotStatus explainRUComponentSnapshotStatus, result readBillingDemoResult) string { note := "weight_version=" + readBillingDemoWeightVersion if snapshotStatus != explainRUComponentSnapshotOK { note = appendExplainRUNote(note, "component_snapshot_"+string(snapshotStatus)) } + for _, op := range result.operators { + if op.status == readBillingDemoStatusPartial && op.reason != "" { + note = appendExplainRUNote(note, "partial_"+op.reason) + } + } return note } @@ -1056,6 +1243,9 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill source: unit.source, note: "input_side=" + unit.side + ",weight_version=" + readBillingDemoWeightVersion, } + if readBillingDemoUnitDiagnosticOnly(unit.unit) { + row.note = appendExplainRUNote(row.note, "diagnostic_only=true") + } if op.hasActRows { row.actRows = op.actRows row.hasActRows = true @@ -1076,10 +1266,25 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill case readBillingDemoUnitInputBytes: row.workBytes = unit.value row.hasWorkBytes = true + case readBillingDemoUnitWriteKeys, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitWriteByte: + row.workBytes = unit.value + row.hasWorkBytes = true } return row } +func readBillingDemoUnitDiagnosticOnly(unit string) bool { + switch unit { + case readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: + return true + default: + return false + } +} + func appendExplainRUNote(note, extra string) string { if note == "" { return extra diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 7bfa7bdb051b3..c0f199f2e201c 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -5719,7 +5719,7 @@ func (b *PlanBuilder) buildExplainPlan(targetPlan base.Plan, format string, expl if format == types.ExplainFormatRU && analyze { // Secondary guard for callers that already have a target plan. The main // SQL path below rejects before optimization or no-delay execution. - if status := explainRUSelectGateStatus(execStmt); status != explainRUStatusSuccess { + if status := explainRUTargetGateStatus(execStmt); status != explainRUStatusSuccess { recordExplainRUStatus(status) return nil, explainRUError(status) } @@ -5831,7 +5831,7 @@ func (b *PlanBuilder) buildExplain(ctx context.Context, explain *ast.ExplainStmt recordExplainRUStatus(explainRUStatusUnsupportedNonSelect) return nil, explainRUError(explainRUStatusUnsupportedNonSelect) } - if status := explainRUSelectGateStatus(explain.Stmt); status != explainRUStatusSuccess { + if status := explainRUTargetGateStatus(explain.Stmt); status != explainRUStatusSuccess { recordExplainRUStatus(status) return nil, explainRUError(status) } diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index 984380d5216d2..9762bae67e186 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -417,6 +417,7 @@ func TestRUV2MetricsSnapshotCalculateRUValues(t *testing.T) { metrics.AddResourceManagerWriteCnt(31) metrics.AddWriteKeys(3) metrics.AddWriteSize(66) + metrics.AddPrewriteRegionNum(4) metrics.AddSessionParserTotal(37) metrics.AddTxnCnt(41) metrics.AddTiKVKVEngineCacheMiss(43) @@ -438,6 +439,7 @@ func TestRUV2MetricsSnapshotCalculateRUValues(t *testing.T) { require.InEpsilon(t, 181980.2851783309, totalRU, 0.01) require.Equal(t, int64(3), metrics.WriteKeys()) require.Equal(t, int64(66), metrics.WriteSize()) + require.Equal(t, int64(4), metrics.PrewriteRegionNum()) t.Run("zero scale stays zero", func(t *testing.T) { zeroScaleWeights := weights @@ -480,26 +482,31 @@ func TestUpdateRUV2MetricsFromCommitDetails(t *testing.T) { beforeRU := metrics.CalculateRUValues(weights) UpdateRUV2MetricsFromCommitDetails(metrics, &util.CommitDetails{ - WriteKeys: 3, - WriteSize: 66, + WriteKeys: 3, + WriteSize: 66, + PrewriteRegionNum: 4, }) require.Equal(t, int64(3), metrics.WriteKeys()) require.Equal(t, int64(66), metrics.WriteSize()) + require.Equal(t, int64(4), metrics.PrewriteRegionNum()) require.InEpsilon(t, beforeRU+float64(3)*weights.WriteKeys*weights.RUScale, metrics.CalculateRUValues(weights), 0.01) detail := FormatRUV2Metrics(metrics, weights, 0, 0) require.Contains(t, detail, "write_keys:3") require.Contains(t, detail, "write_size:66") + require.Contains(t, detail, "prewrite_region_num:4") bypassed := NewRUV2Metrics() bypassed.SetBypass(true) UpdateRUV2MetricsFromCommitDetails(bypassed, &util.CommitDetails{ - WriteKeys: 1, - WriteSize: 2, + WriteKeys: 1, + WriteSize: 2, + PrewriteRegionNum: 3, }) require.Zero(t, bypassed.WriteKeys()) require.Zero(t, bypassed.WriteSize()) + require.Zero(t, bypassed.PrewriteRegionNum()) } func TestRUV2MetricsSnapshotFreezesRUValues(t *testing.T) { diff --git a/pkg/util/execdetails/ruv2_metrics.go b/pkg/util/execdetails/ruv2_metrics.go index 0f6f861cd0f47..08ae039bd2416 100644 --- a/pkg/util/execdetails/ruv2_metrics.go +++ b/pkg/util/execdetails/ruv2_metrics.go @@ -160,6 +160,9 @@ func UpdateRUV2MetricsFromCommitDetails(metrics *RUV2Metrics, commitDetails *tik if commitDetails.WriteSize != 0 { metrics.AddWriteSize(int64(commitDetails.WriteSize)) } + if prewriteRegionNum := atomic.LoadInt32(&commitDetails.PrewriteRegionNum); prewriteRegionNum != 0 { + metrics.AddPrewriteRegionNum(int64(prewriteRegionNum)) + } } // RUV2Metrics stores statement-level RUv2 metrics. @@ -193,6 +196,7 @@ type ruv2MetricsExtra struct { resourceManagerWriteCnt int64 writeKeys int64 writeSize int64 + prewriteRegionNum int64 tikvCoprocessorExecutorIterations int64 tikvCoprocessorResponseBytes int64 @@ -400,6 +404,15 @@ func (m *RUV2Metrics) AddWriteSize(delta int64) { atomic.AddInt64(&m.ensureExtra().writeSize, delta) } +// AddPrewriteRegionNum records commit prewrite region fanout for RUv2 diagnostics. +func (m *RUV2Metrics) AddPrewriteRegionNum(delta int64) { + if m.Bypass() { + return + } + metrics.RUV2PrewriteRegionNum.Add(float64(delta)) + atomic.AddInt64(&m.ensureExtra().prewriteRegionNum, delta) +} + // AddTiKVKVEngineCacheMiss records TiKV kv_engine_cache_miss counters from ExecDetailsV2. func (m *RUV2Metrics) AddTiKVKVEngineCacheMiss(delta int64) { if m.Bypass() { @@ -680,6 +693,7 @@ func cloneRUV2MetricsExtra(dst, src *ruv2MetricsExtra) { addRUV2FixedCounter(&dst.resourceManagerWriteCnt, atomic.LoadInt64(&src.resourceManagerWriteCnt)) addRUV2FixedCounter(&dst.writeKeys, atomic.LoadInt64(&src.writeKeys)) addRUV2FixedCounter(&dst.writeSize, atomic.LoadInt64(&src.writeSize)) + addRUV2FixedCounter(&dst.prewriteRegionNum, atomic.LoadInt64(&src.prewriteRegionNum)) addRUV2FixedCounter(&dst.tikvCoprocessorExecutorIterations, atomic.LoadInt64(&src.tikvCoprocessorExecutorIterations)) addRUV2FixedCounter(&dst.tikvCoprocessorResponseBytes, atomic.LoadInt64(&src.tikvCoprocessorResponseBytes)) addRUV2FixedCounter(&dst.tikvRaftstoreStoreWriteTriggerWB, atomic.LoadInt64(&src.tikvRaftstoreStoreWriteTriggerWB)) @@ -793,6 +807,15 @@ func (m *RUV2Metrics) WriteSize() int64 { return atomic.LoadInt64(&extra.writeSize) } +// PrewriteRegionNum returns commit prewrite region fanout for RUv2 diagnostics. +func (m *RUV2Metrics) PrewriteRegionNum() int64 { + extra := m.loadExtra() + if extra == nil { + return 0 + } + return atomic.LoadInt64(&extra.prewriteRegionNum) +} + // TiKVKVEngineCacheMiss returns TiKV kv_engine_cache_miss counters from ExecDetailsV2. func (m *RUV2Metrics) TiKVKVEngineCacheMiss() int64 { if m == nil { @@ -869,6 +892,7 @@ func (m *RUV2Metrics) IsZero() bool { m.ResourceManagerWriteCnt() == 0 && m.WriteKeys() == 0 && m.WriteSize() == 0 && + m.PrewriteRegionNum() == 0 && m.TiKVCoprocessorExecutorIterations() == 0 && m.TiKVCoprocessorResponseBytes() == 0 && m.TiKVRaftstoreStoreWriteTriggerWB() == 0 && @@ -949,6 +973,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas resourceManagerWriteCnt int64 writeKeys int64 writeSize int64 + prewriteRegionNum int64 tiKVKVEngineCacheMiss int64 tiKVCoprocessorExecutorIterations int64 tiKVCoprocessorResponseBytes int64 @@ -969,6 +994,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas resourceManagerWriteCnt = atomic.LoadInt64(&extra.resourceManagerWriteCnt) writeKeys = atomic.LoadInt64(&extra.writeKeys) writeSize = atomic.LoadInt64(&extra.writeSize) + prewriteRegionNum = atomic.LoadInt64(&extra.prewriteRegionNum) tiKVCoprocessorExecutorIterations = atomic.LoadInt64(&extra.tikvCoprocessorExecutorIterations) tiKVCoprocessorResponseBytes = atomic.LoadInt64(&extra.tikvCoprocessorResponseBytes) tiKVRaftstoreStoreWriteTriggerWB = atomic.LoadInt64(&extra.tikvRaftstoreStoreWriteTriggerWB) @@ -996,6 +1022,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas resourceManagerWriteCnt == 0 && writeKeys == 0 && writeSize == 0 && + prewriteRegionNum == 0 && tiKVKVEngineCacheMiss == 0 && tiKVCoprocessorExecutorIterations == 0 && tiKVCoprocessorResponseBytes == 0 && @@ -1046,6 +1073,7 @@ func FormatRUV2Summary(metrics *RUV2Metrics, weights RUV2Weights, tiKVRU, tiFlas appendInt("resource_manager_write_cnt", resourceManagerWriteCnt) appendInt("write_keys", writeKeys) appendInt("write_size", writeSize) + appendInt("prewrite_region_num", prewriteRegionNum) appendInt("tikv_kv_engine_cache_miss", tiKVKVEngineCacheMiss) appendInt("tikv_coprocessor_executor_iterations", tiKVCoprocessorExecutorIterations) appendInt("tikv_coprocessor_response_bytes", tiKVCoprocessorResponseBytes) From cff80e86df6c44382433db4c8f428895644ded5c Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 10 Jul 2026 05:06:02 +0800 Subject: [PATCH 13/25] planner: add TiDB KV mutation preview RU --- .../references/executor-case-map.md | 2 +- .../references/planner-case-map.md | 2 +- .../references/session-case-map.md | 2 +- .../references/util-case-map.md | 5 +- ...7-08-write-billing-units-weights-design.md | 5 + .../2026-07-10-preview-ru-tidb-kv-mutation.md | 133 ++++ pkg/executor/explain_test.go | 367 +++++++++- pkg/infoschema/tables.go | 1 + pkg/planner/core/BUILD.bazel | 1 + pkg/planner/core/common_plans_test.go | 113 ++- pkg/planner/core/explain_ru.go | 652 +++++++++++++----- pkg/session/BUILD.bazel | 1 + pkg/session/session.go | 24 +- pkg/session/tidb_test.go | 135 ++++ pkg/session/txn.go | 80 +++ pkg/sessionctx/stmtctx/stmtctx.go | 84 +++ pkg/sessionctx/vardef/tidb_vars.go | 2 +- pkg/sessionctx/variable/session.go | 2 +- pkg/util/stmtsummary/BUILD.bazel | 2 +- pkg/util/stmtsummary/read_billing.go | 8 + pkg/util/stmtsummary/read_billing_test.go | 30 + pkg/util/stmtsummary/reader.go | 2 + pkg/util/stmtsummary/v2/reader.go | 2 + pkg/util/stmtsummary/v2/reader_test.go | 18 +- pkg/util/stmtsummary/v2/record_test.go | 3 + 25 files changed, 1451 insertions(+), 225 deletions(-) create mode 100644 docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index 1a14fcc5a97cd..51ceb482efd36 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -29,7 +29,7 @@ - `pkg/executor/executor_failpoint_test.go` - executor: Tests TiDB last txn info commit mode. - `pkg/executor/executor_pkg_test.go` - executor: Tests build KV ranges for index join without CWC. - `pkg/executor/executor_required_rows_test.go` - executor: Tests limit required rows. -- `pkg/executor/explain_test.go` - executor: Tests explain analyze memory. +- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including preview RU DML mutation/write operators, exact index and optimistic-retry mutation counts, join/sort/read-tree preservation, operator-level unavailable-TiKV status rows, restricted/feature-off isolation, RUv2 A/B equivalence, explicit-COMMIT ownership, and diagnostic compatibility. - `pkg/executor/explain_unit_test.go` - executor: Tests explain analyze invoke next and close. - `pkg/executor/explainfor_test.go` - executor: Tests explain for. - `pkg/executor/grant_test.go` - executor: Tests grant global. diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index f7e62a9cd68c3..4fdadb9be9211 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests LOAD DATA line/field defaults. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, weights, DML mutation/write and explicit-COMMIT payload construction, pipelined payload suppression, and ancillary-work partial diagnostics. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/.agents/skills/tidb-test-guidelines/references/session-case-map.md b/.agents/skills/tidb-test-guidelines/references/session-case-map.md index 944b913714e9e..64df94f729bf7 100644 --- a/.agents/skills/tidb-test-guidelines/references/session-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/session-case-map.md @@ -13,7 +13,7 @@ - `pkg/session/bootstrap_test.go` - session: Tests MySQL DB tables. - `pkg/session/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/session/session_test.go` - session: Tests get start mode. -- `pkg/session/tidb_test.go` - session: Tests do map handle nil. +- `pkg/session/tidb_test.go` - session: Tests domain-map basics, RUv2 isolation, all four preview KV mutation APIs including failed attempts, and pipelined COMMIT diagnostics. - `pkg/session/upgrade_test.go` - session: Tests upgrade to ver functions check. ## pkg/session/cursor diff --git a/.agents/skills/tidb-test-guidelines/references/util-case-map.md b/.agents/skills/tidb-test-guidelines/references/util-case-map.md index 95410e5b6185f..6b1d1b380ae2b 100644 --- a/.agents/skills/tidb-test-guidelines/references/util-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/util-case-map.md @@ -528,6 +528,7 @@ ### Tests - `pkg/util/stmtsummary/evicted_test.go` - util/stmtsummary: Tests map to evicted count datum. - `pkg/util/stmtsummary/main_test.go` - Configures default goleak settings and registers testdata. +- `pkg/util/stmtsummary/read_billing_test.go` - util/stmtsummary: Tests preview RU dimension aggregation, caps, and DML-kind separation. - `pkg/util/stmtsummary/statement_summary_test.go` - util/stmtsummary: Tests set up. ## pkg/util/stmtsummary/v2 @@ -535,8 +536,8 @@ ### Tests - `pkg/util/stmtsummary/v2/column_test.go` - util/stmtsummary/v2: Tests column. - `pkg/util/stmtsummary/v2/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/util/stmtsummary/v2/reader_test.go` - util/stmtsummary/v2: Tests time range overlap. -- `pkg/util/stmtsummary/v2/record_test.go` - util/stmtsummary/v2: Tests stmt record. +- `pkg/util/stmtsummary/v2/reader_test.go` - util/stmtsummary/v2: Tests time ranges plus current/history preview RU rows and backward-compatible DML kind reads. +- `pkg/util/stmtsummary/v2/record_test.go` - util/stmtsummary/v2: Tests statement records, including preview RU DML kind aggregation and JSON round trips. - `pkg/util/stmtsummary/v2/stmtsummary_benchmark_test.go` - util/stmtsummary/v2: Tests stmt summary add single workload. - `pkg/util/stmtsummary/v2/stmtsummary_test.go` - util/stmtsummary/v2: Tests stmt window. diff --git a/docs/design/2026-07-08-write-billing-units-weights-design.md b/docs/design/2026-07-08-write-billing-units-weights-design.md index d5df0f6b026d7..9b58f00218205 100644 --- a/docs/design/2026-07-08-write-billing-units-weights-design.md +++ b/docs/design/2026-07-08-write-billing-units-weights-design.md @@ -1,5 +1,10 @@ # TiDB 写入计费 Unit 和初始权重设计 +> **已被取代。** 本文保留为早期多 opclass / rows-cells 方案的历史记录， +> 不再描述当前 preview RU 写侧实现。当前收敛模型见 +> [Preview RU TiDB KV mutation operator](2026-07-10-preview-ru-tidb-kv-mutation.md)。 +> 不得把本文的 L5、rows/cells 或 seed 权重用于 `tidb/kv_mutation`。 + - Author(s): Codex - Discussion PR: N/A - Tracking Issue: N/A diff --git a/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md b/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md new file mode 100644 index 0000000000000..8882457a2ec57 --- /dev/null +++ b/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md @@ -0,0 +1,133 @@ +# Preview RU TiDB KV mutation operator + +## Status and scope + +This document defines the write-side model implemented by the preview RU demo. +It does not change production RUv2 charging, reporting, or configuration. + +The model intentionally covers only the narrow TiDB work that constructs and +encodes record/index KVs and prepares foreground transaction MemDB mutations. +Scan, reader, filter, projection, join, sort, row preparation, comparisons, +constraints, foreign keys, and auto-ID work remain separate plan or future +synthetic operators. Missing coverage is reported as partial; it is not folded +into a rows, cells, or DML-specific mutation formula. + +## Operators and formulas + +TiDB statement-local mutation work is represented by one shared operator: + +```text +site=tidb +op_class=kv_mutation +operator_kind=memdb_mutation +scope=statement_attempted +source=stmt_memdb_mutation_calls + +RU_tidb_mutation = encoded_mutation_count * tidb_mutation_count_weight + + encoded_mutation_bytes * tidb_mutation_byte_weight +``` + +All INSERT, UPDATE, DELETE, INSERT IGNORE, and UPSERT paths use this same weight +key. `dml_kind` is a diagnostic statement-summary dimension and does not +participate in `(site, op_class, weight_version)` lookup. The corresponding +`DML_KIND` information-schema column is appended after the existing base-unit +columns so older column ordinals remain stable. + +The mutation weights are independent calibration slots. They are currently +zero and output is explicitly marked `uncalibrated`; they do not alias RUv2 L5, +RUv2 write-key, scan-byte, or TiKV coefficients. Shadow units `set_count`, +`delete_count`, `key_bytes`, and `value_bytes` have zero weight. + +TiKV transaction payload remains separate: + +```text +site=tikv +op_class=kv_write +operator_kind=txn_prewrite +scope=txn_prewrite_payload +source=commit_detail + +RU_tikv_write = write_keys * tikv_write_key_weight + + write_byte * tikv_write_byte_weight +``` + +`write_byte` is retained for preview-output compatibility and is annotated with +the semantic name `write_bytes`. Region and write-RPC counts remain zero-weight +diagnostics. TiKV coefficients are not used by the TiDB mutation operator. + +## Exact mutation semantics + +- Each attempted `Set`, `SetWithFlags`, `Delete`, or `DeleteWithFlags` counts + once after key/value encoding and before the underlying MemDB call. +- Set bytes are `len(key)+len(value)`; delete bytes are `len(key)`. +- A failed MemDB call still counts because encoding and call preparation have + already occurred. +- Same-key overwrites, pessimistic statement retries, and mutations later + removed by staging cleanup or rollback remain counted. +- `UpdateFlags`, staging release/cleanup, lock-only operations, commit-time net + mutations, and local-temporary-table apply copies are not counted as new + foreground encoded mutations. +- The recorder lives on `StatementContext`. It is allocated only when the demo + flag is on or `EXPLAIN ANALYZE FORMAT='RU'` explicitly requests preview data. +- Both restricted-SQL entry points execute through the internal-statement path, + including their current-session form, so internal/meta DML cannot become a + foreground mutation sample merely because the preview flag is enabled. + +The transaction wrapper dynamically resolves the current statement context. +This preserves per-statement attribution during optimistic history replay. +Because a statement summary already emitted before a later optimistic COMMIT +cannot be rewritten, retryable explicit transactions are marked partial for +optimistic replay attribution. + +## Availability and lifecycle + +- Autocommit DML normally exposes TiDB mutation and TiKV prewrite operators. +- Explicit-transaction DML always retains statement-local TiDB units and its + read tree. Missing commit detail creates a partial TiKV operator only. The + unavailable operator remains visible in `FORMAT='RU'` as an unweighted + status row with its site, class, scope, and reason; it is not reduced to a + statement-level summary note. + The final COMMIT emits the transaction-scoped TiKV prewrite payload with an + empty `dml_kind`; it is never guessed or backfilled onto earlier DML. +- No-op/zero-match DML emits both TiDB primary units with value zero. A missing + TiKV commit detail is partial rather than a successful zero payload. +- Pipelined mutation units remain valid, while missing TiKV logical flush + payload is reported as `pipelined_tikv_payload_unsupported`. Pipelined + transactions allocate a `CommitDetails` object without populating its + `WriteKeys/WriteSize`; those empty fields are not published as zero units. +- Deprecated batch DML keeps one statement recorder across its internal + transaction switches. The TiDB mutation count therefore includes every + batch exactly once; available commit details continue to accumulate on the + statement context. +- Ancillary DML CPU is currently partial for every supported DML kind. The gate + cannot yet distinguish a simple DELETE from one that performs unmodeled FK + checks or cascade orchestration, so DELETE is conservatively partial too. + REPLACE remains rejected by the existing gate. IMPORT INTO and other + non-foreground mutation paths are not represented by this operator. +- Local temporary table encoding is counted once on the foreground transaction + MemDB. Its later copy to session temporary data does not pass through the + recorder, and TiKV commit detail is unavailable/partial. + +## Plan-tree composition + +DML is flattened before result construction. The DML root is a non-billable +wrapper, while its SELECT/CTE/scalar-subquery and scan/filter/reader descendants +continue through the existing preview operator classifier. The mutation and +available TiKV write operators are appended once. Summary RU is the sum of all +weighted unit rows. An unsupported node or a node with missing runtime evidence +is emitted as an unweighted partial status row, and traversal continues so it +cannot hide independently usable descendants. These diagnostic rows do not +enter the summary total. + +## Primary code locations + +- Recorder and snapshot: `pkg/sessionctx/stmtctx/stmtctx.go` +- MemDB/transaction interception and statement lifecycle: + `pkg/session/txn.go`, `pkg/session/session.go` +- Preview composition, weights, status, and output: + `pkg/planner/core/explain_ru.go` +- Structured `dml_kind` diagnostic: + `pkg/util/stmtsummary/read_billing.go`, `pkg/infoschema/tables.go` +- Regression coverage: + `pkg/session/tidb_test.go`, `pkg/planner/core/common_plans_test.go`, + `pkg/executor/explain_test.go` diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 3a05de01482df..79061169702df 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" @@ -686,39 +687,81 @@ func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorC require.Failf(t, "missing weighted FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) } -func requireExplainRUWriteRows(t *testing.T, rows [][]any, operatorKind string) { +func requireExplainRUDMLRows(t *testing.T, rows [][]any, dmlKind string, requireReadTree, requireTiKV, requireMutations bool) { t.Helper() require.NotEmpty(t, rows) require.Len(t, rows[0], 17) require.Equal(t, "summary", rows[0][0]) require.Equal(t, "total_preview_ru", rows[0][2]) require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v1") + require.Contains(t, fmt.Sprint(rows[0][16]), "mutation_weights_uncalibrated=true") + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_dml_ancillary_work_partial") summaryPreviewRU, err := strconv.ParseFloat(fmt.Sprint(rows[0][14]), 64) require.NoError(t, err) - var writeKeysRows, writeByteRows int + var mutationCountRows, mutationByteRows, writeKeysRows, writeByteRows, partialWriteRows, readRows int + var mutationCount float64 var planPreviewRU float64 + seenPlanUnits := make(map[string]struct{}) for _, row := range rows { require.Len(t, row, 17) - if row[0] != "plan" || row[2] != operatorKind || row[3] != "tikv/kv_write" { + if row[0] != "plan" { + continue + } + if unit := fmt.Sprint(row[11]); unit != "" { + key := fmt.Sprintf("%s/%s/%s/%s/%s", row[1], row[3], unit, row[15], row[16]) + require.NotContains(t, seenPlanUnits, key, "DML plan/unit row must not be emitted twice: %v", row) + seenPlanUnits[key] = struct{}{} + } + if fmt.Sprint(row[14]) != "" { + previewRU, err := strconv.ParseFloat(fmt.Sprint(row[14]), 64) + require.NoError(t, err) + planPreviewRU += previewRU + } + if row[3] != "tidb/kv_mutation" && row[3] != "tikv/kv_write" { + readRows++ + continue + } + require.Contains(t, fmt.Sprint(row[16]), "dml_kind="+dmlKind) + if row[3] == "tikv/kv_write" && row[11] == "" { + partialWriteRows++ + require.Contains(t, fmt.Sprint(row[16]), "status=partial") + require.Contains(t, fmt.Sprint(row[16]), "reason=missing_commit_detail") + require.Contains(t, fmt.Sprint(row[16]), "scope=txn_prewrite_payload") continue } - previewRU, err := strconv.ParseFloat(fmt.Sprint(row[14]), 64) - require.NoError(t, err) - planPreviewRU += previewRU switch row[11] { + case "encoded_mutation_count": + mutationCountRows++ + require.Equal(t, "memdb_mutation", row[2]) + require.Equal(t, "tidb/kv_mutation", row[3]) + require.Equal(t, "stmt_memdb_mutation_calls", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "scope=statement_attempted") + require.Contains(t, fmt.Sprint(row[16]), "uncalibrated=true") + mutationCount, err = strconv.ParseFloat(fmt.Sprint(row[12]), 64) + require.NoError(t, err) + case "encoded_mutation_bytes": + mutationByteRows++ + require.Equal(t, "memdb_mutation", row[2]) + require.Equal(t, "tidb/kv_mutation", row[3]) + require.Equal(t, "stmt_memdb_mutation_calls", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "scope=statement_attempted") case "write_keys": writeKeysRows++ + require.Equal(t, "txn_prewrite", row[2]) require.NotEmpty(t, row[12]) require.NotEmpty(t, row[13]) require.NotEmpty(t, row[14]) require.Equal(t, "commit_detail", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "scope=txn_prewrite_payload") case "write_byte": writeByteRows++ + require.Equal(t, "txn_prewrite", row[2]) require.NotEmpty(t, row[10]) require.NotEmpty(t, row[13]) require.NotEmpty(t, row[14]) require.Equal(t, "commit_detail", row[15]) + require.Contains(t, fmt.Sprint(row[16]), "semantic_name=write_bytes") case "prewrite_region_num", "tikv_write_rpc_count": require.NotEmpty(t, row[12]) require.Equal(t, "0.000000", row[13]) @@ -726,9 +769,50 @@ func requireExplainRUWriteRows(t *testing.T, rows [][]any, operatorKind string) require.Contains(t, fmt.Sprint(row[16]), "diagnostic_only=true") } } - require.Equal(t, 1, writeKeysRows, rows) - require.Equal(t, 1, writeByteRows, rows) - require.InEpsilon(t, planPreviewRU, summaryPreviewRU, 0.000001) + require.Equal(t, 1, mutationCountRows, rows) + require.Equal(t, 1, mutationByteRows, rows) + if requireMutations { + require.Positive(t, mutationCount, rows) + } else { + require.Zero(t, mutationCount, rows) + } + if requireReadTree { + require.Positive(t, readRows, rows) + } + if requireTiKV { + require.Equal(t, 1, writeKeysRows, rows) + require.Equal(t, 1, writeByteRows, rows) + } else { + require.Zero(t, writeKeysRows, rows) + require.Zero(t, writeByteRows, rows) + require.Equal(t, 1, partialWriteRows, rows) + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_missing_commit_detail") + } + require.InDelta(t, planPreviewRU, summaryPreviewRU, 0.000001) +} + +func requireExplainRUOperatorClass(t *testing.T, rows [][]any, operatorClass string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" && row[3] == operatorClass { + return + } + } + require.Failf(t, "missing RU operator class", "operatorClass=%s rows=%v", operatorClass, rows) +} + +func explainRUCountUnitValue(t *testing.T, rows [][]any, operatorClass, unit string) float64 { + t.Helper() + for _, row := range rows { + if len(row) != 17 || row[0] != "plan" || row[3] != operatorClass || row[11] != unit { + continue + } + value, err := strconv.ParseFloat(fmt.Sprint(row[12]), 64) + require.NoError(t, err) + return value + } + require.Failf(t, "missing RU count unit", "operatorClass=%s unit=%s rows=%v", operatorClass, unit, rows) + return 0 } func TestExplainAnalyzeFormatRUPlanDigest(t *testing.T) { @@ -768,26 +852,147 @@ func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { tk.MustExec("use test") tk.MustExec("drop table if exists explain_ru_write") tk.MustExec("create table explain_ru_write(a int primary key, b int, c varchar(20), key idx_b(b))") + tk.MustExec("drop table if exists explain_ru_write_src") + tk.MustExec("create table explain_ru_write_src(a int primary key, b int, c varchar(20))") + tk.MustExec("insert into explain_ru_write_src values (2, 20, 'src2'), (3, 30, 'src3')") rows, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write values (1, 10, 'aa')") require.NoError(t, err) - requireExplainRUWriteRows(t, rows, "insert") + requireExplainRUDMLRows(t, rows, "insert", false, true, true) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Zero(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 10 aa")) - rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'bbb' where a = 1") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write select * from explain_ru_write_src where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", true, true, true) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write values (2, 20, 'upsert') on duplicate key update c = values(c)") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "upsert", false, true, true) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert ignore into explain_ru_write values (2, 20, 'ignored')") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert_ignore", false, false, false) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write with cte as (select * from explain_ru_write_src where a = 3) select * from cte") require.NoError(t, err) - requireExplainRUWriteRows(t, rows, "update") - tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 10 bbb")) + requireExplainRUDMLRows(t, rows, "insert", true, true, true) - rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set b = 11 where a = 1") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = (select 'scalar') where a = 3") require.NoError(t, err) - requireExplainRUWriteRows(t, rows, "update") - tk.MustQuery("select a, b, c from explain_ru_write").Check(testkit.Rows("1 11 bbb")) + requireExplainRUDMLRows(t, rows, "update", true, true, true) - rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' delete from explain_ru_write where a = 1") + // Changing an indexed column emits one record Set plus an index Delete/Set. + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set b = 21 where a = 2") require.NoError(t, err) - requireExplainRUWriteRows(t, rows, "delete") - tk.MustQuery("select count(*) from explain_ru_write").Check(testkit.Rows("0")) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + require.Equal(t, 3.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Equal(t, 1.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'bbb' where b >= 10") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + tk.MustQuery("select a, b, c from explain_ru_write order by a").Check(testkit.Rows("1 10 bbb", "2 21 bbb", "3 30 bbb")) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'never' where a = 999") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, false) + + // A matched but unchanged row only contributes read/compare work. The + // unchanged-key lock bookkeeping must not become a MemDB mutation. + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'bbb' where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, false) + + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' delete from explain_ru_write where b = 10") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "delete", true, true, true) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Zero(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + tk.MustQuery("select a, b, c from explain_ru_write order by a").Check(testkit.Rows("2 21 bbb", "3 30 bbb")) + + // Keep the join, projection, sort, reader, filter, and scan descendants of + // INSERT SELECT visible alongside the shared mutation/write operators. + tk.MustExec("create table explain_ru_write_join_dst(a int primary key, b int, c varchar(20))") + tk.MustExec("create table explain_ru_write_join_dim(a int primary key, d int)") + tk.MustExec("insert into explain_ru_write_join_dim values (2, 200), (3, 300)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_join_dst select /*+ hash_join(s, d) */ s.a, s.b, s.c from explain_ru_write_src s join explain_ru_write_join_dim d on s.a = d.a where d.d >= 200 order by d.d + s.b desc") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", true, true, true) + for _, operatorClass := range []string{ + "tidb/join_hash", + "tidb/full_ordering", + "tidb/projection_eval", + "tidb/reader_receive", + "tikv/filter_eval", + "tikv/kv_range_scan", + } { + requireExplainRUOperatorClass(t, rows, operatorClass) + } + tk.MustQuery("select * from explain_ru_write_join_dst order by a").Check(testkit.Rows("2 20 src2", "3 30 src3")) + + // An unsupported reader must make only that operator partial. The scans + // below it still belong to the DML plan tree and must remain in the output. + tk.MustExec("set tidb_enable_index_merge=on") + tk.MustExec("create table explain_ru_write_index_merge(id int primary key, a int, b int, c int, key idx_a(a), key idx_b(b))") + tk.MustExec("insert into explain_ru_write_index_merge values (1, 10, 1, 1), (2, 2, 20, 2)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update /*+ use_index_merge(explain_ru_write_index_merge, idx_a, idx_b) */ explain_ru_write_index_merge set c = c + 1 where a = 10 or b = 20") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, true, true) + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_unsupported_index_merge") + var hasPartialIndexMerge, hasPartialScan bool + for _, row := range rows { + if len(row) != 17 || row[0] != "plan" || !strings.Contains(fmt.Sprint(row[16]), "status=partial") { + continue + } + note := fmt.Sprint(row[16]) + hasPartialIndexMerge = hasPartialIndexMerge || strings.Contains(note, "reason=unsupported_index_merge") + hasPartialScan = hasPartialScan || + (strings.Contains(strings.ToLower(fmt.Sprint(row[2])), "scan") && strings.Contains(note, "reason=missing_scan_detail")) + } + require.True(t, hasPartialIndexMerge, rows) + require.True(t, hasPartialScan, rows) + + tk.MustExec("begin") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'txn' where a = 2") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, true) + + // The next statement gets a fresh recorder even though it shares the same + // explicit transaction and the previous statement attempted mutations. + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'never' where a = 999") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, false) + + tk.MustExec("savepoint explain_ru_write_savepoint") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write set c = 'savepoint' where a = 3") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "update", true, false, true) + tk.MustExec("rollback to savepoint explain_ru_write_savepoint") + tk.MustQuery("select c from explain_ru_write where a = 3").Check(testkit.Rows("bbb")) + tk.MustExec("rollback") + tk.MustQuery("select a, b, c from explain_ru_write order by a").Check(testkit.Rows("2 21 bbb", "3 30 bbb")) + + tk.MustExec("create temporary table explain_ru_write_tmp(a int primary key)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_tmp values (1)") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", false, false, true) + require.Equal(t, 1.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + + originalEnableBatchDML := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_batch_dml").Rows()[0][0]) + defer tk.MustExec("set global tidb_enable_batch_dml=" + originalEnableBatchDML) + tk.MustExec("set global tidb_enable_batch_dml=on") + tk.MustExec("set tidb_batch_insert=on") + tk.MustExec("set tidb_dml_batch_size=1") + tk.MustExec("create table explain_ru_write_batch(a int primary key)") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_batch values (1), (2)") + require.NoError(t, err) + requireExplainRUDMLRows(t, rows, "insert", false, true, true) + require.Equal(t, 2.0, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) } func TestExplainAnalyzeFormatRUUnsupportedTargetsBeforeExecution(t *testing.T) { @@ -877,6 +1082,10 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v2") errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v2") projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v2") + mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v2") + mutationBytes := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v2") + writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_keys", "commit_detail", "all", "v2") + writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_byte", "commit_detail", "all", "v2") tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) require.Equal(t, 0.0, readExecutorCounterValue(t, success)) @@ -890,6 +1099,13 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width = 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_chunk_bytes all v2 v1 1 1 1")) tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) tk.MustQuery(`select column_name from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and ordinal_position = 1`).Check(testkit.Rows("INSTANCE")) + // Append the diagnostic dimension so consumers that depend on the existing + // base-unit column ordinals keep seeing UNIT and every later legacy column + // at the same position. + tk.MustQuery(`select column_name, ordinal_position from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and column_name in ('UNIT', 'DML_KIND') order by ordinal_position`).Check(testkit.Rows( + "UNIT 14", + "DML_KIND 22", + )) tk.MustExec("set tidb_enable_read_billing_demo=on") beforeRestrictedSuccess := readExecutorCounterValue(t, success) @@ -900,17 +1116,63 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, beforeRestrictedSuccess, readExecutorCounterValue(t, success)) require.Equal(t, beforeRestrictedBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + // The statement-taking restricted API must also mark a current-session DML + // as internal. It may modify its target table, but must not become a + // foreground mutation sample merely because the preview flag is enabled. + tk.MustExec("create table read_billing_internal(a int primary key)") + beforeRestrictedSuccess = readExecutorCounterValue(t, success) + beforeRestrictedMutation := readExecutorCounterValue(t, mutationCount) + internalInsert, err := session.ParseWithParams4Test(context.Background(), tk.Session(), "insert into read_billing_internal values (1)") + require.NoError(t, err) + _, _, restrictedErr = tk.Session().GetRestrictedSQLExecutor().ExecRestrictedStmt(internalCtx, internalInsert, sqlexec.ExecOptionUseCurSession) + require.NoError(t, restrictedErr) + tk.MustQuery("select * from read_billing_internal").Check(testkit.Rows("1")) + require.Equal(t, beforeRestrictedSuccess, readExecutorCounterValue(t, success)) + require.Equal(t, beforeRestrictedMutation, readExecutorCounterValue(t, mutationCount)) + + // Preview collection is isolated from production RUv2. Equivalent INSERTs + // with the flag off/on retain identical RUv2 inputs, while only the enabled + // statement publishes preview mutation units. + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustExec("create table read_billing_ruv2_off(a int primary key)") + tk.MustExec("create table read_billing_ruv2_on(a int primary key)") + tk.MustExec("create table read_billing_plain_explain(a int primary key)") + beforeFlagOffMutation := readExecutorCounterValue(t, mutationCount) + tk.MustExec("insert into read_billing_ruv2_off values (1)") + offRUV2 := tk.Session().GetSessionVars().RUV2Metrics.Clone() + require.Nil(t, tk.Session().GetSessionVars().StmtCtx.PreviewKVMutationRecorder) + require.Equal(t, beforeFlagOffMutation, readExecutorCounterValue(t, mutationCount)) + plainExplainRows := tk.MustQuery("explain analyze insert into read_billing_plain_explain values (1)").Rows() + require.NotEmpty(t, plainExplainRows) + require.Nil(t, tk.Session().GetSessionVars().StmtCtx.PreviewKVMutationRecorder) + require.Equal(t, beforeFlagOffMutation, readExecutorCounterValue(t, mutationCount)) + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustExec("insert into read_billing_ruv2_on values (1)") + onRUV2 := tk.Session().GetSessionVars().RUV2Metrics.Clone() + require.Equal(t, offRUV2.ExecutorL5InsertRows(), onRUV2.ExecutorL5InsertRows()) + require.Equal(t, offRUV2.PlanCnt(), onRUV2.PlanCnt()) + require.Equal(t, offRUV2.PlanDeriveStatsPaths(), onRUV2.PlanDeriveStatsPaths()) + require.Equal(t, offRUV2.SessionParserTotal(), onRUV2.SessionParserTotal()) + require.Equal(t, offRUV2.TxnCnt(), onRUV2.TxnCnt()) + require.Equal(t, offRUV2.ResourceManagerReadCnt(), onRUV2.ResourceManagerReadCnt()) + require.Equal(t, offRUV2.ResourceManagerWriteCnt(), onRUV2.ResourceManagerWriteCnt()) + require.Equal(t, offRUV2.WriteKeys(), onRUV2.WriteKeys()) + require.Equal(t, offRUV2.WriteSize(), onRUV2.WriteSize()) + require.Equal(t, offRUV2.PrewriteRegionNum(), onRUV2.PrewriteRegionNum()) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeFlagOffMutation) + tk.MustExec("set tidb_enable_read_billing_demo=off") tk.MustExec("prepare read_billing_demo_stmt from 'select ? + 1'") tk.MustExec("set @read_billing_demo_param := 2") tk.MustExec("set tidb_enable_read_billing_demo=on") + beforePreparedSuccess := readExecutorCounterValue(t, success) tk.MustQuery("execute read_billing_demo_stmt using @read_billing_demo_param").Check(testkit.Rows("3")) - require.Equal(t, 2.0, readExecutorCounterValue(t, success)) + require.Equal(t, beforePreparedSuccess+1, readExecutorCounterValue(t, success)) beforeBaseUnits := readExecutorCounterValue(t, projectionFixedEvents) beforeBaseUnitsTotal := readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) - writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "insert", "write_keys", "commit_detail", "all", "v2") - writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "insert", "write_byte", "commit_detail", "all", "v2") + beforeMutationCount := readExecutorCounterValue(t, mutationCount) + beforeMutationBytes := readExecutorCounterValue(t, mutationBytes) beforeWriteKeys := readExecutorCounterValue(t, writeKeys) beforeWriteByte := readExecutorCounterValue(t, writeByte) beforeWriteSuccess := readExecutorCounterValue(t, success) @@ -919,11 +1181,33 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, beforeUnsupported, readExecutorCounterValue(t, unsupported)) require.Equal(t, beforeWriteSuccess+1, readExecutorCounterValue(t, success)) require.Equal(t, beforeBaseUnits, readExecutorCounterValue(t, projectionFixedEvents)) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeMutationCount) + require.Greater(t, readExecutorCounterValue(t, mutationBytes), beforeMutationBytes) require.Greater(t, readExecutorCounterValue(t, writeKeys), beforeWriteKeys) require.Greater(t, readExecutorCounterValue(t, writeByte), beforeWriteByte) require.Greater(t, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter), beforeBaseUnitsTotal) tk.MustQuery("select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_demo`%' and site = 'statement' group by status, reason").Check(testkit.Rows("success none 1")) - tk.MustQuery("select unit, input_source, input_side, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_demo`%' and site = 'tikv' and op_class = 'kv_write' and operator_kind = 'insert' and unit in ('write_byte', 'write_keys') order by unit").Check(testkit.Rows("write_byte commit_detail all 1", "write_keys commit_detail all 1")) + tk.MustQuery("select site, op_class, operator_kind, dml_kind, unit, input_source, input_side, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_demo`%' and unit in ('encoded_mutation_bytes', 'encoded_mutation_count', 'write_byte', 'write_keys') order by site, unit").Check(testkit.Rows( + "tidb kv_mutation memdb_mutation insert encoded_mutation_bytes stmt_memdb_mutation_calls all 1", + "tidb kv_mutation memdb_mutation insert encoded_mutation_count stmt_memdb_mutation_calls all 1", + "tikv kv_write txn_prewrite insert write_byte commit_detail all 1", + "tikv kv_write txn_prewrite insert write_keys commit_detail all 1", + )) + + // Explicit-transaction DML keeps statement-local TiDB units and reports a + // missing TiKV payload. The final COMMIT owns the transaction-scoped + // prewrite payload without guessing a DML kind or backfilling prior SQL. + tk.MustExec("create table read_billing_explicit_txn(a int primary key)") + tk.MustExec("begin") + tk.MustExec("insert into read_billing_explicit_txn values (1)") + tk.MustQuery("select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_explicit_txn`%' and site = 'tikv'").Check(testkit.Rows("0")) + tk.MustQuery("select status, reason from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_explicit_txn`%' and site = 'tikv' and op_class = 'kv_write'").Check(testkit.Rows("partial missing_commit_detail")) + tk.MustExec("commit") + tk.MustQuery("select unit, input_source, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'commit' and site = 'tikv' and op_class = 'kv_write' and unit in ('write_byte', 'write_keys') order by unit").Check(testkit.Rows( + "write_byte commit_detail 1", + "write_keys commit_detail 1", + )) + tk.MustQuery("select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'commit' and site = 'tikv' and dml_kind <> ''").Check(testkit.Rows("0")) beforeUnknownInput := readExecutorCounterValue(t, unknownInput) beforeBaseUnits = readExecutorCounterValue(t, projectionFixedEvents) @@ -941,6 +1225,37 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustQuery(`select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?' and site = 'statement' group by status, reason`).Check(testkit.Rows("unknown_input missing_scan_detail 1")) tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?'`).Check(testkit.Rows("0")) + // Inject an error after AddRecord has emitted its MemDB mutation. Statement + // staging cleanup removes the table change but must not erase the attempted + // mutation counters. + tk.MustExec("create table read_billing_fail(a int primary key)") + beforeMutationCount = readExecutorCounterValue(t, mutationCount) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/table/tables/corruptMutations", `return("unknown-command")`)) + err = tk.ExecToErr("insert into read_billing_fail values (1)") + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/table/tables/corruptMutations")) + require.Error(t, err) + tk.MustQuery("select count(*) from read_billing_fail").Check(testkit.Rows("0")) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeMutationCount) + tk.MustQuery("select status, reason from information_schema.statements_summary_read_billing_demo_status where digest_text like 'insert into `read_billing_fail`%' and site = 'statement'").Check(testkit.Rows("error statement_error")) + tk.MustQuery("select dml_kind, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_fail`%' and site = 'tidb' and op_class = 'kv_mutation' and unit = 'encoded_mutation_count'").Check(testkit.Rows("insert 1")) + + // A retryable autocommit failure replays the whole statement. Both the + // original record Set and the replayed Set are attempted local mutation + // work, even though the injected commit error ultimately rolls them back. + tk.MustExec("set tidb_txn_mode='optimistic'") + tk.MustExec("set tidb_retry_limit=1") + tk.MustExec("create table read_billing_retry(a int primary key, b int)") + tk.MustExec("insert into read_billing_retry values (1, 1)") + beforeMutationCount = readExecutorCounterValue(t, mutationCount) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/session/mockCommitError8942", `return(true)`)) + err = tk.ExecToErr("update read_billing_retry set b = b + 1 where a = 1") + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/session/mockCommitError8942")) + require.Error(t, err) + require.True(t, kv.ErrTxnRetryable.Equal(err), err) + require.Equal(t, beforeMutationCount+2, readExecutorCounterValue(t, mutationCount)) + tk.MustQuery("select b from read_billing_retry where a = 1").Check(testkit.Rows("1")) + tk.MustQuery("select dml_kind, value from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'update `read_billing_retry`%' and site = 'tidb' and op_class = 'kv_mutation' and unit = 'encoded_mutation_count'").Check(testkit.Rows("update 2")) + tk.MustExec("set tidb_enable_read_billing_demo=off") tk.MustExec("create table read_billing_compile_error(a int)") tk.MustExec("prepare read_billing_compile_error_stmt from 'select * from read_billing_compile_error'") @@ -948,7 +1263,7 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustExec("set tidb_enable_read_billing_demo=on") beforeError := readExecutorCounterValue(t, errorStatus) beforeBaseUnitsTotal = readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter) - err := tk.ExecToErr("execute read_billing_compile_error_stmt") + err = tk.ExecToErr("execute read_billing_compile_error_stmt") require.Error(t, err) require.Equal(t, beforeError+1, readExecutorCounterValue(t, errorStatus)) require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) @@ -974,8 +1289,8 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.NoError(t, rs.Close()) require.Equal(t, beforeEarlyError+2, readExecutorCounterValue(t, errorStatus)) require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) - tk.MustQuery(`select sum(count) from information_schema.statements_summary_read_billing_demo_status where site = 'statement' and status = 'error' and reason = 'statement_error'`).Check(testkit.Rows("3")) - tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units b join information_schema.statements_summary_read_billing_demo_status s on b.digest = s.digest where s.status in ('unsupported', 'error')`).Check(testkit.Rows("0")) + tk.MustQuery(`select sum(count) from information_schema.statements_summary_read_billing_demo_status where site = 'statement' and status = 'error' and reason = 'statement_error'`).Check(testkit.Rows("5")) + tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units b join information_schema.statements_summary_read_billing_demo_status s on b.digest = s.digest where s.status in ('unsupported', 'error') and b.digest_text not like 'insert into ` + "`read_billing_fail`" + `%' and b.digest_text not like 'update ` + "`read_billing_retry`" + `%'`).Check(testkit.Rows("0")) } func readExecutorCounterValue(t *testing.T, counter prometheus.Counter) float64 { diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index a620816e06703..1cfb3c4ef73e9 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -1485,6 +1485,7 @@ var tableStatementsSummaryReadBillingDemoBaseUnitCols = []columnInfo{ {name: stmtsummary.ReadBillingDemoSampleCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Number of unit samples"}, {name: stmtsummary.ReadBillingDemoRowWidthSumStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Sum of row-width samples"}, {name: stmtsummary.ReadBillingDemoAvgRowWidthStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average row-width sample"}, + {name: stmtsummary.ReadBillingDemoDMLKindStr, tp: mysql.TypeVarchar, size: 32, flag: mysql.NotNullFlag, comment: "Diagnostic DML kind; not part of the preview weight key"}, } var tableStatementsSummaryReadBillingDemoStatusCols = []columnInfo{ diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 6732335bba5a2..2e04a1615b1cb 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -301,6 +301,7 @@ go_test( "//pkg/session/sessionapi", "//pkg/session/sessmgr", "//pkg/sessionctx", + "//pkg/sessionctx/stmtctx", "//pkg/sessionctx/vardef", "//pkg/sessionctx/variable", "//pkg/sessiontxn", diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index 1f8da270d47d7..26b5e4962e2e6 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" "github.com/pingcap/tidb/pkg/planner/property" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/mock" @@ -172,6 +173,8 @@ func TestExplainRUTargetGateStatus(t *testing.T) { }{ {"explain analyze format='ru' select 1", explainRUStatusSuccess}, {"explain analyze format='ru' insert into t values (1)", explainRUStatusSuccess}, + {"explain analyze format='ru' insert ignore into t values (1)", explainRUStatusSuccess}, + {"explain analyze format='ru' insert into t values (1) on duplicate key update a = values(a)", explainRUStatusSuccess}, {"explain analyze format='ru' update t set a = 2 where a = 1", explainRUStatusSuccess}, {"explain analyze format='ru' delete from t where a = 1", explainRUStatusSuccess}, {"explain analyze format='ru' replace into t values (1)", explainRUStatusUnsupportedNonSelect}, @@ -185,6 +188,20 @@ func TestExplainRUTargetGateStatus(t *testing.T) { explain := stmt.(*ast.ExplainStmt) require.Equal(t, tc.expected, explainRUTargetGateStatus(explain.Stmt), tc.sql) } + + for sql, expectedKind := range map[string]string{ + "insert into t values (1)": "insert", + "insert ignore into t values (1)": "insert_ignore", + "insert into t values (1) on duplicate key update a = values(a)": "upsert", + "update t set a = 2 where a = 1": "update", + "delete from t where a = 1": "delete", + } { + stmt, err := parser.New().ParseOneStmt(sql, "", "") + require.NoError(t, err) + kind, ok := explainRUWriteDMLKind(stmt) + require.True(t, ok) + require.Equal(t, expectedKind, kind) + } } func TestExplainRURowFormatting(t *testing.T) { @@ -263,6 +280,19 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { require.Zero(t, previewRU) } + mutationWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassKVMutation, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, mutationWeights.mutationCount) + require.Zero(t, mutationWeights.mutationByte) + require.NotEqual(t, writeWeights.writeKey, mutationWeights.mutationCount) + require.NotEqual(t, writeWeights.writeByte, mutationWeights.mutationByte) + for _, mutationUnit := range []string{readBillingDemoUnitEncodedMutationCount, readBillingDemoUnitEncodedMutationBytes} { + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: mutationUnit, value: 99}, mutationWeights) + require.True(t, ok) + require.Zero(t, weight) + require.Zero(t, previewRU) + } + ctx := mock.NewContext() col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) @@ -393,6 +423,41 @@ func TestExplainRUComponentSnapshotStatusAndWeights(t *testing.T) { } func TestReadBillingDemoWriteDMLResult(t *testing.T) { + ctx := mock.NewContext() + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordSet(5, 7) + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordDelete(3) + for _, dmlKind := range []string{"insert", "update", "delete", "upsert"} { + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&result, ctx, dmlKind) + require.NotEmpty(t, result.operators) + op := result.operators[0] + require.Equal(t, readBillingDemoSiteTiDB, op.site) + require.Equal(t, readBillingDemoOpClassKVMutation, op.opClass) + require.Equal(t, readBillingDemoOperatorMemDBMutation, op.operatorKind) + require.Equal(t, dmlKind, op.dmlKind) + require.Equal(t, readBillingDemoScopeStatementAttempted, op.scope) + units := make(map[string]readBillingDemoUnit) + for _, unit := range op.units { + units[unit.unit] = unit + } + require.Equal(t, 2.0, units[readBillingDemoUnitEncodedMutationCount].value) + require.Equal(t, 15.0, units[readBillingDemoUnitEncodedMutationBytes].value) + weights, ok := readBillingDemoResolveWeights(op.site, op.opClass, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, weights.mutationCount) + require.Zero(t, weights.mutationByte) + require.Contains(t, result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonDMLAncillaryPartial)) + } + + ctx.GetSessionVars().SetInTxn(true) + ctx.GetSessionVars().TxnCtx.CouldRetry = true + retryableResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&retryableResult, ctx, "update") + require.Contains(t, retryableResult.operators, readBillingDemoMutationDiagnostic("update", readBillingDemoReasonOptimisticReplayPartial)) + ctx.GetSessionVars().SetInTxn(false) + ctx.GetSessionVars().TxnCtx.CouldRetry = false + ruv2Metrics := execdetails.NewRUV2Metrics() ruv2Metrics.AddResourceManagerWriteCnt(7) commitDetail := &tikvutil.CommitDetails{ @@ -405,6 +470,8 @@ func TestReadBillingDemoWriteDMLResult(t *testing.T) { require.Equal(t, readBillingDemoReasonNone, result.reason) require.Len(t, result.operators, 1) require.Equal(t, readBillingDemoStatusOperatorOK, result.operators[0].status) + require.Equal(t, readBillingDemoOperatorTxnPrewrite, result.operators[0].operatorKind) + require.Equal(t, "insert", result.operators[0].dmlKind) units := make(map[string]readBillingDemoUnit) for _, unit := range result.operators[0].units { @@ -415,6 +482,20 @@ func TestReadBillingDemoWriteDMLResult(t *testing.T) { require.Equal(t, 2.0, units[readBillingDemoUnitPrewriteRegionNum].value) require.Equal(t, 7.0, units[readBillingDemoUnitTiKVWriteRPCCount].value) + ctx.GetSessionVars().StmtCtx.MergeExecDetails(commitDetail) + commitResult := buildTxnCommitBillingDemoResult(ctx, ruv2Metrics, nil) + require.Equal(t, readBillingDemoStatusSuccess, commitResult.status) + require.NotEmpty(t, commitResult.operators) + require.Equal(t, readBillingDemoStatusOperatorOK, commitResult.operators[0].status) + require.Empty(t, commitResult.operators[0].dmlKind) + require.Equal(t, readBillingDemoScopeTxnPrewritePayload, commitResult.operators[0].scope) + commitUnits := make(map[string]readBillingDemoUnit) + for _, unit := range commitResult.operators[0].units { + commitUnits[unit.unit] = unit + } + require.Equal(t, 3.0, commitUnits[readBillingDemoUnitWriteKeys].value) + require.Equal(t, 66.0, commitUnits[readBillingDemoUnitWriteByte].value) + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) require.InEpsilon(t, 3*readBillingDemoWriteKeyWeight+66*readBillingDemoWriteByteWeight, @@ -445,22 +526,38 @@ func TestReadBillingDemoWriteDMLResult(t *testing.T) { require.Contains(t, rows[0].note, "partial_missing_write_rpc_count") missingResult := buildWriteBillingDemoResultFromDetails("update", nil, ruv2Metrics) - require.Equal(t, readBillingDemoStatusUnknownInput, missingResult.status) - require.Equal(t, readBillingDemoReasonMissingCommitDetail, missingResult.reason) + require.Equal(t, readBillingDemoStatusSuccess, missingResult.status) + require.Len(t, missingResult.operators, 1) + require.Equal(t, readBillingDemoStatusPartial, missingResult.operators[0].status) + require.Equal(t, readBillingDemoReasonMissingCommitDetail, missingResult.operators[0].reason) + require.True(t, missingResult.operators[0].emitStatusRow) + // Pipelined transactions expose a non-nil CommitDetails without logical + // WriteKeys/WriteSize. The incomplete payload must not become billable zero + // units merely because the detail object exists. + pipelinedResult := buildTiKVWriteBillingDemoOperators("update", &tikvutil.CommitDetails{}, ruv2Metrics, true) + require.Len(t, pipelinedResult, 1) + require.Equal(t, readBillingDemoStatusPartial, pipelinedResult[0].status) + require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedResult[0].reason) + require.True(t, pipelinedResult[0].emitStatusRow) + require.Empty(t, pipelinedResult[0].units) + require.Empty(t, buildReadBillingDemoStatementStats(readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + operators: pipelinedResult, + }).BaseUnits) missingWriteKeys := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{WriteSize: 2}, ruv2Metrics) - require.Equal(t, readBillingDemoStatusUnknownInput, missingWriteKeys.status) - require.Equal(t, readBillingDemoReasonMissingWriteKeys, missingWriteKeys.reason) + require.Equal(t, readBillingDemoStatusSuccess, missingWriteKeys.status) + require.Contains(t, missingWriteKeys.operators, readBillingDemoWriteDiagnosticStatus("update", readBillingDemoReasonMissingWriteKeys)) missingWriteByte := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{WriteKeys: 1}, ruv2Metrics) - require.Equal(t, readBillingDemoStatusUnknownInput, missingWriteByte.status) - require.Equal(t, readBillingDemoReasonMissingWriteByte, missingWriteByte.reason) + require.Equal(t, readBillingDemoStatusSuccess, missingWriteByte.status) + require.Contains(t, missingWriteByte.operators, readBillingDemoWriteDiagnosticStatus("update", readBillingDemoReasonMissingWriteByte)) zeroResult := buildWriteBillingDemoResultFromDetails("update", &tikvutil.CommitDetails{}, ruv2Metrics) require.Equal(t, readBillingDemoStatusSuccess, zeroResult.status) - require.Len(t, zeroResult.operators, 1) require.Equal(t, readBillingDemoReasonZeroMutation, zeroResult.operators[0].reason) - require.Empty(t, zeroResult.operators[0].units) + require.Equal(t, 0.0, zeroResult.operators[0].units[0].value) + require.Equal(t, 0.0, zeroResult.operators[0].units[1].value) } func extractExplainRUTestSnapshotStatus(stats *execdetails.RURuntimeStats) explainRUComponentSnapshotStatus { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 720f787d45e53..852e3f1e1b6e3 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -53,87 +53,109 @@ const ( explainRUSectionPlan = "plan" explainRUSourceSummaryTotal = "summary_total" - explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" - explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" - explainRUWidthSourceNotApplicable = "not_applicable" - readBillingDemoModelVersion = "v2" - readBillingDemoWeightVersion = "v1" - readBillingDemoStatusSuccess = "success" - readBillingDemoStatusUnsupported = "unsupported" - readBillingDemoStatusUnknownInput = "unknown_input" - readBillingDemoStatusError = "error" - readBillingDemoStatusOperatorOK = "ok" - readBillingDemoStatusPartial = "partial" - readBillingDemoReasonNone = "none" - readBillingDemoReasonStatementError = "statement_error" - readBillingDemoReasonMissingPlan = "missing_plan" - readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" - readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" - readBillingDemoReasonMissingRuntimeBytes = "missing_runtime_bytes" - readBillingDemoReasonMissingInputBytes = "missing_input_bytes" - readBillingDemoReasonMissingScanDetail = "missing_scan_detail" - readBillingDemoReasonUnsupportedOperator = "unsupported_operator" - readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" - readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" - readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" - readBillingDemoReasonUnsupportedLock = "unsupported_lock" - readBillingDemoReasonNonBillable = "non_billable" - readBillingDemoReasonMissingCommitDetail = "missing_commit_detail" - readBillingDemoReasonMissingWriteKeys = "missing_write_keys" - readBillingDemoReasonMissingWriteByte = "missing_write_byte" - readBillingDemoReasonZeroMutation = "zero_mutation" - readBillingDemoReasonMissingPrewriteRegion = "missing_prewrite_region_num" - readBillingDemoReasonMissingWriteRPCCount = "missing_write_rpc_count" - readBillingDemoSiteStatement = "statement" - readBillingDemoSiteTiDB = "tidb" - readBillingDemoSiteTiKV = "tikv" - readBillingDemoOpClassStatement = "statement" - readBillingDemoOpClassFilter = "filter_eval" - readBillingDemoOpClassProjection = "projection_eval" - readBillingDemoOpClassLimit = "row_limit" - readBillingDemoOpClassTopN = "bounded_topn" - readBillingDemoOpClassSort = "full_ordering" - readBillingDemoOpClassWindow = "window_eval" - readBillingDemoOpClassHashAgg = "agg_hash" - readBillingDemoOpClassStreamAgg = "agg_stream" - readBillingDemoOpClassHashJoin = "join_hash" - readBillingDemoOpClassMergeJoin = "join_merge" - readBillingDemoOpClassLookupJoin = "join_lookup" - readBillingDemoOpClassReaderReceive = "reader_receive" - readBillingDemoOpClassLookupReader = "lookup_reader" - readBillingDemoOpClassOverlayReader = "overlay_reader" - readBillingDemoOpClassMetadataReader = "metadata_reader" - readBillingDemoOpClassPointLookup = "kv_point_lookup" - readBillingDemoOpClassRangeScan = "kv_range_scan" - readBillingDemoOpClassKVWrite = "kv_write" - readBillingDemoOpClassWrapper = "wrapper" - readBillingDemoOpClassSynthetic = "synthetic_source" - readBillingDemoOperatorStatement = "statement" - readBillingDemoUnitFixedEvents = "fixed_events" - readBillingDemoUnitInputRows = "input_rows" - readBillingDemoUnitInputBytes = "input_bytes" - readBillingDemoUnitWriteKeys = "write_keys" - readBillingDemoUnitWriteByte = "write_byte" - readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" - readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" - readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" - readBillingDemoInputSourceScanDetail = "scan_detail" - readBillingDemoInputSourceCommitDetail = "commit_detail" - readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" - readBillingDemoInputSideAll = "all" - readBillingDemoInputSideBuild = "build" - readBillingDemoInputSideProbe = "probe" - readBillingDemoInputSideLeft = "left" - readBillingDemoInputSideRight = "right" + explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" + explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" + explainRUWidthSourceNotApplicable = "not_applicable" + readBillingDemoModelVersion = "v2" + readBillingDemoWeightVersion = "v1" + readBillingDemoStatusSuccess = "success" + readBillingDemoStatusUnsupported = "unsupported" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoStatusError = "error" + readBillingDemoStatusOperatorOK = "ok" + readBillingDemoStatusPartial = "partial" + readBillingDemoReasonNone = "none" + readBillingDemoReasonStatementError = "statement_error" + readBillingDemoReasonMissingPlan = "missing_plan" + readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" + readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" + readBillingDemoReasonMissingRuntimeBytes = "missing_runtime_bytes" + readBillingDemoReasonMissingInputBytes = "missing_input_bytes" + readBillingDemoReasonMissingScanDetail = "missing_scan_detail" + readBillingDemoReasonUnsupportedOperator = "unsupported_operator" + readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" + readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" + readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" + readBillingDemoReasonUnsupportedLock = "unsupported_lock" + readBillingDemoReasonNonBillable = "non_billable" + readBillingDemoReasonMissingCommitDetail = "missing_commit_detail" + readBillingDemoReasonMissingWriteKeys = "missing_write_keys" + readBillingDemoReasonMissingWriteByte = "missing_write_byte" + readBillingDemoReasonZeroMutation = "zero_mutation" + readBillingDemoReasonMissingPrewriteRegion = "missing_prewrite_region_num" + readBillingDemoReasonMissingWriteRPCCount = "missing_write_rpc_count" + readBillingDemoReasonMissingMutationRecorder = "missing_mutation_recorder" + readBillingDemoReasonUncalibratedMutation = "uncalibrated_mutation_weights" + readBillingDemoReasonDMLAncillaryPartial = "dml_ancillary_work_partial" + readBillingDemoReasonPipelinedWritePartial = "pipelined_tikv_payload_unsupported" + readBillingDemoReasonOptimisticReplayPartial = "optimistic_replay_attribution_unsupported" + readBillingDemoSiteStatement = "statement" + readBillingDemoSiteTiDB = "tidb" + readBillingDemoSiteTiKV = "tikv" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOpClassFilter = "filter_eval" + readBillingDemoOpClassProjection = "projection_eval" + readBillingDemoOpClassLimit = "row_limit" + readBillingDemoOpClassTopN = "bounded_topn" + readBillingDemoOpClassSort = "full_ordering" + readBillingDemoOpClassWindow = "window_eval" + readBillingDemoOpClassHashAgg = "agg_hash" + readBillingDemoOpClassStreamAgg = "agg_stream" + readBillingDemoOpClassHashJoin = "join_hash" + readBillingDemoOpClassMergeJoin = "join_merge" + readBillingDemoOpClassLookupJoin = "join_lookup" + readBillingDemoOpClassReaderReceive = "reader_receive" + readBillingDemoOpClassLookupReader = "lookup_reader" + readBillingDemoOpClassOverlayReader = "overlay_reader" + readBillingDemoOpClassMetadataReader = "metadata_reader" + readBillingDemoOpClassPointLookup = "kv_point_lookup" + readBillingDemoOpClassRangeScan = "kv_range_scan" + readBillingDemoOpClassKVMutation = "kv_mutation" + readBillingDemoOpClassKVWrite = "kv_write" + readBillingDemoOpClassWrapper = "wrapper" + readBillingDemoOpClassSynthetic = "synthetic_source" + readBillingDemoOperatorStatement = "statement" + readBillingDemoOperatorMemDBMutation = "memdb_mutation" + readBillingDemoOperatorTxnPrewrite = "txn_prewrite" + readBillingDemoUnitFixedEvents = "fixed_events" + readBillingDemoUnitInputRows = "input_rows" + readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoUnitEncodedMutationCount = "encoded_mutation_count" + readBillingDemoUnitEncodedMutationBytes = "encoded_mutation_bytes" + readBillingDemoUnitSetCount = "set_count" + readBillingDemoUnitDeleteCount = "delete_count" + readBillingDemoUnitKeyBytes = "key_bytes" + readBillingDemoUnitValueBytes = "value_bytes" + readBillingDemoUnitWriteKeys = "write_keys" + readBillingDemoUnitWriteByte = "write_byte" + readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" + readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" + readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" + readBillingDemoInputSourceScanDetail = "scan_detail" + readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" + readBillingDemoInputSourceCommitDetail = "commit_detail" + readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" + readBillingDemoInputSideAll = "all" + readBillingDemoInputSideBuild = "build" + readBillingDemoInputSideProbe = "probe" + readBillingDemoInputSideLeft = "left" + readBillingDemoInputSideRight = "right" + readBillingDemoScopeStatementAttempted = "statement_attempted" + readBillingDemoScopeTxnPrewritePayload = "txn_prewrite_payload" // The first write-side preview formula intentionally has no fixed/RPC/region // terms: write keys use the current scaled RUv2 write-key coefficient, and // write bytes use the existing TiKV range-scan byte coefficient as a // calibration seed because RUv2 only shadows commit WriteSize today. - readBillingDemoWriteRUScale = 2.01 - readBillingDemoRUV2WriteKeysWeight = 0.330760861554226 - readBillingDemoWriteKeyWeight = readBillingDemoWriteRUScale * readBillingDemoRUV2WriteKeysWeight - readBillingDemoWriteByteWeight = 0.000020 + readBillingDemoWriteRUScale = 2.01 + readBillingDemoRUV2WriteKeysWeight = 0.330760861554226 + readBillingDemoWriteKeyWeight = readBillingDemoWriteRUScale * readBillingDemoRUV2WriteKeysWeight + readBillingDemoWriteByteWeight = 0.000020 + // Mutation weights are independent preview calibration slots. They stay at + // zero until mutation-only TiDB CPU calibration supplies a defensible seed; + // they must never alias RUv2 or TiKV write coefficients. + readBillingDemoMutationCountWeight = 0.0 + readBillingDemoMutationByteWeight = 0.0 readBillingDemoDiagnosticZeroWeight = 0.0 ) @@ -149,15 +171,19 @@ type readBillingDemoUnit struct { } type readBillingDemoOperatorResult struct { - id string - site string - opClass string - operatorKind string - status string - reason string - actRows int64 - hasActRows bool - units []readBillingDemoUnit + id string + site string + opClass string + operatorKind string + dmlKind string + scope string + uncalibrated bool + emitStatusRow bool + status string + reason string + actRows int64 + hasActRows bool + units []readBillingDemoUnit } type readBillingDemoResult struct { @@ -167,13 +193,19 @@ type readBillingDemoResult struct { } type readBillingDemoOperatorWeights struct { - fixedEvent float64 - row float64 - byte float64 - writeKey float64 - writeByte float64 - writeRPC float64 - region float64 + fixedEvent float64 + row float64 + byte float64 + mutationCount float64 + mutationByte float64 + setCount float64 + deleteCount float64 + keyByte float64 + valueByte float64 + writeKey float64 + writeByte float64 + writeRPC float64 + region float64 } type readBillingDemoWeightKey struct { @@ -183,15 +215,23 @@ type readBillingDemoWeightKey struct { } var readBillingDemoWeights = map[readBillingDemoWeightKey]readBillingDemoOperatorWeights{ - {readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000020}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion}: {writeKey: readBillingDemoWriteKeyWeight, writeByte: readBillingDemoWriteByteWeight, writeRPC: readBillingDemoDiagnosticZeroWeight, region: readBillingDemoDiagnosticZeroWeight}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000040, byte: 0.000006}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000006}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000008, byte: 0.000002}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000075, byte: 0.000012}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000100, byte: 0.000014}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000065, byte: 0.000010}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion}: {fixedEvent: 0.045, row: 0.000030, byte: 0.000012}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000020}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion}: {writeKey: readBillingDemoWriteKeyWeight, writeByte: readBillingDemoWriteByteWeight, writeRPC: readBillingDemoDiagnosticZeroWeight, region: readBillingDemoDiagnosticZeroWeight}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000040, byte: 0.000006}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000006}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000008, byte: 0.000002}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000075, byte: 0.000012}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000100, byte: 0.000014}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000065, byte: 0.000010}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion}: {fixedEvent: 0.045, row: 0.000030, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassKVMutation, readBillingDemoWeightVersion}: { + mutationCount: readBillingDemoMutationCountWeight, + mutationByte: readBillingDemoMutationByteWeight, + setCount: readBillingDemoDiagnosticZeroWeight, + deleteCount: readBillingDemoDiagnosticZeroWeight, + keyByte: readBillingDemoDiagnosticZeroWeight, + valueByte: readBillingDemoDiagnosticZeroWeight, + }, {readBillingDemoSiteTiDB, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000005}, {readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000020, byte: 0.000004}, {readBillingDemoSiteTiDB, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000006, byte: 0.000001}, @@ -264,6 +304,12 @@ func RecordReadBillingDemoForStatement(sctx sessionctx.Context, plan base.Plan, } func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast.StmtNode, execErr error, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + if _, ok := stmt.(*ast.CommitStmt); ok { + return buildTxnCommitBillingDemoResult(sctx, ruv2Metrics, execErr) + } + if dmlKind, ok := explainRUWriteDMLKind(stmt); ok { + return buildWriteBillingDemoResult(sctx, plan, dmlKind, ruv2Metrics, execErr) + } if execErr != nil { return readBillingDemoFailure(readBillingDemoStatusError, readBillingDemoReasonStatementError) } @@ -273,9 +319,6 @@ func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast. if gateStatus := explainRUTargetGateStatus(stmt); gateStatus != explainRUStatusSuccess { return readBillingDemoFailure(readBillingDemoStatusUnsupported, string(gateStatus)) } - if operatorKind, ok := explainRUWriteDMLKind(stmt); ok { - return buildWriteBillingDemoResult(sctx, operatorKind, ruv2Metrics) - } if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx.RuntimeStatsColl == nil { return readBillingDemoFailure(readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingRuntimeStats) } @@ -306,49 +349,203 @@ func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast. return result } -func buildWriteBillingDemoResult(sctx base.PlanContext, operatorKind string, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { +func buildTxnCommitBillingDemoResult(sctx base.PlanContext, ruv2Metrics *execdetails.RUV2Metrics, execErr error) readBillingDemoResult { + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + if execErr != nil { + result.status = readBillingDemoStatusError + result.reason = readBillingDemoReasonStatementError + } + var commitDetail *tikvutil.CommitDetails + var pipelined bool + if sctx != nil && sctx.GetSessionVars() != nil { + vars := sctx.GetSessionVars() + if ruv2Metrics == nil { + ruv2Metrics = vars.RUV2Metrics + } + if vars.StmtCtx != nil { + commitDetail = vars.StmtCtx.GetExecDetails().CommitDetail + if recorder := vars.StmtCtx.PreviewKVMutationRecorder; recorder != nil { + pipelined = recorder.Snapshot().Pipelined + } + } + } + + // A final COMMIT owns the transaction-scoped TiKV payload. Keep dml_kind + // empty rather than guessing how the payload should be split among prior + // statements in the explicit transaction. + result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators("", commitDetail, ruv2Metrics, pipelined)...) + return result +} + +func buildWriteBillingDemoResult(sctx base.PlanContext, plan base.Plan, dmlKind string, ruv2Metrics *execdetails.RUV2Metrics, execErr error) readBillingDemoResult { + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + } + if execErr != nil { + result.status = readBillingDemoStatusError + result.reason = readBillingDemoReasonStatementError + } + + var commitDetail *tikvutil.CommitDetails + var mutationPipelined bool if sctx != nil && sctx.GetSessionVars() != nil { if ruv2Metrics == nil { ruv2Metrics = sctx.GetSessionVars().RUV2Metrics } if sctx.GetSessionVars().StmtCtx != nil { commitDetail = sctx.GetSessionVars().StmtCtx.GetExecDetails().CommitDetail + if recorder := sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder; recorder != nil { + mutationPipelined = recorder.Snapshot().Pipelined + } } } - return buildWriteBillingDemoResultFromDetails(operatorKind, commitDetail, ruv2Metrics) + + appendReadBillingDemoDMLPlan(&result, sctx, plan) + appendReadBillingDemoMutation(&result, sctx, dmlKind) + result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators(dmlKind, commitDetail, ruv2Metrics, mutationPipelined)...) + return result } -func buildWriteBillingDemoResultFromDetails(operatorKind string, commitDetail *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { - operator := readBillingDemoOperatorResult{ - id: "commit_txn", - site: readBillingDemoSiteTiKV, - opClass: readBillingDemoOpClassKVWrite, - operatorKind: operatorKind, +func appendReadBillingDemoDMLPlan(result *readBillingDemoResult, sctx base.PlanContext, plan base.Plan) { + if plan == nil || sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx == nil { + result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingPlan)) + return } - if commitDetail == nil { - return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingCommitDetail)) + runtimeStats := sctx.GetSessionVars().StmtCtx.RuntimeStatsColl + if runtimeStats == nil { + result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingRuntimeStats)) + return + } + flat := FlattenPhysicalPlan(plan, true) + if flat == nil || len(flat.Main) == 0 || flat.InExplain || flat.InExecute { + result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingPlan)) + return + } + appendTree := func(tree FlatPlanTree) { + appendReadBillingDemoDMLTree(result, runtimeStats, tree) + } + appendTree(flat.Main) + for _, tree := range flat.CTEs { + appendTree(tree) + } + for _, tree := range flat.ScalarSubQueries { + appendTree(tree) } +} - result := readBillingDemoResult{ - status: readBillingDemoStatusSuccess, - reason: readBillingDemoReasonNone, +func readBillingDemoPlanDiagnostic(reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "dml_plan", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassWrapper, + operatorKind: "dml_plan", + emitStatusRow: true, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + +func appendReadBillingDemoMutation(result *readBillingDemoResult, sctx base.PlanContext, dmlKind string) { + operator := readBillingDemoOperatorResult{ + id: "stmt_memdb_mutation", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassKVMutation, + operatorKind: readBillingDemoOperatorMemDBMutation, + dmlKind: dmlKind, + scope: readBillingDemoScopeStatementAttempted, + uncalibrated: true, + } + if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx == nil || + sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder == nil { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonMissingMutationRecorder + result.operators = append(result.operators, operator) + return } + + snapshot := sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.Snapshot() operator.status = readBillingDemoStatusOperatorOK operator.reason = readBillingDemoReasonNone + if snapshot.EncodedMutationCount == 0 { + operator.reason = readBillingDemoReasonZeroMutation + } + operator.units = append(operator.units, + readBillingDemoUnit{unit: readBillingDemoUnitEncodedMutationCount, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.EncodedMutationCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitEncodedMutationBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.EncodedMutationBytes), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitSetCount, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.SetCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitDeleteCount, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.DeleteCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitKeyBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.KeyBytes), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitValueBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.ValueBytes), widthSource: explainRUWidthSourceNotApplicable}, + ) + result.operators = append(result.operators, operator) + result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonUncalibratedMutation)) + // The foreground gate does not currently distinguish simple DML from + // schemas that require unmodeled row preparation, constraint checks, or FK + // orchestration. Keep the encoded mutation units available, but report that + // wider DML CPU as partial for every supported DML kind, including DELETE. + result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonDMLAncillaryPartial)) + vars := sctx.GetSessionVars() + if vars.InTxn() && vars.TxnCtx != nil && vars.TxnCtx.CouldRetry { + result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonOptimisticReplayPartial)) + } +} +func readBillingDemoMutationDiagnostic(dmlKind, reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "stmt_memdb_mutation", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassKVMutation, + operatorKind: readBillingDemoOperatorMemDBMutation, + dmlKind: dmlKind, + scope: readBillingDemoScopeStatementAttempted, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + +func buildWriteBillingDemoResultFromDetails(dmlKind string, commitDetail *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + return readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + operators: buildTiKVWriteBillingDemoOperators(dmlKind, commitDetail, ruv2Metrics, false), + } +} + +func buildTiKVWriteBillingDemoOperators(dmlKind string, commitDetail *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics, pipelined bool) []readBillingDemoOperatorResult { + operator := readBillingDemoOperatorResult{ + id: "commit_txn", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassKVWrite, + operatorKind: readBillingDemoOperatorTxnPrewrite, + dmlKind: dmlKind, + scope: readBillingDemoScopeTxnPrewritePayload, + emitStatusRow: true, + } + // Pipelined transactions allocate CommitDetails, but the logical flush + // payload is not accumulated into WriteKeys/WriteSize. Do not publish those + // empty fields as a complete zero payload merely because the detail exists. + if pipelined { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonPipelinedWritePartial + return []readBillingDemoOperatorResult{operator} + } + if commitDetail == nil { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonMissingCommitDetail + return []readBillingDemoOperatorResult{operator} + } + + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone writeKeys := int64(commitDetail.WriteKeys) writeBytes := int64(commitDetail.WriteSize) if writeKeys == 0 && writeBytes == 0 { operator.reason = readBillingDemoReasonZeroMutation - result.operators = append(result.operators, operator) - return result - } - if writeKeys == 0 { - return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingWriteKeys)) - } - if writeBytes == 0 { - return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, operator.withReason(readBillingDemoReasonMissingWriteByte)) } operator.units = append(operator.units, readBillingDemoUnit{ @@ -366,6 +563,13 @@ func buildWriteBillingDemoResultFromDetails(operatorKind string, commitDetail *t widthSource: explainRUWidthSourceNotApplicable, }, ) + operators := []readBillingDemoOperatorResult{operator} + if writeKeys == 0 && writeBytes > 0 { + operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingWriteKeys)) + } + if writeBytes == 0 && writeKeys > 0 { + operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingWriteByte)) + } prewriteRegionNum := int64(atomic.LoadInt32(&commitDetail.PrewriteRegionNum)) if prewriteRegionNum > 0 { operator.units = append(operator.units, readBillingDemoUnit{ @@ -389,22 +593,24 @@ func buildWriteBillingDemoResultFromDetails(operatorKind string, commitDetail *t widthSource: explainRUWidthSourceNotApplicable, }) } - result.operators = append(result.operators, operator) if prewriteRegionNum == 0 { - result.operators = append(result.operators, readBillingDemoWriteDiagnosticStatus(operatorKind, readBillingDemoReasonMissingPrewriteRegion)) + operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingPrewriteRegion)) } if writeRPCCount == 0 { - result.operators = append(result.operators, readBillingDemoWriteDiagnosticStatus(operatorKind, readBillingDemoReasonMissingWriteRPCCount)) + operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingWriteRPCCount)) } - return result + operators[0] = operator + return operators } -func readBillingDemoWriteDiagnosticStatus(operatorKind, reason string) readBillingDemoOperatorResult { +func readBillingDemoWriteDiagnosticStatus(dmlKind, reason string) readBillingDemoOperatorResult { return readBillingDemoOperatorResult{ id: "commit_txn", site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassKVWrite, - operatorKind: operatorKind, + operatorKind: readBillingDemoOperatorTxnPrewrite, + dmlKind: dmlKind, + scope: readBillingDemoScopeTxnPrewritePayload, status: readBillingDemoStatusPartial, reason: reason, } @@ -430,6 +636,18 @@ func readBillingDemoUnitWeight(weights readBillingDemoOperatorWeights, unit stri return weights.row, true case readBillingDemoUnitInputBytes: return weights.byte, true + case readBillingDemoUnitEncodedMutationCount: + return weights.mutationCount, true + case readBillingDemoUnitEncodedMutationBytes: + return weights.mutationByte, true + case readBillingDemoUnitSetCount: + return weights.setCount, true + case readBillingDemoUnitDeleteCount: + return weights.deleteCount, true + case readBillingDemoUnitKeyBytes: + return weights.keyByte, true + case readBillingDemoUnitValueBytes: + return weights.valueByte, true case readBillingDemoUnitWriteKeys: return weights.writeKey, true case readBillingDemoUnitWriteByte: @@ -478,9 +696,6 @@ func readBillingDemoFailedOperator(status string, op readBillingDemoOperatorResu } func summarizeReadBillingDemoBaseUnits(result readBillingDemoResult) stmtsummary.ReadBillingDemoBaseUnitSummary { - if result.status != readBillingDemoStatusSuccess { - return stmtsummary.ReadBillingDemoBaseUnitSummary{} - } var summary stmtsummary.ReadBillingDemoBaseUnitSummary for _, op := range result.operators { if op.status != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { @@ -549,7 +764,7 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar Reason: opReason, }) } - if status != readBillingDemoStatusSuccess || opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + if opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { continue } for _, unit := range op.units { @@ -559,6 +774,7 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar Site: op.site, OpClass: op.opClass, OperatorKind: op.operatorKind, + DMLKind: op.dmlKind, Unit: unit.unit, InputSource: unit.source, InputSide: unit.side, @@ -581,47 +797,71 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar } func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) (string, readBillingDemoOperatorResult) { - for i, op := range tree { - if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { - continue - } - operator, supported, reason := readBillingDemoClassifyOperator(op) - if !supported { - return readBillingDemoStatusUnsupported, operator.withReason(reason) - } - operator.id = op.ExplainID().String() - if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, tree, i, op, operator); ok { - operator.actRows = actRows - operator.hasActRows = true + for i := range tree { + status, operator := appendReadBillingDemoOperator(result, runtimeStats, tree, i) + if status != readBillingDemoStatusSuccess { + return status, operator } - if !readBillingDemoOperatorBillable(operator) { - operator.status = readBillingDemoStatusOperatorOK - result.operators = append(result.operators, operator.withReason(readBillingDemoReasonNonBillable)) + } + return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} +} + +// appendReadBillingDemoDMLTree keeps every independently usable plan operator. +// A missing or unsupported node makes only that node partial; it must not hide +// supported descendants from the DML read/compute tree. +func appendReadBillingDemoDMLTree(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) { + for i := range tree { + status, operator := appendReadBillingDemoOperator(result, runtimeStats, tree, i) + if status == readBillingDemoStatusSuccess { continue } - var units []readBillingDemoUnit - var missingReason string - var ok bool - if op.IsRoot { - units, missingReason, ok = readBillingDemoRootUnits(runtimeStats, tree, i, op, operator) - } else { - units, missingReason, ok = readBillingDemoCopUnits(runtimeStats, tree, i, operator) - } - if !ok { - if missingReason == "" { - if op.IsRoot { - missingReason = readBillingDemoReasonMissingRuntimeBytes - } else { - missingReason = readBillingDemoReasonMissingScanDetail - } + operator.emitStatusRow = true + operator.status = readBillingDemoStatusPartial + result.operators = append(result.operators, operator) + } +} + +func appendReadBillingDemoOperator(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int) (string, readBillingDemoOperatorResult) { + op := tree[idx] + if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { + return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} + } + operator, supported, reason := readBillingDemoClassifyOperator(op) + operator.id = op.ExplainID().String() + if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, tree, idx, op, operator); ok { + operator.actRows = actRows + operator.hasActRows = true + } + if !supported { + return readBillingDemoStatusUnsupported, operator.withReason(reason) + } + if !readBillingDemoOperatorBillable(operator) { + operator.status = readBillingDemoStatusOperatorOK + result.operators = append(result.operators, operator.withReason(readBillingDemoReasonNonBillable)) + return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} + } + var units []readBillingDemoUnit + var missingReason string + var ok bool + if op.IsRoot { + units, missingReason, ok = readBillingDemoRootUnits(runtimeStats, tree, idx, op, operator) + } else { + units, missingReason, ok = readBillingDemoCopUnits(runtimeStats, tree, idx, operator) + } + if !ok { + if missingReason == "" { + if op.IsRoot { + missingReason = readBillingDemoReasonMissingRuntimeBytes + } else { + missingReason = readBillingDemoReasonMissingScanDetail } - return readBillingDemoStatusUnknownInput, operator.withReason(missingReason) } - operator.status = readBillingDemoStatusOperatorOK - operator.reason = readBillingDemoReasonNone - operator.units = units - result.operators = append(result.operators, operator) + return readBillingDemoStatusUnknownInput, operator.withReason(missingReason) } + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + operator.units = units + result.operators = append(result.operators, operator) return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} } @@ -678,6 +918,8 @@ func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorR } switch op.Origin.TP() { + case plancodec.TypeInsert, plancodec.TypeUpdate, plancodec.TypeDelete: + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassWrapper, operatorKind: operatorKind}, true, "" case plancodec.TypeExchangeReceiver, plancodec.TypeExchangeSender: return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedMPP case plancodec.TypeIndexMerge: @@ -953,7 +1195,7 @@ func recordReadBillingDemoResult(result readBillingDemoResult) { reason = readBillingDemoReasonNone } metrics.RecordReadBillingDemoOperatorStatus(op.site, op.opClass, op.operatorKind, opStatus, reason, readBillingDemoModelVersion) - if status != readBillingDemoStatusSuccess { + if opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { continue } for _, unit := range op.units { @@ -981,7 +1223,8 @@ func (e *Explain) recordExplainRUStatus(status explainRUStatus) { // explainRUTargetGateStatus is the pre-execution safety gate for FORMAT='RU'. // SELECT keeps the side-effect-free checks from the read-side demo; write DML -// is limited to statements whose commit details expose foreground write volume. +// is limited to foreground statements with statement-local mutation recording. +// TiKV commit detail availability is handled independently after execution. func explainRUTargetGateStatus(stmt ast.StmtNode) explainRUStatus { if _, ok := explainRUWriteDMLKind(stmt); ok { return explainRUStatusSuccess @@ -995,6 +1238,12 @@ func explainRUWriteDMLKind(stmt ast.StmtNode) (string, bool) { if x == nil || x.IsReplace { return "", false } + if len(x.OnDuplicate) > 0 { + return "upsert", true + } + if x.IgnoreErr { + return "insert_ignore", true + } return "insert", true case *ast.UpdateStmt: if x == nil { @@ -1193,7 +1442,13 @@ func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus }} totalPreviewRU := 0.0 for _, op := range result.operators { - if op.status != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { + if op.status != readBillingDemoStatusOperatorOK { + if op.emitStatusRow { + rows = append(rows, explainRUReadBillingStatusRow(op)) + } + continue + } + if !readBillingDemoOperatorBillable(op) { continue } weights, hasWeights := readBillingDemoResolveWeights(op.site, op.opClass, readBillingDemoWeightVersion) @@ -1217,12 +1472,42 @@ func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus return rows } +func explainRUReadBillingStatusRow(op readBillingDemoOperatorResult) explainRURow { + row := explainRURow{ + section: explainRUSectionPlan, + id: op.id, + component: op.operatorKind, + operatorClass: op.site + "/" + op.opClass, + note: "weight_version=" + readBillingDemoWeightVersion, + } + row.note = appendExplainRUNote(row.note, "status="+op.status) + if op.reason != "" { + row.note = appendExplainRUNote(row.note, "reason="+op.reason) + } + if op.scope != "" { + row.note = appendExplainRUNote(row.note, "scope="+op.scope) + } + if op.dmlKind != "" { + row.note = appendExplainRUNote(row.note, "dml_kind="+op.dmlKind) + } + if op.hasActRows { + row.actRows = op.actRows + row.hasActRows = true + row.outputRows = op.actRows + row.hasOutputRows = true + } + return row +} + func explainRUReadBillingSummaryNote(snapshotStatus explainRUComponentSnapshotStatus, result readBillingDemoResult) string { note := "weight_version=" + readBillingDemoWeightVersion if snapshotStatus != explainRUComponentSnapshotOK { note = appendExplainRUNote(note, "component_snapshot_"+string(snapshotStatus)) } for _, op := range result.operators { + if op.uncalibrated { + note = appendExplainRUNote(note, "mutation_weights_uncalibrated=true") + } if op.status == readBillingDemoStatusPartial && op.reason != "" { note = appendExplainRUNote(note, "partial_"+op.reason) } @@ -1243,6 +1528,18 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill source: unit.source, note: "input_side=" + unit.side + ",weight_version=" + readBillingDemoWeightVersion, } + if op.scope != "" { + row.note = appendExplainRUNote(row.note, "scope="+op.scope) + } + if op.dmlKind != "" { + row.note = appendExplainRUNote(row.note, "dml_kind="+op.dmlKind) + } + if op.uncalibrated { + row.note = appendExplainRUNote(row.note, "uncalibrated=true") + } + if unit.unit == readBillingDemoUnitWriteByte { + row.note = appendExplainRUNote(row.note, "semantic_name=write_bytes") + } if readBillingDemoUnitDiagnosticOnly(unit.unit) { row.note = appendExplainRUNote(row.note, "diagnostic_only=true") } @@ -1266,10 +1563,12 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill case readBillingDemoUnitInputBytes: row.workBytes = unit.value row.hasWorkBytes = true - case readBillingDemoUnitWriteKeys, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: + case readBillingDemoUnitEncodedMutationCount, readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, + readBillingDemoUnitWriteKeys, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: row.count = int64(unit.value) row.hasCount = true - case readBillingDemoUnitWriteByte: + case readBillingDemoUnitEncodedMutationBytes, readBillingDemoUnitKeyBytes, readBillingDemoUnitValueBytes, + readBillingDemoUnitWriteByte: row.workBytes = unit.value row.hasWorkBytes = true } @@ -1278,7 +1577,8 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill func readBillingDemoUnitDiagnosticOnly(unit string) bool { switch unit { - case readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: + case readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitKeyBytes, + readBillingDemoUnitValueBytes, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: return true default: return false diff --git a/pkg/session/BUILD.bazel b/pkg/session/BUILD.bazel index f1013f3bff3bd..87e9dcd4d6d40 100644 --- a/pkg/session/BUILD.bazel +++ b/pkg/session/BUILD.bazel @@ -190,6 +190,7 @@ go_test( "//pkg/parser/mysql", "//pkg/session/sessionapi", "//pkg/sessionctx", + "//pkg/sessionctx/stmtctx", "//pkg/sessionctx/vardef", "//pkg/sessionctx/variable", "//pkg/statistics", diff --git a/pkg/session/session.go b/pkg/session/session.go index 9b32c7da5f6dc..757fd03c80d98 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -987,6 +987,7 @@ func (s *session) CommitTxn(ctx context.Context) error { defer r.End() s.setLastTxnInfoBeforeTxnEnd() + s.markPreviewKVMutationTxnPipelined() var commitDetail *tikvutil.CommitDetails ctx = context.WithValue(ctx, tikvutil.CommitDetailCtxKey, &commitDetail) err := s.doCommitWithRetry(ctx) @@ -1022,6 +1023,16 @@ func (s *session) CommitTxn(ctx context.Context) error { return err } +func (s *session) markPreviewKVMutationTxnPipelined() { + // Pipelined transactions currently do not populate CommitDetails write + // keys/bytes. Preserve that fact on the COMMIT statement recorder before + // doCommitWithRetry invalidates the transaction. + if recorder := s.sessionVars.StmtCtx.PreviewKVMutationRecorder; recorder != nil && + s.txn.Valid() && s.txn.IsPipelined() { + recorder.MarkPipelined() + } +} + func (s *session) RollbackTxn(ctx context.Context) { r, ctx := tracing.StartRegionEx(ctx, "session.RollbackTxn") defer r.End() @@ -2174,7 +2185,7 @@ func (s *session) ExecRestrictedStmt(ctx context.Context, stmtNode ast.StmtNode, startTime := time.Now() metrics.SessionRestrictedSQLCounter.Inc() ctx = execdetails.ContextWithInitializedExecDetails(ctx) - rs, err := se.ExecuteStmt(ctx, stmtNode) + rs, err := se.ExecuteInternalStmt(ctx, stmtNode) if err != nil { se.sessionVars.StmtCtx.AppendError(err) } @@ -2430,6 +2441,9 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (r r, ctx := tracing.StartRegionEx(ctx, "session.ExecuteStmt") defer r.End() ctx = execdetails.ContextWithMissingExecDetailsInitialized(ctx) + // Prevent a recorder from the previous statement from observing any setup + // work before this statement receives a fresh StatementContext. + s.txn.setPreviewKVMutationCollection(nil, false) readBillingDemoHandledByStmt := false defer func() { if err != nil && !readBillingDemoHandledByStmt { @@ -2452,6 +2466,14 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (r if err := executor.ResetContextOfStmt(s, stmtNode); err != nil { return nil, err } + previewKVMutationEnabled := !s.sessionVars.InRestrictedSQL && + (s.sessionVars.EnableReadBillingDemo || + (s.sessionVars.StmtCtx.InExplainStmt && s.sessionVars.StmtCtx.InExplainAnalyzeStmt && + strings.EqualFold(s.sessionVars.StmtCtx.ExplainFormat, "ru"))) + if previewKVMutationEnabled { + s.sessionVars.StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + } + s.txn.setPreviewKVMutationCollection(s.sessionVars, previewKVMutationEnabled) // ResetContextOfStmt clears SQLKiller, so honor a canceled caller before executing the next statement. if err := ctx.Err(); err != nil { return nil, err diff --git a/pkg/session/tidb_test.go b/pkg/session/tidb_test.go index b72430bfed24d..0939d76aa7b92 100644 --- a/pkg/session/tidb_test.go +++ b/pkg/session/tidb_test.go @@ -16,13 +16,16 @@ package session import ( "context" + "errors" "testing" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/vardef" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/store/mockstore" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/execdetails" @@ -30,6 +33,138 @@ import ( "github.com/stretchr/testify/require" ) +type previewMutationTestBuffer struct { + kv.MemBuffer + setErr error + setWithFlagsErr error + deleteErr error + deleteWithFlagsErr error + nextStaging kv.StagingHandle +} + +func (b *previewMutationTestBuffer) Set(_ kv.Key, _ []byte) error { + return b.setErr +} + +func (b *previewMutationTestBuffer) SetWithFlags(_ kv.Key, _ []byte, _ ...kv.FlagsOp) error { + return b.setWithFlagsErr +} + +func (b *previewMutationTestBuffer) Delete(_ kv.Key) error { + return b.deleteErr +} + +func (b *previewMutationTestBuffer) DeleteWithFlags(_ kv.Key, _ ...kv.FlagsOp) error { + return b.deleteWithFlagsErr +} + +func (*previewMutationTestBuffer) UpdateFlags(_ kv.Key, _ ...kv.FlagsOp) {} + +func (b *previewMutationTestBuffer) Staging() kv.StagingHandle { + b.nextStaging++ + return b.nextStaging +} + +func (*previewMutationTestBuffer) Release(kv.StagingHandle) {} + +func (*previewMutationTestBuffer) Cleanup(kv.StagingHandle) {} + +type previewMutationTestTxn struct { + kv.Transaction + buffer kv.MemBuffer + pipelined bool +} + +func (t *previewMutationTestTxn) Set(key kv.Key, value []byte) error { + return t.buffer.Set(key, value) +} + +func (t *previewMutationTestTxn) Delete(key kv.Key) error { + return t.buffer.Delete(key) +} + +func (t *previewMutationTestTxn) GetMemBuffer() kv.MemBuffer { + return t.buffer +} + +func (t *previewMutationTestTxn) IsPipelined() bool { + return t.pipelined +} + +func (*previewMutationTestTxn) Valid() bool { + return true +} + +func TestPreviewKVMutationRecorder(t *testing.T) { + vars := variable.NewSessionVars(nil) + vars.StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + buffer := &previewMutationTestBuffer{} + innerTxn := &previewMutationTestTxn{buffer: buffer} + txn := &LazyTxn{Transaction: innerTxn} + txn.setPreviewKVMutationCollection(vars, true) + + memBuffer := txn.GetMemBuffer() + require.NoError(t, memBuffer.Set(kv.Key("a"), []byte("12"))) + require.NoError(t, memBuffer.SetWithFlags(kv.Key("bb"), []byte("3"), kv.SetNeedLocked)) + require.NoError(t, memBuffer.Delete(kv.Key("ccc"))) + require.NoError(t, memBuffer.DeleteWithFlags(kv.Key("d"), kv.SetNeedLocked)) + memBuffer.UpdateFlags(kv.Key("flags-only"), kv.SetNeedLocked) + + stage := memBuffer.Staging() + require.NoError(t, memBuffer.Set(kv.Key("same"), []byte("v1"))) + require.NoError(t, memBuffer.Set(kv.Key("same"), []byte("v2"))) + memBuffer.Cleanup(stage) + + // Direct Transaction calls use the same recorder but do not pass through + // the MemBuffer wrapper, so each attempted mutation is counted once. + require.NoError(t, txn.Set(kv.Key("z"), []byte("vv"))) + require.NoError(t, txn.Delete(kv.Key("zz"))) + + buffer.setErr = errors.New("injected set failure") + require.Error(t, memBuffer.Set(kv.Key("fail"), []byte("x"))) + buffer.setWithFlagsErr = errors.New("injected set-with-flags failure") + require.Error(t, memBuffer.SetWithFlags(kv.Key("fail-flags"), []byte("vv"), kv.SetNeedLocked)) + buffer.deleteErr = errors.New("injected delete failure") + require.Error(t, memBuffer.Delete(kv.Key("fail-delete"))) + buffer.deleteWithFlagsErr = errors.New("injected delete-with-flags failure") + require.Error(t, memBuffer.DeleteWithFlags(kv.Key("fail-delete-flags"), kv.SetNeedLocked)) + + // A pessimistic statement retry resets execution state but must retain the + // attempted mutation work from the previous attempt. + vars.StmtCtx.ResetForRetry() + buffer.setErr = nil + buffer.setWithFlagsErr = nil + buffer.deleteErr = nil + buffer.deleteWithFlagsErr = nil + require.NoError(t, memBuffer.Set(kv.Key("retry"), []byte("v"))) + + innerTxn.pipelined = true + require.NoError(t, txn.GetMemBuffer().Set(kv.Key("p"), []byte("v"))) + + snapshot := vars.StmtCtx.PreviewKVMutationRecorder.Snapshot() + require.Equal(t, uint64(14), snapshot.EncodedMutationCount) + require.Equal(t, uint64(80), snapshot.EncodedMutationBytes) + require.Equal(t, uint64(9), snapshot.SetCount) + require.Equal(t, uint64(5), snapshot.DeleteCount) + require.Equal(t, uint64(66), snapshot.KeyBytes) + require.Equal(t, uint64(14), snapshot.ValueBytes) + require.True(t, snapshot.Pipelined) + + // The explicit COMMIT statement has no mutation call through which to + // observe pipelined mode. Preserve the transaction mode before commit + // invalidates the transaction so its TiKV payload is reported partial. + vars.StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + se := &session{sessionVars: vars, txn: *txn} + se.markPreviewKVMutationTxnPipelined() + commitSnapshot := vars.StmtCtx.PreviewKVMutationRecorder.Snapshot() + require.True(t, commitSnapshot.Pipelined) + + // Disabling preview collection restores the original hot path. + txn.setPreviewKVMutationCollection(nil, false) + require.NoError(t, txn.GetMemBuffer().Set(kv.Key("ignored"), []byte("ignored"))) + require.Equal(t, commitSnapshot, vars.StmtCtx.PreviewKVMutationRecorder.Snapshot()) +} + func TestDomapHandleNil(t *testing.T) { // this is required for enterprise plugins // ref: https://github.com/pingcap/tidb/issues/37319 diff --git a/pkg/session/txn.go b/pkg/session/txn.go index 2295510b2cd00..6b3c578cc3176 100644 --- a/pkg/session/txn.go +++ b/pkg/session/txn.go @@ -32,6 +32,8 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/session/txninfo" "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" + "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/sessiontxn" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/util/logutil" @@ -57,6 +59,10 @@ type LazyTxn struct { initCnt int stagingHandle kv.StagingHandle writeSLI sli.TxnWriteThroughputSLI + // previewMutationVars is set only while preview RU collection is enabled. + // Reading the current StmtCtx through SessionVars keeps optimistic replay + // mutations attributed to the statement being replayed. + previewMutationVars *variable.SessionVars enterFairLockingOnValid bool @@ -76,6 +82,80 @@ type LazyTxn struct { lastCommitTS uint64 } +type previewMutationMemBuffer struct { + kv.MemBuffer + recorder *stmtctx.PreviewKVMutationRecorder +} + +func (b *previewMutationMemBuffer) Set(key kv.Key, value []byte) error { + b.recorder.RecordSet(len(key), len(value)) + return b.MemBuffer.Set(key, value) +} + +func (b *previewMutationMemBuffer) SetWithFlags(key kv.Key, value []byte, flags ...kv.FlagsOp) error { + b.recorder.RecordSet(len(key), len(value)) + return b.MemBuffer.SetWithFlags(key, value, flags...) +} + +func (b *previewMutationMemBuffer) Delete(key kv.Key) error { + b.recorder.RecordDelete(len(key)) + return b.MemBuffer.Delete(key) +} + +func (b *previewMutationMemBuffer) DeleteWithFlags(key kv.Key, flags ...kv.FlagsOp) error { + b.recorder.RecordDelete(len(key)) + return b.MemBuffer.DeleteWithFlags(key, flags...) +} + +func (txn *LazyTxn) setPreviewKVMutationCollection(vars *variable.SessionVars, enabled bool) { + if !enabled { + txn.previewMutationVars = nil + return + } + txn.previewMutationVars = vars +} + +func (txn *LazyTxn) previewKVMutationRecorder() *stmtctx.PreviewKVMutationRecorder { + vars := txn.previewMutationVars + if vars == nil || vars.StmtCtx == nil || vars.StmtCtx.PreviewKVMutationRecorder == nil { + return nil + } + recorder := vars.StmtCtx.PreviewKVMutationRecorder + if txn.Transaction != nil && txn.Transaction.IsPipelined() { + recorder.MarkPipelined() + } + return recorder +} + +// GetMemBuffer overrides the embedded transaction method so all foreground +// table record/index Set/Delete calls share one preview recorder. The normal +// production path returns the original MemBuffer directly. +func (txn *LazyTxn) GetMemBuffer() kv.MemBuffer { + buffer := txn.Transaction.GetMemBuffer() + if recorder := txn.previewKVMutationRecorder(); recorder != nil { + return &previewMutationMemBuffer{MemBuffer: buffer, recorder: recorder} + } + return buffer +} + +// Set overrides the embedded transaction method for the few foreground paths +// that write through Transaction rather than GetMemBuffer. +func (txn *LazyTxn) Set(key kv.Key, value []byte) error { + if recorder := txn.previewKVMutationRecorder(); recorder != nil { + recorder.RecordSet(len(key), len(value)) + } + return txn.Transaction.Set(key, value) +} + +// Delete overrides the embedded transaction method for record deletes and +// other foreground paths that write through Transaction directly. +func (txn *LazyTxn) Delete(key kv.Key) error { + if recorder := txn.previewKVMutationRecorder(); recorder != nil { + recorder.RecordDelete(len(key)) + } + return txn.Transaction.Delete(key) +} + // GetTableInfo returns the cached index name. func (txn *LazyTxn) GetTableInfo(id int64) *model.TableInfo { return txn.Transaction.GetTableInfo(id) diff --git a/pkg/sessionctx/stmtctx/stmtctx.go b/pkg/sessionctx/stmtctx/stmtctx.go index 42982b6842ed1..74c6d802f1dad 100644 --- a/pkg/sessionctx/stmtctx/stmtctx.go +++ b/pkg/sessionctx/stmtctx/stmtctx.go @@ -89,6 +89,86 @@ type LogicalPlanBuildState struct { planCacheAlwaysWarn bool } +// PreviewKVMutationSnapshot is a statement-local snapshot of attempted encoded +// mutations sent to the foreground transaction MemDB. It deliberately counts +// calls rather than the final distinct write set, so staging cleanup and +// same-key overwrites do not subtract from these values. +type PreviewKVMutationSnapshot struct { + EncodedMutationCount uint64 + EncodedMutationBytes uint64 + SetCount uint64 + DeleteCount uint64 + KeyBytes uint64 + ValueBytes uint64 + Pipelined bool +} + +// PreviewKVMutationRecorder records attempted transaction MemDB mutation calls +// for the preview RU model. A recorder is allocated only for statements that +// explicitly enable preview collection. +type PreviewKVMutationRecorder struct { + encodedMutationCount atomic.Uint64 + encodedMutationBytes atomic.Uint64 + setCount atomic.Uint64 + deleteCount atomic.Uint64 + keyBytes atomic.Uint64 + valueBytes atomic.Uint64 + pipelined atomic.Bool +} + +// RecordSet records one attempted Set or SetWithFlags call before the MemDB is +// invoked. Recording before the call preserves work when the MemDB returns an +// error. +func (r *PreviewKVMutationRecorder) RecordSet(keyBytes, valueBytes int) { + if r == nil { + return + } + keySize, valueSize := uint64(keyBytes), uint64(valueBytes) + r.encodedMutationCount.Add(1) + r.encodedMutationBytes.Add(keySize + valueSize) + r.setCount.Add(1) + r.keyBytes.Add(keySize) + r.valueBytes.Add(valueSize) +} + +// RecordDelete records one attempted Delete or DeleteWithFlags call before the +// MemDB is invoked. +func (r *PreviewKVMutationRecorder) RecordDelete(keyBytes int) { + if r == nil { + return + } + keySize := uint64(keyBytes) + r.encodedMutationCount.Add(1) + r.encodedMutationBytes.Add(keySize) + r.deleteCount.Add(1) + r.keyBytes.Add(keySize) +} + +// MarkPipelined records that at least one mutation used a pipelined MemDB. The +// TiDB mutation units remain valid, while the current TiKV commit payload source +// is incomplete for this path. +func (r *PreviewKVMutationRecorder) MarkPipelined() { + if r != nil { + r.pipelined.Store(true) + } +} + +// Snapshot returns the current statement-local attempted mutation counters. +func (r *PreviewKVMutationRecorder) Snapshot() PreviewKVMutationSnapshot { + if r == nil { + return PreviewKVMutationSnapshot{} + } + return PreviewKVMutationSnapshot{ + EncodedMutationCount: r.encodedMutationCount.Load(), + EncodedMutationBytes: r.encodedMutationBytes.Load(), + SetCount: r.setCount.Load(), + DeleteCount: r.deleteCount.Load(), + KeyBytes: r.keyBytes.Load(), + ValueBytes: r.valueBytes.Load(), + Pipelined: r.pipelined.Load(), + } +} + // ReferenceCount indicates the reference count of StmtCtx. type ReferenceCount int32 @@ -437,6 +517,10 @@ type StatementContext struct { // always created during each statement execution. KvExecCounter *stmtstats.KvExecCounter + // PreviewKVMutationRecorder is non-nil only while the preview RU model needs + // statement-local foreground transaction MemDB mutation counters. + PreviewKVMutationRecorder *PreviewKVMutationRecorder + // WeakConsistency is true when read consistency is weak and in a read statement and not in a transaction. WeakConsistency bool diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index fda9d39bf8d60..79d8eb50ea491 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -768,7 +768,7 @@ const ( // TiDBEnableCollectExecutionInfo indicates that whether execution info is collected. TiDBEnableCollectExecutionInfo = "tidb_enable_collect_execution_info" - // TiDBEnableReadBillingDemo enables the first-version read billing demo base-unit metrics. + // TiDBEnableReadBillingDemo enables preview RU base-unit metrics for foreground statements. TiDBEnableReadBillingDemo = "tidb_enable_read_billing_demo" // TiDBExecutorConcurrency is used for controlling the concurrency of all types of executors. diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index a0816309e0619..ab40040ee2c25 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1231,7 +1231,7 @@ type SessionVars struct { // AllowProjectionPushDown enables pushdown projection on TiKV. AllowProjectionPushDown bool - // EnableReadBillingDemo indicates whether read billing demo metrics are emitted for SELECT reads. + // EnableReadBillingDemo indicates whether preview RU base-unit metrics are emitted for foreground statements. EnableReadBillingDemo bool // EnableStrictNotNullCheck enables strict not-null check for single-row insert in non-strict mode. diff --git a/pkg/util/stmtsummary/BUILD.bazel b/pkg/util/stmtsummary/BUILD.bazel index dc6bf9ad49e60..9a9d6f68ccbf0 100644 --- a/pkg/util/stmtsummary/BUILD.bazel +++ b/pkg/util/stmtsummary/BUILD.bazel @@ -42,7 +42,7 @@ go_test( ], embed = [":stmtsummary"], flaky = True, - shard_count = 32, + shard_count = 33, deps = [ "//pkg/meta/model", "//pkg/metrics", diff --git a/pkg/util/stmtsummary/read_billing.go b/pkg/util/stmtsummary/read_billing.go index c03c4022f6f62..6ac5980078dc8 100644 --- a/pkg/util/stmtsummary/read_billing.go +++ b/pkg/util/stmtsummary/read_billing.go @@ -36,6 +36,7 @@ const ( ReadBillingDemoSiteStr = "SITE" ReadBillingDemoOpClassStr = "OP_CLASS" ReadBillingDemoOperatorKindStr = "OPERATOR_KIND" + ReadBillingDemoDMLKindStr = "DML_KIND" ReadBillingDemoUnitStr = "UNIT" ReadBillingDemoInputSourceStr = "INPUT_SOURCE" ReadBillingDemoInputSideStr = "INPUT_SIDE" @@ -121,6 +122,7 @@ type ReadBillingDemoBaseUnitSample struct { Site string OpClass string OperatorKind string + DMLKind string Unit string InputSource string InputSide string @@ -147,6 +149,7 @@ type ReadBillingDemoBaseUnitKey struct { Site string OpClass string OperatorKind string + DMLKind string Unit string InputSource string InputSide string @@ -167,6 +170,7 @@ type ReadBillingDemoBaseUnitAggEntry struct { Site string `json:"site"` OpClass string `json:"op_class"` OperatorKind string `json:"operator_kind"` + DMLKind string `json:"dml_kind,omitempty"` Unit string `json:"unit"` InputSource string `json:"input_source"` InputSide string `json:"input_side"` @@ -211,6 +215,7 @@ func makeReadBillingDemoBaseUnitKey(sample ReadBillingDemoBaseUnitSample) ReadBi Site: sample.Site, OpClass: sample.OpClass, OperatorKind: sample.OperatorKind, + DMLKind: sample.DMLKind, Unit: sample.Unit, InputSource: sample.InputSource, InputSide: sample.InputSide, @@ -237,6 +242,7 @@ func (e ReadBillingDemoBaseUnitAggEntry) key() ReadBillingDemoBaseUnitKey { Site: e.Site, OpClass: e.OpClass, OperatorKind: e.OperatorKind, + DMLKind: e.DMLKind, Unit: e.Unit, InputSource: e.InputSource, InputSide: e.InputSide, @@ -305,6 +311,7 @@ func readBillingDemoBaseUnitEntry(key ReadBillingDemoBaseUnitKey, agg ReadBillin Site: key.Site, OpClass: key.OpClass, OperatorKind: key.OperatorKind, + DMLKind: key.DMLKind, Unit: key.Unit, InputSource: key.InputSource, InputSide: key.InputSide, @@ -673,6 +680,7 @@ func sortReadBillingDemoBaseUnitEntries(entries []ReadBillingDemoBaseUnitAggEntr cmp.Compare(i.Site, j.Site), cmp.Compare(i.OpClass, j.OpClass), cmp.Compare(i.OperatorKind, j.OperatorKind), + cmp.Compare(i.DMLKind, j.DMLKind), cmp.Compare(i.Unit, j.Unit), cmp.Compare(i.InputSource, j.InputSource), cmp.Compare(i.InputSide, j.InputSide), diff --git a/pkg/util/stmtsummary/read_billing_test.go b/pkg/util/stmtsummary/read_billing_test.go index 58804863f45d9..72118e9bb4072 100644 --- a/pkg/util/stmtsummary/read_billing_test.go +++ b/pkg/util/stmtsummary/read_billing_test.go @@ -83,6 +83,36 @@ func TestReadBillingDemoAggregationCaps(t *testing.T) { require.Zero(t, acceptedSummary.SumReadBillingDemoInputBytes) } +func TestReadBillingDemoDMLKindAggregation(t *testing.T) { + stats := ReadBillingDemoStatementStats{ModelVersion: "v2", WeightVersion: "v1"} + for _, dmlKind := range []string{"insert", "update"} { + stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ + ModelVersion: "v2", + WeightVersion: "v1", + Site: "tidb", + OpClass: "kv_mutation", + OperatorKind: "memdb_mutation", + DMLKind: dmlKind, + Unit: "encoded_mutation_count", + InputSource: "stmt_memdb_mutation_calls", + InputSide: "all", + RowWidthSource: "not_applicable", + Value: 1, + }) + } + + aggs, _, _ := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) + require.Len(t, aggs, 2) + entries := ReadBillingDemoBaseUnitEntriesFromMap(aggs) + require.Equal(t, "insert", entries[0].DMLKind) + require.Equal(t, "update", entries[1].DMLKind) + for _, entry := range entries { + require.Equal(t, "kv_mutation", entry.OpClass) + require.Equal(t, "memdb_mutation", entry.OperatorKind) + require.Equal(t, "encoded_mutation_count", entry.Unit) + } +} + func TestReadBillingDemoReservedStatusMergeBypassesStatusCap(t *testing.T) { fullStatusAggs := make(map[ReadBillingDemoStatusKey]ReadBillingDemoStatusAgg) fullStatusEntries := make([]ReadBillingDemoStatusAggEntry, 0, MaxReadBillingDemoStatusKeysPerRecord) diff --git a/pkg/util/stmtsummary/reader.go b/pkg/util/stmtsummary/reader.go index c0f5ca66b5f8d..25637a3aa3a1f 100644 --- a/pkg/util/stmtsummary/reader.go +++ b/pkg/util/stmtsummary/reader.go @@ -313,6 +313,8 @@ func (ssr *stmtSummaryReader) readBillingDemoBaseUnitColumnValue(col string, ssE return entry.OpClass case ReadBillingDemoOperatorKindStr: return entry.OperatorKind + case ReadBillingDemoDMLKindStr: + return entry.DMLKind case ReadBillingDemoUnitStr: return entry.Unit case ReadBillingDemoInputSourceStr: diff --git a/pkg/util/stmtsummary/v2/reader.go b/pkg/util/stmtsummary/v2/reader.go index 44fa1205b6d38..bb2cef85f37a6 100644 --- a/pkg/util/stmtsummary/v2/reader.go +++ b/pkg/util/stmtsummary/v2/reader.go @@ -669,6 +669,8 @@ func readBillingDemoBaseUnitColumnValue(info columnInfo, col string, record *Stm return entry.OpClass case stmtsummarybase.ReadBillingDemoOperatorKindStr: return entry.OperatorKind + case stmtsummarybase.ReadBillingDemoDMLKindStr: + return entry.DMLKind case stmtsummarybase.ReadBillingDemoUnitStr: return entry.Unit case stmtsummarybase.ReadBillingDemoInputSourceStr: diff --git a/pkg/util/stmtsummary/v2/reader_test.go b/pkg/util/stmtsummary/v2/reader_test.go index e4aa0c4c18431..cd689e180a891 100644 --- a/pkg/util/stmtsummary/v2/reader_test.go +++ b/pkg/util/stmtsummary/v2/reader_test.go @@ -281,6 +281,7 @@ func TestReadBillingDemoMemReader(t *testing.T) { Site: "tidb", OpClass: "projection_eval", OperatorKind: "projection", + DMLKind: "insert", Unit: "fixed_events", InputSource: "runtime_act_rows", InputSide: "all", @@ -330,6 +331,7 @@ func TestReadBillingDemoMemReader(t *testing.T) { {Name: ast.NewCIStr(DigestStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSiteStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoDMLKindStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoUnitStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoValueStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSampleCountStr)}, @@ -337,8 +339,8 @@ func TestReadBillingDemoMemReader(t *testing.T) { baseRows := NewReadBillingDemoMemReader(ss, baseUnitCols, "", timeLocation, nil, false, nil, nil, stmtsummarybase.ReadBillingDemoTableBaseUnits).Rows() require.Len(t, baseRows, 1) require.Equal(t, map[string]string{ - "digest_read_billing/tidb/projection_eval/fixed_events": "2 1", - }, readBillingDemoRowsByKey(baseRows, 4)) + "digest_read_billing/tidb/projection_eval/insert/fixed_events": "2 1", + }, readBillingDemoRowsByKey(baseRows, 5)) statusCols := []*model.ColumnInfo{ {Name: ast.NewCIStr(DigestStr)}, @@ -526,9 +528,11 @@ func TestReadBillingDemoHistoryReader(t *testing.T) { defer func() { require.NoError(t, os.Remove(filename)) }() - _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_base_unit_aggs":[{"model_version":"v1","weight_version":"v1","site":"tidb","op_class":"projection_eval","operator_kind":"projection","unit":"fixed_events","input_source":"runtime_act_rows","input_side":"all","row_width_source":"operator_helper","value":2,"sample_count":1,"row_width_sum":16}],"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"success","reason":"none","count":1}]}` + "\n") + _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_base_unit_aggs":[{"model_version":"v1","weight_version":"v1","site":"tidb","op_class":"projection_eval","operator_kind":"projection","dml_kind":"insert","unit":"fixed_events","input_source":"runtime_act_rows","input_side":"all","row_width_source":"operator_helper","value":2,"sample_count":1,"row_width_sum":16}],"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"success","reason":"none","count":1}]}` + "\n") require.NoError(t, err) - _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing_error","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"error","reason":"statement_error","count":1}]}` + "\n") + // The second persisted record intentionally omits dml_kind to prove older + // JSON remains readable and yields the empty diagnostic dimension. + _, err = file.WriteString(`{"begin":1672129270,"end":1672129280,"digest":"digest_read_billing_error","normalized_sql":"select ?","stmt_type":"Select","auth_users":{"user":{}},"read_billing_demo_base_unit_aggs":[{"model_version":"v1","weight_version":"v1","site":"tidb","op_class":"projection_eval","operator_kind":"projection","unit":"fixed_events","input_source":"runtime_act_rows","input_side":"all","row_width_source":"operator_helper","value":1,"sample_count":1,"row_width_sum":8}],"read_billing_demo_status_aggs":[{"model_version":"v1","weight_version":"v1","site":"statement","op_class":"statement","operator_kind":"statement","status":"error","reason":"statement_error","count":1}]}` + "\n") require.NoError(t, err) require.NoError(t, file.Close()) @@ -538,6 +542,7 @@ func TestReadBillingDemoHistoryReader(t *testing.T) { {Name: ast.NewCIStr(DigestStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoSiteStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoOpClassStr)}, + {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoDMLKindStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoUnitStr)}, {Name: ast.NewCIStr(stmtsummarybase.ReadBillingDemoValueStr)}, } @@ -546,8 +551,9 @@ func TestReadBillingDemoHistoryReader(t *testing.T) { baseRows := readAllRows(t, baseReader) require.NoError(t, baseReader.Close()) require.Equal(t, map[string]string{ - "digest_read_billing/tidb/projection_eval/fixed_events": "2", - }, readBillingDemoRowsByKey(baseRows, 4)) + "digest_read_billing/tidb/projection_eval/insert/fixed_events": "2", + "digest_read_billing_error/tidb/projection_eval//fixed_events": "1", + }, readBillingDemoRowsByKey(baseRows, 5)) statusCols := []*model.ColumnInfo{ {Name: ast.NewCIStr(DigestStr)}, diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index 260004a292a11..c73d99bb6a080 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -143,6 +143,7 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { Site: "tidb", OpClass: "projection_eval", OperatorKind: "projection", + DMLKind: "insert", Unit: "fixed_events", InputSource: "runtime_act_rows", InputSide: "all", @@ -156,6 +157,7 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { Site: "tidb", OpClass: "projection_eval", OperatorKind: "projection", + DMLKind: "insert", Unit: "fixed_events", InputSource: "runtime_act_rows", InputSide: "all", @@ -177,6 +179,7 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { require.Equal(t, 6.0, record.ReadBillingDemoBaseUnitAggs[0].Value) require.Equal(t, uint64(2), record.ReadBillingDemoBaseUnitAggs[0].SampleCount) require.Equal(t, 40.0, record.ReadBillingDemoBaseUnitAggs[0].RowWidthSum) + require.Equal(t, "insert", record.ReadBillingDemoBaseUnitAggs[0].DMLKind) require.Len(t, record.ReadBillingDemoStatusAggs, 1) b, err := marshalStmtRecord(record) From 87528cae8b34a173af9e83e8f58c44e303e54240 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 16 Jul 2026 05:06:03 +0800 Subject: [PATCH 14/25] planner: refine preview RU operator costing --- .../references/distsql-case-map.md | 2 +- .../references/executor-case-map.md | 2 +- .../references/planner-case-map.md | 2 +- .../2026-07-01-read-billing-demo-ru-model.md | 207 ++-- pkg/distsql/BUILD.bazel | 1 + pkg/distsql/select_result_test.go | 63 + pkg/executor/explain_test.go | 34 +- pkg/metrics/explain_ru.go | 9 +- pkg/metrics/metrics_internal_test.go | 16 +- pkg/planner/core/common_plans_test.go | 742 +++++++++++- pkg/planner/core/explain_ru.go | 1064 ++++++++++++++--- 11 files changed, 1828 insertions(+), 314 deletions(-) diff --git a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md index 59c3d6778b9bb..768f00abb342e 100644 --- a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md @@ -14,7 +14,7 @@ - `pkg/distsql/distsql_test.go` - Tests normal select. - `pkg/distsql/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/distsql/request_builder_test.go` - Tests table handles to KV ranges. -- `pkg/distsql/select_result_test.go` - Tests update coprocessor runtime stats. +- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership, multi-response aggregation, and incomplete-summary task counts used by preview RU. ## pkg/distsql/context diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index 51ceb482efd36..84fa2d3c313d9 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -29,7 +29,7 @@ - `pkg/executor/executor_failpoint_test.go` - executor: Tests TiDB last txn info commit mode. - `pkg/executor/executor_pkg_test.go` - executor: Tests build KV ranges for index join without CWC. - `pkg/executor/executor_required_rows_test.go` - executor: Tests limit required rows. -- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including preview RU DML mutation/write operators, exact index and optimistic-retry mutation counts, join/sort/read-tree preservation, operator-level unavailable-TiKV status rows, restricted/feature-off isolation, RUv2 A/B equivalence, explicit-COMMIT ownership, and diagnostic compatibility. +- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including preview RU v3 TiKV cop diagnostics, DML mutation/write operators, exact index and optimistic-retry mutation counts, join/sort/read-tree preservation, operator-level unavailable-TiKV status rows, restricted/feature-off isolation, RUv2 A/B equivalence, explicit-COMMIT ownership, and diagnostic compatibility. - `pkg/executor/explain_unit_test.go` - executor: Tests explain analyze invoke next and close. - `pkg/executor/explainfor_test.go` - executor: Tests explain for. - `pkg/executor/grant_test.go` - executor: Tests grant global. diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index 4fdadb9be9211..aacb778f6aefc 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, weights, DML mutation/write and explicit-COMMIT payload construction, pipelined payload suppression, and ancillary-work partial diagnostics. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, weights, Sort/TopN algorithmic `order_work`, TiKV component/child-row/scan-width input estimation and width barriers, DML mutation/write and explicit-COMMIT payload construction, pipelined payload suppression, and ancillary-work partial diagnostics. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/docs/design/2026-07-01-read-billing-demo-ru-model.md b/docs/design/2026-07-01-read-billing-demo-ru-model.md index bac1fdeeaf31f..745834c4732dc 100644 --- a/docs/design/2026-07-01-read-billing-demo-ru-model.md +++ b/docs/design/2026-07-01-read-billing-demo-ru-model.md @@ -31,8 +31,11 @@ preview_ru = fixed_events * fixed_weight + input_rows * row_weight + input_bytes * byte_weight + + order_work * order_weight ``` +`order_work` 只由比较型排序算子产生：full sort 使用 `n * log2(max(n, 2))`，TopN 使用 `n * log2(max(offset + count, 2))`。它作为 double base unit 展示在 `work_rows` 并进入明细统计，不强制转换为 int64 `COUNT`。排序类仍保留真实 `input_rows` base unit，但其 seed `row_weight` 为 0，避免把线性行项与算法 work 重复计费。 + 第一版是校准 demo，不是最终生产 billing 语义。它和现有 RU v2 resource-control 上报保持隔离，并分成两个输出面： 1. `EXPLAIN ANALYZE FORMAT='RU'`：单语句诊断输出，显示 base units、weight 和 preview RU。 @@ -40,15 +43,17 @@ preview_ru = Prometheus `tidb_read_billing_demo_*` metrics 可以继续作为 bounded observability 辅助，但不能作为 workload testers 的必需接口；校准契约以 SQL 查询面为准。 -### v2 实现说明 +### v3 实现说明 -当前实现的 `model_version = 'v2'` 已经把 `input_bytes` 从 v1 的 planner/schema row-width 估算切换为执行期 byte evidence： +当前实现的 `model_version = 'v3'` 延续 v2 的执行期 byte evidence，并为证据充分的 TiKV 一元 cop 链增加保守的输入边估算： - TiDB root operator 使用 runtime chunk 的 logical live bytes，`input_source = runtime_chunk_bytes`，`row_width_source = runtime_chunk_avg`。 - TiKV range scan 使用 scan detail 中的 `processed_key_size / processed_keys * total_keys`，`row_width_source = scan_detail_processed_key_avg`。 -- 缺少 byte evidence 的路径以 `unknown_input` fail closed，不产出 partial billable base units。 +- TiKV Selection、Projection、Limit、TopN、HashAgg 和 StreamAgg 使用直接 child 在 exact plan ID 下聚合到的 runtime `actRows` 作为 `input_rows`，再乘同一 TiKV component 的 `ProcessedKeysSize / ProcessedKeys` 作为估算 `input_bytes`；这类样本标记为 `input_source = runtime_child_act_rows`、`row_width_source = scan_detail_processed_key_avg_estimate`。这里的 exact 只表示 plan-ID attribution，不表示所有 response summary 必然无缺口。 +- Selection、Limit、TopN 保持行形状并向父节点传播平均宽度；Projection、HashAgg 和 StreamAgg 可以消费 child 宽度，但其输出是 width barrier，不能继续向父节点传播。 +- 缺少 exact child summary、唯一 scan/detail 归属或可传播宽度时，以 bounded reason fail closed；SELECT 不产出 partial billable units，DML 只保留其他独立成功算子的 units。 -本文后续保留的 `runtime_act_rows`、`plan_stats`、`schema_type_width`、`schema_fallback`、`operator_helper` 等描述只适用于 v1 历史估算模型或背景说明，不再代表 v2 的 `input_bytes` 构造规则。 +这里的 estimated bytes 不是 TiKV executor edge 上真实编码字节数，而是 runtime child rows 与 storage KV 平均宽度的混合 proxy。本文后续保留的 `runtime_act_rows`、`plan_stats`、`schema_type_width`、`schema_fallback`、`operator_helper` 等描述只适用于 v1 历史估算模型或背景说明，不代表 v2/v3 的当前构造规则。v2 的历史契约只覆盖 TiDB runtime bytes 与 TiKV range-scan detail；v3 才增加上述非 scan cop 输入估算。 ## 背景和目标 @@ -70,7 +75,7 @@ read billing demo 的目标是： - 不把 per-SQL log 作为校准接口； - 不要求 downstream testers 搭 Prometheus 或依赖 metrics retention； - 不在第一版覆盖 TiFlash、MPP、IndexMerge 或复杂 storage path； -- 不把当前 `v1` weights 解释成生产校准值。 +- 不把当前 `weight_version = 'v2'` 的 seed weights 解释成生产校准值。 ## 当前代码边界 @@ -124,13 +129,15 @@ fixed_events = 1 - `input_rows`：该 operator 消耗的行数或 key 数； - `input_bytes`：该 operator 消耗的 byte-shaped 输入。 +Sort 和 TopN 额外产生 `order_work`，用于保存基于 runtime input rows 的算法规模项。 + 每个 operator 结果都有以下身份字段： - `site`：`tidb`、`tikv`，statement-level status 用 `statement`； - `op_class`：用于解析权重的有限 class； - `operator_kind`：真实 plan/operator 名称，只用于观测和调参，不参与 weight lookup； -- `model_version`：当前统计模型版本，当前是 `v1`； -- `weight_version`：当前 weight table 版本，当前是 `v1`。 +- `model_version`：当前统计模型版本，当前是 `v3`； +- `weight_version`：当前 weight table 版本，当前是 `v2`；v1 是采用线性 Sort/TopN row 项的历史表。 权重按下面的 key 解析： @@ -146,30 +153,42 @@ TiKV 侧不是只按 scan 计费，而是把 cop executor 也纳入模型。第 | TiKV opclass | 覆盖的 operator 形态 | base-unit 输入 | |---|---|---| -| `kv_range_scan` | table/index range scan | `input_rows = ScanDetail.TotalKeys`；`input_bytes = ScanDetail.ProcessedKeysSize` 优先，缺失时回退 `TotalKeys * row_width` | +| `kv_range_scan` | table/index range scan | `input_rows = ScanDetail.TotalKeys`；`input_bytes = TotalKeys * (ProcessedKeysSize / ProcessedKeys)` | | `kv_point_lookup` | point get / batch point get | point lookup work；当前 variable input 基于 runtime rows，fixed event 覆盖 operator setup | -| `filter_eval` | selection | 直接 cop child runtime rows 和 modeled row width | -| `projection_eval` | projection | 直接 cop child runtime rows 和 modeled row width | -| `row_limit` | limit | 直接 cop child runtime rows 和 modeled row width | -| `bounded_topn` | TopN | 直接 cop child runtime rows 和 modeled row width | -| `agg_hash` | hash aggregation | aggregation input rows 和 modeled input bytes | -| `agg_stream` | stream aggregation | aggregation input rows 和 modeled input bytes | +| `filter_eval` | selection | direct child exact actRows × attributable scan-detail average；向父节点传播宽度 | +| `projection_eval` | projection | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | +| `row_limit` | limit | direct child exact actRows × attributable scan-detail average；向父节点传播宽度 | +| `bounded_topn` | TopN | direct child exact actRows × attributable scan-detail average；另有 `order_work = input_rows * log2(max(count, 2))`，其中 pushed plan 必须 `offset=0` 且 `count` 已折叠原 `offset+count`；向父节点传播宽度 | +| `agg_hash` | hash aggregation | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | +| `agg_stream` | stream aggregation | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | -range scan 的 byte 输入优先使用 `ProcessedKeysSize`，因为 scan 成本不仅和输出行数有关，还和 key/value bytes、MVCC 数据移动、row decode 有关。如果 runtime 只有 `TotalKeys` 而没有 processed size，则回退为： +range scan 的 byte 输入使用 processed-key average 外推到 `TotalKeys`，因为 scan 成本不仅和输出行数有关，还和 key/value bytes、MVCC 数据移动、row decode 有关： ```text -input_bytes = TotalKeys * row_width +input_bytes = TotalKeys * (ProcessedKeysSize / ProcessedKeys) ``` -如果是 mock-store 或其他没有 scan detail 的路径，则回退为： +TiKV 非 scan 一元算子的输入估算为： ```text -input_rows = actRows -input_bytes = actRows * row_width -input_source = runtime_act_rows +input_rows = direct_child_exact_actRows +input_bytes = input_rows * (ProcessedKeysSize / ProcessedKeys) +input_source = runtime_child_act_rows +row_width_source = scan_detail_processed_key_avg_estimate ``` -这个 fallback 会通过 `input_source` 明确标出，避免把 runtime rows 伪装成 scan detail。 +TiKV TopN 在上述输入证据成立后额外生成： + +```text +order_work = input_rows * log2(max(count, 2)) +input_source = runtime_ordering_work +``` + +TiKV TopN protobuf 只发送 `Limit=Count`，所以 pushed-down physical TopN 的规范形态是 `Offset=0`、`Count=原始 offset+count`。非零 cop Offset 会以 `invalid_ordering_work` 失败关闭，不按 root 公式猜测。 + +该公式只适用于结构可证明为单 scan、单 detail holder 的 TiKV component。multi-child、多 scan、多 detail holder、child summary 缺失或 task count 可检测为不完整、以及跨 Projection/Agg 的父边都会失败关闭；当前实现不回退 planner/schema width，也不跨 sibling component 借用 scan detail。 + +runtime rows 的数据质量契约是：stats 不存在或 `GetTasks() == 0` 为 missing；`GetTasks() > 0` 且 rows 为 0 是有效零行观测；负 rows 为 invalid；direct child task count 小于同 component 可见最大 task count 时为 incomplete。`GetTasks() > 0` 只能证明至少合并过一条完整 summary；如果同一 response 的 summary 在 component 所有 plan ID 上同步缺失，现有 `RuntimeStatsColl` 无法检测，因此 v3 rows 仍是带完整性边界的 estimate，校准时必须结合 coverage 与 failure reason。 ### TiDB Root Executor 模型 @@ -177,21 +196,21 @@ TiDB 侧覆盖 root executor CPU、row movement、reader materialization、join | TiDB opclass | 覆盖的 operator 形态 | base-unit 输入 | |---|---|---| -| `filter_eval` | root selection | local child rows 和 modeled input bytes | -| `projection_eval` | root projection | local child rows 和 modeled input bytes | -| `row_limit` | root limit | local child rows 和 modeled input bytes | -| `bounded_topn` | root TopN | local child rows 和 modeled input bytes | -| `full_ordering` | sort | local child rows 和 modeled input bytes | -| `window_eval` | window executor | local child rows 和 modeled input bytes | -| `agg_hash` | hash aggregation | aggregation input rows 和 modeled input bytes | -| `agg_stream` | stream aggregation | aggregation input rows 和 modeled input bytes | +| `filter_eval` | root selection | local child runtime rows 和 logical live bytes | +| `projection_eval` | root projection | local child runtime rows 和 logical live bytes | +| `row_limit` | root limit | local child runtime rows 和 logical live bytes | +| `bounded_topn` | root TopN | local child runtime rows、logical live bytes，以及 `order_work = n * log2(max(offset + count, 2))` | +| `full_ordering` | sort | local child runtime rows、logical live bytes，以及 `order_work = n * log2(max(n, 2))` | +| `window_eval` | window executor | local child runtime rows 和 logical live bytes | +| `agg_hash` | hash aggregation | aggregation input rows 和 logical live bytes | +| `agg_stream` | stream aggregation | aggregation input rows 和 logical live bytes | | `join_hash` | hash join | build/probe 两侧 input rows 和 bytes | | `join_merge` | merge join | left/right 两侧 input rows 和 bytes | | `join_lookup` | index lookup join family | left/right 两侧 input rows 和 bytes | -| `reader_receive` | table/index reader receive | runtime rows 和 modeled received bytes | -| `lookup_reader` | index lookup reader orchestration | runtime rows 和 modeled bytes | -| `overlay_reader` | union scan / local overlay | runtime rows 和 modeled bytes | -| `metadata_reader` | memory / cluster metadata reads | runtime rows 和 modeled bytes | +| `reader_receive` | table/index reader receive | runtime output rows 和 logical live bytes | +| `lookup_reader` | index lookup reader orchestration | runtime output rows 和 logical live bytes | +| `overlay_reader` | union scan / local overlay | runtime rows 和 logical live bytes | +| `metadata_reader` | memory / cluster metadata reads | runtime rows 和 logical live bytes | Join 的 side 会通过 `input_side` 保留下来： @@ -231,10 +250,10 @@ Join 的 side 会通过 `input_side` 保留下来： | `SITE` | varchar | `tidb` / `tikv` | | `OP_CLASS` | varchar | bounded opclass | | `OPERATOR_KIND` | varchar | bounded plan/operator kind | -| `UNIT` | varchar | `fixed_events` / `input_rows` / `input_bytes` | -| `INPUT_SOURCE` | varchar | `scan_detail` / `runtime_act_rows` | +| `UNIT` | varchar | `fixed_events` / `input_rows` / `input_bytes` / `order_work` / write-side units | +| `INPUT_SOURCE` | varchar | bounded source，例如 `runtime_chunk_bytes` / `scan_detail` / `runtime_child_act_rows` / `runtime_ordering_work` / write-side sources | | `INPUT_SIDE` | varchar | `all` / `build` / `probe` / `left` / `right` | -| `ROW_WIDTH_SOURCE` | varchar | `plan_stats` / `schema_type_width` / `schema_fallback` / `operator_helper` | +| `ROW_WIDTH_SOURCE` | varchar | 当前值包括 `runtime_chunk_avg` / `scan_detail_processed_key_avg` / `scan_detail_processed_key_avg_estimate` / `not_applicable`；v1 历史值保留原标签 | | `VALUE` | double unsigned | 该 key 的 base-unit total | | `SAMPLE_COUNT` | bigint unsigned | 汇入该 key 的 unit samples 数 | | `ROW_WIDTH_SUM` | double unsigned | row-width sample sum，用于诊断 modeled bytes | @@ -262,7 +281,7 @@ Join 的 side 会通过 `input_side` 保留下来： | `OP_CLASS` | varchar | statement or operator opclass | | `OPERATOR_KIND` | varchar | statement or plan/operator kind | | `STATUS` | varchar | `success` / `unsupported` / `unknown_input` / `error` / `ok` | -| `REASON` | varchar | bounded reason，例如 `unsupported_mpp`、`missing_scan_detail`、`statement_error` | +| `REASON` | varchar | bounded reason，例如 `unsupported_mpp`、`missing_scan_width_evidence`、`unsupported_cop_width_transform`、`statement_error` | | `COUNT` | bigint unsigned | 该 status key 的出现次数 | 失败 statement 只写 status table，不写 base-unit table。成功 statement 写一条 statement-level `success` status，并为每个 billable operator 写 operator-level `ok` status。 @@ -371,7 +390,8 @@ SELECT FROM information_schema.cluster_statements_summary_history_read_billing_demo_status WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' - AND model_version = 'v1' + AND model_version = 'v3' + AND weight_version = 'v2' GROUP BY status, reason ORDER BY count DESC; ``` @@ -391,7 +411,8 @@ SELECT FROM information_schema.cluster_statements_summary_history_read_billing_demo_status WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' - AND model_version = 'v1' + AND model_version = 'v3' + AND weight_version = 'v2' GROUP BY site, op_class, operator_kind, status, reason ORDER BY count DESC; ``` @@ -411,7 +432,8 @@ SELECT FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' - AND model_version = 'v1' + AND model_version = 'v3' + AND weight_version = 'v2' GROUP BY site, op_class, unit, input_source, input_side ORDER BY site, op_class, unit, input_source, input_side; ``` @@ -422,6 +444,24 @@ ORDER BY site, op_class, unit, input_source, input_side; X[site, op_class, unit] = 当前时间窗口内观测到的 base-unit total ``` +校准 v3 TiKV 非 scan estimator 时必须按 unit 使用不同的 source/width-source predicate，避免把 `fixed_events` 或 TopN 的 `order_work` 过滤掉： + +```sql +SELECT op_class, unit, input_source, row_width_source, SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE model_version = 'v3' + AND weight_version = 'v2' + AND site = 'tikv' + AND op_class IN ('filter_eval', 'projection_eval', 'row_limit', 'bounded_topn', 'agg_hash', 'agg_stream') + AND ( + (input_source = 'runtime_child_act_rows' AND unit = 'fixed_events' AND row_width_source = 'not_applicable') + OR (input_source = 'runtime_child_act_rows' AND unit IN ('input_rows', 'input_bytes') AND row_width_source = 'scan_detail_processed_key_avg_estimate') + OR (op_class = 'bounded_topn' AND input_source = 'runtime_ordering_work' AND unit = 'order_work' AND row_width_source = 'not_applicable') + ) +GROUP BY op_class, unit, input_source, row_width_source +ORDER BY op_class, unit, input_source, row_width_source; +``` + 如果需要区分 plan shape，可以保留 `DIGEST` 和 `PLAN_DIGEST`： ```sql @@ -435,13 +475,14 @@ SELECT FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' - AND model_version = 'v1' + AND model_version = 'v3' + AND weight_version = 'v2' GROUP BY digest, plan_digest, site, op_class, unit; ``` ### 第三步：检查 row-width evidence -row width 是模型 factor，不是 runtime 真实 byte sample。拟合前需要检查它是否稳定： +v3 row width 既可能来自 TiDB runtime chunk average，也可能是 TiKV scan-detail KV average proxy；后者不是 cop executor edge 的真实 byte sample。拟合前需要按 `row_width_source` 检查它是否稳定： ```sql SELECT @@ -454,12 +495,13 @@ FROM information_schema.cluster_statements_summary_history_read_billing_demo_bas WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' AND unit = 'input_bytes' - AND model_version = 'v1' + AND model_version = 'v3' + AND weight_version = 'v2' GROUP BY site, op_class, row_width_source ORDER BY site, op_class, row_width_source; ``` -如果 `schema_fallback` 占比高，或某些 opclass 的 row width 异常，拟合出来的 byte weight 可能是在补偿 row-width 估计误差，而不是真实 byte cost。 +如果 `scan_detail_processed_key_avg_estimate` 的分布异常，或 residual 随 projection shape、group NDV、task count 系统漂移，拟合出的 byte weight 可能是在补偿 proxy 误差，而不是真实 byte cost。v3 不使用 `schema_fallback`。 ### 第四步：用任意 weights 重算 RU @@ -471,18 +513,21 @@ WITH observed AS ( FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units WHERE summary_begin_time >= TIMESTAMP '2026-07-02 10:00:00' AND summary_end_time <= TIMESTAMP '2026-07-02 11:00:00' - AND model_version = 'v1' + AND model_version = 'v3' + AND weight_version = 'v2' GROUP BY site, op_class, unit ), -weights(site, op_class, fixed_weight, row_weight, byte_weight) AS ( - SELECT 'tikv', 'kv_range_scan', 0.070, 0.000045, 0.000020 UNION ALL - SELECT 'tidb', 'projection_eval', 0.020, 0.000020, 0.000004 +weights(site, op_class, fixed_weight, row_weight, byte_weight, order_weight) AS ( + SELECT 'tikv', 'kv_range_scan', 0.070, 0.000045, 0.000020, 0.000000 UNION ALL + SELECT 'tidb', 'projection_eval', 0.020, 0.000020, 0.000004, 0.000000 UNION ALL + SELECT 'tidb', 'full_ordering', 0.080, 0.000000, 0.000012, 0.000070 ) SELECT SUM(CASE observed.unit WHEN 'fixed_events' THEN observed.value * weights.fixed_weight WHEN 'input_rows' THEN observed.value * weights.row_weight WHEN 'input_bytes' THEN observed.value * weights.byte_weight + WHEN 'order_work' THEN observed.value * weights.order_weight ELSE 0 END) AS recalculated_ru FROM observed @@ -518,55 +563,55 @@ subject to w >= 0 拟合规则建议： -1. 按 `site/op_class` 分别拟合 `fixed`、`row`、`byte` 三类系数。 +1. 按 `site/op_class` 分别拟合 `fixed`、`row`、`byte`、`order` 系数；只有 Sort/TopN 产生 `order_work`。 2. 保持权重非负。 3. 使用 hold-out workload window 验证预测误差。 4. 默认保留符合源码成本形态的单调关系，除非数据强烈反证，例如： - `agg_hash.row >= agg_stream.row`； - - `bounded_topn.row >= row_limit.row`； + - `bounded_topn.order > 0`，同时保持 `bounded_topn.row = 0` 以避免和 order work 双算； - `kv_range_scan.byte >= filter_eval.byte`； - `join_hash.row >= join_merge.row`。 -5. 拟合后的新权重应该发布为新的 `weight_version`，不要静默改变 `v1` 的含义。 +5. 拟合后的新权重应该发布为新的 `weight_version`，不要静默改变当前 `v2` 或历史 `v1` 的含义。 ## 初始权重 -当前 `v1` weights 是 demo calibration seed，不是 production-calibrated billing constants。它们来自源码成本形态和相对排序： +当前 `v2` weights 是 demo calibration seed，不是 production-calibrated billing constants。v2 将 Sort/TopN 的历史线性 row coefficient 移到独立 `order_weight`，因此没有复用 v1 的权重语义。它们来自源码成本形态和相对排序： - scan 同时受 row/key 数和 bytes 影响，比纯 expression eval 更重； - hash aggregation 和 hash join 需要维护 hash map / hash table / aggregate state，row weight 更高； - stream aggregation 和 merge join 依赖有序输入，通常低于 hash 版本； - limit 是最轻的 row-shaped operator； -- TopN 需要 order expression 和 heap maintenance，比 limit 重； +- TopN 需要 order expression 和 heap maintenance，算法 work 是 `n * log2(k)`；full sort 是 `n * log2(n)`； - TiDB reader class 对 bytes 更敏感，因为它负责接收、materialize chunk 和协调 lookup； - TiDB 与 TiKV 即使 opclass 名称相同，也运行不同代码路径，因此权重分开设置。 -当前 `v1` 权重如下： - -| site | opclass | fixed_weight | row_weight | byte_weight | 初始设置原因 | -|---|---|---:|---:|---:|---| -| `tikv` | `kv_range_scan` | 0.070 | 0.000045 | 0.000020 | range iterator、MVCC/key-value movement、row decode 和 scan limiter 使 scan 同时对 row/key 和 bytes 敏感。 | -| `tikv` | `kv_point_lookup` | 0.045 | 0.000030 | 0.000012 | point get 有 per-key dispatch 和 decode，但没有 range iterator loop。 | -| `tikv` | `filter_eval` | 0.020 | 0.000040 | 0.000006 | RPN predicate eval 主要按 logical rows 消耗 CPU，byte movement 权重较低。 | -| `tikv` | `projection_eval` | 0.020 | 0.000030 | 0.000006 | projection 会执行表达式或 materialize column reference，row 成本低于 filter。 | -| `tikv` | `row_limit` | 0.010 | 0.000008 | 0.000002 | 主要是 pass-through counter / gate，是最轻的 executor。 | -| `tikv` | `bounded_topn` | 0.060 | 0.000075 | 0.000012 | order expression eval 加 bounded heap maintenance。 | -| `tikv` | `agg_hash` | 0.080 | 0.000100 | 0.000014 | group expression eval、hash group lookup/insert、aggregate state update。 | -| `tikv` | `agg_stream` | 0.060 | 0.000065 | 0.000010 | ordered group compare 加 state update，不构建 hash table。 | -| `tidb` | `filter_eval` | 0.020 | 0.000030 | 0.000005 | TiDB chunk 上的 expression eval，低于 scan 和 hash 类 operator。 | -| `tidb` | `projection_eval` | 0.020 | 0.000020 | 0.000004 | projection / materialization，per-row 成本较低。 | -| `tidb` | `row_limit` | 0.010 | 0.000006 | 0.000001 | 轻量 pass-through control。 | -| `tidb` | `bounded_topn` | 0.060 | 0.000060 | 0.000010 | heap / order expression work，比 full sort 范围小。 | -| `tidb` | `full_ordering` | 0.080 | 0.000070 | 0.000012 | full sort 需要处理完整输入。 | -| `tidb` | `window_eval` | 0.070 | 0.000070 | 0.000010 | partition / order / frame state maintenance。 | -| `tidb` | `agg_hash` | 0.070 | 0.000085 | 0.000012 | hash grouping 和 aggregate state update。 | -| `tidb` | `agg_stream` | 0.050 | 0.000055 | 0.000008 | ordered streaming aggregate，低于 hash agg。 | -| `tidb` | `join_hash` | 0.110 | 0.000115 | 0.000020 | build/probe hash table work，是 TiDB 侧最高 row cost 之一。 | -| `tidb` | `join_merge` | 0.090 | 0.000075 | 0.000012 | ordered comparison / merge，低于 hash join。 | -| `tidb` | `join_lookup` | 0.120 | 0.000120 | 0.000020 | lookup task orchestration 加 join-side work，固定成本和 row 成本都较高。 | -| `tidb` | `reader_receive` | 0.040 | 0.000025 | 0.000014 | network/result receive 和 chunk materialization，对 bytes 敏感。 | -| `tidb` | `lookup_reader` | 0.070 | 0.000045 | 0.000016 | index/table two-phase lookup reader coordination。 | -| `tidb` | `overlay_reader` | 0.050 | 0.000035 | 0.000012 | UnionScan/local overlay merge。 | -| `tidb` | `metadata_reader` | 0.020 | 0.000008 | 0.000002 | metadata / memory-table read，故意设得较轻。 | +当前 `v2` 权重如下： + +| site | opclass | fixed_weight | row_weight | byte_weight | order_weight | 初始设置原因 | +|---|---|---:|---:|---:|---:|---| +| `tikv` | `kv_range_scan` | 0.070 | 0.000045 | 0.000020 | 0 | range iterator、MVCC/key-value movement、row decode 和 scan limiter 使 scan 同时对 row/key 和 bytes 敏感。 | +| `tikv` | `kv_point_lookup` | 0.045 | 0.000030 | 0.000012 | 0 | point get 有 per-key dispatch 和 decode，但没有 range iterator loop。 | +| `tikv` | `filter_eval` | 0.020 | 0.000040 | 0.000006 | 0 | RPN predicate eval 主要按 logical rows 消耗 CPU，byte movement 权重较低。 | +| `tikv` | `projection_eval` | 0.020 | 0.000030 | 0.000006 | 0 | projection 会执行表达式或 materialize column reference，row 成本低于 filter。 | +| `tikv` | `row_limit` | 0.010 | 0.000008 | 0.000002 | 0 | 主要是 pass-through counter / gate，是最轻的 executor。 | +| `tikv` | `bounded_topn` | 0.060 | 0 | 0.000012 | 0.000075 | order expression eval 加 `n * log2(k)` bounded heap maintenance。 | +| `tikv` | `agg_hash` | 0.080 | 0.000100 | 0.000014 | 0 | group expression eval、hash group lookup/insert、aggregate state update。 | +| `tikv` | `agg_stream` | 0.060 | 0.000065 | 0.000010 | 0 | ordered group compare 加 state update，不构建 hash table。 | +| `tidb` | `filter_eval` | 0.020 | 0.000030 | 0.000005 | 0 | TiDB chunk 上的 expression eval，低于 scan 和 hash 类 operator。 | +| `tidb` | `projection_eval` | 0.020 | 0.000020 | 0.000004 | 0 | projection / materialization，per-row 成本较低。 | +| `tidb` | `row_limit` | 0.010 | 0.000006 | 0.000001 | 0 | 轻量 pass-through control。 | +| `tidb` | `bounded_topn` | 0.060 | 0 | 0.000010 | 0.000060 | heap / order expression 的 `n * log2(k)` work。 | +| `tidb` | `full_ordering` | 0.080 | 0 | 0.000012 | 0.000070 | full sort 的 `n * log2(n)` work。 | +| `tidb` | `window_eval` | 0.070 | 0.000070 | 0.000010 | 0 | partition / order / frame state maintenance。 | +| `tidb` | `agg_hash` | 0.070 | 0.000085 | 0.000012 | 0 | hash grouping 和 aggregate state update。 | +| `tidb` | `agg_stream` | 0.050 | 0.000055 | 0.000008 | 0 | ordered streaming aggregate，低于 hash agg。 | +| `tidb` | `join_hash` | 0.110 | 0.000115 | 0.000020 | 0 | build/probe hash table work，是 TiDB 侧最高 row cost 之一。 | +| `tidb` | `join_merge` | 0.090 | 0.000075 | 0.000012 | 0 | ordered comparison / merge，低于 hash join。 | +| `tidb` | `join_lookup` | 0.120 | 0.000120 | 0.000020 | 0 | lookup task orchestration 加 join-side work，固定成本和 row 成本都较高。 | +| `tidb` | `reader_receive` | 0.040 | 0.000025 | 0.000014 | 0 | network/result receive 和 chunk materialization，对 bytes 敏感。 | +| `tidb` | `lookup_reader` | 0.070 | 0.000045 | 0.000016 | 0 | index/table two-phase lookup reader coordination。 | +| `tidb` | `overlay_reader` | 0.050 | 0.000035 | 0.000012 | 0 | UnionScan/local overlay merge。 | +| `tidb` | `metadata_reader` | 0.020 | 0.000008 | 0.000002 | 0 | metadata / memory-table read，故意设得较轻。 | ## 性能和容量控制 @@ -708,7 +753,7 @@ per_record_read_billing_keys <= 256 + 128 - 新表会增加 statement summary 内存占用；必须用单 digest/window key 上限和 overflow status 控制。 - v2 persistent history 会扩大 statement summary JSON record；需要 benchmark 和 history-reader 兼容测试。 -- row-width factor 仍然是 modeled value，如果大量来自 `schema_fallback`，byte weight 可能补偿 row-width 误差。 +- TiKV 非 scan row width 是 storage KV average proxy，不是 executor edge 真值；byte weight 可能补偿 projection/aggregation 行形状、编码和 task 分布误差。 - evicted aggregate 会丢 digest/plan 细节，不适合作为精细校准输入；用户需要查询 status 和 evicted 表判断窗口质量。 - 当前 point lookup miss 的 variable row cost 证据仍有限；没有 requested-key 证据时主要由 fixed event 覆盖。 diff --git a/pkg/distsql/BUILD.bazel b/pkg/distsql/BUILD.bazel index 05416ed307778..cfd5e77f3a4b7 100644 --- a/pkg/distsql/BUILD.bazel +++ b/pkg/distsql/BUILD.bazel @@ -101,6 +101,7 @@ go_test( "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//util", "@org_uber_go_goleak//:goleak", ], ) diff --git a/pkg/distsql/select_result_test.go b/pkg/distsql/select_result_test.go index e08a4ecea5061..886ae3a3fe70c 100644 --- a/pkg/distsql/select_result_test.go +++ b/pkg/distsql/select_result_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" + tikvutil "github.com/tikv/client-go/v2/util" ) func TestUpdateCopRuntimeStats(t *testing.T) { @@ -72,6 +73,68 @@ func TestUpdateCopRuntimeStats(t *testing.T) { sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", BackoffSleep: backOffSleep}}, 0, false) require.Equal(t, "tikv_task:{time:1ns, loops:1}", ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(1234).String()) require.Equal(t, sr.stats.backoffSleep["RegionMiss"], time.Duration(500)) + + newSummary := func(rows uint64) *tipb.ExecutorExecutionSummary { + one := uint64(1) + return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &one, NumProducedRows: &rows, NumIterations: &one} + } + update := func(scanRows, parentRows uint64, detail *tikvutil.ScanDetail) { + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{ + newSummary(scanRows), + newSummary(parentRows), + }} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", ScanDetail: detail}, + }, 0, false)) + } + + const scanPlanID = 2001 + const parentPlanID = 2002 + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + sr.copPlanIDs = []int{scanPlanID, parentPlanID} + update(4, 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + scanStats := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + require.Equal(t, int64(4), scanStats.GetActRows()) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Zero(t, scanStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int64(2), parentStats.GetActRows()) + require.Equal(t, int32(1), parentStats.GetTasks()) + require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int64(100), parentStats.GetScanDetail().ProcessedKeysSize) + + update(3, 1, &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}) + require.Equal(t, int64(7), scanStats.GetActRows()) + require.Equal(t, int32(2), scanStats.GetTasks()) + require.Equal(t, int64(3), parentStats.GetActRows()) + require.Equal(t, int32(2), parentStats.GetTasks()) + require.Equal(t, int64(7), parentStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int64(140), parentStats.GetScanDetail().ProcessedKeysSize) + + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{newSummary(4), nil}} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", ScanDetail: &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}}, + }, 0, false)) + scanStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int32(0), parentStats.GetTasks()) + require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) + + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + update(4, 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{nil, newSummary(1)}} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee", ScanDetail: &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}}, + }, 0, false)) + scanStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int32(2), parentStats.GetTasks()) } func TestNewSelRespChannelIter(t *testing.T) { diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 79061169702df..5febd0339198c 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -545,7 +545,7 @@ func TestExplainAnalyzeFormatRUOutput(t *testing.T) { _, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' select * from explain_ru_t where a > 0") require.Error(t, err) require.Contains(t, err.Error(), "status=unknown_input") - require.Contains(t, err.Error(), "reason=missing_scan_detail") + require.Contains(t, err.Error(), "reason=missing_scan_width_evidence") require.Contains(t, err.Error(), "operator=tikv/kv_range_scan") rows = tk.MustQuery("explain analyze format='ru' select * from explain_ru_t where a = 1").Rows() @@ -593,7 +593,7 @@ func TestExplainAnalyzeFormatRUTiKVCopOperatorClasses(t *testing.T) { rows, err := queryExplainRURowsOrErr(t, tk, tc.sql) if err != nil { require.Contains(t, err.Error(), "status=unknown_input", tc.sql) - require.Contains(t, err.Error(), "reason=missing_runtime_bytes", tc.sql) + require.Contains(t, err.Error(), "reason=missing_scan_width_evidence", tc.sql) require.Contains(t, err.Error(), "operator="+tc.nonScanOpClass, tc.sql) continue } @@ -656,7 +656,7 @@ func requireExplainRUPlanRow(t *testing.T, rows [][]any) { require.NotEmpty(t, row[13]) require.NotEmpty(t, row[14]) require.NotEmpty(t, row[15]) - require.Contains(t, fmt.Sprint(row[16]), "weight_version=v1") + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v2") return } require.Fail(t, "missing FORMAT='RU' plan row") @@ -681,7 +681,7 @@ func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorC } require.NotEmpty(t, row[13], "missing weight for %s row %v", operatorClass, row) require.NotEmpty(t, row[14], "missing preview RU for %s row %v", operatorClass, row) - require.Contains(t, fmt.Sprint(row[16]), "weight_version=v1") + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v2") return } require.Failf(t, "missing weighted FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) @@ -693,7 +693,7 @@ func requireExplainRUDMLRows(t *testing.T, rows [][]any, dmlKind string, require require.Len(t, rows[0], 17) require.Equal(t, "summary", rows[0][0]) require.Equal(t, "total_preview_ru", rows[0][2]) - require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v1") + require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v2") require.Contains(t, fmt.Sprint(rows[0][16]), "mutation_weights_uncalibrated=true") require.Contains(t, fmt.Sprint(rows[0][16]), "partial_dml_ancillary_work_partial") summaryPreviewRU, err := strconv.ParseFloat(fmt.Sprint(rows[0][14]), 64) @@ -952,7 +952,7 @@ func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { note := fmt.Sprint(row[16]) hasPartialIndexMerge = hasPartialIndexMerge || strings.Contains(note, "reason=unsupported_index_merge") hasPartialScan = hasPartialScan || - (strings.Contains(strings.ToLower(fmt.Sprint(row[2])), "scan") && strings.Contains(note, "reason=missing_scan_detail")) + (strings.Contains(strings.ToLower(fmt.Sprint(row[2])), "scan") && strings.Contains(note, "reason=missing_scan_width_evidence")) } require.True(t, hasPartialIndexMerge, rows) require.True(t, hasPartialScan, rows) @@ -1077,15 +1077,15 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustExec("create table read_billing_demo(a int primary key)") tk.MustExec("insert into read_billing_demo values (1), (2)") - success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2") - unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v2") - unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v2") - errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v2") - projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v2") - mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v2") - mutationBytes := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v2") - writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_keys", "commit_detail", "all", "v2") - writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_byte", "commit_detail", "all", "v2") + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v3") + unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v3") + unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v3") + errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v3") + projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v3") + mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v3") + mutationBytes := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v3") + writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_keys", "commit_detail", "all", "v3") + writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_byte", "commit_detail", "all", "v3") tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) require.Equal(t, 0.0, readExecutorCounterValue(t, success)) @@ -1096,7 +1096,7 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { require.Equal(t, 1.0, readExecutorCounterValue(t, projectionFixedEvents)) tk.MustExec("set tidb_enable_read_billing_demo=off") tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events > 0, sum_read_billing_demo_input_rows > 0, sum_read_billing_demo_input_bytes = 0 from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 1 1 1")) - tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width = 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_chunk_bytes all v2 v1 1 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0, avg_row_width = 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and site = 'tidb' and op_class = 'projection_eval' and operator_kind = 'projection' and unit = 'fixed_events'`).Check(testkit.Rows("tidb projection_eval projection fixed_events runtime_chunk_bytes all v3 v2 1 1 1")) tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) tk.MustQuery(`select column_name from information_schema.columns where table_schema = 'INFORMATION_SCHEMA' and table_name = 'CLUSTER_STATEMENTS_SUMMARY_READ_BILLING_DEMO_BASE_UNITS' and ordinal_position = 1`).Check(testkit.Rows("INSTANCE")) // Append the diagnostic dimension so consumers that depend on the existing @@ -1222,7 +1222,7 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustQuery("select * from read_billing_demo where a > 0").Sort().Check(testkit.Rows("1", "2", "3")) require.Equal(t, beforeUnknownInput+1, readExecutorCounterValue(t, unknownInput)) require.Equal(t, beforeBaseUnitsTotal, readExecutorCounterVecTotal(t, metrics.ReadBillingDemoBaseUnitsCounter)) - tk.MustQuery(`select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?' and site = 'statement' group by status, reason`).Check(testkit.Rows("unknown_input missing_scan_detail 1")) + tk.MustQuery(`select status, reason, sum(count) from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?' and site = 'statement' group by status, reason`).Check(testkit.Rows("unknown_input missing_scan_width_evidence 1")) tk.MustQuery(`select count(*) from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select * from ` + "`read_billing_demo`" + ` where ` + "`a`" + ` > ?'`).Check(testkit.Rows("0")) // Inject an error after AddRecord has emitted its MemDB mutation. Statement diff --git a/pkg/metrics/explain_ru.go b/pkg/metrics/explain_ru.go index 42d48e101beb7..e25d6e4b9f4fc 100644 --- a/pkg/metrics/explain_ru.go +++ b/pkg/metrics/explain_ru.go @@ -22,6 +22,7 @@ import ( const ( explainRULabelStatus = "status" explainRULabelComponentSnapshotStatus = "component_snapshot_status" + explainRULabelWeightVersion = "weight_version" readBillingDemoLabelModelVersion = "model_version" ) @@ -51,7 +52,7 @@ func InitExplainRUMetrics() { Subsystem: "explain_ru", Name: "preview_ru_total", Help: "Counter of read billing preview RU values generated by EXPLAIN ANALYZE FORMAT='RU'.", - }, []string{"section", "component", "operator", "source"}, + }, []string{"section", "component", "operator", "source", explainRULabelWeightVersion}, ) ExplainRUWorkRowsCounter = metricscommon.NewCounterVec( prometheus.CounterOpts{ @@ -163,11 +164,11 @@ func RecordExplainRUComponentSnapshot(status string) { } // ObserveExplainRURow records bounded numeric samples from generated FORMAT='RU' rows. -func ObserveExplainRURow(section, component, operator, source, rowWidthSource string, previewRU, workRows, workBytes, rowWidth float64) { +func ObserveExplainRURow(section, component, operator, source, rowWidthSource, weightVersion string, previewRU, workRows, workBytes, rowWidth float64) { // Negative values are sentinels for "not present" so callers can use one // path for summary and plan rows without emitting fake zeros. - if ExplainRUPreviewRUCounter != nil && previewRU >= 0 { - ExplainRUPreviewRUCounter.WithLabelValues(section, component, operator, source).Add(previewRU) + if ExplainRUPreviewRUCounter != nil && previewRU >= 0 && weightVersion != "" { + ExplainRUPreviewRUCounter.WithLabelValues(section, component, operator, source, weightVersion).Add(previewRU) } if ExplainRUWorkRowsCounter != nil && workRows >= 0 { ExplainRUWorkRowsCounter.WithLabelValues(section, component, operator, source).Add(workRows) diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index 535809eedc10e..889c7cd42cd8a 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -101,7 +101,7 @@ func TestExplainRUMetrics(t *testing.T) { RecordExplainRUStatus("success") ObserveExplainRURenderDuration("success", 0.01) RecordExplainRUComponentSnapshot("ok") - ObserveExplainRURow("plan", "", "projection", "read_billing_model", "runtime_chunk_avg", 1.25, 3, 24, 8) + ObserveExplainRURow("plan", "", "projection", "read_billing_model", "runtime_chunk_avg", "v2", 1.25, 3.5, 24, 8) RecordReadBillingDemoStatement("success", "v2") RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2") AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 3) @@ -109,8 +109,8 @@ func TestExplainRUMetrics(t *testing.T) { require.Equal(t, 1.0, readCounterValue(t, ExplainRUStatementsCounter.WithLabelValues("success"))) require.Equal(t, 1.0, readCounterValue(t, ExplainRUComponentSnapshotCounter.WithLabelValues("ok"))) - require.Equal(t, 1.25, readCounterValue(t, ExplainRUPreviewRUCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) - require.Equal(t, 3.0, readCounterValue(t, ExplainRUWorkRowsCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) + require.Equal(t, 1.25, readCounterValue(t, ExplainRUPreviewRUCounter.WithLabelValues("plan", "", "projection", "read_billing_model", "v2"))) + require.Equal(t, 3.5, readCounterValue(t, ExplainRUWorkRowsCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) require.Equal(t, 24.0, readCounterValue(t, ExplainRUWorkBytesCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2"))) require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v2"))) @@ -130,7 +130,10 @@ func TestExplainRUMetrics(t *testing.T) { require.NoError(t, registry.Register(ReadBillingDemoRowWidthHistogram)) families, err := registry.Gather() require.NoError(t, err) - require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_preview_ru_total")) + previewRUFamily := findMetricFamily(families, "tidb_explain_ru_preview_ru_total") + require.NotNil(t, previewRUFamily) + requireMetricFamilyHasLabels(t, previewRUFamily, "section", "component", "operator", "source", "weight_version") + require.True(t, metricHasLabelValue(previewRUFamily.GetMetric()[0], "weight_version", "v2")) require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_work_rows_total")) require.NotNil(t, findMetricFamily(families, "tidb_explain_ru_work_bytes_total")) rowWidthFamily := findMetricFamily(families, "tidb_explain_ru_row_width_bytes") @@ -164,8 +167,9 @@ func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { RecordExplainRUStatus("") ObserveExplainRURenderDuration("", 0.01) RecordExplainRUComponentSnapshot("") - ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", 0, -1, -1, -1) - ObserveExplainRURow("plan", "", "", "read_billing_model", "runtime_chunk_avg", -1, -1, -1, 32) + ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", "", 1, -1, -1, -1) + ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", "v2", 0, -1, -1, -1) + ObserveExplainRURow("plan", "", "", "read_billing_model", "runtime_chunk_avg", "v2", -1, -1, -1, 32) RecordReadBillingDemoStatement("", "v2") RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v2") AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 0) diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index 26b5e4962e2e6..b1710ae416c5f 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -25,10 +25,12 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" "github.com/pingcap/tidb/pkg/planner/property" + plannerutil "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" tikvutil "github.com/tikv/client-go/v2/util" rmclient "github.com/tikv/pd/client/resource_group/controller" @@ -229,14 +231,15 @@ func TestExplainRURowFormatting(t *testing.T) { previewRU: 6, hasPreviewRU: true, source: readBillingDemoInputSourceRuntimeChunkBytes, - note: "input_side=all,weight_version=v1", + note: "input_side=all,weight_version=v2", } require.Equal(t, []string{ - "plan", "Projection_1", "projection", "tidb/projection_eval", "1", "2", "1", "8.000000", "runtime_chunk_avg", "2", "", "input_rows", "2", "0.250000", "6.000000", "runtime_chunk_bytes", "input_side=all,weight_version=v1", + "plan", "Projection_1", "projection", "tidb/projection_eval", "1", "2", "1", "8.000000", "runtime_chunk_avg", "2", "", "input_rows", "2", "0.250000", "6.000000", "runtime_chunk_bytes", "input_side=all,weight_version=v2", }, row.toStrings()) } func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { + require.Equal(t, "v2", readBillingDemoWeightVersion) tidbWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) require.True(t, ok) tikvWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) @@ -254,6 +257,18 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { require.True(t, ok) require.Equal(t, tidbWeights.byte, weight) require.Equal(t, 4096*tidbWeights.byte, previewRU) + topNWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassTopN, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, topNWeights.row) + require.NotZero(t, topNWeights.orderWork) + weight, previewRU, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitOrderWork, value: 12}, topNWeights) + require.True(t, ok) + require.Equal(t, topNWeights.orderWork, weight) + require.Equal(t, 12*topNWeights.orderWork, previewRU) + tikvTopNWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, tikvTopNWeights.row) + require.NotZero(t, tikvTopNWeights.orderWork) _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: "scan_total_keys", value: 4}, tidbWeights) require.False(t, ok) @@ -374,6 +389,108 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { Origin: physicalop.PhysicalExchangeReceiver{}.Init(ctx, stats), IsRoot: true, }, readBillingDemoSiteTiDB, readBillingDemoOpClassReaderReceive, false, readBillingDemoReasonUnsupportedMPP) + + t.Run("root sort and topn preserve algorithmic work", func(t *testing.T) { + child := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + child.SetSchema(schema) + sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + topN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0) + topN.SetSchema(schema) + for _, tc := range []struct { + name string + plan *FlatOperator + expectedClass string + inputRows int + inputBytes int64 + expectedWork float64 + expectedWorkText string + }{ + {name: "sort n log n preserves fractional work", plan: &FlatOperator{Origin: sort, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, expectedClass: readBillingDemoOpClassSort, inputRows: 3, inputBytes: 30, expectedWork: 4.754887502163469, expectedWorkText: "4.754887502163469"}, + {name: "topn n log k", plan: &FlatOperator{Origin: topN, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, expectedClass: readBillingDemoOpClassTopN, inputRows: 8, inputBytes: 80, expectedWork: 16, expectedWorkText: "16"}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := FlatPlanTree{ + tc.plan, + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + for _, planID := range []int{tc.plan.Origin.ID(), child.ID()} { + basic := runtimeStats.GetBasicRuntimeStats(planID, true) + basic.Record(time.Millisecond, tc.inputRows) + basic.RecordBytes(0, tc.inputBytes) + } + operator, supported, reason := readBillingDemoClassifyOperator(tree[0]) + require.True(t, supported) + require.Empty(t, reason) + require.Equal(t, tc.expectedClass, operator.opClass) + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], operator) + require.True(t, ok) + require.Empty(t, reason) + require.Equal(t, float64(tc.inputRows), readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.InDelta(t, tc.expectedWork, readBillingDemoUnitValue(units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll), 1e-12) + require.Equal(t, readBillingDemoInputSourceRuntimeOrderingWork, readBillingDemoUnitSource(units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) + operator.status = readBillingDemoStatusOperatorOK + operator.units = units + operator.id = tc.plan.ExplainID().String() + result := readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + operators: []readBillingDemoOperatorResult{operator}, + } + weights, ok := readBillingDemoResolveWeights(operator.site, operator.opClass, readBillingDemoWeightVersion) + require.True(t, ok) + require.Zero(t, weights.row) + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.InEpsilon(t, weights.fixedEvent+float64(tc.inputBytes)*weights.byte+tc.expectedWork*weights.orderWork, rows[0].previewRU, 0.000001) + var orderRows int + for _, row := range rows { + if row.unit == readBillingDemoUnitOrderWork { + orderRows++ + require.InDelta(t, tc.expectedWork, row.workRows, 1e-12) + require.Equal(t, tc.expectedWorkText, row.toStrings()[9]) + } + } + require.Equal(t, 1, orderRows) + + stats := buildReadBillingDemoStatementStats(result) + require.Equal(t, "v2", stats.WeightVersion) + var orderSamples int + for _, sample := range stats.BaseUnits { + if sample.Unit == readBillingDemoUnitOrderWork { + orderSamples++ + require.InDelta(t, tc.expectedWork, sample.Value, 1e-12) + require.Equal(t, readBillingDemoInputSourceRuntimeOrderingWork, sample.InputSource) + } + } + require.Equal(t, 1, orderSamples) + }) + } + }) + + t.Run("ordering work boundaries fail only on invalid evidence", func(t *testing.T) { + sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + sortOp := &FlatOperator{Origin: sort, IsRoot: true, StoreType: kv.TiDB} + unit, ok := readBillingDemoOrderingWorkUnit(sortOp, readBillingDemoOpClassSort, 0) + require.True(t, ok) + require.Zero(t, unit.value) + _, ok = readBillingDemoOrderingWorkUnit(sortOp, readBillingDemoOpClassSort, -1) + require.False(t, ok) + + maxInt64 := int64(^uint64(0) >> 1) + unit, ok = readBillingDemoOrderingWorkUnit(sortOp, readBillingDemoOpClassSort, maxInt64/2) + require.True(t, ok) + require.Greater(t, unit.value, float64(maxInt64)) + + hugeTopN := physicalop.PhysicalTopN{Offset: ^uint64(0), Count: ^uint64(0)}.Init(ctx, stats, 0) + unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: hugeTopN, IsRoot: true, StoreType: kv.TiDB}, readBillingDemoOpClassTopN, 1) + require.True(t, ok) + require.Greater(t, unit.value, 64.0) + require.LessOrEqual(t, unit.value, 65.0) + + malformedCopTopN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0) + _, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: malformedCopTopN, IsRoot: false, StoreType: kv.TiKV}, readBillingDemoOpClassTopN, 8) + require.False(t, ok) + }) indexMerge := &physicalop.PhysicalIndexMergeReader{} indexMerge.BasePhysicalPlan = physicalop.NewBasePhysicalPlan(ctx, "IndexMerge", indexMerge, 0) requireReadBillingDemoClass(t, &FlatOperator{ @@ -578,40 +695,42 @@ func TestReadBillingDemoNonScanCopWithoutBytesFailsClosed(t *testing.T) { col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) stats := &property.StatsInfo{RowCount: 5} + reader := physicalop.PhysicalTableReader{}.Init(ctx, 0) proj := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + reader.SetSchema(schema) proj.SetSchema(schema) + scan.SetSchema(schema) tree := FlatPlanTree{ - {Origin: proj, IsRoot: false, StoreType: kv.TiKV}, + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: proj, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, } runtimeStats := execdetails.NewRuntimeStatsColl(nil) - runtimeStats.RecordCopStats(proj.ID(), kv.TiKV, &tikvutil.ScanDetail{}, tikvutil.TimeDetail{}, nil) - operator, supported, reason := readBillingDemoClassifyOperator(&FlatOperator{ - Origin: proj, - IsRoot: false, - StoreType: kv.TiKV, - }) + operator, supported, reason := readBillingDemoClassifyOperator(tree[1]) require.True(t, supported) require.Empty(t, reason) require.Equal(t, readBillingDemoSiteTiKV, operator.site) require.Equal(t, readBillingDemoOpClassProjection, operator.opClass) - units, actualReason, ok := readBillingDemoCopUnits(runtimeStats, tree, 0, operator) - require.False(t, ok) - require.Nil(t, units) - require.Equal(t, readBillingDemoReasonMissingRuntimeBytes, actualReason) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.False(t, outcome.success) + require.Nil(t, outcome.units) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, outcome.failure.reason) } func TestReadBillingDemoRangeScanUsesProcessedKeyAverage(t *testing.T) { ctx := mock.NewContext() col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) - scan := physicalop.PhysicalTableScan{}.Init(ctx, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) scan.SetSchema(schema) - scan.StoreType = kv.TiKV - scan.TblCols = []*expression.Column{col} + reader := physicalop.PhysicalTableReader{}.Init(ctx, 0) + reader.SetSchema(schema) tree := FlatPlanTree{ - {Origin: scan, IsRoot: false, StoreType: kv.TiKV}, + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, } rows, bytes, ok := readBillingDemoRangeScanInput(10, 5, 100) @@ -631,30 +750,597 @@ func TestReadBillingDemoRangeScanUsesProcessedKeyAverage(t *testing.T) { require.False(t, ok) } - buildUnits := func(scanDetail *tikvutil.ScanDetail) ([]readBillingDemoUnit, string, bool) { + buildUnits := func(scanDetail *tikvutil.ScanDetail) readBillingDemoCopUnitOutcome { runtimeStats := execdetails.NewRuntimeStatsColl(nil) runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, scanDetail, tikvutil.TimeDetail{}, nil) return readBillingDemoCopUnits( - runtimeStats, - tree, - 0, + newReadBillingDemoCopEstimator(tree, runtimeStats), + 1, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "tablescan"}, ) } - units, actualReason, ok := buildUnits(&tikvutil.ScanDetail{TotalKeys: 10, ProcessedKeys: 5, ProcessedKeysSize: 100}) - require.True(t, ok) - require.Empty(t, actualReason) + outcome := buildUnits(&tikvutil.ScanDetail{TotalKeys: 10, ProcessedKeys: 5, ProcessedKeysSize: 100}) + require.True(t, outcome.success) + units := outcome.units require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) require.Equal(t, 10.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) require.Equal(t, 200.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) require.Equal(t, explainRUWidthSourceScanDetailProcessedAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) - units, actualReason, ok = buildUnits(&tikvutil.ScanDetail{}) - require.False(t, ok) - require.Nil(t, units) - require.Equal(t, readBillingDemoReasonMissingScanDetail, actualReason) + outcome = buildUnits(&tikvutil.ScanDetail{}) + require.False(t, outcome.success) + require.Nil(t, outcome.units) + require.Equal(t, readBillingDemoReasonMissingScanWidthEvidence, outcome.failure.reason) +} + +func TestReadBillingDemoCopInputEstimator(t *testing.T) { + ctx := mock.NewContext() + col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} + schema := expression.NewSchema(col) + stats := &property.StatsInfo{RowCount: 10} + reader := physicalop.PhysicalTableReader{}.Init(ctx, 0) + limit := physicalop.PhysicalLimit{}.Init(ctx, stats, 0) + projection := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + selection := physicalop.PhysicalSelection{}.Init(ctx, stats, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + reader.SetSchema(schema) + limit.SetSchema(schema) + projection.SetSchema(schema) + scan.SetSchema(schema) + + recordSummary := func(runtimeStats *execdetails.RuntimeStatsColl, planID int, rows uint64, detail *tikvutil.ScanDetail) { + one := uint64(1) + runtimeStats.RecordCopStats(planID, kv.TiKV, detail, tikvutil.TimeDetail{}, &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &rows, + NumIterations: &one, + }) + } + + t.Run("selection uses direct child rows and component scan width", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + operator, supported, _ := readBillingDemoClassifyOperator(tree[1]) + require.True(t, supported) + outcome := readBillingDemoCopUnits(estimator, 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeChildActRows, readBillingDemoUnitSource(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceScanDetailProcessedEstimate, readBillingDemoUnitWidthSource(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.LessOrEqual(t, estimator.nodeVisits, 6*len(tree)) + require.LessOrEqual(t, estimator.edgeVisits, 4) + require.LessOrEqual(t, estimator.auxiliaryEntryCount(), 12*len(tree)) + }) + + t.Run("supported unary operator width propagation matrix", func(t *testing.T) { + topN := physicalop.PhysicalTopN{Count: 8}.Init(ctx, stats, 0) + topN.SetSchema(schema) + hashAgg := (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema) + streamAgg := (&physicalop.BasePhysicalAgg{}).InitForStream(ctx, stats, 0, schema) + cases := []struct { + name string + node *FlatOperator + widthState readBillingDemoCopWidthState + expectedOrderWork float64 + }{ + {name: "selection", node: &FlatOperator{Origin: selection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, + {name: "limit", node: &FlatOperator{Origin: limit, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, + {name: "topn", node: &FlatOperator{Origin: topN, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: 12}, + {name: "projection", node: &FlatOperator{Origin: projection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, + {name: "hashagg", node: &FlatOperator{Origin: hashAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, + {name: "streamagg", node: &FlatOperator{Origin: streamAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.node.ChildrenIdx = []int{2} + tc.node.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + tc.node, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, tc.node.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + operator, supported, _ := readBillingDemoClassifyOperator(tc.node) + require.True(t, supported) + outcome := readBillingDemoCopUnits(estimator, 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, tc.expectedOrderWork, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) + require.Equal(t, tc.widthState, estimator.outputEstimate(1).widthState) + }) + } + }) + + t.Run("selection output feeds projection input", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: projection, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: selection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 5, nil) + recordSummary(runtimeStats, selection.ID(), 3, nil) + recordSummary(runtimeStats, projection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(tree[1]) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 3.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 60.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("multi scan and multi detail components are ambiguous", func(t *testing.T) { + otherScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + otherScan.SetSchema(schema) + multiScanTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2, 3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: otherScan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, otherScan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(multiScanTree, runtimeStats) + require.Equal(t, readBillingDemoCopWidthAmbiguous, estimator.outputEstimate(2).widthState) + scanOperator, _, _ := readBillingDemoClassifyOperator(multiScanTree[2]) + require.Equal(t, readBillingDemoReasonAmbiguousCopScanWidth, readBillingDemoCopUnits(estimator, 2, scanOperator).failure.reason) + + multiDetailTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats = execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 3, ProcessedKeys: 3, ProcessedKeysSize: 60}) + estimator = newReadBillingDemoCopEstimator(multiDetailTree, runtimeStats) + require.Equal(t, readBillingDemoCopWidthAmbiguous, estimator.outputEstimate(2).widthState) + }) + + t.Run("projection consumes width but blocks its parent", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: limit, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: projection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, projection.ID(), 3, nil) + recordSummary(runtimeStats, limit.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + projectionOperator, _, _ := readBillingDemoClassifyOperator(tree[2]) + require.True(t, readBillingDemoCopUnits(estimator, 2, projectionOperator).success) + limitOperator, _, _ := readBillingDemoClassifyOperator(tree[1]) + limitOutcome := readBillingDemoCopUnits(estimator, 1, limitOperator) + require.False(t, limitOutcome.success) + require.Equal(t, readBillingDemoReasonUnsupportedCopWidthTransform, limitOutcome.failure.reason) + require.Equal(t, readBillingDemoCopFailureCurrent, limitOutcome.failure.kind) + }) + + t.Run("missing scan summary becomes projection cause across output edge", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: limit, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: projection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, projection.ID(), 3, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + recordSummary(runtimeStats, limit.ID(), 2, nil) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + limitOperator, _, _ := readBillingDemoClassifyOperator(tree[1]) + outcome := readBillingDemoCopUnits(estimator, 1, limitOperator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoCopFailureIntrinsicCause, outcome.failure.kind) + require.Equal(t, 2, outcome.failure.failingIdx) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, outcome.failure.reason) + }) + + t.Run("select and DML materialize projection cause without skipping scan", func(t *testing.T) { + limit.SetChildren(projection) + projection.SetChildren(scan) + reader.TablePlan = limit + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, projection.ID(), 3, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + recordSummary(runtimeStats, limit.ID(), 2, nil) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + selectResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusUnknownInput, selectResult.status) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, selectResult.reason) + require.Len(t, selectResult.operators, 1) + require.Equal(t, flat.Main[2].ExplainID().String(), selectResult.operators[0].id) + + dmlResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoDMLTree(&dmlResult, runtimeStats, flat.Main) + var dependentCount, projectionPartialCount, scanOKCount int + for _, operator := range dmlResult.operators { + switch { + case operator.id == flat.Main[1].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonDependentCopInputUnavailable: + dependentCount++ + case operator.id == flat.Main[2].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonMissingCopChildRuntimeRows: + projectionPartialCount++ + case operator.id == flat.Main[3].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + scanOKCount++ + } + } + require.Equal(t, 1, dependentCount) + require.Equal(t, 1, projectionPartialCount) + require.Equal(t, 1, scanOKCount) + }) + + t.Run("DML keeps projection units when its output summary is missing", func(t *testing.T) { + limit.SetChildren(projection) + projection.SetChildren(scan) + reader.TablePlan = limit + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, limit.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + selectResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusUnknownInput, selectResult.status) + require.Equal(t, readBillingDemoReasonMissingCopChildRuntimeRows, selectResult.reason) + require.Len(t, selectResult.operators, 1) + require.Equal(t, flat.Main[1].ExplainID().String(), selectResult.operators[0].id) + + dmlResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoDMLTree(&dmlResult, runtimeStats, flat.Main) + var limitPartial, projectionOK, scanOK int + for _, operator := range dmlResult.operators { + switch { + case operator.id == flat.Main[1].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonMissingCopChildRuntimeRows: + limitPartial++ + case operator.id == flat.Main[2].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + projectionOK++ + require.Equal(t, 4.0, readBillingDemoUnitValue(operator.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(operator.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + case operator.id == flat.Main[3].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + scanOK++ + } + } + require.Equal(t, 1, limitPartial) + require.Equal(t, 1, projectionOK) + require.Equal(t, 1, scanOK) + }) + + t.Run("unsupported descendant cause is preserved", func(t *testing.T) { + sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + limit.SetChildren(sort) + sort.SetChildren(scan) + reader.TablePlan = limit + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, sort.ID(), 3, nil) + recordSummary(runtimeStats, limit.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + selectResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusUnsupported, selectResult.status) + require.Equal(t, readBillingDemoReasonUnsupportedOperator, selectResult.reason) + require.Len(t, selectResult.operators, 1) + require.Equal(t, flat.Main[2].ExplainID().String(), selectResult.operators[0].id) + + dmlResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoDMLTree(&dmlResult, runtimeStats, flat.Main) + var limitDependent, sortUnsupported, scanOK int + for _, operator := range dmlResult.operators { + switch { + case operator.id == flat.Main[1].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonDependentCopInputUnavailable: + limitDependent++ + case operator.id == flat.Main[2].ExplainID().String() && operator.status == readBillingDemoStatusPartial && operator.reason == readBillingDemoReasonUnsupportedOperator: + sortUnsupported++ + case operator.id == flat.Main[3].ExplainID().String() && operator.status == readBillingDemoStatusOperatorOK: + scanOK++ + } + } + require.Equal(t, 1, limitDependent) + require.Equal(t, 1, sortUnsupported) + require.Equal(t, 1, scanOK) + }) + + t.Run("invalid child rows and multi child arity fail explicitly", func(t *testing.T) { + invalidTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), ^uint64(0), nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(invalidTree[1]) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(invalidTree, runtimeStats), 1, operator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoCopFailureIntrinsicCause, outcome.failure.kind) + require.Equal(t, 2, outcome.failure.failingIdx) + require.Equal(t, readBillingDemoReasonInvalidCopRuntimeRows, outcome.failure.reason) + + otherScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + otherScan.SetSchema(schema) + multiChildTree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2, 3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: otherScan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + outcome = readBillingDemoCopUnits(newReadBillingDemoCopEstimator(multiChildTree, execdetails.NewRuntimeStatsColl(nil)), 1, operator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoStatusUnsupported, outcome.failure.status) + require.Equal(t, readBillingDemoReasonUnsupportedCopMultiChild, outcome.failure.reason) + }) + + t.Run("malformed references stay linear and fail structurally", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1, 1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, + } + estimator := newReadBillingDemoCopEstimator(tree, execdetails.NewRuntimeStatsColl(nil)) + failure, ok := estimator.firstTreeFailure() + require.True(t, ok) + require.Equal(t, readBillingDemoReasonUnsupportedCopStructure, failure.reason) + require.LessOrEqual(t, estimator.nodeVisits, 6*len(tree)) + require.LessOrEqual(t, estimator.edgeVisits, 4) + + const siblingCount = 128 + wideTree := make(FlatPlanTree, siblingCount+1) + children := make([]int, siblingCount) + wideTree[0] = &FlatOperator{Origin: reader, ChildrenIdx: children, ChildrenEndIdx: siblingCount, IsRoot: true, StoreType: kv.TiDB} + for i := 1; i <= siblingCount; i++ { + children[i-1] = i + wideTree[i] = &FlatOperator{Origin: scan, ChildrenEndIdx: siblingCount, IsRoot: false, StoreType: kv.TiKV} + } + estimator = newReadBillingDemoCopEstimator(wideTree, execdetails.NewRuntimeStatsColl(nil)) + failure, ok = estimator.firstTreeFailure() + require.True(t, ok) + require.Equal(t, readBillingDemoReasonUnsupportedCopStructure, failure.reason) + require.LessOrEqual(t, estimator.nodeVisits, 6*len(wideTree)) + require.LessOrEqual(t, estimator.edgeVisits, 2*siblingCount) + }) + + t.Run("special tree root may omit children end index", func(t *testing.T) { + // CTE-definition and scalar-subquery synthetic roots populate ChildrenIdx + // but intentionally leave ChildrenEndIdx at its zero value. + tree := FlatPlanTree{ + {Origin: projection, ChildrenIdx: []int{1}, IsRoot: true, StoreType: kv.TiDB, Label: SeedPart}, + {Origin: reader, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{3}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + _, failed := estimator.firstTreeFailure() + require.False(t, failed) + operator, _, _ := readBillingDemoClassifyOperator(tree[2]) + outcome := readBillingDemoCopUnits(estimator, 2, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("zero rows are observed and partial summaries fail closed", func(t *testing.T) { + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 0, nil) + recordSummary(runtimeStats, selection.ID(), 0, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(tree[1]) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 0.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 0.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + + recordSummary(runtimeStats, selection.ID(), 0, nil) + outcome = readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, outcome.failure.reason) + }) + + t.Run("sibling components keep scan detail isolated", func(t *testing.T) { + indexScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + indexScan.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1, 2}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, IsRoot: false, StoreType: kv.TiKV}, + {Origin: indexScan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 50}, tikvutil.TimeDetail{}, nil) + runtimeStats.RecordCopStats(indexScan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 150}, tikvutil.TimeDetail{}, nil) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + require.Equal(t, 10.0, estimator.outputEstimate(1).avgRowWidth) + require.Equal(t, 30.0, estimator.outputEstimate(2).avgRowWidth) + }) + + t.Run("full builder wires flattened table reader component", func(t *testing.T) { + selection.SetChildren(scan) + reader.TablePlan = selection + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 3) + require.Equal(t, selection.ID(), flat.Main[1].Origin.ID()) + require.Equal(t, scan.ID(), flat.Main[2].Origin.ID()) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, selection.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + var selectionResult readBillingDemoOperatorResult + for _, operator := range result.operators { + if operator.id == flat.Main[1].ExplainID().String() { + selectionResult = operator + break + } + } + require.Equal(t, readBillingDemoStatusOperatorOK, selectionResult.status) + require.Equal(t, 4.0, readBillingDemoUnitValue(selectionResult.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(selectionResult.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + statementStats := buildReadBillingDemoStatementStats(result) + require.NotEmpty(t, statementStats.BaseUnits) + for _, sample := range statementStats.BaseUnits { + require.Equal(t, "v3", sample.ModelVersion) + } + }) + + t.Run("full builder wires root sort order work", func(t *testing.T) { + rootSort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) + scan.SetStats(stats) + reader.TablePlan = scan + rootSort.SetChildren(reader) + flat := FlattenPhysicalPlan(rootSort, true) + require.Len(t, flat.Main, 3) + require.Equal(t, rootSort.ID(), flat.Main[0].Origin.ID()) + require.Equal(t, reader.ID(), flat.Main[1].Origin.ID()) + require.Equal(t, scan.ID(), flat.Main[2].Origin.ID()) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + for _, rootEvidence := range []struct { + planID int + rows int + bytes int64 + }{ + {planID: rootSort.ID(), rows: 8, bytes: 80}, + {planID: reader.ID(), rows: 8, bytes: 80}, + } { + basic := runtimeStats.GetBasicRuntimeStats(rootEvidence.planID, true) + basic.Record(time.Millisecond, rootEvidence.rows) + basic.RecordBytes(0, rootEvidence.bytes) + } + recordSummary(runtimeStats, scan.ID(), 8, &tikvutil.ScanDetail{TotalKeys: 8, ProcessedKeys: 8, ProcessedKeysSize: 160}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, rootSort, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + var sortResult readBillingDemoOperatorResult + for _, operator := range result.operators { + if operator.site == readBillingDemoSiteTiDB && operator.opClass == readBillingDemoOpClassSort { + sortResult = operator + break + } + } + require.Equal(t, readBillingDemoStatusOperatorOK, sortResult.status) + require.Equal(t, 8.0, readBillingDemoUnitValue(sortResult.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(sortResult.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 24.0, readBillingDemoUnitValue(sortResult.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) + }) + + t.Run("full builder wires root and pushed topn order work", func(t *testing.T) { + reqProp := property.NewPhysicalProperty(property.RootTaskType, nil, false, 0, false) + rootTopN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0, reqProp) + scan.SetStats(stats) + copTopN, globalTopN := getPushedDownTopN(rootTopN, scan, kv.TiKV) + require.NotNil(t, copTopN) + require.Nil(t, globalTopN) + require.Zero(t, copTopN.Offset) + require.Equal(t, uint64(4), copTopN.Count) + copTopN.SetSchema(schema) + rootTopN.SetSchema(schema) + reader.TablePlan = copTopN + rootTopN.SetChildren(reader) + flat := FlattenPhysicalPlan(rootTopN, true) + require.Len(t, flat.Main, 4) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + for _, rootEvidence := range []struct { + planID int + rows int + bytes int64 + }{ + {planID: rootTopN.ID(), rows: 3, bytes: 30}, + {planID: reader.ID(), rows: 4, bytes: 40}, + } { + basic := runtimeStats.GetBasicRuntimeStats(rootEvidence.planID, true) + basic.Record(time.Millisecond, rootEvidence.rows) + basic.RecordBytes(0, rootEvidence.bytes) + } + recordSummary(runtimeStats, scan.ID(), 8, nil) + recordSummary(runtimeStats, copTopN.ID(), 4, &tikvutil.ScanDetail{TotalKeys: 8, ProcessedKeys: 8, ProcessedKeysSize: 160}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, rootTopN, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + orderWorkByClass := make(map[string]float64) + for _, operator := range result.operators { + if operator.opClass == readBillingDemoOpClassTopN { + orderWorkByClass[operator.site+"/"+operator.opClass] = readBillingDemoUnitValue(operator.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll) + } + } + require.Equal(t, 8.0, orderWorkByClass[readBillingDemoSiteTiDB+"/"+readBillingDemoOpClassTopN]) + require.Equal(t, 16.0, orderWorkByClass[readBillingDemoSiteTiKV+"/"+readBillingDemoOpClassTopN]) + }) + + t.Run("full builder keeps index lookup components isolated", func(t *testing.T) { + indexScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tableScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + indexScan.SetSchema(schema) + tableScan.SetSchema(schema) + lookup := (physicalop.PhysicalIndexLookUpReader{IndexPlan: indexScan, TablePlan: tableScan}).Init(ctx, 0, plannerutil.IndexLookUpPushDownNone) + flat := FlattenPhysicalPlan(lookup, true) + require.Len(t, flat.Main, 3) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + rootStats := runtimeStats.GetBasicRuntimeStats(lookup.ID(), true) + rootStats.Record(time.Millisecond, 2) + rootStats.RecordBytes(0, 40) + runtimeStats.RecordCopStats(indexScan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 50}, tikvutil.TimeDetail{}, nil) + runtimeStats.RecordCopStats(tableScan.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 150}, tikvutil.TimeDetail{}, nil) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, lookup, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + bytesByID := make(map[string]float64) + for _, operator := range result.operators { + if operator.opClass == readBillingDemoOpClassRangeScan { + bytesByID[operator.id] = readBillingDemoUnitValue(operator.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll) + } + } + require.Equal(t, 50.0, bytesByID[flat.Main[1].ExplainID().String()]) + require.Equal(t, 150.0, bytesByID[flat.Main[2].ExplainID().String()]) + }) } func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 852e3f1e1b6e3..63972859cb9ae 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -15,6 +15,7 @@ package core import ( + "math" "strconv" "strings" "sync/atomic" @@ -25,6 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" @@ -53,95 +55,109 @@ const ( explainRUSectionPlan = "plan" explainRUSourceSummaryTotal = "summary_total" - explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" - explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" - explainRUWidthSourceNotApplicable = "not_applicable" - readBillingDemoModelVersion = "v2" - readBillingDemoWeightVersion = "v1" - readBillingDemoStatusSuccess = "success" - readBillingDemoStatusUnsupported = "unsupported" - readBillingDemoStatusUnknownInput = "unknown_input" - readBillingDemoStatusError = "error" - readBillingDemoStatusOperatorOK = "ok" - readBillingDemoStatusPartial = "partial" - readBillingDemoReasonNone = "none" - readBillingDemoReasonStatementError = "statement_error" - readBillingDemoReasonMissingPlan = "missing_plan" - readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" - readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" - readBillingDemoReasonMissingRuntimeBytes = "missing_runtime_bytes" - readBillingDemoReasonMissingInputBytes = "missing_input_bytes" - readBillingDemoReasonMissingScanDetail = "missing_scan_detail" - readBillingDemoReasonUnsupportedOperator = "unsupported_operator" - readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" - readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" - readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" - readBillingDemoReasonUnsupportedLock = "unsupported_lock" - readBillingDemoReasonNonBillable = "non_billable" - readBillingDemoReasonMissingCommitDetail = "missing_commit_detail" - readBillingDemoReasonMissingWriteKeys = "missing_write_keys" - readBillingDemoReasonMissingWriteByte = "missing_write_byte" - readBillingDemoReasonZeroMutation = "zero_mutation" - readBillingDemoReasonMissingPrewriteRegion = "missing_prewrite_region_num" - readBillingDemoReasonMissingWriteRPCCount = "missing_write_rpc_count" - readBillingDemoReasonMissingMutationRecorder = "missing_mutation_recorder" - readBillingDemoReasonUncalibratedMutation = "uncalibrated_mutation_weights" - readBillingDemoReasonDMLAncillaryPartial = "dml_ancillary_work_partial" - readBillingDemoReasonPipelinedWritePartial = "pipelined_tikv_payload_unsupported" - readBillingDemoReasonOptimisticReplayPartial = "optimistic_replay_attribution_unsupported" - readBillingDemoSiteStatement = "statement" - readBillingDemoSiteTiDB = "tidb" - readBillingDemoSiteTiKV = "tikv" - readBillingDemoOpClassStatement = "statement" - readBillingDemoOpClassFilter = "filter_eval" - readBillingDemoOpClassProjection = "projection_eval" - readBillingDemoOpClassLimit = "row_limit" - readBillingDemoOpClassTopN = "bounded_topn" - readBillingDemoOpClassSort = "full_ordering" - readBillingDemoOpClassWindow = "window_eval" - readBillingDemoOpClassHashAgg = "agg_hash" - readBillingDemoOpClassStreamAgg = "agg_stream" - readBillingDemoOpClassHashJoin = "join_hash" - readBillingDemoOpClassMergeJoin = "join_merge" - readBillingDemoOpClassLookupJoin = "join_lookup" - readBillingDemoOpClassReaderReceive = "reader_receive" - readBillingDemoOpClassLookupReader = "lookup_reader" - readBillingDemoOpClassOverlayReader = "overlay_reader" - readBillingDemoOpClassMetadataReader = "metadata_reader" - readBillingDemoOpClassPointLookup = "kv_point_lookup" - readBillingDemoOpClassRangeScan = "kv_range_scan" - readBillingDemoOpClassKVMutation = "kv_mutation" - readBillingDemoOpClassKVWrite = "kv_write" - readBillingDemoOpClassWrapper = "wrapper" - readBillingDemoOpClassSynthetic = "synthetic_source" - readBillingDemoOperatorStatement = "statement" - readBillingDemoOperatorMemDBMutation = "memdb_mutation" - readBillingDemoOperatorTxnPrewrite = "txn_prewrite" - readBillingDemoUnitFixedEvents = "fixed_events" - readBillingDemoUnitInputRows = "input_rows" - readBillingDemoUnitInputBytes = "input_bytes" - readBillingDemoUnitEncodedMutationCount = "encoded_mutation_count" - readBillingDemoUnitEncodedMutationBytes = "encoded_mutation_bytes" - readBillingDemoUnitSetCount = "set_count" - readBillingDemoUnitDeleteCount = "delete_count" - readBillingDemoUnitKeyBytes = "key_bytes" - readBillingDemoUnitValueBytes = "value_bytes" - readBillingDemoUnitWriteKeys = "write_keys" - readBillingDemoUnitWriteByte = "write_byte" - readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" - readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" - readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" - readBillingDemoInputSourceScanDetail = "scan_detail" - readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" - readBillingDemoInputSourceCommitDetail = "commit_detail" - readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" - readBillingDemoInputSideAll = "all" - readBillingDemoInputSideBuild = "build" - readBillingDemoInputSideProbe = "probe" - readBillingDemoInputSideLeft = "left" - readBillingDemoInputSideRight = "right" - readBillingDemoScopeStatementAttempted = "statement_attempted" - readBillingDemoScopeTxnPrewritePayload = "txn_prewrite_payload" + explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" + explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" + explainRUWidthSourceScanDetailProcessedEstimate = "scan_detail_processed_key_avg_estimate" + explainRUWidthSourceNotApplicable = "not_applicable" + readBillingDemoModelVersion = "v3" + readBillingDemoWeightVersion = "v2" + readBillingDemoStatusSuccess = "success" + readBillingDemoStatusUnsupported = "unsupported" + readBillingDemoStatusUnknownInput = "unknown_input" + readBillingDemoStatusError = "error" + readBillingDemoStatusOperatorOK = "ok" + readBillingDemoStatusPartial = "partial" + readBillingDemoReasonNone = "none" + readBillingDemoReasonStatementError = "statement_error" + readBillingDemoReasonMissingPlan = "missing_plan" + readBillingDemoReasonMissingRuntimeStats = "missing_runtime_stats" + readBillingDemoReasonMissingRuntimeRows = "missing_runtime_rows" + readBillingDemoReasonMissingRuntimeBytes = "missing_runtime_bytes" + readBillingDemoReasonMissingInputBytes = "missing_input_bytes" + readBillingDemoReasonMissingScanDetail = "missing_scan_detail" + readBillingDemoReasonUnsupportedOperator = "unsupported_operator" + readBillingDemoReasonUnsupportedTiFlash = "unsupported_tiflash" + readBillingDemoReasonUnsupportedMPP = "unsupported_mpp" + readBillingDemoReasonUnsupportedIndexMerge = "unsupported_index_merge" + readBillingDemoReasonUnsupportedLock = "unsupported_lock" + readBillingDemoReasonNonBillable = "non_billable" + readBillingDemoReasonMissingCommitDetail = "missing_commit_detail" + readBillingDemoReasonMissingWriteKeys = "missing_write_keys" + readBillingDemoReasonMissingWriteByte = "missing_write_byte" + readBillingDemoReasonZeroMutation = "zero_mutation" + readBillingDemoReasonMissingPrewriteRegion = "missing_prewrite_region_num" + readBillingDemoReasonMissingWriteRPCCount = "missing_write_rpc_count" + readBillingDemoReasonMissingMutationRecorder = "missing_mutation_recorder" + readBillingDemoReasonUncalibratedMutation = "uncalibrated_mutation_weights" + readBillingDemoReasonDMLAncillaryPartial = "dml_ancillary_work_partial" + readBillingDemoReasonPipelinedWritePartial = "pipelined_tikv_payload_unsupported" + readBillingDemoReasonOptimisticReplayPartial = "optimistic_replay_attribution_unsupported" + readBillingDemoReasonMissingCopChildRuntimeRows = "missing_cop_child_runtime_rows" + readBillingDemoReasonMissingScanWidthEvidence = "missing_scan_width_evidence" + readBillingDemoReasonAmbiguousCopScanWidth = "ambiguous_cop_scan_width" + readBillingDemoReasonUnsupportedCopMultiChild = "unsupported_cop_multi_child" + readBillingDemoReasonUnsupportedCopWidthTransform = "unsupported_cop_width_transform" + readBillingDemoReasonUnsupportedCopStructure = "unsupported_cop_structure" + readBillingDemoReasonInvalidCopRuntimeRows = "invalid_cop_runtime_rows" + readBillingDemoReasonIncompleteCopRuntimeRows = "incomplete_cop_runtime_rows" + readBillingDemoReasonDependentCopInputUnavailable = "dependent_cop_input_unavailable" + readBillingDemoReasonInvalidOrderingWork = "invalid_ordering_work" + readBillingDemoSiteStatement = "statement" + readBillingDemoSiteTiDB = "tidb" + readBillingDemoSiteTiKV = "tikv" + readBillingDemoOpClassStatement = "statement" + readBillingDemoOpClassFilter = "filter_eval" + readBillingDemoOpClassProjection = "projection_eval" + readBillingDemoOpClassLimit = "row_limit" + readBillingDemoOpClassTopN = "bounded_topn" + readBillingDemoOpClassSort = "full_ordering" + readBillingDemoOpClassWindow = "window_eval" + readBillingDemoOpClassHashAgg = "agg_hash" + readBillingDemoOpClassStreamAgg = "agg_stream" + readBillingDemoOpClassHashJoin = "join_hash" + readBillingDemoOpClassMergeJoin = "join_merge" + readBillingDemoOpClassLookupJoin = "join_lookup" + readBillingDemoOpClassReaderReceive = "reader_receive" + readBillingDemoOpClassLookupReader = "lookup_reader" + readBillingDemoOpClassOverlayReader = "overlay_reader" + readBillingDemoOpClassMetadataReader = "metadata_reader" + readBillingDemoOpClassPointLookup = "kv_point_lookup" + readBillingDemoOpClassRangeScan = "kv_range_scan" + readBillingDemoOpClassKVMutation = "kv_mutation" + readBillingDemoOpClassKVWrite = "kv_write" + readBillingDemoOpClassWrapper = "wrapper" + readBillingDemoOpClassSynthetic = "synthetic_source" + readBillingDemoOperatorStatement = "statement" + readBillingDemoOperatorMemDBMutation = "memdb_mutation" + readBillingDemoOperatorTxnPrewrite = "txn_prewrite" + readBillingDemoUnitFixedEvents = "fixed_events" + readBillingDemoUnitInputRows = "input_rows" + readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoUnitOrderWork = "order_work" + readBillingDemoUnitEncodedMutationCount = "encoded_mutation_count" + readBillingDemoUnitEncodedMutationBytes = "encoded_mutation_bytes" + readBillingDemoUnitSetCount = "set_count" + readBillingDemoUnitDeleteCount = "delete_count" + readBillingDemoUnitKeyBytes = "key_bytes" + readBillingDemoUnitValueBytes = "value_bytes" + readBillingDemoUnitWriteKeys = "write_keys" + readBillingDemoUnitWriteByte = "write_byte" + readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" + readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" + readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" + readBillingDemoInputSourceScanDetail = "scan_detail" + readBillingDemoInputSourceRuntimeChildActRows = "runtime_child_act_rows" + readBillingDemoInputSourceRuntimeOrderingWork = "runtime_ordering_work" + readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" + readBillingDemoInputSourceCommitDetail = "commit_detail" + readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" + readBillingDemoInputSideAll = "all" + readBillingDemoInputSideBuild = "build" + readBillingDemoInputSideProbe = "probe" + readBillingDemoInputSideLeft = "left" + readBillingDemoInputSideRight = "right" + readBillingDemoScopeStatementAttempted = "statement_attempted" + readBillingDemoScopeTxnPrewritePayload = "txn_prewrite_payload" // The first write-side preview formula intentionally has no fixed/RPC/region // terms: write keys use the current scaled RUv2 write-key coefficient, and @@ -192,10 +208,106 @@ type readBillingDemoResult struct { operators []readBillingDemoOperatorResult } +type readBillingDemoCopWidthState uint8 + +const ( + readBillingDemoCopWidthMissing readBillingDemoCopWidthState = iota + readBillingDemoCopWidthKnown + readBillingDemoCopWidthBarrier + readBillingDemoCopWidthAmbiguous +) + +type readBillingDemoCopFailureKind uint8 + +const ( + readBillingDemoCopFailureCurrent readBillingDemoCopFailureKind = iota + readBillingDemoCopFailureIntrinsicCause +) + +type readBillingDemoCopRowsState uint8 + +const ( + readBillingDemoCopRowsMissing readBillingDemoCopRowsState = iota + readBillingDemoCopRowsObserved + readBillingDemoCopRowsInvalid +) + +type readBillingDemoCopFailure struct { + present bool + kind readBillingDemoCopFailureKind + status string + reason string + failingIdx int +} + +type readBillingDemoCopRowsEvidence struct { + state readBillingDemoCopRowsState + rows int64 + tasks int32 +} + +type readBillingDemoCopOutputEstimate struct { + widthState readBillingDemoCopWidthState + avgRowWidth float64 + widthSource string + scanPlanID int + failure readBillingDemoCopFailure +} + +type readBillingDemoCopInputEstimate struct { + rows int64 + inputBytes float64 + avgRowWidth float64 + inputSource string + widthSource string + failure readBillingDemoCopFailure +} + +type readBillingDemoCopComponentEvidence struct { + scanCount int + scanIdx int + detailHolderCount int + scanDetail tikvutil.ScanDetail + maxSummaryTasks int32 +} + +type readBillingDemoCopEstimator struct { + tree FlatPlanTree + runtimeStats *execdetails.RuntimeStatsColl + parentIdx []int + componentID []int + components []readBillingDemoCopComponentEvidence + nodeFailures map[int]readBillingDemoCopFailure + treeFailures []readBillingDemoCopFailure + treeFailureSeen map[int]struct{} + inputMemo map[int]readBillingDemoCopInputEstimate + inputMemoSet map[int]struct{} + outputMemo map[int]readBillingDemoCopOutputEstimate + outputMemoSet map[int]struct{} + visiting map[int]bool + nodeVisits int + edgeVisits int + auxiliaryEntries int +} + +type readBillingDemoCopUnitOutcome struct { + success bool + units []readBillingDemoUnit + failure readBillingDemoCopFailure +} + +type readBillingDemoAppendOutcome struct { + success bool + status string + current readBillingDemoOperatorResult + cause readBillingDemoCopFailure +} + type readBillingDemoOperatorWeights struct { fixedEvent float64 row float64 byte float64 + orderWork float64 mutationCount float64 mutationByte float64 setCount float64 @@ -220,7 +332,7 @@ var readBillingDemoWeights = map[readBillingDemoWeightKey]readBillingDemoOperato {readBillingDemoSiteTiKV, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000040, byte: 0.000006}, {readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000006}, {readBillingDemoSiteTiKV, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000008, byte: 0.000002}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000075, byte: 0.000012}, + {readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, byte: 0.000012, orderWork: 0.000075}, {readBillingDemoSiteTiKV, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000100, byte: 0.000014}, {readBillingDemoSiteTiKV, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000065, byte: 0.000010}, {readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion}: {fixedEvent: 0.045, row: 0.000030, byte: 0.000012}, @@ -235,8 +347,8 @@ var readBillingDemoWeights = map[readBillingDemoWeightKey]readBillingDemoOperato {readBillingDemoSiteTiDB, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000005}, {readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000020, byte: 0.000004}, {readBillingDemoSiteTiDB, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000006, byte: 0.000001}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000060, byte: 0.000010}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassSort, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000070, byte: 0.000012}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, byte: 0.000010, orderWork: 0.000060}, + {readBillingDemoSiteTiDB, readBillingDemoOpClassSort, readBillingDemoWeightVersion}: {fixedEvent: 0.080, byte: 0.000012, orderWork: 0.000070}, {readBillingDemoSiteTiDB, readBillingDemoOpClassWindow, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000070, byte: 0.000010}, {readBillingDemoSiteTiDB, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000085, byte: 0.000012}, {readBillingDemoSiteTiDB, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.050, row: 0.000055, byte: 0.000008}, @@ -267,7 +379,7 @@ type explainRURow struct { rowWidth float64 hasRowWidth bool rowWidthSource string - workRows int64 + workRows float64 hasWorkRows bool workBytes float64 hasWorkBytes bool @@ -636,6 +748,8 @@ func readBillingDemoUnitWeight(weights readBillingDemoOperatorWeights, unit stri return weights.row, true case readBillingDemoUnitInputBytes: return weights.byte, true + case readBillingDemoUnitOrderWork: + return weights.orderWork, true case readBillingDemoUnitEncodedMutationCount: return weights.mutationCount, true case readBillingDemoUnitEncodedMutationBytes: @@ -796,12 +910,496 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar return stats } +func readBillingDemoCopFailureAt(idx int, kind readBillingDemoCopFailureKind, status, reason string) readBillingDemoCopFailure { + return readBillingDemoCopFailure{ + present: true, + kind: kind, + status: status, + reason: reason, + failingIdx: idx, + } +} + +func readBillingDemoPromoteCopFailure(failure readBillingDemoCopFailure) readBillingDemoCopFailure { + if failure.present { + failure.kind = readBillingDemoCopFailureIntrinsicCause + } + return failure +} + +func readBillingDemoExactCopRowsEvidence(runtimeStats *execdetails.RuntimeStatsColl, planID int) readBillingDemoCopRowsEvidence { + if runtimeStats == nil { + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsMissing} + } + stats := runtimeStats.GetCopStats(planID) + if stats == nil || stats.GetTasks() <= 0 { + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsMissing} + } + rows := stats.GetActRows() + if rows < 0 { + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsInvalid, rows: rows, tasks: stats.GetTasks()} + } + return readBillingDemoCopRowsEvidence{state: readBillingDemoCopRowsObserved, rows: rows, tasks: stats.GetTasks()} +} + +func newReadBillingDemoCopEstimator(tree FlatPlanTree, runtimeStats *execdetails.RuntimeStatsColl) *readBillingDemoCopEstimator { + estimator := &readBillingDemoCopEstimator{ + tree: tree, + runtimeStats: runtimeStats, + parentIdx: make([]int, len(tree)), + componentID: make([]int, len(tree)), + nodeFailures: make(map[int]readBillingDemoCopFailure), + treeFailureSeen: make(map[int]struct{}), + inputMemo: make(map[int]readBillingDemoCopInputEstimate), + inputMemoSet: make(map[int]struct{}), + outputMemo: make(map[int]readBillingDemoCopOutputEstimate), + outputMemoSet: make(map[int]struct{}), + visiting: make(map[int]bool), + } + for i := range tree { + estimator.parentIdx[i] = -1 + estimator.componentID[i] = -1 + } + addStructuralFailure := func(idx int) { + if idx < 0 || idx >= len(tree) { + return + } + failure := readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + estimator.nodeFailures[idx] = failure + if _, ok := estimator.treeFailureSeen[idx]; !ok { + estimator.treeFailureSeen[idx] = struct{}{} + estimator.treeFailures = append(estimator.treeFailures, failure) + } + } + + // Build the reverse direct-edge index and reject malformed references. This + // pass is O(n+m), where m is the number of explicit ChildrenIdx entries. + for idx, node := range tree { + estimator.nodeVisits++ + if node == nil || node.Origin == nil { + continue + } + previousChild := idx + for _, childIdx := range node.ChildrenIdx { + estimator.edgeVisits++ + if childIdx <= previousChild || childIdx <= idx || childIdx >= len(tree) { + addStructuralFailure(idx) + continue + } + previousChild = childIdx + if tree[childIdx] == nil || tree[childIdx].Origin == nil { + addStructuralFailure(idx) + continue + } + if previousParent := estimator.parentIdx[childIdx]; previousParent >= 0 { + addStructuralFailure(previousParent) + addStructuralFailure(idx) + addStructuralFailure(childIdx) + continue + } + estimator.parentIdx[childIdx] = idx + } + } + + componentRoots := make([]int, 0) + for idx, node := range tree { + estimator.nodeVisits++ + if !readBillingDemoIsTiKVCopNode(node) { + continue + } + parentIdx := estimator.parentIdx[idx] + if parentIdx < 0 || parentIdx >= len(tree) || tree[parentIdx] == nil || tree[parentIdx].Origin == nil { + addStructuralFailure(idx) + continue + } + parent := tree[parentIdx] + if parent.IsRoot { + componentRoots = append(componentRoots, idx) + continue + } + if !readBillingDemoIsTiKVCopNode(parent) { + addStructuralFailure(idx) + addStructuralFailure(parentIdx) + } + } + + lastComponentRoot := -1 + lastComponentEnd := -1 + for _, rootIdx := range componentRoots { + rootEnd := tree[rootIdx].ChildrenEndIdx + if rootEnd < rootIdx || rootEnd >= len(tree) { + addStructuralFailure(rootIdx) + continue + } + // Component roots are discovered in preorder. Validate their intervals + // with one ordered sweep so malformed, overlapping roots cannot make us + // rescan the same suffix once per sibling component. + if rootIdx <= lastComponentEnd { + addStructuralFailure(lastComponentRoot) + addStructuralFailure(rootIdx) + continue + } + lastComponentRoot = rootIdx + lastComponentEnd = rootEnd + estimator.validateReadBillingDemoCopComponent(rootIdx, addStructuralFailure) + } + for idx, node := range tree { + estimator.nodeVisits++ + if readBillingDemoIsTiKVCopNode(node) && estimator.componentID[idx] < 0 { + addStructuralFailure(idx) + } + } + + // Gather component evidence and intrinsic invalid-row failures in one + // exact-plan-ID pass. Detail ownership and row ownership intentionally stay + // separate because distsql attaches response ScanDetail to the last plan ID. + for idx, node := range tree { + estimator.nodeVisits++ + if !readBillingDemoIsTiKVCopNode(node) { + continue + } + if evidence := readBillingDemoExactCopRowsEvidence(runtimeStats, node.Origin.ID()); evidence.state == readBillingDemoCopRowsInvalid { + if _, structural := estimator.nodeFailures[idx]; !structural { + estimator.nodeFailures[idx] = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidCopRuntimeRows) + } + } + componentID := estimator.componentID[idx] + if componentID < 0 || componentID >= len(estimator.components) { + continue + } + component := &estimator.components[componentID] + operator, supported, _ := readBillingDemoClassifyOperator(node) + if supported && operator.opClass == readBillingDemoOpClassRangeScan { + component.scanCount++ + component.scanIdx = idx + } + if runtimeStats == nil { + continue + } + stats := runtimeStats.GetCopStats(node.Origin.ID()) + if stats == nil { + continue + } + if tasks := stats.GetTasks(); tasks > component.maxSummaryTasks { + component.maxSummaryTasks = tasks + } + detail := stats.GetScanDetail() + if detail.ProcessedKeys > 0 && detail.ProcessedKeysSize > 0 { + component.detailHolderCount++ + component.scanDetail = detail + } + } + estimator.auxiliaryEntries = len(estimator.parentIdx) + len(estimator.componentID) + len(estimator.components) + len(estimator.nodeFailures) + len(estimator.treeFailures) + return estimator +} + +func readBillingDemoIsTiKVCopNode(node *FlatOperator) bool { + return node != nil && node.Origin != nil && !node.IsRoot && node.StoreType == kv.TiKV +} + +func (e *readBillingDemoCopEstimator) validateReadBillingDemoCopComponent(rootIdx int, addStructuralFailure func(int)) { + if rootIdx < 0 || rootIdx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[rootIdx]) { + addStructuralFailure(rootIdx) + return + } + rootEnd := e.tree[rootIdx].ChildrenEndIdx + if rootEnd < rootIdx || rootEnd >= len(e.tree) { + addStructuralFailure(rootIdx) + return + } + componentID := len(e.components) + e.components = append(e.components, readBillingDemoCopComponentEvidence{scanIdx: -1}) + visited := make(map[int]struct{}) + + var visit func(int, int) int + visit = func(idx, expectedParent int) int { + e.nodeVisits++ + if idx < rootIdx || idx > rootEnd || idx >= len(e.tree) { + addStructuralFailure(rootIdx) + return idx + } + node := e.tree[idx] + if !readBillingDemoIsTiKVCopNode(node) { + addStructuralFailure(idx) + return idx + } + if _, duplicate := visited[idx]; duplicate { + addStructuralFailure(idx) + return node.ChildrenEndIdx + } + visited[idx] = struct{}{} + if expectedParent >= 0 && e.parentIdx[idx] != expectedParent { + addStructuralFailure(expectedParent) + addStructuralFailure(idx) + } + if previousComponent := e.componentID[idx]; previousComponent >= 0 && previousComponent != componentID { + addStructuralFailure(idx) + return idx + } + e.componentID[idx] = componentID + nodeEnd := node.ChildrenEndIdx + if nodeEnd < idx || nodeEnd > rootEnd { + addStructuralFailure(idx) + return idx + } + if len(node.ChildrenIdx) == 0 { + if nodeEnd != idx { + addStructuralFailure(idx) + } + return nodeEnd + } + expectedChild := idx + 1 + for _, childIdx := range node.ChildrenIdx { + e.edgeVisits++ + if childIdx != expectedChild { + addStructuralFailure(idx) + } + childEnd := visit(childIdx, idx) + if childEnd < childIdx { + addStructuralFailure(childIdx) + childEnd = childIdx + } + expectedChild = childEnd + 1 + } + if expectedChild-1 != nodeEnd { + addStructuralFailure(idx) + } + return nodeEnd + } + + if visit(rootIdx, e.parentIdx[rootIdx]) != rootEnd { + addStructuralFailure(rootIdx) + } + for idx := rootIdx; idx <= rootEnd; idx++ { + e.nodeVisits++ + if e.componentID[idx] != componentID { + addStructuralFailure(idx) + } + } +} + +func (e *readBillingDemoCopEstimator) firstTreeFailure() (readBillingDemoCopFailure, bool) { + if len(e.treeFailures) == 0 { + return readBillingDemoCopFailure{}, false + } + failure := e.treeFailures[0] + for _, candidate := range e.treeFailures[1:] { + if candidate.failingIdx < failure.failingIdx { + failure = candidate + } + } + return failure, true +} + +func (e *readBillingDemoCopEstimator) componentOutputWidth(idx int) readBillingDemoCopOutputEstimate { + if idx < 0 || idx >= len(e.componentID) { + return readBillingDemoCopOutputEstimate{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} + } + componentID := e.componentID[idx] + if componentID < 0 || componentID >= len(e.components) { + return readBillingDemoCopOutputEstimate{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} + } + component := e.components[componentID] + if component.scanCount > 1 || component.detailHolderCount > 1 { + return readBillingDemoCopOutputEstimate{widthState: readBillingDemoCopWidthAmbiguous} + } + if component.scanCount != 1 || component.detailHolderCount != 1 { + return readBillingDemoCopOutputEstimate{widthState: readBillingDemoCopWidthMissing} + } + rowWidth := float64(component.scanDetail.ProcessedKeysSize) / float64(component.scanDetail.ProcessedKeys) + if rowWidth <= 0 || math.IsNaN(rowWidth) || math.IsInf(rowWidth, 0) { + return readBillingDemoCopOutputEstimate{widthState: readBillingDemoCopWidthMissing} + } + return readBillingDemoCopOutputEstimate{ + widthState: readBillingDemoCopWidthKnown, + avgRowWidth: rowWidth, + widthSource: explainRUWidthSourceScanDetailProcessedEstimate, + scanPlanID: e.tree[component.scanIdx].Origin.ID(), + } +} + +func (e *readBillingDemoCopEstimator) directCopChild(idx int) (int, readBillingDemoCopFailure, bool) { + if idx < 0 || idx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[idx]) { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure), false + } + children := e.tree[idx].ChildrenIdx + if len(children) == 0 { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure), false + } + if len(children) > 1 { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopMultiChild), false + } + childIdx := children[0] + if childIdx < 0 || childIdx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[childIdx]) || e.componentID[childIdx] != e.componentID[idx] { + return 0, readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure), false + } + return childIdx, readBillingDemoCopFailure{}, true +} + +func readBillingDemoCopOperatorPreservesWidth(opClass string) bool { + switch opClass { + case readBillingDemoOpClassFilter, readBillingDemoOpClassLimit, readBillingDemoOpClassTopN: + return true + default: + return false + } +} + +func (e *readBillingDemoCopEstimator) inputEstimate(idx int) readBillingDemoCopInputEstimate { + if _, ok := e.inputMemoSet[idx]; ok { + return e.inputMemo[idx] + } + estimate := readBillingDemoCopInputEstimate{} + defer func() { + e.inputMemo[idx] = estimate + e.inputMemoSet[idx] = struct{}{} + }() + if failure, ok := e.nodeFailures[idx]; ok { + estimate.failure = failure + return estimate + } + if idx < 0 || idx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[idx]) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + operator, supported, reason := readBillingDemoClassifyOperator(e.tree[idx]) + if !supported { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, reason) + return estimate + } + if operator.opClass == readBillingDemoOpClassRangeScan { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + childIdx, failure, ok := e.directCopChild(idx) + if !ok { + estimate.failure = failure + return estimate + } + childOutput := e.outputEstimate(childIdx) + if childOutput.failure.present { + estimate.failure = readBillingDemoPromoteCopFailure(childOutput.failure) + return estimate + } + rowsEvidence := readBillingDemoExactCopRowsEvidence(e.runtimeStats, e.tree[childIdx].Origin.ID()) + switch rowsEvidence.state { + case readBillingDemoCopRowsInvalid: + estimate.failure = readBillingDemoCopFailureAt(childIdx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidCopRuntimeRows) + return estimate + case readBillingDemoCopRowsMissing: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingCopChildRuntimeRows) + return estimate + } + componentID := e.componentID[idx] + if componentID < 0 || componentID >= len(e.components) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + if maxTasks := e.components[componentID].maxSummaryTasks; maxTasks > 0 && rowsEvidence.tasks < maxTasks { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows) + return estimate + } + switch childOutput.widthState { + case readBillingDemoCopWidthBarrier: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonUnsupportedCopWidthTransform) + return estimate + case readBillingDemoCopWidthAmbiguous: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonAmbiguousCopScanWidth) + return estimate + case readBillingDemoCopWidthMissing: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence) + return estimate + case readBillingDemoCopWidthKnown: + default: + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence) + return estimate + } + inputBytes := float64(rowsEvidence.rows) * childOutput.avgRowWidth + if inputBytes < 0 || math.IsNaN(inputBytes) || math.IsInf(inputBytes, 0) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence) + return estimate + } + estimate.rows = rowsEvidence.rows + estimate.inputBytes = inputBytes + estimate.avgRowWidth = childOutput.avgRowWidth + estimate.inputSource = readBillingDemoInputSourceRuntimeChildActRows + estimate.widthSource = childOutput.widthSource + return estimate +} + +func (e *readBillingDemoCopEstimator) outputEstimate(idx int) readBillingDemoCopOutputEstimate { + if _, ok := e.outputMemoSet[idx]; ok { + return e.outputMemo[idx] + } + estimate := readBillingDemoCopOutputEstimate{} + defer func() { + e.outputMemo[idx] = estimate + e.outputMemoSet[idx] = struct{}{} + }() + if e.visiting[idx] { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + e.visiting[idx] = true + defer delete(e.visiting, idx) + if failure, ok := e.nodeFailures[idx]; ok { + estimate.failure = failure + return estimate + } + if idx < 0 || idx >= len(e.tree) || !readBillingDemoIsTiKVCopNode(e.tree[idx]) { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + operator, supported, reason := readBillingDemoClassifyOperator(e.tree[idx]) + if !supported { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, reason) + return estimate + } + if operator.opClass == readBillingDemoOpClassRangeScan { + if len(e.tree[idx].ChildrenIdx) != 0 { + estimate.failure = readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure) + return estimate + } + estimate = e.componentOutputWidth(idx) + return estimate + } + input := e.inputEstimate(idx) + if input.failure.present { + estimate.failure = readBillingDemoPromoteCopFailure(input.failure) + return estimate + } + if readBillingDemoCopOperatorPreservesWidth(operator.opClass) { + estimate.widthState = readBillingDemoCopWidthKnown + estimate.avgRowWidth = input.avgRowWidth + estimate.widthSource = input.widthSource + childIdx, _, ok := e.directCopChild(idx) + if ok { + estimate.scanPlanID = e.outputEstimate(childIdx).scanPlanID + } + return estimate + } + estimate.widthState = readBillingDemoCopWidthBarrier + return estimate +} + +func (e *readBillingDemoCopEstimator) auxiliaryEntryCount() int { + return e.auxiliaryEntries + len(e.inputMemo) + len(e.inputMemoSet) + len(e.outputMemo) + len(e.outputMemoSet) + len(e.visiting) +} + func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanContext, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) (string, readBillingDemoOperatorResult) { + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + if failure, ok := estimator.firstTreeFailure(); ok { + return failure.status, readBillingDemoMaterializeCopFailure(runtimeStats, tree, failure) + } for i := range tree { - status, operator := appendReadBillingDemoOperator(result, runtimeStats, tree, i) - if status != readBillingDemoStatusSuccess { - return status, operator + outcome := appendReadBillingDemoOperator(result, runtimeStats, tree, estimator, i) + if outcome.success { + continue } + if outcome.cause.present { + return outcome.cause.status, readBillingDemoMaterializeCopFailure(runtimeStats, tree, outcome.cause) + } + return outcome.status, outcome.current } return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} } @@ -810,35 +1408,66 @@ func appendReadBillingDemoTree(result *readBillingDemoResult, sctx base.PlanCont // A missing or unsupported node makes only that node partial; it must not hide // supported descendants from the DML read/compute tree. func appendReadBillingDemoDMLTree(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree) { - for i := range tree { - status, operator := appendReadBillingDemoOperator(result, runtimeStats, tree, i) - if status == readBillingDemoStatusSuccess { - continue + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + type diagnosticKey struct { + idx int + reason string + } + reported := make(map[diagnosticKey]struct{}) + appendDiagnostic := func(idx int, operator readBillingDemoOperatorResult, reason string) { + key := diagnosticKey{idx: idx, reason: reason} + if _, ok := reported[key]; ok { + return } + reported[key] = struct{}{} operator.emitStatusRow = true operator.status = readBillingDemoStatusPartial + operator.reason = reason result.operators = append(result.operators, operator) } + for _, failure := range estimator.treeFailures { + appendDiagnostic(failure.failingIdx, readBillingDemoMaterializeCopFailure(runtimeStats, tree, failure), failure.reason) + } + for i := range tree { + outcome := appendReadBillingDemoOperator(result, runtimeStats, tree, estimator, i) + if outcome.success { + continue + } + if outcome.cause.present && outcome.cause.failingIdx != i { + appendDiagnostic(i, outcome.current, readBillingDemoReasonDependentCopInputUnavailable) + appendDiagnostic( + outcome.cause.failingIdx, + readBillingDemoMaterializeCopFailure(runtimeStats, tree, outcome.cause), + outcome.cause.reason, + ) + continue + } + appendDiagnostic(i, outcome.current, outcome.current.reason) + } } -func appendReadBillingDemoOperator(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int) (string, readBillingDemoOperatorResult) { +func appendReadBillingDemoOperator(result *readBillingDemoResult, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, estimator *readBillingDemoCopEstimator, idx int) readBillingDemoAppendOutcome { op := tree[idx] if op == nil || op.Origin == nil || op.Origin.ExplainID().String() == "_0" { - return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} + return readBillingDemoAppendOutcome{success: true, status: readBillingDemoStatusSuccess} + } + if failure, ok := estimator.nodeFailures[idx]; ok { + operator := readBillingDemoMaterializeCopFailure(runtimeStats, tree, failure) + return readBillingDemoAppendOutcome{status: failure.status, current: operator, cause: failure} } operator, supported, reason := readBillingDemoClassifyOperator(op) operator.id = op.ExplainID().String() - if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, tree, idx, op, operator); ok { + if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, op); ok { operator.actRows = actRows operator.hasActRows = true } if !supported { - return readBillingDemoStatusUnsupported, operator.withReason(reason) + return readBillingDemoAppendOutcome{status: readBillingDemoStatusUnsupported, current: operator.withReason(reason)} } if !readBillingDemoOperatorBillable(operator) { operator.status = readBillingDemoStatusOperatorOK result.operators = append(result.operators, operator.withReason(readBillingDemoReasonNonBillable)) - return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} + return readBillingDemoAppendOutcome{success: true, status: readBillingDemoStatusSuccess} } var units []readBillingDemoUnit var missingReason string @@ -846,7 +1475,23 @@ func appendReadBillingDemoOperator(result *readBillingDemoResult, runtimeStats * if op.IsRoot { units, missingReason, ok = readBillingDemoRootUnits(runtimeStats, tree, idx, op, operator) } else { - units, missingReason, ok = readBillingDemoCopUnits(runtimeStats, tree, idx, operator) + outcome := readBillingDemoCopUnits(estimator, idx, operator) + if outcome.success { + units, ok = outcome.units, true + } else { + failure := outcome.failure + currentReason := failure.reason + cause := readBillingDemoCopFailure{} + if failure.kind == readBillingDemoCopFailureIntrinsicCause && failure.failingIdx != idx { + currentReason = readBillingDemoReasonDependentCopInputUnavailable + cause = failure + } + return readBillingDemoAppendOutcome{ + status: failure.status, + current: operator.withReason(currentReason), + cause: cause, + } + } } if !ok { if missingReason == "" { @@ -856,13 +1501,34 @@ func appendReadBillingDemoOperator(result *readBillingDemoResult, runtimeStats * missingReason = readBillingDemoReasonMissingScanDetail } } - return readBillingDemoStatusUnknownInput, operator.withReason(missingReason) + return readBillingDemoAppendOutcome{status: readBillingDemoStatusUnknownInput, current: operator.withReason(missingReason)} } operator.status = readBillingDemoStatusOperatorOK operator.reason = readBillingDemoReasonNone operator.units = units result.operators = append(result.operators, operator) - return readBillingDemoStatusSuccess, readBillingDemoOperatorResult{} + return readBillingDemoAppendOutcome{success: true, status: readBillingDemoStatusSuccess} +} + +func readBillingDemoMaterializeCopFailure(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, failure readBillingDemoCopFailure) readBillingDemoOperatorResult { + if failure.failingIdx < 0 || failure.failingIdx >= len(tree) || tree[failure.failingIdx] == nil || tree[failure.failingIdx].Origin == nil { + return readBillingDemoOperatorResult{ + id: "cop_structure", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassRangeScan, + operatorKind: "cop_structure", + reason: failure.reason, + } + } + op := tree[failure.failingIdx] + operator, _, _ := readBillingDemoClassifyOperator(op) + operator.id = op.ExplainID().String() + operator.reason = failure.reason + if actRows, ok := readBillingDemoOperatorActRows(runtimeStats, op); ok { + operator.actRows = actRows + operator.hasActRows = true + } + return operator } func (op readBillingDemoOperatorResult) withReason(reason string) readBillingDemoOperatorResult { @@ -874,15 +1540,16 @@ func readBillingDemoOperatorBillable(op readBillingDemoOperatorResult) bool { return op.opClass != readBillingDemoOpClassWrapper && op.opClass != readBillingDemoOpClassSynthetic } -func readBillingDemoOperatorActRows(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) (int64, bool) { +func readBillingDemoOperatorActRows(runtimeStats *execdetails.RuntimeStatsColl, op *FlatOperator) (int64, bool) { if op == nil { return 0, false } if op.IsRoot { return readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) } - if copStats := readBillingDemoCopStats(runtimeStats, tree, idx, operator.opClass); copStats != nil { - return copStats.GetActRows(), true + evidence := readBillingDemoExactCopRowsEvidence(runtimeStats, op.Origin.ID()) + if evidence.state == readBillingDemoCopRowsObserved { + return evidence.rows, true } return 0, false } @@ -986,6 +1653,13 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F return nil, reason, false } units = append(units, readBillingDemoRuntimeChunkInputUnits(inputRows, inputBytes, readBillingDemoInputSideAll)...) + orderWork, ok := readBillingDemoOrderingWorkUnit(op, operator.opClass, inputRows) + if !ok { + return nil, readBillingDemoReasonInvalidOrderingWork, false + } + if orderWork.unit != "" { + units = append(units, orderWork) + } return units, "", true } } @@ -1015,6 +1689,46 @@ func readBillingDemoAverageRowWidth(rows int64, bytes float64) float64 { return bytes / float64(rows) } +func readBillingDemoOrderingWorkUnit(op *FlatOperator, opClass string, inputRows int64) (readBillingDemoUnit, bool) { + if opClass != readBillingDemoOpClassTopN && opClass != readBillingDemoOpClassSort { + return readBillingDemoUnit{}, true + } + if op == nil || op.Origin == nil || inputRows < 0 { + return readBillingDemoUnit{}, false + } + + logWidth := max(float64(inputRows), 2) + if opClass == readBillingDemoOpClassTopN { + topN, ok := op.Origin.(*physicalop.PhysicalTopN) + if !ok { + return readBillingDemoUnit{}, false + } + if op.IsRoot { + // Add after conversion so an extreme OFFSET + COUNT cannot wrap uint64. + logWidth = max(float64(topN.Offset)+float64(topN.Count), 2) + } else { + // Pushdown folds the original OFFSET + COUNT into Count. TiKV's TopN + // protobuf has only Limit, so a non-zero cop Offset is not executable + // evidence for the heap bound used here. + if topN.Offset != 0 { + return readBillingDemoUnit{}, false + } + logWidth = max(float64(topN.Count), 2) + } + } + work := float64(inputRows) * math.Log2(logWidth) + if work < 0 || math.IsNaN(work) || math.IsInf(work, 0) { + return readBillingDemoUnit{}, false + } + return readBillingDemoUnit{ + unit: readBillingDemoUnitOrderWork, + source: readBillingDemoInputSourceRuntimeOrderingWork, + side: readBillingDemoInputSideAll, + value: work, + widthSource: explainRUWidthSourceNotApplicable, + }, true +} + func readBillingDemoUseOutputRowsAsInput(opClass string) bool { switch opClass { case readBillingDemoOpClassReaderReceive, readBillingDemoOpClassLookupReader, readBillingDemoOpClassMetadataReader, readBillingDemoOpClassPointLookup: @@ -1101,26 +1815,58 @@ func readBillingDemoRootOutputRowsAndBytes(runtimeStats *execdetails.RuntimeStat return basic.GetActRows(), basic.GetOutputBytes(), true } -func readBillingDemoCopUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { - if operator.opClass != readBillingDemoOpClassRangeScan { - return nil, readBillingDemoReasonMissingRuntimeBytes, false +func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, operator readBillingDemoOperatorResult) readBillingDemoCopUnitOutcome { + if failure, ok := estimator.nodeFailures[idx]; ok { + return readBillingDemoCopUnitOutcome{failure: failure} } - copStats := readBillingDemoCopStats(runtimeStats, tree, idx, operator.opClass) - if copStats == nil { - return nil, readBillingDemoReasonMissingScanDetail, false + if operator.opClass == readBillingDemoOpClassRangeScan { + if idx < 0 || idx >= len(estimator.tree) || len(estimator.tree[idx].ChildrenIdx) != 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} + } + width := estimator.componentOutputWidth(idx) + if width.failure.present { + return readBillingDemoCopUnitOutcome{failure: width.failure} + } + switch width.widthState { + case readBillingDemoCopWidthAmbiguous: + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonAmbiguousCopScanWidth)} + case readBillingDemoCopWidthMissing: + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} + case readBillingDemoCopWidthKnown: + default: + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} + } + component := estimator.components[estimator.componentID[idx]] + scanDetail := component.scanDetail + scanInputRows, scanInputBytes, ok := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeys, scanDetail.ProcessedKeysSize) + if !ok { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} + } + units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceScanDetail)} + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: width.avgRowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: width.avgRowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + ) + return readBillingDemoCopUnitOutcome{success: true, units: units} } - scanDetail := copStats.GetScanDetail() - scanInputRows, scanInputBytes, ok := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeys, scanDetail.ProcessedKeysSize) - if !ok { - return nil, readBillingDemoReasonMissingScanDetail, false + + input := estimator.inputEstimate(idx) + if input.failure.present { + return readBillingDemoCopUnitOutcome{failure: input.failure} } - rowWidth := readBillingDemoAverageRowWidth(scanInputRows, scanInputBytes) - units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceScanDetail)} + units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChildActRows)} units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: input.inputSource, side: readBillingDemoInputSideAll, value: float64(input.rows), rowWidth: input.avgRowWidth, widthSource: input.widthSource}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: input.inputSource, side: readBillingDemoInputSideAll, value: input.inputBytes, rowWidth: input.avgRowWidth, widthSource: input.widthSource}, ) - return units, "", true + orderWork, ok := readBillingDemoOrderingWorkUnit(estimator.tree[idx], operator.opClass, input.rows) + if !ok { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidOrderingWork)} + } + if orderWork.unit != "" { + units = append(units, orderWork) + } + return readBillingDemoCopUnitOutcome{success: true, units: units} } func readBillingDemoRangeScanInput(totalKeys, processedKeys, processedKeysSize int64) (int64, float64, bool) { @@ -1128,52 +1874,10 @@ func readBillingDemoRangeScanInput(totalKeys, processedKeys, processedKeysSize i return 0, 0, false } inputBytes := float64(processedKeysSize) / float64(processedKeys) * float64(totalKeys) - return totalKeys, inputBytes, true -} - -func readBillingDemoCopStats(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, opClass string) *execdetails.CopRuntimeStats { - if runtimeStats == nil || idx < 0 || idx >= len(tree) || tree[idx] == nil || tree[idx].Origin == nil { - return nil - } - exact := runtimeStats.GetCopStats(tree[idx].Origin.ID()) - if readBillingDemoCopStatsUsable(exact, opClass) { - return exact - } - // distsql may attach scan detail to the last cop plan ID in a task, not - // necessarily to the scan node. Search the same cop subtree before failing. - if end := tree[idx].ChildrenEndIdx; end > idx { - if end >= len(tree) { - end = len(tree) - 1 - } - for i := end; i > idx; i-- { - if tree[i] == nil || tree[i].IsRoot || tree[i].Origin == nil || tree[i].StoreType != tree[idx].StoreType { - continue - } - if stats := runtimeStats.GetCopStats(tree[i].Origin.ID()); readBillingDemoCopStatsUsable(stats, opClass) { - return stats - } - } - } - for i := idx - 1; i >= 0; i-- { - if tree[i] == nil || tree[i].IsRoot || tree[i].Origin == nil || tree[i].StoreType != tree[idx].StoreType || tree[i].ChildrenEndIdx < idx { - continue - } - if stats := runtimeStats.GetCopStats(tree[i].Origin.ID()); readBillingDemoCopStatsUsable(stats, opClass) { - return stats - } - } - return exact -} - -func readBillingDemoCopStatsUsable(copStats *execdetails.CopRuntimeStats, opClass string) bool { - if copStats == nil { - return false - } - if opClass != readBillingDemoOpClassRangeScan { - return true + if inputBytes < 0 || math.IsNaN(inputBytes) || math.IsInf(inputBytes, 0) { + return 0, 0, false } - scanDetail := copStats.GetScanDetail() - return scanDetail.TotalKeys > 0 && scanDetail.ProcessedKeys > 0 && scanDetail.ProcessedKeysSize > 0 + return totalKeys, inputBytes, true } func recordReadBillingDemoResult(result readBillingDemoResult) { @@ -1556,13 +2260,16 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill case readBillingDemoUnitInputRows: row.inputRows = int64(unit.value) row.hasInputRows = true - row.workRows = int64(unit.value) + row.workRows = unit.value row.hasWorkRows = true row.count = int64(unit.value) row.hasCount = true case readBillingDemoUnitInputBytes: row.workBytes = unit.value row.hasWorkBytes = true + case readBillingDemoUnitOrderWork: + row.workRows = unit.value + row.hasWorkRows = true case readBillingDemoUnitEncodedMutationCount, readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitWriteKeys, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: row.count = int64(unit.value) @@ -1632,7 +2339,7 @@ func (row explainRURow) toStrings() []string { formatOptionalInt(row.outputRows, row.hasOutputRows), formatOptionalFloat(row.rowWidth, row.hasRowWidth), row.rowWidthSource, - formatOptionalInt(row.workRows, row.hasWorkRows), + formatOptionalCompactFloat(row.workRows, row.hasWorkRows), formatOptionalFloat(row.workBytes, row.hasWorkBytes), row.unit, formatOptionalInt(row.count, row.hasCount), @@ -1657,6 +2364,13 @@ func formatOptionalFloat(v float64, ok bool) string { return strconv.FormatFloat(v, 'f', 6, 64) } +func formatOptionalCompactFloat(v float64, ok bool) string { + if !ok { + return "" + } + return strconv.FormatFloat(v, 'f', -1, 64) +} + func explainRUObserveRow(row explainRURow) { // Metrics are emitted from rendered rows so the Prometheus view matches the // SQL output and avoids reading live counters after render-side accounting. @@ -1666,7 +2380,7 @@ func explainRUObserveRow(row explainRURow) { } workRows := -1.0 if row.hasWorkRows { - workRows = float64(row.workRows) + workRows = row.workRows } workBytes := -1.0 if row.hasWorkBytes { @@ -1677,7 +2391,7 @@ func explainRUObserveRow(row explainRURow) { rowWidth = row.rowWidth } component, operator := explainRUMetricComponentOperator(row) - metrics.ObserveExplainRURow(row.section, component, operator, row.source, row.rowWidthSource, previewRU, workRows, workBytes, rowWidth) + metrics.ObserveExplainRURow(row.section, component, operator, row.source, row.rowWidthSource, readBillingDemoWeightVersion, previewRU, workRows, workBytes, rowWidth) } func explainRUMetricComponentOperator(row explainRURow) (component, operator string) { From 567942be98961c784bad13458623dcaa228bd309 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 16 Jul 2026 21:30:06 +0800 Subject: [PATCH 15/25] planner: add join and aggregation output RU units --- .../references/distsql-case-map.md | 2 +- .../references/metrics-case-map.md | 2 +- .../references/planner-case-map.md | 2 +- .../2026-07-01-read-billing-demo-ru-model.md | 69 +++- pkg/distsql/select_result.go | 44 ++- pkg/distsql/select_result_test.go | 89 ++++++ pkg/metrics/explain_ru.go | 2 +- pkg/metrics/metrics_internal_test.go | 12 +- pkg/planner/core/common_plans_test.go | 294 +++++++++++++++++- pkg/planner/core/explain_ru.go | 125 +++++++- pkg/util/execdetails/runtime_stats.go | 40 ++- 11 files changed, 638 insertions(+), 43 deletions(-) diff --git a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md index 768f00abb342e..b7df4b7e2e64b 100644 --- a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md @@ -14,7 +14,7 @@ - `pkg/distsql/distsql_test.go` - Tests normal select. - `pkg/distsql/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/distsql/request_builder_test.go` - Tests table handles to KV ranges. -- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership, multi-response aggregation, and incomplete-summary task counts used by preview RU. +- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership, multi-response aggregation, independent expected-task coverage, and close-time stats-only handling that cannot replay stale summaries. ## pkg/distsql/context diff --git a/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md b/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md index 98fb8e2664a31..648fe5daf2299 100644 --- a/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md @@ -10,7 +10,7 @@ ### Tests - `pkg/metrics/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/metrics/metrics_internal_test.go` - Tests retained labels. +- `pkg/metrics/metrics_internal_test.go` - Tests retained labels and preview-RU bounded metric propagation, including measured zero base-unit samples. - `pkg/metrics/metrics_test.go` - Tests metrics. ## pkg/metrics/common diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index aacb778f6aefc..bf8b89be6b1e8 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, weights, Sort/TopN algorithmic `order_work`, TiKV component/child-row/scan-width input estimation and width barriers, DML mutation/write and explicit-COMMIT payload construction, pipelined payload suppression, and ancillary-work partial diagnostics. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, weights, Sort/TopN algorithmic `order_work`, TiDB Join/Agg and guarded TiKV Agg diagnostic output shadows, TiKV component/child-row/scan-width input estimation and width barriers, DML mutation/write and explicit-COMMIT payload construction, pipelined payload suppression, and ancillary-work partial diagnostics. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/docs/design/2026-07-01-read-billing-demo-ru-model.md b/docs/design/2026-07-01-read-billing-demo-ru-model.md index 745834c4732dc..15a06e3d95bff 100644 --- a/docs/design/2026-07-01-read-billing-demo-ru-model.md +++ b/docs/design/2026-07-01-read-billing-demo-ru-model.md @@ -36,6 +36,8 @@ preview_ru = `order_work` 只由比较型排序算子产生：full sort 使用 `n * log2(max(n, 2))`，TopN 使用 `n * log2(max(offset + count, 2))`。它作为 double base unit 展示在 `work_rows` 并进入明细统计，不强制转换为 int64 `COUNT`。排序类仍保留真实 `input_rows` base unit，但其 seed `row_weight` 为 0，避免把线性行项与算法 work 重复计费。 +Join 和 Agg 还会在证据充分时产生 `output_rows`、`output_bytes` shadow units，用于观察结果基数和宽度。它们标记为 `diagnostic_only=true`，不注册 weight，因而不属于上述公式：EXPLAIN 中没有 weight/preview RU，`total_preview_ru` 也不会变化。 + 第一版是校准 demo，不是最终生产 billing 语义。它和现有 RU v2 resource-control 上报保持隔离，并分成两个输出面： 1. `EXPLAIN ANALYZE FORMAT='RU'`：单语句诊断输出，显示 base units、weight 和 preview RU。 @@ -51,6 +53,7 @@ Prometheus `tidb_read_billing_demo_*` metrics 可以继续作为 bounded observa - TiKV range scan 使用 scan detail 中的 `processed_key_size / processed_keys * total_keys`，`row_width_source = scan_detail_processed_key_avg`。 - TiKV Selection、Projection、Limit、TopN、HashAgg 和 StreamAgg 使用直接 child 在 exact plan ID 下聚合到的 runtime `actRows` 作为 `input_rows`，再乘同一 TiKV component 的 `ProcessedKeysSize / ProcessedKeys` 作为估算 `input_bytes`；这类样本标记为 `input_source = runtime_child_act_rows`、`row_width_source = scan_detail_processed_key_avg_estimate`。这里的 exact 只表示 plan-ID attribution，不表示所有 response summary 必然无缺口。 - Selection、Limit、TopN 保持行形状并向父节点传播平均宽度；Projection、HashAgg 和 StreamAgg 可以消费 child 宽度，但其输出是 width barrier，不能继续向父节点传播。 +- TiDB Join/Agg 的 output shadow units 来自算子自身 runtime chunk；TiKV HashAgg/StreamAgg 的 `output_rows` 来自算子自身 plan ID 下的 cop summary，并且只在该 plan 的 observed summary task 数等于每个已收到 TiKV response 独立登记的 expected task 数时生成。TiKV summary 没有 per-executor output bytes，所以只有这项 coverage 成立、top-cop Agg 直接连接单一普通 reader、且 reader/Agg output rows 相等时，才把 reader logical output bytes 归因成 Agg `output_bytes`。 - 缺少 exact child summary、唯一 scan/detail 归属或可传播宽度时，以 bounded reason fail closed；SELECT 不产出 partial billable units，DML 只保留其他独立成功算子的 units。 这里的 estimated bytes 不是 TiKV executor edge 上真实编码字节数，而是 runtime child rows 与 storage KV 平均宽度的混合 proxy。本文后续保留的 `runtime_act_rows`、`plan_stats`、`schema_type_width`、`schema_fallback`、`operator_helper` 等描述只适用于 v1 历史估算模型或背景说明，不代表 v2/v3 的当前构造规则。v2 的历史契约只覆盖 TiDB runtime bytes 与 TiKV range-scan detail；v3 才增加上述非 scan cop 输入估算。 @@ -131,6 +134,13 @@ fixed_events = 1 Sort 和 TopN 额外产生 `order_work`，用于保存基于 runtime input rows 的算法规模项。 +TiDB Join/Agg 以及 TiKV Agg 还会按可用证据产生以下 shadow units： + +- `output_rows`：该 operator 实际产生的行数； +- `output_bytes`：可归因到该 operator 输出边界的 logical live bytes。 + +这两个 unit 只进入 EXPLAIN、bounded metrics 和 statement-summary 明细，不进入 weight lookup 或 preview RU 公式。缺少 shadow evidence 只会省略相应 unit，不会让原本完整的 fixed/input/order 公式失败。 + 每个 operator 结果都有以下身份字段： - `site`：`tidb`、`tikv`，statement-level status 用 `statement`； @@ -159,8 +169,8 @@ TiKV 侧不是只按 scan 计费，而是把 cop executor 也纳入模型。第 | `projection_eval` | projection | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | | `row_limit` | limit | direct child exact actRows × attributable scan-detail average；向父节点传播宽度 | | `bounded_topn` | TopN | direct child exact actRows × attributable scan-detail average；另有 `order_work = input_rows * log2(max(count, 2))`，其中 pushed plan 必须 `offset=0` 且 `count` 已折叠原 `offset+count`；向父节点传播宽度 | -| `agg_hash` | hash aggregation | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | -| `agg_stream` | stream aggregation | direct child exact actRows × attributable scan-detail average；输出形成 width barrier | +| `agg_hash` | hash aggregation | direct child exact actRows × attributable scan-detail average；另在独立 expected/observed task coverage 完整时记录 own `output_rows`，可证明时记录 reader-boundary `output_bytes`；输出仍形成 width barrier | +| `agg_stream` | stream aggregation | direct child exact actRows × attributable scan-detail average；另在独立 expected/observed task coverage 完整时记录 own `output_rows`，可证明时记录 reader-boundary `output_bytes`；输出仍形成 width barrier | range scan 的 byte 输入使用 processed-key average 外推到 `TotalKeys`，因为 scan 成本不仅和输出行数有关，还和 key/value bytes、MVCC 数据移动、row decode 有关： @@ -186,9 +196,22 @@ input_source = runtime_ordering_work TiKV TopN protobuf 只发送 `Limit=Count`，所以 pushed-down physical TopN 的规范形态是 `Offset=0`、`Count=原始 offset+count`。非零 cop Offset 会以 `invalid_ordering_work` 失败关闭，不按 root 公式猜测。 +TiKV Agg 的 output shadow evidence 使用独立契约： + +```text +output_rows = current_agg_plan_id_actRows +input_source = runtime_operator_act_rows + +output_bytes = direct_root_reader_runtime_output_bytes +input_source = runtime_reader_output_chunks +row_width_source = runtime_reader_output_chunk_avg +``` + +两个 output shadow 都先要求 Agg observed summary task 数等于 `RuntimeStatsColl` 对每个已收到 TiKV cop response 独立记录的 expected task 数。`selectResult.close` 先关闭底层 iterator 并等待 worker 退出，再且只再快照一次 unconsumed stats，避免漏掉 cancel/wait 期间完成的失败请求。close 收集到的 unconsumed task 没有新 `SelectResponse`，因此只合并 statement-level RPC/scan/time stats 并增加 expected count，绝不重放最后一个已消费 response 的 executor summaries；这样 output shadows 会因 observed/expected 不等而省略。`output_bytes` 还只在 Agg 是普通 TableReader/IndexReader 的唯一直接 cop child、reader 有 byte record、且 reader output rows 等于 Agg output rows 时生成。这里的完整性边界是“所有已收到 response 的 summary 都可见”，不是 TiKV 集群内绝对 task inventory；plan-ID attribution 也不应被解读成协议级 per-executor byte 观测。Agg 会改变 schema 和 value representation，因此绝不使用 `output_rows * (ProcessedKeysSize / ProcessedKeys)`：scan average 描述输入 KV 宽度，不是 aggregate result 宽度。该 reader-boundary 观测也不会解除 Agg 的 width barrier 或被反向用于 input formula。 + 该公式只适用于结构可证明为单 scan、单 detail holder 的 TiKV component。multi-child、多 scan、多 detail holder、child summary 缺失或 task count 可检测为不完整、以及跨 Projection/Agg 的父边都会失败关闭；当前实现不回退 planner/schema width，也不跨 sibling component 借用 scan detail。 -runtime rows 的数据质量契约是：stats 不存在或 `GetTasks() == 0` 为 missing；`GetTasks() > 0` 且 rows 为 0 是有效零行观测；负 rows 为 invalid；direct child task count 小于同 component 可见最大 task count 时为 incomplete。`GetTasks() > 0` 只能证明至少合并过一条完整 summary；如果同一 response 的 summary 在 component 所有 plan ID 上同步缺失，现有 `RuntimeStatsColl` 无法检测，因此 v3 rows 仍是带完整性边界的 estimate，校准时必须结合 coverage 与 failure reason。 +formula input rows 的数据质量契约是：stats 不存在或 `GetTasks() == 0` 为 missing；`GetTasks() > 0` 且 rows 为 0 是有效零行观测；负 rows 为 invalid；direct child task count 小于同 component 可见最大 task count 时为 incomplete。formula estimator 仍只做 component-relative 检查；如果同一 response 的 summary 在 component 所有 plan ID 上同步缺失，它可能无法发现，因此 v3 input rows 仍是带完整性边界的 estimate，校准时必须结合 coverage 与 failure reason。上述独立 expected-response 计数只收紧 Agg output shadows，不参与、也不改变当前 formula estimator：同步缺失时 formula units 可以继续成功，而 output shadows 会被省略。 ### TiDB Root Executor 模型 @@ -202,11 +225,11 @@ TiDB 侧覆盖 root executor CPU、row movement、reader materialization、join | `bounded_topn` | root TopN | local child runtime rows、logical live bytes，以及 `order_work = n * log2(max(offset + count, 2))` | | `full_ordering` | sort | local child runtime rows、logical live bytes，以及 `order_work = n * log2(max(n, 2))` | | `window_eval` | window executor | local child runtime rows 和 logical live bytes | -| `agg_hash` | hash aggregation | aggregation input rows 和 logical live bytes | -| `agg_stream` | stream aggregation | aggregation input rows 和 logical live bytes | -| `join_hash` | hash join | build/probe 两侧 input rows 和 bytes | -| `join_merge` | merge join | left/right 两侧 input rows 和 bytes | -| `join_lookup` | index lookup join family | left/right 两侧 input rows 和 bytes | +| `agg_hash` | hash aggregation | aggregation input rows 和 logical live bytes；自身实际 `output_rows/output_bytes` shadows | +| `agg_stream` | stream aggregation | aggregation input rows 和 logical live bytes；自身实际 `output_rows/output_bytes` shadows | +| `join_hash` | hash join | build/probe 两侧 input rows 和 bytes；自身实际 `output_rows/output_bytes` shadows | +| `join_merge` | merge join | left/right 两侧 input rows 和 bytes；自身实际 `output_rows/output_bytes` shadows | +| `join_lookup` | index lookup join family | left/right 两侧 input rows 和 bytes；自身实际 `output_rows/output_bytes` shadows | | `reader_receive` | table/index reader receive | runtime output rows 和 logical live bytes | | `lookup_reader` | index lookup reader orchestration | runtime output rows 和 logical live bytes | | `overlay_reader` | union scan / local overlay | runtime rows 和 logical live bytes | @@ -250,10 +273,10 @@ Join 的 side 会通过 `input_side` 保留下来： | `SITE` | varchar | `tidb` / `tikv` | | `OP_CLASS` | varchar | bounded opclass | | `OPERATOR_KIND` | varchar | bounded plan/operator kind | -| `UNIT` | varchar | `fixed_events` / `input_rows` / `input_bytes` / `order_work` / write-side units | -| `INPUT_SOURCE` | varchar | bounded source，例如 `runtime_chunk_bytes` / `scan_detail` / `runtime_child_act_rows` / `runtime_ordering_work` / write-side sources | +| `UNIT` | varchar | `fixed_events` / `input_rows` / `input_bytes` / `output_rows` / `output_bytes` / `order_work` / write-side units | +| `INPUT_SOURCE` | varchar | bounded source，例如 `runtime_chunk_bytes` / `scan_detail` / `runtime_child_act_rows` / `runtime_operator_act_rows` / `runtime_reader_output_chunks` / `runtime_ordering_work` / write-side sources | | `INPUT_SIDE` | varchar | `all` / `build` / `probe` / `left` / `right` | -| `ROW_WIDTH_SOURCE` | varchar | 当前值包括 `runtime_chunk_avg` / `scan_detail_processed_key_avg` / `scan_detail_processed_key_avg_estimate` / `not_applicable`；v1 历史值保留原标签 | +| `ROW_WIDTH_SOURCE` | varchar | 当前值包括 `runtime_chunk_avg` / `runtime_reader_output_chunk_avg` / `scan_detail_processed_key_avg` / `scan_detail_processed_key_avg_estimate` / `not_applicable`；v1 历史值保留原标签 | | `VALUE` | double unsigned | 该 key 的 base-unit total | | `SAMPLE_COUNT` | bigint unsigned | 汇入该 key 的 unit samples 数 | | `ROW_WIDTH_SUM` | double unsigned | row-width sample sum，用于诊断 modeled bytes | @@ -462,6 +485,28 @@ GROUP BY op_class, unit, input_source, row_width_source ORDER BY op_class, unit, input_source, row_width_source; ``` +`output_rows/output_bytes` 不应混入上面的 formula feature query。需要分析 Agg output shadows 时单独查询，并保留 source 以区分 coverage-guarded cop rows 与 guarded reader bytes： + +```sql +SELECT op_class, unit, input_source, row_width_source, + SUM(sample_count) AS samples, SUM(value) AS value +FROM information_schema.cluster_statements_summary_history_read_billing_demo_base_units +WHERE model_version = 'v3' + AND weight_version = 'v2' + AND site = 'tikv' + AND op_class IN ('agg_hash', 'agg_stream') + AND ( + (unit = 'output_rows' AND input_source = 'runtime_operator_act_rows' AND row_width_source = 'not_applicable') + OR (unit = 'output_bytes' AND input_source = 'runtime_reader_output_chunks' AND row_width_source = 'runtime_reader_output_chunk_avg') + ) +GROUP BY op_class, unit, input_source, row_width_source +ORDER BY op_class, unit, input_source, row_width_source; +``` + +TiDB Join/Agg 使用 `input_source = 'runtime_chunk_bytes'`；同样应作为 shadow diagnostics 单独分析，而不是给当前 weight table 增加 output coefficient。 + +`model_version` 仍是 `v3`，因为 output shadows 不改变 formula 或 weights。这个版本号也意味着历史窗口本身不能标识功能部署边界：部署前的 `v3` 记录没有 output-unit 行，缺行不能解释为零。校准查询必须把 `SUMMARY_BEGIN_TIME` 限定在所有目标 TiDB 节点均已部署该能力之后，并另外检查节点/窗口覆盖率；只有实际存在、且 `VALUE = 0` 的 unit sample 才表示观测到零输出。 + 如果需要区分 plan shape，可以保留 `DIGEST` 和 `PLAN_DIGEST`： ```sql @@ -538,6 +583,8 @@ JOIN weights 因为统计面保留的是 base units，重算时只需要替换 `weights` 数据，不需要重新跑 workload。 +这里的 `ELSE 0` 是有意的：`output_rows/output_bytes` 没有 weight，不参与当前公式或 weight-version 语义。 + ### 第五步：拟合新 weights 对每个校准窗口构造： diff --git a/pkg/distsql/select_result.go b/pkg/distsql/select_result.go index fe53353333744..2f55030c3d8eb 100644 --- a/pkg/distsql/select_result.go +++ b/pkg/distsql/select_result.go @@ -658,6 +658,21 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr } r.stats.mergeCopRuntimeStats(copStats, respTime) + if copStats.TimeDetail.ProcessTime > 0 { + r.ctx.CPUUsage.MergeTikvCPUTime(copStats.TimeDetail.ProcessTime) + } + if r.storeType == kv.TiKV { + // Record the independent expectation before validating individual + // summaries so a response with all summaries missing is still visible. + r.ctx.RuntimeStatsColl.RecordExpectedCopTasks(r.copPlanIDs) + } + if forUnconsumedStats { + // Close-time runtime stats do not have a newly unmarshaled SelectResponse. + // Keep the generic RPC/scan/time details above, but never replay summaries + // from the last consumed response. + return + } + // If hasExecutor is true, it means the summary is returned from TiFlash. hasExecutor := false for _, detail := range r.selectResp.GetExecutionSummaries() { @@ -680,9 +695,6 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr } } } - if copStats.TimeDetail.ProcessTime > 0 { - r.ctx.CPUUsage.MergeTikvCPUTime(copStats.TimeDetail.ProcessTime) - } if hasExecutor { if len(r.copPlanIDs) > 0 { r.ctx.RuntimeStatsColl.RecordCopStats(r.copPlanIDs[len(r.copPlanIDs)-1], r.storeType, copStats.ScanDetail, copStats.TimeDetail, nil) @@ -704,7 +716,7 @@ func (r *selectResult) updateCopRuntimeStats(ctx context.Context, copStats *copr // for TiFlash streaming call(BatchCop and MPP), it is by design that only the last response will // carry the execution summaries, so it is ok if some responses have no execution summaries, should // not trigger an error log in this case. - if !forUnconsumedStats && !(r.storeType == kv.TiFlash && len(r.selectResp.GetExecutionSummaries()) == 0) { + if !(r.storeType == kv.TiFlash && len(r.selectResp.GetExecutionSummaries()) == 0) { logutil.Logger(ctx).Warn("invalid cop task execution summaries length", zap.Int("expected", len(r.copPlanIDs)), zap.Int("received", len(r.selectResp.GetExecutionSummaries()))) @@ -764,16 +776,10 @@ func (r *selectResult) close() error { r.memConsume(-respSize) } if r.ctx != nil { - if unconsumed, ok := r.resp.(copr.HasUnconsumedCopRuntimeStats); ok && unconsumed != nil { - unconsumedCopStats := unconsumed.CollectUnconsumedCopRuntimeStats() - for _, copStats := range unconsumedCopStats { - _ = r.updateCopRuntimeStats(context.Background(), copStats, time.Duration(0), true) - r.ctx.ExecDetails.MergeCopExecDetails(&copStats.CopExecDetails, 0) - } - } - } - if r.stats != nil && r.ctx != nil { defer func() { + if r.stats == nil { + return + } if ci, ok := r.resp.(copr.CopInfo); ok { r.stats.buildTaskDuration = ci.GetBuildTaskElapsed() batched, fallback := ci.GetStoreBatchInfo() @@ -788,7 +794,17 @@ func (r *selectResult) close() error { r.ctx.RuntimeStatsColl.RegisterStats(r.rootPlanID, r.stats) }() } - return r.resp.Close() + closeErr := r.resp.Close() + if r.ctx != nil { + if unconsumed, ok := r.resp.(copr.HasUnconsumedCopRuntimeStats); ok && unconsumed != nil { + unconsumedCopStats := unconsumed.CollectUnconsumedCopRuntimeStats() + for _, copStats := range unconsumedCopStats { + _ = r.updateCopRuntimeStats(context.Background(), copStats, time.Duration(0), true) + r.ctx.ExecDetails.MergeCopExecDetails(&copStats.CopExecDetails, 0) + } + } + } + return closeErr } type selRespChannelIter struct { diff --git a/pkg/distsql/select_result_test.go b/pkg/distsql/select_result_test.go index 886ae3a3fe70c..44dacb5731992 100644 --- a/pkg/distsql/select_result_test.go +++ b/pkg/distsql/select_result_test.go @@ -32,9 +32,37 @@ import ( "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikv" + "github.com/tikv/client-go/v2/tikvrpc" tikvutil "github.com/tikv/client-go/v2/util" ) +type closeAppendingRuntimeStatsResponse struct { + stats []*copr.CopRuntimeStats + statsOnClose *copr.CopRuntimeStats + closeErr error + closed bool + collectCalls int +} + +func (*closeAppendingRuntimeStatsResponse) Next(context.Context) (kv.ResultSubset, error) { + return nil, nil +} + +func (r *closeAppendingRuntimeStatsResponse) Close() error { + r.closed = true + r.stats = append(r.stats, r.statsOnClose) + return r.closeErr +} + +func (r *closeAppendingRuntimeStatsResponse) CollectUnconsumedCopRuntimeStats() []*copr.CopRuntimeStats { + r.collectCalls++ + if !r.closed { + return nil + } + return r.stats +} + func TestUpdateCopRuntimeStats(t *testing.T) { ctx := mock.NewContext() ctx.GetSessionVars().StmtCtx = stmtctx.NewStmtCtx() @@ -103,6 +131,8 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int32(1), parentStats.GetTasks()) require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) require.Equal(t, int64(100), parentStats.GetScanDetail().ProcessedKeysSize) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) update(3, 1, &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}) require.Equal(t, int64(7), scanStats.GetActRows()) @@ -111,6 +141,8 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int32(2), parentStats.GetTasks()) require.Equal(t, int64(7), parentStats.GetScanDetail().ProcessedKeys) require.Equal(t, int64(140), parentStats.GetScanDetail().ProcessedKeysSize) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) sr.ctx = ctx.GetDistSQLCtx() @@ -123,6 +155,8 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int32(1), scanStats.GetTasks()) require.Equal(t, int32(0), parentStats.GetTasks()) require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) sr.ctx = ctx.GetDistSQLCtx() @@ -135,6 +169,61 @@ func TestUpdateCopRuntimeStats(t *testing.T) { parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) require.Equal(t, int32(1), scanStats.GetTasks()) require.Equal(t, int32(2), parentStats.GetTasks()) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + + sr.selectResp = &tipb.SelectResponse{ExecutionSummaries: []*tipb.ExecutorExecutionSummary{nil, nil}} + require.NoError(t, sr.updateCopRuntimeStats(context.Background(), &copr.CopRuntimeStats{ + CopExecDetails: execdetails.CopExecDetails{CalleeAddress: "callee"}, + }, 0, false)) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int32(2), parentStats.GetTasks()) + require.Equal(t, int32(3), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(3), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) + sr.ctx = ctx.GetDistSQLCtx() + sr.stats = &selectResultRuntimeStats{} + update(4, 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + scanStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(scanPlanID) + parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) + reqStats := tikv.NewRegionRequestRuntimeStats() + reqStats.RecordRPCRuntimeStats(tikvrpc.CmdCop, time.Millisecond) + closeErr := fmt.Errorf("close failed") + resp := &closeAppendingRuntimeStatsResponse{ + statsOnClose: &copr.CopRuntimeStats{ReqStats: reqStats}, + closeErr: closeErr, + } + sr.resp = resp + require.Equal(t, closeErr, sr.close()) + require.True(t, resp.closed) + require.Equal(t, 1, resp.collectCalls) + require.Equal(t, uint32(1), sr.stats.reqStat.GetCmdRPCCount(tikvrpc.CmdCop)) + require.Equal(t, int64(4), scanStats.GetActRows()) + require.Equal(t, int32(1), scanStats.GetTasks()) + require.Equal(t, int64(2), parentStats.GetActRows()) + require.Equal(t, int32(1), parentStats.GetTasks()) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) + require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + + onlyReqStats := tikv.NewRegionRequestRuntimeStats() + onlyReqStats.RecordRPCRuntimeStats(tikvrpc.CmdCop, time.Millisecond) + onlyResp := &closeAppendingRuntimeStatsResponse{ + statsOnClose: &copr.CopRuntimeStats{ReqStats: onlyReqStats}, + } + const onlyUnconsumedRootID = 3001 + const onlyUnconsumedCopID = 3002 + onlyUnconsumed := selectResult{ + ctx: ctx.GetDistSQLCtx(), + rootPlanID: onlyUnconsumedRootID, + copPlanIDs: []int{onlyUnconsumedCopID}, + storeType: kv.TiKV, + resp: onlyResp, + } + require.NoError(t, onlyUnconsumed.close()) + require.NotNil(t, onlyUnconsumed.stats) + require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(onlyUnconsumedCopID)) + require.True(t, ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.ExistsRootStats(onlyUnconsumedRootID)) } func TestNewSelRespChannelIter(t *testing.T) { diff --git a/pkg/metrics/explain_ru.go b/pkg/metrics/explain_ru.go index e25d6e4b9f4fc..a3dc234e5f6f9 100644 --- a/pkg/metrics/explain_ru.go +++ b/pkg/metrics/explain_ru.go @@ -202,7 +202,7 @@ func RecordReadBillingDemoOperatorStatus(site, opClass, operatorKind, status, re func AddReadBillingDemoBaseUnits(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion string, value float64) { if ReadBillingDemoBaseUnitsCounter == nil || site == "" || opClass == "" || operatorKind == "" || unit == "" || inputSource == "" || inputSide == "" || modelVersion == "" || - value <= 0 { + value < 0 { return } ReadBillingDemoBaseUnitsCounter.WithLabelValues(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion).Add(value) diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index 889c7cd42cd8a..7c4c73e02b5b8 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -105,6 +105,7 @@ func TestExplainRUMetrics(t *testing.T) { RecordReadBillingDemoStatement("success", "v2") RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2") AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 3) + AddReadBillingDemoBaseUnits("tikv", "agg_hash", "hashagg", "output_rows", "runtime_operator_act_rows", "all", "v3", 0) ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", 8) require.Equal(t, 1.0, readCounterValue(t, ExplainRUStatementsCounter.WithLabelValues("success"))) @@ -156,6 +157,15 @@ func TestExplainRUMetrics(t *testing.T) { readBillingBaseUnitFamily := findMetricFamily(families, "tidb_read_billing_demo_base_units_total") require.NotNil(t, readBillingBaseUnitFamily) requireMetricFamilyHasLabels(t, readBillingBaseUnitFamily, "site", "op_class", "operator_kind", "unit", "input_source", "input_side", "model_version") + var zeroOutputRows *dto.Metric + for _, metric := range readBillingBaseUnitFamily.GetMetric() { + if metricHasLabelValue(metric, "unit", "output_rows") { + zeroOutputRows = metric + break + } + } + require.NotNil(t, zeroOutputRows) + require.Zero(t, zeroOutputRows.GetCounter().GetValue()) readBillingRowWidthFamily := findMetricFamily(families, "tidb_read_billing_demo_row_width_bytes") require.NotNil(t, readBillingRowWidthFamily) requireMetricFamilyHasLabels(t, readBillingRowWidthFamily, "site", "op_class", "operator_kind", "row_width_source", "model_version") @@ -172,7 +182,7 @@ func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { ObserveExplainRURow("plan", "", "", "read_billing_model", "runtime_chunk_avg", "v2", -1, -1, -1, 32) RecordReadBillingDemoStatement("", "v2") RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v2") - AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 0) + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", -1) ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", 0) require.Equal(t, 0, countCollectedMetrics(ExplainRUStatementsCounter)) diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index b1710ae416c5f..a23dd4e827670 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -272,6 +272,21 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: "scan_total_keys", value: 4}, tidbWeights) require.False(t, ok) + for _, outputUnit := range []string{readBillingDemoUnitOutputRows, readBillingDemoUnitOutputBytes} { + _, _, ok = readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: outputUnit, value: 4}, tidbWeights) + require.False(t, ok) + require.True(t, readBillingDemoUnitDiagnosticOnly(outputUnit)) + } + for _, outputClass := range []string{ + readBillingDemoOpClassHashAgg, + readBillingDemoOpClassStreamAgg, + readBillingDemoOpClassHashJoin, + readBillingDemoOpClassMergeJoin, + readBillingDemoOpClassLookupJoin, + } { + require.True(t, readBillingDemoOperatorHasOutputShadows(outputClass)) + } + require.False(t, readBillingDemoOperatorHasOutputShadows(readBillingDemoOpClassProjection)) writeWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion) require.True(t, ok) @@ -467,6 +482,92 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { } }) + t.Run("root aggregations expose diagnostic output units without changing formula", func(t *testing.T) { + child := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) + child.SetSchema(schema) + for _, tc := range []struct { + name string + opClass string + agg func() *FlatOperator + }{ + {name: "hash agg", opClass: readBillingDemoOpClassHashAgg, agg: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema), ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB} + }}, + {name: "stream agg", opClass: readBillingDemoOpClassStreamAgg, agg: func() *FlatOperator { + return &FlatOperator{Origin: (&physicalop.BasePhysicalAgg{}).InitForStream(ctx, stats, 0, schema), ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := FlatPlanTree{ + tc.agg(), + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + aggStats := runtimeStats.GetBasicRuntimeStats(tree[0].Origin.ID(), true) + aggStats.Record(time.Millisecond, 2) + aggStats.RecordBytes(80, 30) + childStats := runtimeStats.GetBasicRuntimeStats(child.ID(), true) + childStats.Record(time.Millisecond, 8) + childStats.RecordBytes(0, 80) + + operator, supported, reason := readBillingDemoClassifyOperator(tree[0]) + require.True(t, supported) + require.Empty(t, reason) + require.Equal(t, tc.opClass, operator.opClass) + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], operator) + require.True(t, ok) + require.Empty(t, reason) + require.Equal(t, 8.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 30.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + + operator.id = tree[0].ExplainID().String() + operator.status = readBillingDemoStatusOperatorOK + operator.reason = readBillingDemoReasonNone + operator.actRows = 2 + operator.hasActRows = true + operator.units = units + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone, operators: []readBillingDemoOperatorResult{operator}} + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + weights, ok := readBillingDemoResolveWeights(operator.site, operator.opClass, readBillingDemoWeightVersion) + require.True(t, ok) + require.InEpsilon(t, weights.fixedEvent+8*weights.row+80*weights.byte, rows[0].previewRU, 0.000001) + seenOutputUnits := make(map[string]bool) + for _, row := range rows { + if row.unit != readBillingDemoUnitOutputRows && row.unit != readBillingDemoUnitOutputBytes { + continue + } + seenOutputUnits[row.unit] = true + require.False(t, row.hasWeight) + require.False(t, row.hasPreviewRU) + require.Contains(t, row.note, "diagnostic_only=true") + if row.unit == readBillingDemoUnitOutputRows { + require.Equal(t, 2.0, row.workRows) + require.Equal(t, int64(2), row.count) + } else { + require.Equal(t, 30.0, row.workBytes) + } + rendered := row.toStrings() + require.Empty(t, rendered[13]) + require.Empty(t, rendered[14]) + } + require.Equal(t, map[string]bool{readBillingDemoUnitOutputRows: true, readBillingDemoUnitOutputBytes: true}, seenOutputUnits) + + statementStats := buildReadBillingDemoStatementStats(result) + seenOutputUnits = make(map[string]bool) + for _, sample := range statementStats.BaseUnits { + if sample.Unit == readBillingDemoUnitOutputRows || sample.Unit == readBillingDemoUnitOutputBytes { + seenOutputUnits[sample.Unit] = true + } + } + require.Equal(t, map[string]bool{readBillingDemoUnitOutputRows: true, readBillingDemoUnitOutputBytes: true}, seenOutputUnits) + }) + } + }) + t.Run("ordering work boundaries fail only on invalid evidence", func(t *testing.T) { sort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) sortOp := &FlatOperator{Origin: sort, IsRoot: true, StoreType: kv.TiDB} @@ -792,6 +893,7 @@ func TestReadBillingDemoCopInputEstimator(t *testing.T) { recordSummary := func(runtimeStats *execdetails.RuntimeStatsColl, planID int, rows uint64, detail *tikvutil.ScanDetail) { one := uint64(1) + runtimeStats.RecordExpectedCopTasks([]int{planID}) runtimeStats.RecordCopStats(planID, kv.TiKV, detail, tikvutil.TimeDetail{}, &tipb.ExecutorExecutionSummary{ TimeProcessedNs: &one, NumProducedRows: &rows, @@ -832,13 +934,14 @@ func TestReadBillingDemoCopInputEstimator(t *testing.T) { node *FlatOperator widthState readBillingDemoCopWidthState expectedOrderWork float64 + expectOutputUnits bool }{ {name: "selection", node: &FlatOperator{Origin: selection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, {name: "limit", node: &FlatOperator{Origin: limit, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, {name: "topn", node: &FlatOperator{Origin: topN, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: 12}, {name: "projection", node: &FlatOperator{Origin: projection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, - {name: "hashagg", node: &FlatOperator{Origin: hashAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, - {name: "streamagg", node: &FlatOperator{Origin: streamAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, + {name: "hashagg", node: &FlatOperator{Origin: hashAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1, expectOutputUnits: true}, + {name: "streamagg", node: &FlatOperator{Origin: streamAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1, expectOutputUnits: true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -850,6 +953,9 @@ func TestReadBillingDemoCopInputEstimator(t *testing.T) { {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, } runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) recordSummary(runtimeStats, scan.ID(), 4, nil) recordSummary(runtimeStats, tc.node.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) @@ -861,10 +967,147 @@ func TestReadBillingDemoCopInputEstimator(t *testing.T) { require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) require.Equal(t, tc.expectedOrderWork, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOrderWork, readBillingDemoInputSideAll)) require.Equal(t, tc.widthState, estimator.outputEstimate(1).widthState) + if tc.expectOutputUnits { + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 30.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeOperatorActRows, readBillingDemoUnitSource(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeReaderOutput, readBillingDemoUnitSource(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceRuntimeReaderOutputChunkAvg, readBillingDemoUnitWidthSource(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + } else { + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + } }) } }) + t.Run("aggregation output shadows fail open without attributable bytes", func(t *testing.T) { + newHashAgg := func() *FlatOperator { + agg := (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema) + return &FlatOperator{Origin: agg, IsRoot: false, StoreType: kv.TiKV} + } + t.Run("reader row mismatch keeps exact output rows only", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{2} + agg.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 3) + readerStats.RecordBytes(0, 33) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("zero rows and bytes remain observed evidence", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{2} + agg.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 0) + readerStats.RecordBytes(0, 0) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 0, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.True(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.True(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Zero(t, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Zero(t, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("non top aggregation never borrows reader bytes", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{3} + agg.ChildrenEndIdx = 3 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + agg, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + recordSummary(runtimeStats, selection.ID(), 2, nil) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 2, operator) + require.True(t, outcome.success) + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("incomplete own summary omits both output units", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{2} + agg.ChildrenEndIdx = 2 + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 2, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordSummary(runtimeStats, scan.ID(), 2, nil) + recordSummary(runtimeStats, scan.ID(), 2, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 1, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + + t.Run("missing expected response summaries omit shadows without changing formula inputs", func(t *testing.T) { + agg := newHashAgg() + agg.ChildrenIdx = []int{3} + agg.ChildrenEndIdx = 3 + tree := FlatPlanTree{ + {Origin: limit, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + {Origin: reader, ChildrenIdx: []int{2}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, + agg, + {Origin: scan, ChildrenEndIdx: 3, IsRoot: false, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, scan.ID(), 4, nil) + recordSummary(runtimeStats, agg.Origin.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + // This is the state after either a consumed response omitted every + // summary or an unconsumed task contributed only its expectation. The + // expected count must not be inferred from visible summaries. + runtimeStats.RecordExpectedCopTasks([]int{scan.ID(), agg.Origin.ID()}) + operator, _, _ := readBillingDemoClassifyOperator(agg) + outcome := readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 2, operator) + require.True(t, outcome.success) + require.Equal(t, 4.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.False(t, readBillingDemoUnitExists(outcome.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + }) + }) + t.Run("selection output feeds projection input", func(t *testing.T) { tree := FlatPlanTree{ {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 3, IsRoot: true, StoreType: kv.TiDB}, @@ -1226,6 +1469,49 @@ func TestReadBillingDemoCopInputEstimator(t *testing.T) { } }) + t.Run("full builder preserves pushed aggregation output shadows", func(t *testing.T) { + aggScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + aggScan.SetSchema(schema) + agg := (&physicalop.BasePhysicalAgg{}).InitForHash(ctx, stats, 0, schema) + agg.SetChildren(aggScan) + reader.TablePlan = agg + flat := FlattenPhysicalPlan(reader, true) + require.Len(t, flat.Main, 3) + require.Equal(t, agg.ID(), flat.Main[1].Origin.ID()) + require.Equal(t, aggScan.ID(), flat.Main[2].Origin.ID()) + + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + readerStats := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + readerStats.Record(time.Millisecond, 2) + readerStats.RecordBytes(0, 30) + recordSummary(runtimeStats, aggScan.ID(), 4, nil) + recordSummary(runtimeStats, agg.ID(), 2, &tikvutil.ScanDetail{TotalKeys: 5, ProcessedKeys: 5, ProcessedKeysSize: 100}) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + + result := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, nil) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + var aggResult readBillingDemoOperatorResult + for _, operator := range result.operators { + if operator.site == readBillingDemoSiteTiKV && operator.opClass == readBillingDemoOpClassHashAgg { + aggResult = operator + break + } + } + require.Equal(t, readBillingDemoStatusOperatorOK, aggResult.status) + require.Equal(t, 4.0, readBillingDemoUnitValue(aggResult.units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(aggResult.units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 30.0, readBillingDemoUnitValue(aggResult.units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + + seenStatementUnits := make(map[string]bool) + for _, sample := range buildReadBillingDemoStatementStats(result).BaseUnits { + if sample.Site == readBillingDemoSiteTiKV && sample.OpClass == readBillingDemoOpClassHashAgg && + (sample.Unit == readBillingDemoUnitOutputRows || sample.Unit == readBillingDemoUnitOutputBytes) { + seenStatementUnits[sample.Unit] = true + } + } + require.Equal(t, map[string]bool{readBillingDemoUnitOutputRows: true, readBillingDemoUnitOutputBytes: true}, seenStatementUnits) + }) + t.Run("full builder wires root sort order work", func(t *testing.T) { rootSort := physicalop.PhysicalSort{}.Init(ctx, stats, 0, nil) scan.SetStats(stats) @@ -1384,8 +1670,12 @@ func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideProbe)) require.Equal(t, 40.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideBuild)) require.Equal(t, 60.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideProbe)) + require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) + require.Equal(t, 60.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideBuild)) require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideProbe)) + require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) + require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) } func readBillingDemoUnitValue(units []readBillingDemoUnit, unitName, side string) float64 { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 63972859cb9ae..78d609b641a0e 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -56,6 +56,7 @@ const ( explainRUSourceSummaryTotal = "summary_total" explainRUWidthSourceRuntimeChunkAvg = "runtime_chunk_avg" + explainRUWidthSourceRuntimeReaderOutputChunkAvg = "runtime_reader_output_chunk_avg" explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" explainRUWidthSourceScanDetailProcessedEstimate = "scan_detail_processed_key_avg_estimate" explainRUWidthSourceNotApplicable = "not_applicable" @@ -133,6 +134,8 @@ const ( readBillingDemoUnitFixedEvents = "fixed_events" readBillingDemoUnitInputRows = "input_rows" readBillingDemoUnitInputBytes = "input_bytes" + readBillingDemoUnitOutputRows = "output_rows" + readBillingDemoUnitOutputBytes = "output_bytes" readBillingDemoUnitOrderWork = "order_work" readBillingDemoUnitEncodedMutationCount = "encoded_mutation_count" readBillingDemoUnitEncodedMutationBytes = "encoded_mutation_bytes" @@ -147,6 +150,8 @@ const ( readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" readBillingDemoInputSourceScanDetail = "scan_detail" readBillingDemoInputSourceRuntimeChildActRows = "runtime_child_act_rows" + readBillingDemoInputSourceRuntimeOperatorActRows = "runtime_operator_act_rows" + readBillingDemoInputSourceRuntimeReaderOutput = "runtime_reader_output_chunks" readBillingDemoInputSourceRuntimeOrderingWork = "runtime_ordering_work" readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" readBillingDemoInputSourceCommitDetail = "commit_detail" @@ -1635,7 +1640,8 @@ func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorR } func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { - if _, _, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, op.Origin.ID()); !ok { + outputRows, outputBytes, hasOutput := readBillingDemoRootOutputRowsAndBytes(runtimeStats, op.Origin.ID()) + if !hasOutput { if _, rowsOK := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()); !rowsOK { return nil, readBillingDemoReasonMissingRuntimeRows, false } @@ -1644,9 +1650,19 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChunkBytes)} switch operator.opClass { case readBillingDemoOpClassHashJoin: - return appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, true) + var reason string + var ok bool + units, reason, ok = appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, true) + if !ok { + return nil, reason, false + } case readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: - return appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, false) + var reason string + var ok bool + units, reason, ok = appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, false) + if !ok { + return nil, reason, false + } default: inputRows, inputBytes, reason, ok := readBillingDemoDirectLocalInputRowsAndBytes(runtimeStats, tree, idx, operator.opClass) if !ok { @@ -1660,8 +1676,11 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F if orderWork.unit != "" { units = append(units, orderWork) } - return units, "", true } + if readBillingDemoOperatorHasOutputShadows(operator.opClass) { + units = append(units, readBillingDemoRuntimeChunkOutputUnits(outputRows, outputBytes)...) + } + return units, "", true } func readBillingDemoFixedEventUnit(inputSource string) readBillingDemoUnit { @@ -1682,6 +1701,32 @@ func readBillingDemoRuntimeChunkInputUnits(rows, bytes int64, side string) []rea } } +func readBillingDemoRuntimeChunkOutputUnits(rows, bytes int64) []readBillingDemoUnit { + if rows < 0 || bytes < 0 { + return nil + } + return []readBillingDemoUnit{ + {unit: readBillingDemoUnitOutputRows, source: readBillingDemoInputSourceRuntimeChunkBytes, side: readBillingDemoInputSideAll, value: float64(rows), widthSource: explainRUWidthSourceNotApplicable}, + {unit: readBillingDemoUnitOutputBytes, source: readBillingDemoInputSourceRuntimeChunkBytes, side: readBillingDemoInputSideAll, value: float64(bytes), rowWidth: readBillingDemoAverageRowWidth(rows, float64(bytes)), widthSource: explainRUWidthSourceRuntimeChunkAvg}, + } +} + +func readBillingDemoIsAggClass(opClass string) bool { + return opClass == readBillingDemoOpClassHashAgg || opClass == readBillingDemoOpClassStreamAgg +} + +func readBillingDemoOperatorHasOutputShadows(opClass string) bool { + if readBillingDemoIsAggClass(opClass) { + return true + } + switch opClass { + case readBillingDemoOpClassHashJoin, readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: + return true + default: + return false + } +} + func readBillingDemoAverageRowWidth(rows int64, bytes float64) float64 { if rows <= 0 || bytes <= 0 { return 0 @@ -1815,6 +1860,64 @@ func readBillingDemoRootOutputRowsAndBytes(runtimeStats *execdetails.RuntimeStat return basic.GetActRows(), basic.GetOutputBytes(), true } +func (e *readBillingDemoCopEstimator) aggOutputShadowUnits(idx int, opClass string) []readBillingDemoUnit { + if !readBillingDemoIsAggClass(opClass) || idx < 0 || idx >= len(e.tree) || e.tree[idx] == nil || e.tree[idx].Origin == nil { + return nil + } + rowsEvidence := readBillingDemoExactCopRowsEvidence(e.runtimeStats, e.tree[idx].Origin.ID()) + if rowsEvidence.state != readBillingDemoCopRowsObserved { + return nil + } + expectedTasks := e.runtimeStats.GetExpectedCopTasks(e.tree[idx].Origin.ID()) + if expectedTasks <= 0 || rowsEvidence.tasks != expectedTasks { + return nil + } + componentID := e.componentID[idx] + if componentID < 0 || componentID >= len(e.components) { + return nil + } + if maxTasks := e.components[componentID].maxSummaryTasks; maxTasks > 0 && rowsEvidence.tasks < maxTasks { + return nil + } + units := []readBillingDemoUnit{{ + unit: readBillingDemoUnitOutputRows, + source: readBillingDemoInputSourceRuntimeOperatorActRows, + side: readBillingDemoInputSideAll, + value: float64(rowsEvidence.rows), + widthSource: explainRUWidthSourceNotApplicable, + }} + + parentIdx := e.parentIdx[idx] + if parentIdx < 0 || parentIdx >= len(e.tree) { + return units + } + parent := e.tree[parentIdx] + if parent == nil || parent.Origin == nil || !parent.IsRoot || len(parent.ChildrenIdx) != 1 || parent.ChildrenIdx[0] != idx { + return units + } + parentOperator, supported, _ := readBillingDemoClassifyOperator(parent) + if !supported || parentOperator.opClass != readBillingDemoOpClassReaderReceive { + return units + } + switch parent.Origin.TP() { + case plancodec.TypeTableReader, plancodec.TypeIndexReader: + default: + return units + } + readerRows, readerBytes, ok := readBillingDemoRootOutputRowsAndBytes(e.runtimeStats, parent.Origin.ID()) + if !ok || readerRows != rowsEvidence.rows || readerRows < 0 || readerBytes < 0 { + return units + } + return append(units, readBillingDemoUnit{ + unit: readBillingDemoUnitOutputBytes, + source: readBillingDemoInputSourceRuntimeReaderOutput, + side: readBillingDemoInputSideAll, + value: float64(readerBytes), + rowWidth: readBillingDemoAverageRowWidth(readerRows, float64(readerBytes)), + widthSource: explainRUWidthSourceRuntimeReaderOutputChunkAvg, + }) +} + func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, operator readBillingDemoOperatorResult) readBillingDemoCopUnitOutcome { if failure, ok := estimator.nodeFailures[idx]; ok { return readBillingDemoCopUnitOutcome{failure: failure} @@ -1866,6 +1969,7 @@ func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, op if orderWork.unit != "" { units = append(units, orderWork) } + units = append(units, estimator.aggOutputShadowUnits(idx, operator.opClass)...) return readBillingDemoCopUnitOutcome{success: true, units: units} } @@ -2267,6 +2371,16 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill case readBillingDemoUnitInputBytes: row.workBytes = unit.value row.hasWorkBytes = true + case readBillingDemoUnitOutputRows: + row.outputRows = int64(unit.value) + row.hasOutputRows = true + row.workRows = unit.value + row.hasWorkRows = true + row.count = int64(unit.value) + row.hasCount = true + case readBillingDemoUnitOutputBytes: + row.workBytes = unit.value + row.hasWorkBytes = true case readBillingDemoUnitOrderWork: row.workRows = unit.value row.hasWorkRows = true @@ -2285,7 +2399,8 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill func readBillingDemoUnitDiagnosticOnly(unit string) bool { switch unit { case readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitKeyBytes, - readBillingDemoUnitValueBytes, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: + readBillingDemoUnitValueBytes, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount, + readBillingDemoUnitOutputRows, readBillingDemoUnitOutputBytes: return true default: return false diff --git a/pkg/util/execdetails/runtime_stats.go b/pkg/util/execdetails/runtime_stats.go index b109429b5b6f1..020125a82473d 100644 --- a/pkg/util/execdetails/runtime_stats.go +++ b/pkg/util/execdetails/runtime_stats.go @@ -502,10 +502,11 @@ func (e *BasicRuntimeStats) GetTime() int64 { // RuntimeStatsColl collects executors's execution info. type RuntimeStatsColl struct { - rootStats map[int]*RootRuntimeStats - copStats map[int]*CopRuntimeStats - stmtCopStats StmtCopRuntimeStats - mu sync.Mutex + rootStats map[int]*RootRuntimeStats + copStats map[int]*CopRuntimeStats + expectedCopTasks map[int]int32 + stmtCopStats StmtCopRuntimeStats + mu sync.Mutex } // NewRuntimeStatsColl creates new executor collector. @@ -522,11 +523,15 @@ func NewRuntimeStatsColl(reuse *RuntimeStatsColl) *RuntimeStatsColl { for k := range reuse.copStats { delete(reuse.copStats, k) } + for k := range reuse.expectedCopTasks { + delete(reuse.expectedCopTasks, k) + } return reuse } return &RuntimeStatsColl{ - rootStats: make(map[int]*RootRuntimeStats), - copStats: make(map[int]*CopRuntimeStats), + rootStats: make(map[int]*RootRuntimeStats), + copStats: make(map[int]*CopRuntimeStats), + expectedCopTasks: make(map[int]int32), } } @@ -628,6 +633,29 @@ func (e *RuntimeStatsColl) GetCopCountAndRows(planID int) (int32, int64) { return copStats.GetTasks(), copStats.GetActRows() } +// RecordExpectedCopTasks records that one received TiKV cop response is expected +// to carry an execution summary for every listed plan. +func (e *RuntimeStatsColl) RecordExpectedCopTasks(planIDs []int) { + e.mu.Lock() + defer e.mu.Unlock() + if e.expectedCopTasks == nil { + e.expectedCopTasks = make(map[int]int32) + } + for _, planID := range planIDs { + if planID > 0 { + e.expectedCopTasks[planID]++ + } + } +} + +// GetExpectedCopTasks returns the number of received TiKV cop responses that +// were expected to carry an execution summary for the specified plan. +func (e *RuntimeStatsColl) GetExpectedCopTasks(planID int) int32 { + e.mu.Lock() + defer e.mu.Unlock() + return e.expectedCopTasks[planID] +} + func getPlanIDFromExecutionSummary(summary *tipb.ExecutorExecutionSummary) (int, bool) { if summary.GetExecutorId() != "" { strs := strings.Split(summary.GetExecutorId(), "_") From 16a542007492f3ec167a8b9b6261e0a9807598c2 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Thu, 16 Jul 2026 21:30:27 +0800 Subject: [PATCH 16/25] executor: log preview RU units in general log --- .../references/executor-case-map.md | 3 +- pkg/executor/BUILD.bazel | 1 + pkg/executor/adapter.go | 153 +++++++- pkg/executor/adapter_internal_test.go | 130 +++++++ pkg/executor/adapter_test.go | 338 ++++++++++++++++++ pkg/session/session.go | 24 +- 6 files changed, 635 insertions(+), 14 deletions(-) diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index 84fa2d3c313d9..d0356aadec4f6 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -9,7 +9,8 @@ ## pkg/executor ### Tests -- `pkg/executor/adapter_test.go` - executor: Tests format SQL. +- `pkg/executor/adapter_internal_test.go` - executor: Tests private adapter runtime accounting, including preview RU General Log gates, reduced unit schema, aggregation, and deterministic ordering. +- `pkg/executor/adapter_test.go` - executor: Tests statement adapter behavior, including RU v2 finalization and self-describing preview RU General Log completion after lazy SELECT close, DML/early errors, compile-error suppression, cursor-detach gating, repeated finish calls, and EXPLAIN ANALYZE DML commit errors. - `pkg/executor/analyze_test.go` - executor: Tests analyze index extract top n. - `pkg/executor/analyze_utils_test.go` - executor: Tests get analyze panic err. - `pkg/executor/batch_point_get_test.go` - executor: Tests batch point get lock exist key. diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel index 9b26968e75bc8..ac1c2856843c5 100644 --- a/pkg/executor/BUILD.bazel +++ b/pkg/executor/BUILD.bazel @@ -528,6 +528,7 @@ go_test( "//pkg/util/set", "//pkg/util/sqlexec", "//pkg/util/sqlkiller", + "//pkg/util/stmtsummary", "//pkg/util/stmtsummary/v2:stmtsummary", "//pkg/util/syncutil", "//pkg/util/tableutil", diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index 79c20aff81045..a5aef4afc9941 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -19,6 +19,7 @@ import ( "fmt" "math" "runtime/trace" + "slices" "strconv" "strings" "sync" @@ -283,14 +284,33 @@ func (a *recordSet) Finish() error { } func (a *recordSet) Close() error { + return a.close(nil) +} + +func (a *recordSet) close(lastErr error) error { err := a.Finish() if err != nil { logutil.BgLogger().Error("close recordSet error", zap.Error(err)) } - a.stmt.CloseRecordSet(a.txnStartTS, errors.Join(a.lastErrs...)) + a.stmt.CloseRecordSet(a.txnStartTS, errors.Join(errors.Join(a.lastErrs...), lastErr)) return err } +// CloseRecordSetWithError closes an executor-owned record set and forwards an +// execution error to its single statement-completion path. +func CloseRecordSetWithError(rs sqlexec.RecordSet, lastErr error) error { + switch recordSet := rs.(type) { + case *recordSet: + return recordSet.close(lastErr) + case *chunkRowRecordSet: + recordSet.execStmt.CloseRecordSet(recordSet.execStmt.Ctx.GetSessionVars().TxnCtx.StartTS, lastErr) + return nil + default: + closeErr := rs.Close() + return errors.Join(closeErr, errors.New("record set does not support closing with an execution error")) + } +} + // OnFetchReturned implements commandLifeCycle#OnFetchReturned func (a *recordSet) OnFetchReturned() { a.stmt.LogSlowQuery(a.txnStartTS, len(a.lastErrs) == 0, true) @@ -298,6 +318,11 @@ func (a *recordSet) OnFetchReturned() { // TryDetach creates a new `RecordSet` which doesn't depend on the current session context. func (a *recordSet) TryDetach() (sqlexec.RecordSet, bool, error) { + // A detached static record set no longer reaches ExecStmt completion. Keep + // the live record set only when the preview RU completion event is enabled. + if a.stmt != nil && readBillingDemoGeneralLogEnabled(a.stmt.Ctx) { + return nil, false, nil + } e, ok := Detach(a.executor) if !ok { return nil, false, nil @@ -366,12 +391,13 @@ type ExecStmt struct { Ctx sessionctx.Context // LowerPriority represents whether to lower the execution priority of a query. - LowerPriority bool - isPreparedStmt bool - isSelectForUpdate bool - resolvedPreparedStmt ast.StmtNode - retryCount uint - retryStartTime time.Time + LowerPriority bool + isPreparedStmt bool + isSelectForUpdate bool + resolvedPreparedStmt ast.StmtNode + retryCount uint + retryStartTime time.Time + readBillingDemoGeneralLogEmitted atomic.Bool // Phase durations are splited into two parts: 1. trying to lock keys (but // failed); 2. the final iteration of the retry loop. Here we use @@ -1702,6 +1728,7 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo // client abandons a result without closing/draining it, this first demo can // miss that statement instead of guessing an incomplete status. readBillingDemoStats := plannercore.RecordReadBillingDemoForStatement(a.Ctx, a.Plan, a.readBillingDemoStmtNode(), err) + a.logReadBillingDemoGeneralLog(readBillingDemoStats) a.updateNetworkTrafficStatsAndMetrics() // `LowSlowQuery` and `SummaryStmt` must be called before recording `PrevStmt`. a.LogSlowQuery(txnTS, succ, hasMoreResults) @@ -1764,6 +1791,118 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, err error, hasMoreResults boo a.Ctx.ReportUsageStats() } +const readBillingDemoGeneralLogMessage = "GENERAL_LOG_RU_UNITS" + +type readBillingDemoGeneralLogUnit struct { + site string + opClass string + operatorKind string + unit string + value float64 +} + +func (u readBillingDemoGeneralLogUnit) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("site", u.site) + enc.AddString("op_class", u.opClass) + enc.AddString("operator_kind", u.operatorKind) + enc.AddString("unit", u.unit) + enc.AddFloat64("value", u.value) + return nil +} + +type readBillingDemoGeneralLogUnits []readBillingDemoGeneralLogUnit + +func (units readBillingDemoGeneralLogUnits) MarshalLogArray(enc zapcore.ArrayEncoder) error { + for _, unit := range units { + if err := enc.AppendObject(unit); err != nil { + return err + } + } + return nil +} + +func buildReadBillingDemoGeneralLogUnits(stats stmtsummary.ReadBillingDemoStatementStats) readBillingDemoGeneralLogUnits { + totals := make(map[readBillingDemoGeneralLogUnit]float64, len(stats.BaseUnits)) + for _, sample := range stats.BaseUnits { + key := readBillingDemoGeneralLogUnit{ + site: sample.Site, + opClass: sample.OpClass, + operatorKind: sample.OperatorKind, + unit: sample.Unit, + } + totals[key] += sample.Value + } + + units := make(readBillingDemoGeneralLogUnits, 0, len(totals)) + for unit, value := range totals { + unit.value = value + units = append(units, unit) + } + slices.SortFunc(units, func(left, right readBillingDemoGeneralLogUnit) int { + if cmp := strings.Compare(left.site, right.site); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.opClass, right.opClass); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.operatorKind, right.operatorKind); cmp != 0 { + return cmp + } + return strings.Compare(left.unit, right.unit) + }) + return units +} + +func readBillingDemoGeneralLogEnabled(sctx sessionctx.Context) bool { + if sctx == nil || !vardef.ProcessGeneralLog.Load() { + return false + } + sessVars := sctx.GetSessionVars() + if sessVars == nil || !sessVars.EnableReadBillingDemo || sessVars.InRestrictedSQL || + sessVars.StmtCtx == nil || sessVars.StmtCtx.InRestrictedSQL { + return false + } + return true +} + +func canLogReadBillingDemoGeneralLog(sctx sessionctx.Context, stats stmtsummary.ReadBillingDemoStatementStats) bool { + return readBillingDemoGeneralLogEnabled(sctx) && stats.ModelVersion != "" && stats.WeightVersion != "" +} + +// LogReadBillingDemoGeneralLog emits a preview RU completion event for paths +// that finish before an ExecStmt reaches FinishExecuteStmt. +func LogReadBillingDemoGeneralLog(sctx sessionctx.Context, stats stmtsummary.ReadBillingDemoStatementStats) { + if !canLogReadBillingDemoGeneralLog(sctx, stats) { + return + } + emitReadBillingDemoGeneralLog(sctx, stats) +} + +func emitReadBillingDemoGeneralLog(sctx sessionctx.Context, stats stmtsummary.ReadBillingDemoStatementStats) { + sessVars := sctx.GetSessionVars() + normalizedSQL, digest := sessVars.StmtCtx.SQLDigest() + digestString := "" + if digest != nil { + digestString = digest.String() + } + logutil.GeneralLogger.Info(readBillingDemoGeneralLogMessage, + zap.Uint64("conn", sessVars.ConnectionID), + zap.String("model_version", stats.ModelVersion), + zap.String("weight_version", stats.WeightVersion), + zap.String("normalized_sql", normalizedSQL), + zap.String("sql_digest", digestString), + zap.Array("units", buildReadBillingDemoGeneralLogUnits(stats)), + ) +} + +func (a *ExecStmt) logReadBillingDemoGeneralLog(stats stmtsummary.ReadBillingDemoStatementStats) { + if !canLogReadBillingDemoGeneralLog(a.Ctx, stats) || + !a.readBillingDemoGeneralLogEmitted.CompareAndSwap(false, true) { + return + } + emitReadBillingDemoGeneralLog(a.Ctx, stats) +} + func (a *ExecStmt) recordAffectedRows2Metrics() { sessVars := a.Ctx.GetSessionVars() if affectedRows := sessVars.StmtCtx.AffectedRows(); affectedRows > 0 { diff --git a/pkg/executor/adapter_internal_test.go b/pkg/executor/adapter_internal_test.go index 7aa2237ece5e4..015ca5e59a27e 100644 --- a/pkg/executor/adapter_internal_test.go +++ b/pkg/executor/adapter_internal_test.go @@ -29,11 +29,14 @@ import ( "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/auth" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/execdetails" + "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/pingcap/tidb/pkg/util/topsql" topsqlmock "github.com/pingcap/tidb/pkg/util/topsql/collector/mock" topsqlstate "github.com/pingcap/tidb/pkg/util/topsql/state" @@ -44,6 +47,8 @@ import ( metastorage "github.com/tikv/pd/client/clients/metastorage" "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) type stmtStatsTestContext struct { @@ -97,6 +102,131 @@ func resetTopProfilingStateForTest(t *testing.T) { }) } +func TestReadBillingDemoGeneralLogUnits(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + sctx := mock.NewContext() + sessVars := sctx.GetSessionVars() + sessVars.ConnectionID = 42 + sessVars.StmtCtx.OriginalSQL = "select 12345, 'secret_literal'" + expectedNormalizedSQL, expectedDigest := parser.NormalizeDigest(sessVars.StmtCtx.OriginalSQL) + stats := stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v2", + BaseUnits: []stmtsummary.ReadBillingDemoBaseUnitSample{ + { + Site: "tikv", OpClass: "join_hash", OperatorKind: "hashjoin", Unit: "input_rows", Value: 2, + DMLKind: "insert", InputSource: "child", InputSide: "left", RowWidthSource: "scan", RowWidth: 4, + }, + { + Site: "tikv", OpClass: "join_hash", OperatorKind: "hashjoin", Unit: "input_rows", Value: 3, + DMLKind: "update", InputSource: "runtime", InputSide: "right", RowWidthSource: "schema", RowWidth: 8, + }, + {Site: "tidb", OpClass: "agg_hash", OperatorKind: "hashagg", Unit: "output_rows", Value: 1}, + {Site: "tidb", OpClass: "agg_hash", OperatorKind: "hashagg", Unit: "input_bytes", Value: 8}, + }, + } + + // Neither individual gate is sufficient. + sessVars.EnableReadBillingDemo = true + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + vardef.ProcessGeneralLog.Store(true) + sessVars.EnableReadBillingDemo = false + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + + // Internal SQL is excluded whether marked at session or statement scope. + sessVars.EnableReadBillingDemo = true + sessVars.InRestrictedSQL = true + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + sessVars.InRestrictedSQL = false + sessVars.StmtCtx.InRestrictedSQL = true + LogReadBillingDemoGeneralLog(sctx, stats) + require.Zero(t, recorded.Len()) + sessVars.StmtCtx.InRestrictedSQL = false + + LogReadBillingDemoGeneralLog(sctx, stats) + entries := recorded.TakeAll() + require.Len(t, entries, 1) + require.Equal(t, readBillingDemoGeneralLogMessage, entries[0].Message) + fields := entries[0].ContextMap() + require.Len(t, fields, 6) + require.Equal(t, uint64(42), fields["conn"]) + require.Equal(t, "v3", fields["model_version"]) + require.Equal(t, "v2", fields["weight_version"]) + require.Equal(t, expectedNormalizedSQL, fields["normalized_sql"]) + require.Equal(t, expectedDigest.String(), fields["sql_digest"]) + require.NotContains(t, fields["normalized_sql"], "12345") + require.NotContains(t, fields["normalized_sql"], "secret_literal") + require.NotContains(t, fields, "sql") + rawUnits, ok := fields["units"].([]any) + require.True(t, ok) + require.Len(t, rawUnits, 3) + + units := make([]map[string]any, 0, len(rawUnits)) + for _, rawUnit := range rawUnits { + unit, ok := rawUnit.(map[string]any) + require.True(t, ok) + require.Len(t, unit, 5) + for _, field := range []string{"site", "op_class", "operator_kind", "unit", "value"} { + require.Contains(t, unit, field) + } + for _, internalField := range []string{"dml_kind", "input_source", "input_side", "row_width_source", "row_width", "operator_id"} { + require.NotContains(t, unit, internalField) + } + units = append(units, unit) + } + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "unit": "input_bytes", "value": float64(8), + }, units[0]) + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "unit": "output_rows", "value": float64(1), + }, units[1]) + require.Equal(t, map[string]any{ + "site": "tikv", "op_class": "join_hash", "operator_kind": "hashjoin", "unit": "input_rows", "value": float64(5), + }, units[2]) + + // A valid completed snapshot with no unit samples stays structured and does + // not invent one statement status from the snapshot's multi-status model. + LogReadBillingDemoGeneralLog(sctx, stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v2", + }) + entries = recorded.TakeAll() + require.Len(t, entries, 1) + fields = entries[0].ContextMap() + require.Len(t, fields, 6) + require.NotContains(t, fields, "status") + require.NotContains(t, fields, "reason") + rawUnits, ok = fields["units"].([]any) + require.True(t, ok) + require.Empty(t, rawUnits) + + // Explicitly initialized empty identity is safe and does not require a + // digest object to be present. + emptyIdentityCtx := mock.NewContext() + emptyIdentityCtx.GetSessionVars().EnableReadBillingDemo = true + emptyIdentityCtx.GetSessionVars().StmtCtx.InitSQLDigest("", nil) + LogReadBillingDemoGeneralLog(emptyIdentityCtx, stmtsummary.ReadBillingDemoStatementStats{ + ModelVersion: "v3", + WeightVersion: "v2", + }) + entries = recorded.TakeAll() + require.Len(t, entries, 1) + fields = entries[0].ContextMap() + require.Equal(t, "", fields["normalized_sql"]) + require.Equal(t, "", fields["sql_digest"]) +} + func newExecStmtWithStmtStatsForTest(goCtx context.Context, t *testing.T) (*ExecStmt, *stmtstats.StatementStats) { t.Helper() diff --git a/pkg/executor/adapter_test.go b/pkg/executor/adapter_test.go index 75bf763df0100..24dcf21706a5d 100644 --- a/pkg/executor/adapter_test.go +++ b/pkg/executor/adapter_test.go @@ -30,10 +30,12 @@ import ( "github.com/pingcap/tidb/pkg/config" distsqlctx "github.com/pingcap/tidb/pkg/distsql/context" "github.com/pingcap/tidb/pkg/executor" + "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/auth" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/resourcegroup" "github.com/pingcap/tidb/pkg/session" "github.com/pingcap/tidb/pkg/sessionctx/slowlogrule" @@ -43,6 +45,7 @@ import ( "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tidb/pkg/util/sqlexec" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -58,6 +61,15 @@ type mockRUV2ConsumptionReporter struct { tiflashRU float64 } +func requireReadBillingDemoGeneralLogIdentity(t *testing.T, entry observer.LoggedEntry, sql string) { + t.Helper() + normalizedSQL, digest := parser.NormalizeDigest(sql) + fields := entry.ContextMap() + require.Equal(t, normalizedSQL, fields["normalized_sql"]) + require.Equal(t, digest.String(), fields["sql_digest"]) + require.NotContains(t, fields, "sql") +} + func (*mockRUV2ConsumptionReporter) ReportConsumption(_ string, _ *rmpb.Consumption) {} func (m *mockRUV2ConsumptionReporter) ReportRUV2Consumption(resourceGroupName string, tikvRUV2, tidbRUV2, tiflashRUV2 float64) { @@ -644,6 +656,332 @@ func TestFinishExecuteStmtSyncsTiDBRUV2FromRUDetails(t *testing.T) { close(done) wg.Wait() }) + + t.Run("preview RU general log waits for SELECT close", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + selectSQL := "select 12345, 'secret_literal'" + rs, err := tk.Exec(selectSQL) + require.NoError(t, err) + require.NotNil(t, rs) + beforeClose := recorded.TakeAll() + require.Len(t, beforeClose, 1) + require.Equal(t, "GENERAL_LOG", beforeClose[0].Message) + startFields := beforeClose[0].ContextMap() + // Keep the existing start record's field contract unchanged; preview + // units belong only to the separate completion record. + require.Len(t, startFields, 11) + require.Equal(t, selectSQL, startFields["sql"]) + require.NotContains(t, startFields, "model_version") + require.NotContains(t, startFields, "weight_version") + require.NotContains(t, startFields, "units") + + require.NoError(t, rs.Close()) + afterClose := recorded.TakeAll() + require.Len(t, afterClose, 1) + require.Equal(t, "GENERAL_LOG_RU_UNITS", afterClose[0].Message) + completionFields := afterClose[0].ContextMap() + require.Len(t, completionFields, 6) + require.Contains(t, completionFields, "conn") + require.Contains(t, completionFields, "model_version") + require.Contains(t, completionFields, "weight_version") + require.Contains(t, completionFields, "normalized_sql") + require.Contains(t, completionFields, "sql_digest") + require.Contains(t, completionFields, "units") + require.NotContains(t, completionFields, "sql") + requireReadBillingDemoGeneralLogIdentity(t, afterClose[0], selectSQL) + require.NotContains(t, completionFields["normalized_sql"], "12345") + require.NotContains(t, completionFields["normalized_sql"], "secret_literal") + }) + + t.Run("preview RU general log DML completion is self describing", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table dml_ru_units_identity (id int primary key, v int)") + tk.MustExec("insert into dml_ru_units_identity values (12345, 1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + updateSQL := "update dml_ru_units_identity set v = 98765 where id = 12345" + tk.MustExec(updateSQL) + entries := recorded.TakeAll() + require.Len(t, entries, 2) + // DML currently completes inside runStmt before its deferred legacy + // GENERAL_LOG. Identity makes the completion independent of this order. + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[0].Message) + require.Equal(t, "GENERAL_LOG", entries[1].Message) + requireReadBillingDemoGeneralLogIdentity(t, entries[0], updateSQL) + normalizedSQL := entries[0].ContextMap()["normalized_sql"] + require.NotContains(t, normalizedSQL, "98765") + require.NotContains(t, normalizedSQL, "12345") + }) + + t.Run("preview RU general log skips pre-defer compile error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + _, err := tk.Exec("select * from missing_ru_units_identity") + require.Error(t, err) + // Compile fails before legacy GENERAL_LOG is registered. Do not emit an + // orphan completion that has no corresponding executed statement. + require.Empty(t, recorded.TakeAll()) + }) + + t.Run("preview RU general log covers prepared point get setup error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table point_get_setup_error_ru_units (id int primary key, v int)") + tk.MustExec("insert into point_get_setup_error_ru_units values (12345, 1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + querySQL := "select v from point_get_setup_error_ru_units where id = ?" + failpointName := "github.com/pingcap/tidb/pkg/session/mockGetTSFail" + require.NoError(t, failpoint.Enable(failpointName, "return")) + failpointEnabled := true + t.Cleanup(func() { + if failpointEnabled { + require.NoError(t, failpoint.Disable(failpointName)) + } + }) + ctx := failpoint.WithHook(context.Background(), func(_ context.Context, name string) bool { + return name == failpointName + }) + _, err := tk.ExecWithContext(ctx, querySQL, 12345) + require.NoError(t, failpoint.Disable(failpointName)) + failpointEnabled = false + require.Error(t, err) + + entries := recorded.TakeAll() + require.Len(t, entries, 2) + require.Equal(t, "GENERAL_LOG", entries[0].Message) + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[1].Message) + requireReadBillingDemoGeneralLogIdentity(t, entries[1], querySQL) + rawUnits, ok := entries[1].ContextMap()["units"].([]any) + require.True(t, ok) + require.Empty(t, rawUnits) + }) + + t.Run("preview RU general log disables cursor detach only when needed", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table detach_ru_units (a int)") + tk.MustExec("insert into detach_ru_units values (1), (2)") + tk.Session().GetSessionVars().SetStatusFlag(mysql.ServerStatusCursorExists, true) + + // General Log alone preserves the existing detach behavior. + vardef.ProcessGeneralLog.Store(true) + rs, err := tk.Exec("select * from detach_ru_units where a > ?", 0) + require.NoError(t, err) + detachable := rs.(sqlexec.DetachableRecordSet) + detached, ok, err := detachable.TryDetach() + require.NoError(t, err) + require.True(t, ok) + require.NotNil(t, detached) + require.NoError(t, detached.Close()) + + // With both gates enabled, keep the live record set so Close can freeze + // runtime evidence and emit exactly one completion event. + vardef.ProcessGeneralLog.Store(false) + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + rs, err = tk.Exec("select * from detach_ru_units where a > ?", 0) + require.NoError(t, err) + detachable = rs.(sqlexec.DetachableRecordSet) + detached, ok, err = detachable.TryDetach() + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, detached) + beforeClose := recorded.TakeAll() + require.Len(t, beforeClose, 1) + require.Equal(t, "GENERAL_LOG", beforeClose[0].Message) + require.NoError(t, rs.Close()) + afterClose := recorded.TakeAll() + require.Len(t, afterClose, 1) + require.Equal(t, "GENERAL_LOG_RU_UNITS", afterClose[0].Message) + }) + + t.Run("preview RU general log covers early statement error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table early_error_ru_units (a int primary key)") + tk.MustExec("insert into early_error_ru_units values (1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + tk.MustExec("begin pessimistic") + tk.MustQuery("select * from early_error_ru_units where a = 1 for update").Check(testkit.Rows("1")) + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + atomic.StoreUint32(&tk.Session().GetSessionVars().TxnCtx.LockExpire, 1) + t.Cleanup(func() { + atomic.StoreUint32(&tk.Session().GetSessionVars().TxnCtx.LockExpire, 0) + }) + + _, err := tk.Exec("select 1") + atomic.StoreUint32(&tk.Session().GetSessionVars().TxnCtx.LockExpire, 0) + require.Error(t, err) + entries := recorded.TakeAll() + require.Len(t, entries, 2) + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[0].Message) + require.Equal(t, "GENERAL_LOG", entries[1].Message) + require.Equal(t, "select 1", entries[1].ContextMap()["sql"]) + requireReadBillingDemoGeneralLogIdentity(t, entries[0], "select 1") + rawUnits, ok := entries[0].ContextMap()["units"].([]any) + require.True(t, ok) + require.Empty(t, rawUnits) + }) + + t.Run("preview RU general log covers explain analyze DML commit error", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + tk.MustExec("set tidb_retry_limit = 0") + tk.MustExec("create table explain_commit_error_ru_units (id int primary key, v int)") + tk.MustExec("insert into explain_commit_error_ru_units values (1, 1)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + recorded.TakeAll() + + explainSQL := "explain analyze update explain_commit_error_ru_units set v = v + 1 where id = 1" + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/session/mockCommitError8942", `return(true)`)) + failpointEnabled := true + t.Cleanup(func() { + if failpointEnabled { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/session/mockCommitError8942")) + } + }) + rs, err := tk.Exec(explainSQL) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/session/mockCommitError8942")) + failpointEnabled = false + require.Nil(t, rs) + require.Error(t, err) + require.True(t, kv.ErrTxnRetryable.Equal(err), err) + entries := recorded.TakeAll() + require.Len(t, entries, 2) + require.Equal(t, "GENERAL_LOG_RU_UNITS", entries[0].Message) + require.Equal(t, "GENERAL_LOG", entries[1].Message) + requireReadBillingDemoGeneralLogIdentity(t, entries[0], explainSQL) + digest := entries[0].ContextMap()["sql_digest"] + + vardef.ProcessGeneralLog.Store(false) + tk.MustQuery("select v from explain_commit_error_ru_units where id = 1").Check(testkit.Rows("1")) + tk.MustQuery("select status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest = ? and site = 'statement'", digest). + Check(testkit.Rows("error statement_error 1")) + }) + + t.Run("preview RU general log emits once across repeated finish", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(true) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + ctx := mock.NewContext() + sessVars := ctx.GetSessionVars() + sessVars.EnableReadBillingDemo = true + sessVars.StartTime = time.Now() + sessVars.StmtCtx.StmtType = "Select" + sessVars.StmtCtx.OriginalSQL = "select 1" + sessVars.StmtCtx.ResetSQLDigest(sessVars.StmtCtx.OriginalSQL) + goCtx := execdetails.ContextWithInitializedExecDetails(context.Background()) + sessVars.RUV2Metrics = execdetails.RUV2MetricsFromContext(goCtx) + execStmt := &executor.ExecStmt{ + Ctx: ctx, + GoCtx: goCtx, + StmtNode: &ast.SelectStmt{}, + } + + execStmt.FinishExecuteStmt(0, nil, false) + execStmt.FinishExecuteStmt(0, nil, false) + entries := recorded.TakeAll() + completionCount := 0 + for _, entry := range entries { + if entry.Message == "GENERAL_LOG_RU_UNITS" { + completionCount++ + } + } + require.Equal(t, 1, completionCount) + }) } func TestSlowLogMaxPerSec(t *testing.T) { diff --git a/pkg/session/session.go b/pkg/session/session.go index 757fd03c80d98..54ce556716920 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -2445,9 +2445,13 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (r // work before this statement receives a fresh StatementContext. s.txn.setPreviewKVMutationCollection(nil, false) readBillingDemoHandledByStmt := false + readBillingDemoLegacyGeneralLogRegistered := false defer func() { if err != nil && !readBillingDemoHandledByStmt { - s.recordReadBillingDemoEarlyError(stmtNode, err) + readBillingDemoStats := s.recordReadBillingDemoEarlyError(stmtNode, err) + if readBillingDemoLegacyGeneralLogRegistered { + executor.LogReadBillingDemoGeneralLog(s, readBillingDemoStats) + } } }() @@ -2661,6 +2665,7 @@ func (s *session) executeStmtImpl(ctx context.Context, stmtNode ast.StmtNode) (r // Execute the physical plan. defer logStmt(stmt, s) // defer until txnStartTS is set + readBillingDemoLegacyGeneralLogRegistered = true if sessVars.MemArbitrator.WaitAverse == variable.MemArbitratorNolimit { metrics.GlobalMemArbitratorSubTasks.NoLimit.Inc() @@ -2795,9 +2800,9 @@ func resolvePreparedStmt(stmt ast.StmtNode, vars *variable.SessionVars) (ast.Stm return prepareStmt.PreparedAst.Stmt, nil } -func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err error) { +func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err error) stmtsummary.ReadBillingDemoStatementStats { if err == nil { - return + return stmtsummary.ReadBillingDemoStatementStats{} } metricsStmt := stmtNode if resolvedStmt, resolveErr := resolvePreparedStmt(stmtNode, s.sessionVars); resolveErr == nil && resolvedStmt != nil { @@ -2805,7 +2810,7 @@ func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err err } readBillingDemoStats := plannercore.RecordReadBillingDemoForStatement(s, nil, metricsStmt, err) if readBillingDemoStats.IsEmpty() || s.sessionVars == nil || s.sessionVars.StmtCtx == nil { - return + return stmtsummary.ReadBillingDemoStatementStats{} } userString := "" if s.sessionVars.User != nil { @@ -2833,6 +2838,7 @@ func (s *session) recordReadBillingDemoEarlyError(stmtNode ast.StmtNode, err err ResourceGroupName: stmtCtx.ResourceGroupName, ReadBillingDemoStats: readBillingDemoStats, }) + return readBillingDemoStats } func shouldBypass(ctx context.Context, stmtNode ast.StmtNode, sessVars *variable.SessionVars) bool { @@ -3061,7 +3067,8 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. origTxnCtx := sessVars.TxnCtx err = se.checkTxnAborted(s) if err != nil { - se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) + readBillingDemoStats := se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) + executor.LogReadBillingDemoGeneralLog(se, readBillingDemoStats) return nil, err } if sessVars.TxnCtx.CouldRetry && !s.IsReadOnly(sessVars) { @@ -3069,7 +3076,8 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. // otherwise, the stmt won't be add into stmt history, and also don't need check. // About `stmt-count-limit`, see more in https://docs.pingcap.com/tidb/stable/tidb-configuration-file#stmt-count-limit if err := checkStmtLimit(ctx, se, false); err != nil { - se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) + readBillingDemoStats := se.recordReadBillingDemoEarlyError(s.GetStmtNode(), err) + executor.LogReadBillingDemoGeneralLog(se, readBillingDemoStats) return nil, err } } @@ -3095,6 +3103,10 @@ func runStmt(ctx context.Context, se *session, s sqlexec.Statement) (rs sqlexec. if !sessVars.InTxn() { se.StmtCommit(ctx) if err := se.CommitTxn(ctx); err != nil { + if closeErr := executor.CloseRecordSetWithError(rs, err); closeErr != nil { + logutil.Logger(ctx).Error("close EXPLAIN ANALYZE DML record set after commit error failed", + zap.Error(closeErr), zap.NamedError("commitError", err)) + } return nil, err } } From 9205384d4f995253df222303beaddd998eab0887 Mon Sep 17 00:00:00 2001 From: Yiding Date: Wed, 22 Jul 2026 02:14:06 +0800 Subject: [PATCH 17/25] docs: stabilize preview RU resource formula plan --- ...-07-22-preview-ru-resource-formula-plan.md | 469 ++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 docs/design/2026-07-22-preview-ru-resource-formula-plan.md diff --git a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md new file mode 100644 index 0000000000000..26337b6805fd4 --- /dev/null +++ b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md @@ -0,0 +1,469 @@ +# Stabilize preview RU around resource-shaped operator formulas + +This ExecPlan is a living document. Keep `Progress`, `Surprises & Discoveries`, `Decision Log`, and `Outcomes & Retrospective` up to date as work proceeds. + +Reference: `PLANS.md` at repository root. This plan must be maintained according to it. + +## Purpose / Big Picture + +The current preview RU demo uses an operator-specific matrix of fixed, row, byte, and ordering weights. That matrix is useful for collecting evidence, but it does not express the intended model directly: expression evaluation should consume one shared CPU weight, scans should consume one scan-byte weight, transport should consume network-byte and request weights, and stateful operators should add small hash-table or join-output terms. + +After this plan is implemented, `EXPLAIN ANALYZE FORMAT='RU'`, the bounded metrics, the general-log detail, and the statement-summary detail tables will expose the same coefficient-free resource work units. A reader can reproduce every operator's preview RU with six named weights and can see exactly which current runtime detail supplied each input. The preview remains isolated from production RUv2 charging and resource-control reporting. + +This plan deliberately does not implement production code. It freezes the model, field mappings, failure behavior, migration, and validation needed by the later implementation worktree. + +## Progress + +- [x] (2026-07-22 09:00Z) Inspected the current preview RU implementation and its closest tests in `pkg/planner/core/explain_ru.go`, `pkg/planner/core/common_plans_test.go`, and `pkg/executor/explain_test.go`. +- [x] (2026-07-22 09:20Z) Verified current row, byte, scan-detail, cop-task, RUv2 response-byte/RPC, aggregation-output, inline Projection, and reader-child evidence in code. +- [x] (2026-07-22 09:35Z) Resolved the formula, expression-count, reader-attribution, write-unit, failure, and version-migration decisions in this document. +- [x] (2026-07-22 10:05Z) Incorporated the explicit de-duplication constraints for Sort/TopN inline Projection CPU and IndexJoin inner-reader requests. +- [x] (2026-07-22 11:15Z) Incorporated the final ordering clarification: Sort owns `n*log(n)`, TopN owns `n*log(k)` with `k` derived from offset plus count, and expression evaluation remains Projection-owned. +- [x] (2026-07-22 11:40Z) A fresh-context release reviewer re-read the updated user handoff, plan, and code and concluded: “当前版本无需必要修改”. +- [x] (2026-07-22 11:45Z) Applied the Ready profile to this design-only diff: no Go/Bazel trigger was present and the staged Markdown patch passed `git diff --cached --check`. +- [ ] Implement Milestone 1: introduce v4 work units and expression-count helpers without changing production RUv2. +- [ ] Implement Milestone 2: construct operator units from current runtime details and add only the permitted HashJoin state-row detail. +- [ ] Implement Milestone 3: migrate all preview outputs atomically and add regression coverage. +- [ ] After implementation, run the implementation Ready validation profile and record concise evidence here. + +## Surprises & Discoveries + +- Observation: root executor byte accounting is logical live chunk bytes, not encoded network bytes. + Evidence: `pkg/util/execdetails/runtime_stats.go` defines `BasicRuntimeStats.inputBytes/outputBytes`, and `pkg/executor/internal/exec/executor.go` records child/output chunk bytes around `Next`. + +- Observation: exact TiKV response bytes and read/write RPC counters already exist, but only at statement scope. + Evidence: `pkg/util/execdetails/ruv2_metrics.go` exposes `TiKVCoprocessorResponseBytes`, `ResourceManagerReadCnt`, and `ResourceManagerWriteCnt`; none is keyed by reader plan ID. + +- Observation: `selectResultRuntimeStats` already counts cop responses internally, but proportional or per-reader network-byte attribution is not available from current exec details. + Evidence: `pkg/distsql/select_result.go` keeps `copRespTime` and `reqStat` inside the unexported `selectResultRuntimeStats`, while response bytes are drained into statement-level `RUV2Metrics`. + +- Observation: HashAgg runtime details do not expose a separate group-count counter, but the physical Agg node's own actual output rows are the number of materialized group states for this simple model. + Evidence: `RuntimeStatsColl` provides own-plan `GetActRows`; the current preview implementation already publishes Agg output-row shadows from this value and verifies TiKV expected/observed task coverage. + +- Observation: HashJoin state is row-backed rather than one-entry-per-distinct-key, and the exact admitted row count exists below the exec-details boundary. + Evidence: `pkg/executor/join/hash_table_v1.go::Len` counts ordinary inserted row pointers, while NAAJ null-key rows are separately retained in `hashNANullBucket.entries`; `pkg/executor/join/join_row_table.go::validKeyCount` counts v2 rows eligible for the hash lookup structure. Current HashJoin runtime stats expose timing/collision data but not the total admitted lookup-state row count. + +- Observation: IndexJoin's inner lookup work is executed by dynamically built reader children, whose physical network requests already enter statement-level read RPC details. + Evidence: `pkg/executor/join/index_lookup_join.go::fetchInnerResults` drains `task.innerExec`, while the reader paths use DistSQL and feed `RUV2Metrics.ResourceManagerReadCnt`; adding the private inner task count as another request term would charge the same lookup twice. + +- Observation: inline Projection materializes scalar Sort/TopN `ByItems`, while ordinary column keys need no scalar evaluation; injection is not universal for pushed cop plans. + Evidence: `pkg/planner/core/rule_inject_extra_projection.go::InjectProjBelowSort` injects only when a `ByItems` expression is a `ScalarFunction`, and root post-optimization does not rewrite every pushed cop plan. V4 therefore assigns expression evaluation exclusively to an actual Projection, assigns one aggregate sorting-complexity term to Sort/TopN regardless of key count, and fails closed if a scalar ordering expression remains unmaterialized. + +- Observation: completed IndexHashJoin plans do not retain the ordinary IndexJoin equality representation. + Evidence: `pkg/planner/core/exhaust_physical_plans.go::completePhysicalIndexJoin` clears `EqualConditions` after deriving `OuterHashKeys` and `InnerHashKeys`; IndexMergeJoin instead exposes executable `CompareFuncs` and optional `OuterCompareFuncs`. Counting only the embedded IndexJoin fields would undercount these subtypes. + +- Observation: zero-valued RUv2 getters do not prove that their payload was present, and the read-RPC counter is broader than cop response bytes. + Evidence: absent/bypassed `RUV2Metrics` reads as zero; `ResourceManagerReadCnt` covers TiKV read RPC producers including unsupported point/ancillary paths, while `TiKVCoprocessorResponseBytes` covers cop responses. Runtime task coverage and a closed producer set are therefore required before zero or statement-wide totals are attributable. + +## Decision Log + +- Decision: replace, rather than layer on top of, the v3 fixed/row/byte opclass matrix for new samples. + Rationale: keeping both billable models would double count and make `cpu_weight` cease to mean one expression-slot evaluation. Historical v3 samples remain queryable under their model version. + Date/Author: 2026-07-22 / Codex. + +- Decision: use six semantic RU weights: `cpu_weight`, `scan_weight`, `net_weight`, `request_weight`, `hash_table_weight`, and `join_weight`. + Rationale: these are exactly the independent resource terms in the requested formulas. Operator identity remains a diagnostic dimension, not a weight key. + Date/Author: 2026-07-22 / Codex. + +- Decision: Sort CPU work is `rows * log2(max(rows, 2))`; TopN CPU work is `rows * log2(max(k, 2))`, where `k = offset + count`. + Rationale: this is the final requested ownership split. Inline Projection alone charges scalar expression evaluation; the ordering node charges aggregate algorithmic work, with no expression/key-count multiplier. TopN uses its heap bound rather than full input cardinality, and offset is part of that bound. The implementation rejects `offset + count` overflow instead of wrapping; zero/one rows and zero/one `k` use base two so the work is finite and deterministic. + Date/Author: 2026-07-22 / Codex. + +- Decision: count top-level executable expression slots, not recursive AST nodes. + Rationale: plan fields provide a stable, cheap, deterministic count. Recursive counts would make rewrites of an equivalent scalar expression change billing without runtime evidence and would blur the meaning of one calibrated slot. + Date/Author: 2026-07-22 / Codex. + +- Decision: aggregate reader transport once per statement under `reader_transport`, while retaining TableReader, IndexReader, IndexLookup, and IndexMerge kinds as bounded diagnostics. + Rationale: the existing byte and RPC details are statement-scoped and all listed reader kinds share the same weights. Algebraically, charging the totals once equals summing per-reader formulas. Proportional allocation by logical output bytes is rejected because it invents attribution and is badly biased for IndexLookup/IndexMerge internal legs. + Date/Author: 2026-07-22 / Codex. + +- Decision: interpret the requested HashJoin `distinct_rows` term as runtime `hash_state_rows`, the cumulative rows actually admitted into hash lookup state; interpret the HashAgg term as `group_rows = own output rows`. + Rationale: duplicate join keys still allocate/probe row-backed entries, while null/filter-rejected build rows do not enter that structure. Exact admitted rows are therefore more faithful than all build-child rows, and counting unique keys would require a second hash set solely for billing. This is the one permitted Join-only exec-details addition. + Date/Author: 2026-07-22 / Codex. + +- Decision: IndexJoin has no additional request term at the Join node and requires no request-related runtime datum. + Rationale: its extra lookups execute through inner reader children and are already included in the statement reader-transport request counter. Charging inner lookup tasks again would duplicate request cost. The optional Join exec-detail permission is therefore not exercised for IndexJoin/request accounting; v4 uses it only for HashJoin `hash_state_rows`. + Date/Author: 2026-07-22 / Codex. + +- Decision: keep calibrated-weight injection private to `pkg/planner/core` formula tests; package-external end-to-end tests exercise the production-default uncalibrated contract. + Rationale: `pkg/executor/explain_test.go` is `package executor_test`, while the preview weight container intentionally remains private and has no session/global knob. A public or failpoint configuration surface solely for tests would weaken the initial contract. + Date/Author: 2026-07-22 / Codex. + +- Decision: freeze the three legacy statement-summary convenience totals as v3-only and leave them zero for v4 samples. + Rationale: `fixed_events`, `input_rows`, and `input_bytes` cannot safely represent the new semantic units. V4 consumers must use the versioned detail table; adding a parallel convenience schema and its v1/v2 persistence migration is outside this minimal model change. + Date/Author: 2026-07-22 / Codex. + +- Decision: ship `model_version='v4'` with `weight_version='v3-resource-formula-uncalibrated'` and delete the executable v3 weight map. + Rationale: historical rows already contain values and versions, while private per-statement results do not survive process upgrade. Keeping a second calculation path would create an unnecessary mixed-model state; an eventual calibrated set must receive another immutable version. + Date/Author: 2026-07-22 / Codex. + +- Decision: combine write mutation count and bytes into `mutation_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit`, then apply `cpu_weight` once. + Rationale: this preserves one CPU-priced unit without adding independent mutation weights. `mutation_bytes_per_cpu_unit` is a versioned normalization constant, not an RU coefficient; it must be calibrated and validated as positive before a weighted total is published. + Date/Author: 2026-07-22 / Codex. + +- Decision: do not guess numerical v4 weights or silently map the heterogeneous v3 opclass weights. + Rationale: there is no evidence-backed one-to-one mapping. The implementation first publishes v4 base units with `uncalibrated_weights` and no total until an explicit v4 weight set and positive mutation normalization are installed. The preview flag is off by default, so this is safer than presenting arbitrary numbers as RU. + Date/Author: 2026-07-22 / Codex. + +## Outcomes & Retrospective + +The design phase is stable and independently reviewed. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(offset+count)` work without repeating Projection expression evaluation, removes IndexJoin request double charging, limits new runtime data to one HashJoin state-row count, and defines a no-guess migration. A final reviewer with no inherited conversation context concluded that the current version needs no necessary modification. Implementation is intentionally pending and must happen in the separate worktree specified below. + +## Context and Orientation + +Preview RU is an observational model. It emits coefficient-free work units and optionally multiplies them by preview-only weights. It must not modify the RU charged by TiKV, the RUv2 total reported to resource control, or scheduling decisions. + +The current constructor and renderer live in `pkg/planner/core/explain_ru.go`. `buildReadBillingDemoResult` freezes one statement result, `readBillingDemoRootUnits` derives TiDB root units, `readBillingDemoCopUnits` derives TiKV cop units, and `readBillingDemoUnitPreviewRU` applies weights. The same result feeds `EXPLAIN ANALYZE FORMAT='RU'`, metrics, statement summary, and logging. The later implementation must keep one frozen result as the sole source for all outputs. + +`RuntimeStatsColl` in `pkg/util/execdetails/runtime_stats.go` maps physical plan IDs to root and cop runtime details. Root `BasicRuntimeStats` gives actual output rows and logical chunk bytes. Cop `CopRuntimeStats` gives executor-produced rows, task count, and `ScanDetail`. `RUV2Metrics` in `pkg/util/execdetails/ruv2_metrics.go` gives statement-level TiKV response bytes and read/write RPC counts. Physical plan structs in `pkg/planner/core/operator/physicalop` give expression lists; these are static plan metadata, not new runtime observations. + +In this document, a work unit is a coefficient-free non-negative finite number. A weight converts one work unit into preview RU. A physical operator is one node in the executed physical plan. An expression slot is one top-level expression/function/key comparison that the physical operator evaluates per input row. `cpu_weight` prices one expression-equivalent CPU-work unit; expression evaluation, ordering comparisons, Limit row handling, and normalized mutation preparation can all produce such units. A missing value differs from an observed zero; missing required evidence fails closed, while observed zero remains billable as zero. + +## Stable formula contract + +For one statement, weighted preview RU is the sum of the following unit families: + + preview_ru = + cpu_work * cpu_weight + + scan_bytes * scan_weight + + net_bytes * net_weight + + request_count * request_weight + + hash_state_rows * hash_table_weight + + join_output_rows * join_weight + +All arithmetic is float64 with explicit negative, NaN, infinity, and overflow rejection. Integer counters are converted only after validating that they are non-negative. Each pre-aggregation operator result retains `site`, `op_class`, `operator_kind`, `operator_id`, `source`, and, for joins, `input_side`. Physical results use the executed plan's Explain ID. Statement-scoped synthetic results use reserved, non-plan IDs: `reader_transport@statement`, `mutation@statement`, and `txn_write@statement`. Statement-summary detail intentionally aggregates away `operator_id`; its remaining bounded dimensions and version still preserve formula provenance. + +There is no billable fixed-event term in v4. Setup costs that correlate with remote fanout use `request_count`; other constant setup costs stay outside this intentionally simple model. + +### Operator formulas + +| Operator | v4 formula | Required inputs | +|---|---|---| +| Selection | `rows * n_expr * cpu_weight` | direct child actual rows; selection expression slots | +| Projection | `rows * n_expr * cpu_weight` | direct child actual rows; projected expression slots | +| Sort | `rows * log2(max(rows,2)) * cpu_weight` | direct child actual rows; scalar expression evaluation belongs to inline Projection | +| TopN | `rows * log2(max(k,2)) * cpu_weight` | direct child actual rows; checked `k=offset+count`; scalar expression evaluation belongs to inline Projection | +| TableScan / IndexScan | `scan_bytes * scan_weight` | attributable TiKV `ScanDetail` | +| TableReader / IndexReader / IndexLookup / IndexMerge transport | `net_bytes * net_weight + request_count * request_weight` | statement `RUV2Metrics`, emitted once | +| StreamAgg | `rows * n_expr * cpu_weight` | direct child actual rows; group and aggregate slots | +| HashAgg | `rows * n_expr * cpu_weight + group_rows * hash_table_weight` | StreamAgg inputs plus own actual output rows | +| MergeJoin | `(left_rows + right_rows) * n_expr * cpu_weight + output_rows * join_weight` | both child rows, join slots, own output rows | +| HashJoin | `(left_rows + right_rows) * n_expr * cpu_weight + hash_state_rows * hash_table_weight + output_rows * join_weight` | both child rows, join slots, one Join runtime state count, own output rows | +| IndexJoin family | `(left_rows + right_rows) * n_expr * cpu_weight + output_rows * join_weight` | both child rows, join slots, own output rows; inner reader already owns requests | +| Limit | `rows * cpu_weight` | direct child actual rows | +| Window | `rows * n_expr * cpu_weight` | direct child rows and the refined Window slot count below | +| Write mutation and commit | `mutation_work * cpu_weight + write_request_count * request_weight` | existing mutation recorder and statement/commit RUv2 details | + +The table is normative. Diagnostic `input_rows`, logical chunk bytes, output bytes, mutation component counters, and operator status may still be emitted, but they have no weight and never enter `preview_ru`. + +### Expression-slot count + +`n_expr` is a non-negative integer derived from the executed physical plan. It is stored as a diagnostic unit so offline recomputation does not need the original plan object. + +Selection uses `len(PhysicalSelection.Conditions)`. Projection uses `len(PhysicalProjection.Exprs)`, including inline ordering expressions materialized for Sort/TopN and column pass-through expressions because the executor still materializes an output column. + +Sort and TopN do not emit an expression-count unit and do not multiply algorithmic work by `len(ByItems)` or `len(PartitionBy)`. Ordinary column, constant, and multi-key ordering are covered by the single aggregate sorting term. Before publishing that term, inspect `ByItems`: a remaining `ScalarFunction` proves that expression evaluation was not materialized into a child Projection, so the operator fails closed with `missing_ordering_projection` rather than charging expression CPU at Sort/TopN. If `ByItems` references columns produced by an aligned child `PhysicalProjection`, that Projection's normal formula owns all of its `Exprs` once. A missing/misaligned Projection schema also fails closed. For TopN, compute `k=offset+count` with checked unsigned addition and convert it to float only after validation; do not use `count` alone. + +StreamAgg and HashAgg use `len(GroupByItems) + len(AggFuncs)`. One aggregate function descriptor is one slot regardless of partial/final/complete mode and regardless of argument count. This intentionally avoids a two-phase special model. `COUNT(*)` therefore has one slot rather than zero. Ordered aggregate arguments remain inside their aggregate function slot for v4. + +For joins, one join-key pair or executable comparison function is one slot, not two column slots. Add the lengths of the remaining `LeftConditions`, `RightConditions`, and `OtherConditions`. HashJoin key pairs come from `EqualConditions + NAEqualConditions`; MergeJoin key comparisons come from `CompareFuncs`. + +The IndexJoin family is counted by concrete subtype, because its completed physical plans do not share one key representation: + +- `PhysicalIndexJoin`: count aligned `OuterJoinKeys`/`InnerJoinKeys`, the remaining left/right/other conditions, and `len(CompareFilters.OpType)`. +- `PhysicalIndexHashJoin`: count aligned `OuterHashKeys`/`InnerHashKeys` rather than cleared `EqualConditions`, then the remaining left/right/other conditions and `len(CompareFilters.OpType)`. +- `PhysicalIndexMergeJoin`: count `CompareFuncs`, plus `OuterCompareFuncs` only when `NeedOuterSort` is true, then the remaining left/right/other conditions and `len(CompareFilters.OpType)`. + +The implementation must centralize this in one helper. It must reject mismatched aligned key slices, a `NeedOuterSort`/`OuterCompareFuncs` structural inconsistency, or another impossible subtype layout instead of selecting an arbitrary side or falling back to stale embedded fields. + +Window refines the original direction without adding a new weight: + + n_expr = + len(WindowFuncDescs) + + len(PartitionBy) + + len(OrderBy) + + len(Frame.Start.CalcFuncs, if present) + + len(Frame.End.CalcFuncs, if present) + +This covers per-row function evaluation, partition/order comparisons, and dynamic frame-bound calculation. It does not add a partition-size, frame-width, or buffering term because current exec details do not expose those values and the requested model should remain simple. + +For every expression-based operator, `n_expr == 0` is valid only when the physical operator genuinely contains no expression slot. The resulting CPU work is zero. The implementation must not silently clamp `n_expr` to one; intrinsic work is represented only where the formula explicitly supplies it, such as Limit. + +### Row and state semantics + +For a TiDB root operator, `rows` is the sum of direct root children's own actual output rows from `BasicRuntimeStats.GetActRows`. Unary operators normally have one child. A missing child statistic is not zero. Reader-like leaf nodes do not reuse their own output rows as an input for the CPU formulas because reader transport has a separate formula. + +For a pushed TiKV unary operator, `rows` is the exact-plan-ID actual output rows of its direct cop child. Existing v3 expected/observed task coverage rules remain: no tasks means missing; negative rows are invalid; fewer observed tasks than the component's known coverage is incomplete. The model must not fall back to optimizer estimates. + +HashAgg `group_rows` is the operator's own actual output rows. Each physical partial or final Agg node is charged independently from its own input and output; there is no phase multiplier. TiKV Agg keeps the existing independent expected-response versus observed-summary coverage gate before its own rows are accepted. + +HashJoin `hash_state_rows` is the cumulative number of build rows actually admitted into hash lookup structures. Duplicate keys count once per admitted row; rows rejected by build filters and ordinary null-key rows that are not retained for lookup do not count. V1 null-aware anti join is the exception: its null-key rows are stored and probed through `hashNANullBucket.entries`, so they do count. If spilling rebuilds hash state in another round, admissions in that round count again because the state construction work repeats. This value comes from the HashJoin executor, not from build-child output rows. + +Expose it through one narrow read-only interface in `pkg/util/execdetails`: + + type HashTableRuntimeStats interface { + RuntimeStats + HashTableRows() int64 + } + +Both private v1 and v2 HashJoin runtime stats implement the getter. V1 records successful `hashTable.Len() + len(hashNANullBucket.entries)` for NAAJ, with a nil bucket contributing zero; v2 records the sum of `validKeyCount` admitted for each completed build round. Clone/Merge preserve the cumulative counter. Tests cover ordinary null rejection and V1 NAAJ null-bucket inclusion. Missing or negative state rows fail with `missing_hash_state_rows` or `invalid_hash_state_rows`. No unique-key set is introduced. + +MergeJoin, HashJoin, and IndexJoin `output_rows` is the join node's own `BasicRuntimeStats.GetActRows`. It is the only permitted output term; output bytes remain diagnostic only. Both join inputs and the output are required even when observed as zero. + +### Scan bytes + +TableScan and IndexScan use the current v3 scan-byte proxy: + + if TotalKeys == 0 and ProcessedKeys == 0 and ProcessedKeysSize == 0: + scan_bytes = 0 + otherwise: + scan_bytes = TotalKeys * ProcessedKeysSize / ProcessedKeys + +The all-zero case is an observed zero only when the cop plan has at least one observed task and coverage is complete. Otherwise it is missing. In the nonzero case, all three values must be positive and the result must be finite. This proxy preserves scan work for MVCC/skipped keys while using only current `ScanDetail`; it is labeled `scan_detail_processed_key_avg_estimate`, not presented as encoded response bytes. + +Each scan detail must be attributable to exactly one scan component. Multi-scan IndexMerge is supported by evaluating each partial scan component separately; it must not share one detail across siblings. Ambiguous or absent attribution fails the affected statement according to the atomicity rules below. + +### Reader transport + +The four named reader families share one statement-level transport formula. The constructor identifies all executed TableReader, IndexReader, IndexLookup, and IndexMerge nodes, then emits exactly one `id=reader_transport@statement`, `site=tidb`, `op_class=reader_transport` operator with a bounded `operator_kind` set: `table_reader`, `index_reader`, `index_lookup`, `index_merge`, or `mixed_reader`. + +`net_bytes` is `RUV2Metrics.TiKVCoprocessorResponseBytes()`. `request_count` is `RUV2Metrics.ResourceManagerReadCnt()`. Both are snapshots from the same frozen statement details used by existing preview outputs. The unit source is `ruv2_metrics`; logical `BasicRuntimeStats.GetOutputBytes` remains a diagnostic and is never substituted for transport bytes. + +The statement-wide counters are publishable only when the executed flat plan proves a closed producer set: every possible TiKV read-RPC producer for the statement belongs to those four supported cop-reader families. The initial implementation uses this exact algorithm: + +1. Only a read-only `SELECT` can pass the closed-set gate. If a DML flat plan contains any supported reader, its reader-transport component is always `unknown_input/ambiguous_reader_transport_producers`; v4 has no DML allowlist because current details cannot exclude uniqueness, locking, FK, transaction, or other ancillary reads. +2. Walk the complete executed `FlatPhysicalPlan`, including IndexJoin inner plans. Classify `*physicalop.PhysicalTableReader` with `StoreType == kv.TiKV`, `*physicalop.PhysicalIndexReader`, `*physicalop.PhysicalIndexLookUpReader`, and `*physicalop.PhysicalIndexMergeReader` as supported producers. +3. Reject `*physicalop.PointGetPlan`, `*physicalop.BatchPointGetPlan`, any `PhysicalTableReader` whose `StoreType != kv.TiKV`, `*physicalop.PhysicalExchangeReceiver`, and `*physicalop.PhysicalExchangeSender`. Also reject any node that the existing preview classifier marks as a reader/store-access class but that is not one of the four supported types. This catches TiFlash/MPP and future external reader types without treating a new producer as free. +4. Other already-supported CPU, join, aggregation, wrapper, scan-descendant, UnionScan, MemTable, and TableDual nodes are not independent TiKV read-RPC producers and do not open the set. Any structurally unknown plan node continues to fail through the existing unsupported-operator gate, so transport is never published alongside an unknown tree. + +A supported reader mixed with any rejected producer is not partially charged from the total counter: SELECT fails atomically; DML marks only its reader-transport component unknown. This is conservative because the current statement counter cannot subtract unsupported RPCs. + +Presence and zero handling are normative: + +- A nil, bypassed, or otherwise unavailable RUv2 snapshot is missing, even though public getters return zero. +- Inspect `GetTasks()` and `GetExpectedCopTasks()` for every supported reader/cop descendant. If any observed or expected cop task exists, `net_bytes == 0 && request_count == 0` is missing rather than free. +- `net_bytes > 0 && request_count == 0` is invalid. `request_count > 0 && net_bytes == 0` is valid for an empty cop response. +- `net_bytes == 0 && request_count == 0` is an observed zero only when a present, non-bypassed frozen RUv2 snapshot exists, no supported descendant has an observed or expected cop task, every supported reader root has observed zero output rows, and the producer set is closed. This represents an empty range/no-request execution. + +If no supported reader executed, no reader-transport operator is emitted. Unsupported producers retain explicit bounded status rows until an attributable mapping is designed. The bounded transport reasons are `missing_reader_transport_details` for presence/coverage failure and `ambiguous_reader_transport_producers` for an open producer set. + +### IndexJoin request de-duplication + +IndexJoin, IndexHashJoin, and IndexMergeJoin do not emit a Join-local `request_count`. Their dynamic inner executors use TableReader, IndexReader, or IndexLookup paths, so the resulting physical read RPCs are already present in the statement-level reader-transport `ResourceManagerReadCnt`. PhysicalIndexMergeReader remains a supported standalone transport producer, but `dataReaderBuilder.BuildExecutorForIndexJoin` does not construct it as an IndexJoin inner path. + +This is the explicit de-duplication refinement to the initial simple IndexJoin formula: + + initial: IndexJoin CPU + lookup requests + output rows + v4: IndexJoin CPU + output rows + reader_transport already charges all inner physical requests once + +The existing private inner task counter remains an EXPLAIN timing diagnostic and is not converted into request RU. IndexJoin adds no detail; the only v4 runtime extension is HashJoin's state-row getter above. + +### Write work + +The existing statement-local mutation recorder remains authoritative. Its complete semantics for this plan are: + +- Count each attempted foreground `Set`, `SetWithFlags`, `Delete`, or `DeleteWithFlags` once after encoding and before calling MemDB. A failed MemDB call still counts. Set bytes are `len(key)+len(value)`; delete bytes are `len(key)`. +- Same-key overwrites, pessimistic statement retries, and mutations later removed by staging cleanup or ROLLBACK remain counted because their encoding/preparation CPU already occurred. `UpdateFlags`, staging release/cleanup, lock-only operations, commit-time net mutations, and local-temporary-table apply copies do not create another foreground mutation. +- The recorder is statement-local and dynamically follows the current `StatementContext`, including optimistic history replay. Restricted/internal SQL never becomes a foreground sample. Retryable explicit transactions whose already-emitted statements cannot be rewritten are marked `optimistic_replay_attribution_unsupported`/partial rather than pretending exact attribution. +- No-op or zero-match DML has a present recorder with zero count and bytes. Deprecated batch DML keeps one recorder across its internal transaction switches, so every batch attempt is counted once. Local-temporary-table encoding is counted once at its foreground MemDB write. + +`docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md` remains background evidence, but the bullets above are the normative v4 contract. + +The v4 combined unit is: + + mutation_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit + +`mutation_bytes_per_cpu_unit` is a positive, finite, versioned normalization constant with units bytes per expression-equivalent CPU-work unit. It is stored alongside the v4 weights and included in output metadata. It is not independently multiplied by RU. If it is unset, zero, negative, NaN, or infinite, mutation base components remain visible but weighted v4 total is unavailable with `uncalibrated_weights`. + +`write_request_count` is `RUV2Metrics.ResourceManagerWriteCnt()` and uses the same `request_weight` as reads. The frozen snapshot must exist and be non-bypassed. Autocommit DML normally emits mutation and write-request work together; a nonzero mutation with a zero/missing request payload is partial. Explicit-transaction DML emits statement-local mutation work only; the eventual COMMIT emits transaction-scoped write requests with empty `dml_kind`, without back-attributing them to earlier DML. A known empty transaction COMMIT emits observed zero request work; an absent lifecycle snapshot is missing, not zero. SQL `ROLLBACK` remains unsupported in v4 and emits neither a zero unit nor a total: current routing has no complete rollback-RPC attribution, and declaring it free would be unsafe. + +Pipelined transactions retain valid mutation units but mark the write-request component `pipelined_tikv_payload_unsupported` until current details prove a complete logical flush request count. Deprecated batch DML accumulates available write-request snapshots across internal transaction switches; any missing switch makes the request component partial. Optimistic retry/replay keeps the mutation behavior above and marks unavailable request attribution partial. Thus no retry, pipeline, or batch path publishes a known-incomplete zero. + +Mutation count and bytes continue to be emitted as zero-weight diagnostics so calibration can change `mutation_bytes_per_cpu_unit` offline. `CommitDetails.WriteKeys/WriteSize` are not substituted for the TiDB mutation unit. + +## Availability, atomicity, and degraded behavior + +The existing preview gates remain: the feature is default off; `EXPLAIN ANALYZE FORMAT='RU'` explicitly enables collection; unsupported side-effecting/locking/internal paths are rejected; production resource control is untouched. `*ast.RollbackStmt` is explicitly routed to `unsupported/unsupported_statement` before the ordinary SELECT gate, so it cannot be mistaken for a missing-plan SELECT or an observed-zero write. + +For side-effect-free SELECT, billing is statement-atomic. If any executed supported operator lacks a required input, the statement records status and reason but emits no billable v4 units and no `total_preview_ru`. This prevents a partial plan from looking cheap. Diagnostic status rows may still identify every missing operator. + +For DML, read-tree, mutation, and transaction-request components keep independent status because explicit transactions separate their lifetimes. Complete components may retain coefficient-free units for calibration, but a statement-level weighted total is absent unless every component expected at that lifecycle point is complete and the weight set is calibrated. + +Observed zero is accepted only with presence evidence: an existing root stat, a cop stat with complete task coverage, a HashJoin state-row runtime stat, a mutation recorder snapshot, or a frozen RUv2 snapshot plus the reader consistency checks above. Missing, negative, overflowed, NaN, and infinite inputs have bounded reasons. New reasons are `missing_expression_count`, `missing_ordering_projection`, `invalid_topn_bound`, `missing_reader_transport_details`, `ambiguous_reader_transport_producers`, `missing_hash_state_rows`, `invalid_hash_state_rows`, and `uncalibrated_weights`. + +No fallback may use optimizer estimated rows, schema-estimated widths, plan `netDataSize`, string parsing of `EXPLAIN` runtime text, or proportional allocation of statement counters. + +## Weight units and migration + +The v4 weight container is preview-only: + + type previewRUWeights struct { + Version string + CPUPerWorkUnit float64 + ScanPerByte float64 + NetworkPerByte float64 + Request float64 + HashTablePerRow float64 + JoinPerOutputRow float64 + MutationBytesPerCPUUnit float64 + Calibrated bool + } + +The six RU fields have units stated by their names. `MutationBytesPerCPUUnit` is a normalization, not RU. Validation requires a nonempty new version, finite non-negative RU weights, a positive finite mutation normalization, and `Calibrated=true` before any weighted total is published. Formula tests in `pkg/planner/core` use the private container directly with small deterministic values. No exported setter, session/global variable, or failpoint is added. Package-external executor tests see the production default and assert `uncalibrated_weights`, coefficient-free units, and no `total_preview_ru` until a later calibration change supplies production values. + +Set the exact constants `model_version='v4'` and `weight_version='v3-resource-formula-uncalibrated'`. Do not reuse the old model `v3` or weight `v2` labels. The weight-version string intentionally describes the shipped state; a later calibrated weight set must use another immutable version rather than changing values behind this label. Existing statement-summary detail already carries model and weight versions, so old rows remain self-describing and are not rewritten. Queries that compare workload windows must group by both versions. + +The existing `ReadBillingDemoBaseUnitSummary` and its infoschema convenience columns (`fixed_events`, `input_rows`, and `input_bytes`) are frozen legacy-v3 views. V4 samples contribute zero to all three, and v4 consumers use the versioned base-unit detail rows instead. Do not reinterpret an old column as `cpu_work`, and do not merge v3 and v4 in those totals. This avoids a cross-version semantic lie and avoids expanding the v1/v2 statement-summary persistence schema in this milestone. Tests must cover v3 legacy aggregation unchanged, v4 legacy totals zero, and v4 detail surviving memory/history readers. + +There is no migration of `config.RUV2`, TiKV client RU coefficients, or resource-group settings. Those configure production RUv2 and have different semantics. Delete the current internal `readBillingDemoWeights` map, `readBillingDemoResolveWeights`, and v3 formula application in the same atomic implementation change. `readBillingDemoResult` is constructed and rendered within one statement in one process; no frozen result crosses a process-upgrade boundary, while historical statement-summary rows already store their unit values and versions and are never recomputed from this map. Therefore no v3 calculation compatibility branch is needed or permitted. + +All outputs must switch together: EXPLAIN unit rows, `total_preview_ru`, Prometheus base units, statement-summary detail, and general log. A mixed state where logs use v4 while EXPLAIN uses v3 is not accepted. General-log aggregation must extend its key and serialized object to retain `DMLKind`, `InputSource`, and `InputSide`; otherwise distinct v4 units with the same operator/unit label would collapse and lose their provenance. `operator_id` remains an internal/EXPLAIN identity and is intentionally absent from the bounded statement-summary and general-log aggregation keys. + +## Plan of Work + +### Milestone 1: represent v4 work without changing collection + +In `pkg/planner/core/explain_ru.go`, add bounded v4 unit names (`cpu_work`, `expression_count`, `scan_bytes`, `net_bytes`, `request_count`, `hash_state_rows`, `join_output_rows`, and `mutation_work`), the validated weight container, and one formula application function. Replace opclass-specific billable lookup for new results with semantic-unit lookup. Keep diagnostic legacy units zero-weight. + +Add physical-plan helpers that return expression-slot count for every supported concrete type. Unit-test exact counts for simple and compound Selection, Projection, Agg, Join, and Window plans, including the distinct ordinary IndexJoin, IndexHashJoin, and IndexMergeJoin key representations. For Sort/TopN, test that a root scalar expression backed by inline Projection is evaluated only at Projection, ordinary-column/multi-key plans still receive exactly one aggregate sorting term, and a TiKV pushed TopN with an unmaterialized scalar `ByItems` fails `missing_ordering_projection`. Test checked `offset+count`, a nonzero offset, overflow rejection, and zero/one boundaries. At this milestone, custom unit fixtures in internal `package core` tests use the private calibrated weight container to prove exact algebra and invalid-number rejection before runtime constructors change. + +Acceptance: formula tests with injected weights reproduce hand-calculated totals; no production RUv2 API or configuration changes. + +### Milestone 2: construct units from authoritative details + +Refactor root and cop constructors in `pkg/planner/core/explain_ru.go` around the field mappings in this document. Preserve current exact child-plan attribution and task coverage code where it satisfies the new rows/scan rules. Add statement-scope reader transport from the frozen `RUV2Metrics` snapshot only after proving the closed producer set and the presence/task gates above; support multiple IndexMerge scan components without allocating transport twice. + +Use the existing flat-plan build/probe/left/right labels for join rows and the join node's own rows for output. Do not expose or add an IndexJoin lookup-task counter: its dynamic inner readers are already included by the statement-scope reader transport unit. + +Add `HashTableRuntimeStats` in `pkg/util/execdetails/runtime_stats.go`. Implement it for both HashJoin runtime-stat versions in `pkg/executor/join/hash_join_stats.go`, recording successful v1/v2 state admissions at the existing build completion points in `hash_join_v1.go` and `hash_join_v2.go`. Do not add a unique-key collector or any non-Join runtime field. + +Change write construction in `pkg/planner/core/explain_ru.go` to derive `mutation_work` and use statement/COMMIT write RPC count. Retain raw mutation units as diagnostics. + +Acceptance: every required formula input can be traced to the source table below; source searches show exactly one new Join-only state-row counter and no other runtime field. + +### Milestone 3: output migration and behavioral coverage + +Bump the constants and renderer in `pkg/planner/core/explain_ru.go`, including `buildReadBillingDemoStatementStats`, `summarizeReadBillingDemoBaseUnits`, the EXPLAIN row builders, and `recordReadBillingDemoMetrics`. Use `pkg/metrics/explain_ru.go::{RecordReadBillingDemoStatement, RecordReadBillingDemoOperatorStatus, AddReadBillingDemoBaseUnits, ObserveExplainRURow}` for the bounded v4 labels; their public signatures need change only if a required existing provenance dimension is absent. + +Update aggregation keys and entry conversion in `pkg/util/stmtsummary/read_billing.go`, statement accumulation plus the legacy-v3-only `ReadBillingDemoBaseUnitSummary` behavior in `pkg/util/stmtsummary/statement_summary.go`, and verify `pkg/util/stmtsummary/v2/record.go` persistence/merge plus `pkg/util/stmtsummary/v2/reader.go::{readBillingDemoRowsFromRecord, readBillingDemoBaseUnitColumnValue}` preserve all v4 detail dimensions. Do not add new convenience columns. In `pkg/infoschema/tables.go`, keep the three legacy columns but change their comments to state v3-only/zero-for-v4 semantics; retain the versioned detail table schema unless the existing columns cannot carry one of the frozen dimensions. + +In `pkg/executor/adapter.go`, extend `readBillingDemoGeneralLogUnit`, `buildReadBillingDemoGeneralLogUnits`, and `readBillingDemoGeneralLogUnit.MarshalLogObject` so `DMLKind`, `InputSource`, and `InputSide` participate in aggregation, sorting, and serialization. Apply the exact model/weight versions and output semantics atomically across EXPLAIN, metrics, statement summary, and general log. Update `docs/design/2026-07-01-read-billing-demo-ru-model.md` and `docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md` to point to this v4 contract rather than retaining contradictory current-model claims. + +Extend the existing internal suites instead of creating a new planner casetest category. Keep formula and constructor tests near `pkg/planner/core/common_plans_test.go`; extend `pkg/executor/explain_test.go` for end-to-end RU output and de-duplication; keep transaction lifecycle tests in `pkg/session/tidb_test.go`. Update corresponding `.agents/skills/tidb-test-guidelines/references/*-case-map.md` files when test files change. + +Acceptance: internal formula tests observe exact calibrated totals. EXPLAIN, metrics hooks, statement summary, and log tests observe identical v4 coefficient-free units and the production-default `uncalibrated_weights`/absent-total state; unsupported and missing-evidence cases also have no weighted total, for their specific bounded reasons. + +## Authoritative field map + +| Formula input | Source today | Attribution and validation | New runtime data? | +|---|---|---|---| +| root `rows` | `RuntimeStatsColl` direct child `BasicRuntimeStats.GetActRows` | exact child plan ID must exist | no | +| cop `rows` | direct child `CopRuntimeStats.GetActRows/GetTasks` | exact plan ID and coverage checks | no | +| `n_expr` | concrete `physicalop` plan fields | centralized type switch, structural validation | no; immutable plan metadata | +| `scan_bytes` | `CopRuntimeStats.GetScanDetail` | unique scan component and complete tasks | no | +| `net_bytes` | `RUV2Metrics.TiKVCoprocessorResponseBytes` | once per statement; non-bypassed presence, descendant task gate, closed read producer set | no | +| reader `request_count` | `RUV2Metrics.ResourceManagerReadCnt` | once per statement only when every read-RPC producer is attributable to supported cop readers | no | +| HashAgg `group_rows` | Agg node own runtime rows | TiKV additionally needs expected/observed coverage | no | +| HashJoin `hash_state_rows` | v1 hash-table `Len` plus NAAJ null-bucket entries; v2 row-table `validKeyCount` | completed build round, cumulative across rebuilds | **yes, Join only** | +| Join `output_rows` | Join node own `BasicRuntimeStats.GetActRows` | executed root stat required | no | +| `mutation_count/bytes` | `StatementContext` preview mutation recorder | current attempted-call semantics | no | +| `write_request_count` | `RUV2Metrics.ResourceManagerWriteCnt` | DML/COMMIT lifecycle snapshot | no | + +## Concrete Steps + +The design loop must first commit this file and hand the implementation loop the exact commit hash as ``. From the original repository root, create the required independent branch/worktree with these commands; the committed plan arrives through Git, so no untracked file copy is permitted: + + preview_ru_design_commit= + preview_ru_impl_worktree=/DATA/disk4/yiding/gocode/tidb.worktrees/preview-ru-v4-impl + git cat-file -e "${preview_ru_design_commit}^{commit}" + git worktree add -b preview-ru-v4-impl "$preview_ru_impl_worktree" "$preview_ru_design_commit" + cd "$preview_ru_impl_worktree" + test "$(git rev-parse HEAD)" = "$preview_ru_design_commit" + test -f docs/design/2026-07-22-preview-ru-resource-formula-plan.md + pwd + git branch --show-current + git status --short + +The final `git status --short` must be empty before implementation begins, `pwd` must be `/DATA/disk4/yiding/gocode/tidb.worktrees/preview-ru-v4-impl`, and `git branch --show-current` must print `preview-ru-v4-impl`. If an external orchestration loop creates the worktree, these three facts plus `` are mandatory handoff evidence; the implementation loop must stop rather than reuse the design-loop worktree or guess another base revision. + +Then inspect local changes and apply the Bazel preparation gate: + + git status --short + git diff --name-status + git diff -U0 -- '*.go' + +Run `make bazel_prepare` if the actual diff changes a Go import section, adds/moves/removes a Go file, adds a top-level `func TestXxx(t *testing.T)`, changes Bazel targets, or hits another trigger in `AGENTS.md`. The implementation should normally extend existing tests, but it must use the actual diff rather than assume the gate result. If run, review generated `BUILD.bazel`/`.bzl` changes and include only those caused by the implementation. + +Implement milestones in order. During WIP, run the smallest targeted tests. The affected packages use failpoints, so use the cleanup-safe wrapper rather than raw `go test` where the package scan finds failpoint use: + + ./tools/check/failpoint-go-test.sh pkg/planner/core -run 'TestExplainRU(PlanFormulaAndOperatorClasses|ComponentSnapshotStatusAndWeights)|TestReadBillingDemo' + ./tools/check/failpoint-go-test.sh pkg/executor -run 'TestExplainAnalyzeFormatRU|TestReadBillingDemoMetricsHook|TestReadBillingDemoGeneralLogUnits|TestWriteSlowLog' + ./tools/check/failpoint-go-test.sh pkg/executor/join -run 'Test.*HashJoin.*RuntimeStats' + ./tools/check/failpoint-go-test.sh pkg/session -run 'TestPreviewKVMutationRecorder|TestRUV2MetricsIsolatedPerStatementInExplicitTxn' + ./tools/check/failpoint-go-test.sh pkg/util/stmtsummary -run 'TestReadBillingDemo(BaseUnitsToDatum|StructuredRowsToDatum|AggregationCaps|DMLKindAggregation|ReservedStatusMergeBypassesStatusCap)' + ./tools/check/failpoint-go-test.sh pkg/util/stmtsummary/v2 -run 'TestStmtRecordReadBillingDemoStructuredStats|TestReadBillingDemo(MemReader|HistoryReader)' + go test -tags=intest,deadlock ./pkg/metrics -run 'TestExplainRUMetrics|TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues' + +The metrics package currently has no failpoint use, hence the raw targeted command above. If a package scan changes at implementation time, follow `docs/agents/testing-flow.md`, switch to the matching wrapper/raw form, and record that evidence. + +At Ready, run the minimum targeted set again after any required `make bazel_prepare`, then run: + + make lint + +Do not run `make bazel_lint_changed` or RealTiKV tests unless a discovered dependency, CI reproduction, or explicit request expands the scope. This model consumes already-collected TiKV details and does not require a live TiKV cluster for its formula unit tests; end-to-end mockstore tests remain necessary for rendered output. + +## Validation and Acceptance + +The implementation is accepted only when all of the following are observable. + +With private test weights set to simple values inside `pkg/planner/core`, table-driven formula tests show exact results for every operator row in the formula table, including zero rows, one row, multiple expressions, multi-key joins, all three IndexJoin-family key representations, V1 NAAJ null-bucket state, and Window frame expressions. Sort uses `log2(max(rows,2))`; TopN uses `log2(max(offset+count,2))`, with checked addition and cases where offset is nonzero. Neither ordering operator has an expression/key-count multiplier, and unmaterialized scalar ordering expressions fail closed rather than being charged there. + +End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. + +A multi-reader or IndexMerge case proves that statement `net_bytes` and read `request_count` appear once, while every scan retains its own `scan_bytes`. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. + +Missing root stats, incomplete cop summaries, ambiguous scan details, missing reader transport, invalid expression structure, invalid mutation normalization, negative inputs, overflow, NaN, and infinity all fail closed with bounded reasons. SELECT produces no partial billable total. DML preserves complete independent units but does not claim a complete statement total. + +Search and API review prove that HashJoin state rows are the only runtime field added. `config.RUV2`, TiKV request charging, `ReportRUV2Consumption`, and resource-control behavior are unchanged. + +Statement-summary detail, Prometheus metrics hooks, EXPLAIN rows, and general-log details are built from the same frozen result and agree on unit values. General-log records retain DML kind, input source, and input side. Historical v3 rows remain distinguishable by version; legacy three-column convenience totals keep v3 behavior and remain zero for v4, whose memory/history-reader details remain queryable. + +## Idempotence and Recovery + +All formula construction is read-only over frozen plan/runtime snapshots and must be safe to call repeatedly. Unit construction must not drain `RUDetails`; use the already synchronized/frozen `RUV2Metrics` snapshot exactly as the current preview path does. + +`make bazel_prepare`, formatting, and targeted tests are safe to rerun. The failpoint test wrapper always disables failpoints during cleanup. If a milestone leaves mixed output versions, revert only that milestone's focused changes or finish all output consumers before running behavioral tests; never commit a mixed v3/v4 renderer state. + +If the runtime source cannot prove a required input, add a bounded status reason and keep the formula unavailable. Do not recover by parsing runtime strings or by introducing an estimate not recorded in this plan. Any newly discovered need for another runtime field requires revisiting this design before coding it; the current v4 plan consumes the optional Join-only allowance solely for HashJoin state rows. + +## Artifacts and Notes + +Current evidence commands used while drafting this plan included: + + rg -n 'type (BasicRuntimeStats|CopRuntimeStats|RuntimeStatsColl)' pkg/util/execdetails/runtime_stats.go + rg -n 'TiKVCoprocessorResponseBytes|ResourceManagerReadCnt|ResourceManagerWriteCnt|Bypass' pkg/util/execdetails/ruv2_metrics.go + rg -n 'innerWorker.task|type indexLookUpJoinRuntimeStats' pkg/executor/join + rg -n 'type Physical(Selection|Projection|TopN|Sort|HashAgg|StreamAgg|HashJoin|MergeJoin|IndexJoin|Window)' pkg/planner/core/operator/physicalop + +The evidence establishes availability and location, not completion of the later implementation. + +## Interfaces and Dependencies + +Keep all v4 model types private to `pkg/planner/core` except the narrow `execdetails.HashTableRuntimeStats` read interface. Do not export the preview weight container, concrete executor-private Join stats, or add public session/global variables in the initial implementation. + +The implementation depends only on existing TiDB packages: `physicalop` for immutable plan expressions, `execdetails` for runtime rows/scan/RUv2 details, the statement mutation recorder for write inputs, and existing statement-summary/metrics/log renderers. It adds no third-party dependency and no protocol field. + +At milestone completion, the key internal interfaces should have these conceptual signatures: + + func previewRUExpressionCount(plan base.Plan) (int64, bool) + func previewRUFormulaUnits(plan base.Plan, details previewRUDetails) ([]previewRUUnit, previewRUStatus) + func previewRUForUnit(unit previewRUUnit, weights previewRUWeights) (weight, ru float64, ok bool) + type HashTableRuntimeStats interface { + RuntimeStats + HashTableRows() int64 + } + +The exact private names may follow nearby conventions, but the semantics, data sources, one-Join-runtime-datum boundary, and de-duplication rules in this plan are mandatory. + +Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, TopN owns `n*log(k)` with checked `k=offset+count`, and inner readers own IndexJoin request cost. HashJoin uses the sole permitted runtime addition to expose actual admitted hash-state rows instead of approximating them with all build rows. From e4d6f46572d09e08121bcad12feaf172ec19055e Mon Sep 17 00:00:00 2001 From: Yiding Date: Wed, 22 Jul 2026 02:18:08 +0800 Subject: [PATCH 18/25] docs: add FTS score special column design --- .../fts_score_special_column_design.md | 989 ++++++++++++++++++ 1 file changed, 989 insertions(+) create mode 100644 docs/note/planner/fts_score_special_column_design.md diff --git a/docs/note/planner/fts_score_special_column_design.md b/docs/note/planner/fts_score_special_column_design.md new file mode 100644 index 0000000000000..098a0d7e5f136 --- /dev/null +++ b/docs/note/planner/fts_score_special_column_design.md @@ -0,0 +1,989 @@ +# FTS / BM25 / Vector Score 特殊列映射设计 + +## 1. 背景 + +在全文检索和向量检索场景中，常见查询形态如下： + +```sql +select fts_match(title, 'database') as score +from t +where fts_match(title, 'database') +order by fts_match(title, 'database') desc; +``` + +或者： + +```sql +select vec_l2_distance(vec, '[1,2,3]') +from t +where ... +order by vec_l2_distance(vec, '[1,2,3]'); +``` + +这些函数有一个共同点： + +- 真正的计算必须发生在索引侧，而不是 TiDB 本地。 +- 上层 `SELECT`、`ORDER BY`、后续可能的 `TopN`，都需要复用这个计算结果。 +- 同一个函数可能在同一棵计划树里出现多次，所有出现点都必须映射到同一个返回值，而不是重复计算，也不能按“出现位置”猜测绑定关系。 + +当前 TiDB 在 vector distance 路径上已经有一套“索引返回特殊列，planner 重写父节点表达式”的成熟模式；FTS/BM25 也应当沿用同样的范式。 + +本设计文档讨论： + +- 如何在当前代码结构下引入 `id = -2050` 的 score 特殊列。 +- 如何确保“同一个函数出现多次时，全部正确映射到同一个值”。 +- 如何分阶段落地，先做单读，再考虑回表和索引侧 TopK。 + +## 2. 当前实现概况 + +### 2.1 FTS builtin 的语义 + +当前 FTS builtin 的返回类型就是 `ETReal`： + +- `pkg/expression/builtin_fts.go:113` +- `pkg/expression/builtin_fts.go:187` + +这意味着： + +- FTS 表达式本来就不是纯布尔函数。 +- `WHERE fts_match(...)` 只是上层通过 `IS TRUE` 语义把 `real` 转成布尔条件。 +- 这也正是后续 score / BM25 的预留口。 + +`expr_to_pb.go` 中已经有对应注释： + +- `pkg/expression/expr_to_pb.go:276` + +注释明确写了：TiDB 的 FTS functions 返回 float，是为了潜在的 BM25 score 场景。 + +### 2.2 TiCI FTS path 当前只支持 filter，不支持 score + +`DataSource.analyzeTiCIIndex()` 在 `PredicatePushDown` 阶段尝试构造 TiCI FTS path： + +- `pkg/planner/core/operator/logicalop/logical_datasource.go:173` +- `pkg/planner/core/operator/logicalop/logical_datasource.go:656` + +最终在 `buildTiCIFTSPathAndCleanUp()` 里构建 `FtsQueryInfo`： + +- `pkg/planner/core/operator/logicalop/logical_datasource.go:902` +- `pkg/planner/core/operator/logicalop/logical_datasource.go:951` + +当前固定写死： + +```go +QueryType: tipb.FTSQueryType_FTSQueryTypeNoScore +``` + +因此当前 path 本质上是： + +- TiCI 负责做 FTS filter +- TiDB 不接收 score 列 + +### 2.3 当前 validation 直接禁止 SELECT / ORDER BY 中使用 FTS + +当前逻辑规则 `ftsFuncValidation` 直接禁止： + +- `SELECT fts_match(...)` +- `ORDER BY fts_match(...)` + +对应位置： + +- `pkg/planner/core/rule_ftsfunc_validation.go:44` +- `pkg/planner/core/rule_ftsfunc_validation.go:52` + +这也是为什么现在用户写： + +```sql +select fts_match() from ... +where fts_match() +order by fts_match() +``` + +根本进不到后续物理计划阶段。 + +### 2.4 当前物理路径选择也直接拒绝 “FTS path + sort property” + +即便放开 validation，当前物理路径选择仍会把这类计划判 invalid： + +- `pkg/planner/core/find_best_task.go:2275` + +代码含义是： + +- 只要 `candidate.path.FtsQueryInfo != nil` +- 且当前父节点请求了 sort property +- 直接放弃该 path + +这意味着 V1 方案至少要允许： + +- 无序 TiCI scan +- TiDB root sort / root TopN + +不然 `ORDER BY score` 仍然无法落地。 + +### 2.5 TiCI 当前 row layout 对尾部列顺序有强假设 + +TiCI index scan 当前的特殊 row layout 在： + +- `pkg/planner/core/operator/physicalop/physical_index_scan.go:453` + +其尾部逻辑为： + +- 可选 `ExtraPhysTblID` +- 必定追加 `ExtraVersion` + +executor 侧又依赖两个重要假设： + +- `pkg/executor/builder.go:4574` +- `pkg/executor/distsql.go:796` + +具体假设是： + +- 最后一列一定是 `_tidb_mvcc_version` +- 如果有 `ExtraPhysTblID`，它被认为在 version 前一个位置 + +因此 `-2050` score 列如果要插入 TiCI 返回 schema，必须避开这两个假设。 + +### 2.6 vector distance 已经有成熟的“特殊列映射”范式 + +vector distance 优化的核心逻辑位于： + +- `pkg/planner/core/task.go:883` +- `pkg/planner/core/task.go:1011` + +尤其是 `tryReturnDistanceFromIndex()`： + +- 发现 index 侧已有 distance +- 注入一个特殊列 `VirtualColVecSearchDistanceID` +- 把上层 `TopN/Projection` 中的函数重写成这个列 + +这正是 FTS/BM25 score 需要复用的基本范式。 + +## 3. 问题本质 + +这里真正要解决的问题不是“传一个 `-2050` 列”本身，而是： + +1. planner 必须识别哪些表达式需要复用 score。 +2. planner 必须判断哪些表达式在语义上是同一个函数。 +3. planner 必须保证这些语义等价表达式全部绑定到同一个特殊列。 +4. executor 必须稳定接收到该列，并且列顺序不能破坏现有 TiCI 假设。 + +如果只做“索引返回一个 `-2050` 列”但不做语义绑定，会出现以下问题： + +- `SELECT` 和 `ORDER BY` 中各自都有一个 `fts_match()`，TiDB 不知道它们应不应该共用同一个返回值。 +- 如果存在 rewrite，文本不同但语义等价的表达式可能无法匹配。 +- 如果靠出现顺序绑定，一旦 projection / sort / topN 形态变化，绑定就会漂移。 + +因此，绑定必须基于表达式语义，而不是文本或位置。 + +## 4. 设计目标 + +### 4.1 功能目标 + +- 支持 `SELECT score_expr` +- 支持 `ORDER BY score_expr` +- 支持同一个 score expr 在同一棵计划树里重复出现多次 +- 所有重复出现点都映射到同一个 `-2050` 返回列 +- 避免 TiDB 本地重新执行 FTS/BM25/vector score + +### 4.2 非目标 + +V1 不追求： + +- 同一个 `DataSource` 上多个不同 score expr 同时返回 +- `JOIN ON`、`GROUP BY`、`HAVING`、window 中使用 score +- 回表路径 `IndexLookUpReader` 上直接输出 score +- TiCI index side 的 score TopK pushdown + +### 4.3 兼容性目标 + +- 不污染 `DataSource` 的逻辑 schema +- 不破坏 `ExtraVersion` 在最后一列的约定 +- 不破坏现有 `ExtraPhysTblID` 假设 +- 不影响普通 TiCI FTS filter-only 路径 + +## 5. 总体方案 + +总体采用两层结构： + +- 逻辑层：放松限制，但不改逻辑 schema，只保留 shape 校验。 +- 物理层：在 `Attach2Task` 阶段完成 score 绑定、特殊列注入、父节点表达式重写。 + +更具体地说： + +1. 新增特殊列 `ExtraFTSScoreID = -2050` +2. 当物理父节点发现自己消费了 FTS score expr 时： + - 向下定位单读 TiCI path + - 判断这些 expr 是否都语义等价 + - 判断它们是否与 index access cond 中的 FTS expr 语义一致 + - 成功则把 `FtsQueryInfo.QueryType` 切成 `WithScore` + - 向 `PhysicalIndexScan` / `PhysicalIndexReader` 注入 score 列 + - 把父节点表达式中的所有等价 `fts_match()` 重写成 `Column{-2050}` + +这样： + +- 同一个函数出现多次，不再重复求值 +- 也不需要在 TiDB 本地 eval FTS builtin +- 所有映射都绑定到同一个 score 列 + +## 6. 为什么绑定放在物理阶段，而不是逻辑阶段 + +这是本方案的核心取舍。 + +### 6.1 逻辑阶段可见信息不足 + +`analyzeTiCIIndex()` 运行于 `PredicatePushDown`： + +- `pkg/planner/core/operator/logicalop/logical_datasource.go:173` + +当时 `DataSource` 只知道： + +- `PushedDownConds` +- 以及能否构造 TiCI FTS path + +它并不知道： + +- 上层 `Projection` 是否真的要输出 score +- 上层 `Sort/TopN` 是否要消费 score +- 最终是否走单读还是双读 + +### 6.2 逻辑 schema 改写侵入面过大 + +如果在逻辑阶段把 score 列塞进 `DataSource.Schema()`，会牵涉： + +- `PruneColumns` +- index covering +- single/double read 判定 +- 各类 rule 对 schema 的假设 + +这会显著放大改动范围。 + +### 6.3 物理阶段已经有现成范式 + +vector distance 在物理阶段完成： + +- 特殊列注入 +- 父节点表达式替换 + +而且已经证明这样做能稳定工作： + +- `pkg/planner/core/task.go:1011` + +因此 FTS/BM25 score 沿用相同范式是更自然的。 + +## 7. V1 范围与边界 + +V1 明确只支持： + +- 单个 `DataSource` +- 单个 distinct score expr +- `PhysicalIndexReader` / 单读 TiCI path +- `Projection`、`Sort`、`TopN` 消费 score + +V1 不支持： + +- `IndexLookUpReader` +- 多个不同 score expr +- 没有对应 access expr 的纯 score 查询 +- JOIN / AGG / HAVING / WINDOW / CTE + +### 7.1 为什么 V1 只做单读 + +`PhysicalIndexReader` 的数据流更简单： + +- index side 返回什么 +- reader 就可以直接输出什么 + +而 `PhysicalIndexLookUpReader` 的 schema 默认是 table plan schema： + +- `pkg/planner/core/operator/physicalop/physical_indexlookup_reader.go:221` + +它不会自动把 index side 额外列带到最终输出，需要显式把 score 从 index rows 穿过 lookup task 再拼回最终 row。复杂度高很多，适合第二阶段处理。 + +## 8. 特殊列建模 + +### 8.1 model 常量 + +建议在 `pkg/meta/model/table.go` 中新增： + +```go +const ExtraFTSScoreID int64 = -2050 +var ExtraFTSScoreName = ast.NewCIStr("_tidb_fts_score") +``` + +位置建议靠近： + +- `ExtraVersionID` +- `VirtualColVecSearchDistanceID` + +理由： + +- 它和 `ExtraVersionID` 一样，会进入 PB 和 executor 特殊列路径 +- 它不像 vector distance 那样纯粹局限于 table scan 场景 + +### 8.2 ColumnInfo helper + +建议在 `pkg/meta/model/column.go` 中新增： + +```go +func NewExtraFTSScoreColInfo() *ColumnInfo +``` + +字段建议： + +- `ID: ExtraFTSScoreID` +- `Name: ExtraFTSScoreName` +- `Type: mysql.TypeDouble` +- `Flag: 0` 或按需要设置 `NotNullFlag` +- binary charset / collation + +### 8.3 为什么类型建议用 DOUBLE + +当前 FTS builtin 的返回类型是 `ETReal`，更接近 `DOUBLE`，而不是强行缩成 `FLOAT`： + +- `pkg/expression/builtin_fts.go:113` +- `pkg/expression/builtin_fts.go:187` + +如果 score 列类型与 builtin 返回类型不一致，会产生潜在问题： + +- projection type 不一致 +- sort 比较类型不一致 +- PB 侧列类型和父节点表达式预期不一致 + +因此 V1 直接使用 `mysql.TypeDouble` 更稳。 + +## 9. 语义等价判定 + +### 9.1 判定工具 + +必须复用现有表达式语义等价工具： + +- `pkg/expression/scalar_function.go:591` +- `pkg/expression/scalar_function.go:600` + +也就是： + +- `CanonicalHashCode()` +- `ExpressionsSemanticEqual()` + +不能做的事情： + +- 不能用 `expr.String()` +- 不能用文本匹配 +- 不能按出现位置绑定 + +### 9.2 归一化步骤 + +对所有候选 score expr，需要做统一归一化，伪代码如下： + +```go +func normalizeFTSScoreExpr(expr expression.Expression) expression.Expression { + // 1. 如果是 MATCH ... AGAINST，先 rewrite 到内部 helper + // 2. 如果外层包了 IsTruthWithNull / IsTruthWithoutNull，把壳去掉 + // 3. 返回规范化后的 expr +} +``` + +原因： + +- `WHERE fts_match(...)` 往往会包 `IS TRUE` +- `SELECT fts_match(...)` 则通常不会 +- 如果不剥壳，这两个位置虽然语义同源，但 hash 可能不一致 + +归一化逻辑应尽量和： + +- `pkg/planner/core/operator/logicalop/logical_datasource.go:875` +- `pkg/expression/expr_to_pb.go:276` + +保持同一语义。 + +### 9.3 V1 的 distinct expr 限制 + +V1 要求： + +- 同一条 path 上所有 score consumer 归一化后，只能得到 1 个 distinct hash + +支持： + +```sql +select f(), f() +from t +where f() +order by f(); +``` + +不支持： + +```sql +select f1(), f2() +from t +where f1() and f2(); +``` + +后一类在 V1 直接报错： + +- `multiple distinct FTS score expressions are not supported yet` + +## 10. 物理阶段的绑定与重写 + +### 10.1 关键思想 + +score 的绑定不在逻辑层做，而在以下父节点的 `Attach2Task` 中做： + +- `PhysicalProjection` +- `PhysicalSort` +- `PhysicalTopN` + +对应位置： + +- `pkg/planner/core/task.go:1472` +- `pkg/planner/core/task.go:864` +- `pkg/planner/core/task.go:1294` + +做法是： + +1. 父节点扫描自己的表达式，看是否包含 FTS score consumer。 +2. 如果没有，走原逻辑。 +3. 如果有，强制把 child task 转成 root task。 +4. 在 root task 中向下找到 `PhysicalIndexReader + PhysicalIndexScan`。 +5. 检查是否为单读 TiCI FTS path。 +6. 成功则注入 `-2050`，并把父节点中所有等价函数都改成同一个 score 列。 + +### 10.2 为什么强制先转 root task + +V1 中，如果父节点消费了 score，我建议不要尝试把这个节点继续 pushdown 到 cop 侧，而是： + +- 直接转 root +- 在 root 上消费 score 列 + +理由： + +- 改动更小 +- 避免和现有 projection/topN pushdown 逻辑交叉 +- 避免出现 “cop 侧想 push 一个 FTS function，但我们其实只想要 score 列”的混合状态 + +因此 V1 的策略很简单： + +- 发现 score consumer +- 先 root 化 +- 再 rewrite 为列 + +## 11. 单读路径上的 plan shape + +`CopTask.ConvertToRootTask()` 在单读 index path 下，会构造： + +- `PhysicalIndexReader` + +对应位置： + +- `pkg/planner/core/operator/physicalop/task_base.go:485` +- `pkg/planner/core/operator/physicalop/task_base.go:544` + +也就是说，单读 TiCI path 的根形态通常是： + +```text +Projection / Sort / TopN + └─ IndexReader + └─ IndexScan(TiCI) +``` + +因此 V1 的核心 helper 可以明确只处理这条链： + +- 向下找到 `PhysicalIndexReader` +- 再找到 `PhysicalIndexScan` + +如果不是这条链，V1 直接返回“不支持”。 + +## 12. 建议新增的 helper + +### 12.1 找路径 helper + +建议新增： + +```go +func findSingleReadTiCIFTSReader(plan base.PhysicalPlan) (*physicalop.PhysicalIndexReader, *physicalop.PhysicalIndexScan, bool) +``` + +约束： + +- 只接受单读 `PhysicalIndexReader` +- 允许中间存在一层轻量 wrapper，但最终必须能定位到唯一 `PhysicalIndexScan` +- `scan.StoreType == kv.TiCI` +- `scan.FtsQueryInfo != nil` + +### 12.2 score consumer 收集 + +```go +func collectFTSScoreExprs(exprs []expression.Expression) []expression.Expression +``` + +用于： + +- `Projection.Exprs` +- `TopN.ByItems` +- `Sort.ByItems` + +### 12.3 归一化与 distinct 检查 + +```go +func buildFTSScoreBinding( + scoreExprs []expression.Expression, + accessConds []expression.Expression, + parserType model.FullTextParserType, +) (*FTSScoreBinding, error) +``` + +该函数应完成： + +- consumer expr 归一化 +- access cond expr 归一化 +- consumer distinct hash 统计 +- consumer 是否出现在 access cond hash 集合中的校验 + +### 12.4 score 列注入 + +```go +func ensureFTSScoreInjected( + reader *physicalop.PhysicalIndexReader, + scan *physicalop.PhysicalIndexScan, + binding *FTSScoreBinding, +) (*expression.Column, error) +``` + +职责： + +- 若 scan 已有 `-2050`，复用 +- 否则向 scan schema 注入 +- 同步向 reader schema / outputColumns 注入 +- 返回“供父节点 rewrite 使用的 reader schema 中的 score 列” + +### 12.5 父节点表达式重写 + +```go +func rewriteFTSScoreExpr( + expr expression.Expression, + binding *FTSScoreBinding, + scoreCol *expression.Column, +) expression.Expression +``` + +规则： + +- 若 `normalize(expr)` 与 `binding.ExprHash` 相同，替换为 `scoreCol.Clone()` +- 若是 scalar function，递归处理参数 +- 否则原样返回 + +## 13. `ensureFTSScoreInjected()` 的精确行为 + +这是整套方案最关键的实现点之一。 + +### 13.1 先检查 scan schema 中是否已存在 score 列 + +如果已存在： + +- 直接复用 +- 保证多个父节点共用同一个 injected score 列 + +这正是“同一个函数可能出现多次，他们要都能正确映射到那个值”的核心保证。 + +### 13.2 注入到 `PhysicalIndexScan.Schema()` + +当前 TiCI row layout 在： + +- `pkg/planner/core/operator/physicalop/physical_index_scan.go:459` + +V1 要求插入规则为： + +- 没有 `ExtraPhysTblID`：`[..., score, ExtraVersion]` +- 有 `ExtraPhysTblID`：`[..., score, ExtraPhysTblID, ExtraVersion]` + +不能做成： + +- `[..., ExtraPhysTblID, score, ExtraVersion]` +- `[..., ExtraPhysTblID, ExtraVersion, score]` + +原因： + +- [builder.go:4578](/DATA/disk4/yiding/gocode/tidb/pkg/executor/builder.go#L4578) 假定 `ExtraPhysTblID` 在 version 前一列 +- [distsql.go:800](/DATA/disk4/yiding/gocode/tidb/pkg/executor/distsql.go#L800) 假定最后一列是 version + +### 13.3 注入到 `PhysicalIndexReader.Schema()` 与 `OutputColumns` + +`PhysicalIndexReader` 默认 schema 不是 index scan schema，而是 `DataSourceSchema`： + +- `pkg/planner/core/operator/physicalop/physical_index_reader.go:77` + +因此仅仅修改 `scan.Schema()` 不够，还必须： + +- 在 `reader.Schema()` 中 append 一个 score 列 +- 在 `reader.OutputColumns` 中 append 对应列 + +不然 executor builder 仍然不会请求这个列： + +- `pkg/executor/builder.go:4379` + +### 13.4 `scoreCol.Index` 的处理 + +`IndexReader` 在执行前会对 `OutputColumns` 做 `ResolveIndices()`： + +- `pkg/planner/core/operator/physicalop/physical_index_reader.go:185` + +因此建议： + +- 注入时先构造列对象并挂入 schema +- `ResolveIndices()` 自然会把它解析到 scan schema 中的实际位置 +- 避免手工写死 index + +## 14. `ToPB()` 与 PB schema + +在 `PhysicalIndexScan.ToPB()` 中，目前只识别： + +- `ExtraHandleID` +- `ExtraPhysTblID` +- `ExtraVersionID` + +位置： + +- `pkg/planner/core/operator/physicalop/physical_index_scan.go:628` + +V1 需要新增： + +```go +else if col.ID == model.ExtraFTSScoreID { + columns = append(columns, model.NewExtraFTSScoreColInfo()) +} +``` + +否则： + +- `-2050` 会被误当成普通 table column 去查 `FindColumnInfoByID` +- 直接失败 + +## 15. `Projection` 的处理细节 + +### 15.1 触发条件 + +`PhysicalProjection` 中只要 `Exprs` 里存在 FTS score consumer，就走特殊路径。 + +### 15.2 处理流程 + +1. `t := tasks[0].Copy()` +2. `t = t.ConvertToRootTask(...)` +3. 在 root plan 中定位 `IndexReader + IndexScan` +4. 构建 `FTSScoreBinding` +5. `scan.FtsQueryInfo.QueryType = WithScore` +6. 注入 score 列 +7. 将 `p.Exprs` 中所有等价 score expr 替换为 `scoreCol` +8. 再正常 `attachPlan2Task(p, t)` + +### 15.3 替换后的收益 + +替换完成后： + +- `Projection` 不再包含 FTS builtin +- 只包含一个普通列 + +因此： + +- executor 不会本地 eval FTS function +- 多个 `SELECT f()` 会直接复用同一个 score 列 + +## 16. `Sort` / `TopN` 的处理细节 + +### 16.1 `Sort` + +`PhysicalSort` 的逻辑和 `Projection` 基本一致： + +- 发现 `ByItems` 中存在 score expr +- root 化 +- 绑定、注入、rewrite +- 最终 root sort 按 score 列排序 + +### 16.2 `TopN` + +`TopN` 比 `Sort` 更复杂，因为它当前有一套 pushdown 和 heavy-function 优化逻辑： + +- `pkg/planner/core/task.go:883` +- `pkg/planner/core/task.go:1294` + +V1 建议不要尝试复用 `getPushedDownTopN()` 去做 FTS score pushdown，而是： + +- 只要 `TopN.ByItems` 包含 score expr +- 直接转 root +- rewrite 成 score 列 +- root 上执行 TopN + +这样可以规避两个复杂问题： + +- TiCI score TopK pushdown 语义 +- 与现有 vector heavy-function 优化逻辑交织 + +### 16.3 为什么 V1 不做 TiCI score TopK pushdown + +当前 `TryToPassTiCITopN()`： + +- `pkg/planner/core/operator/physicalop/physical_index_scan.go:726` + +只支持 hybrid index sort columns 与普通列排序匹配，不支持函数排序。把 score TopK pushdown 一起做，会引入更多协议和执行语义约束。V1 不需要承担这部分复杂度。 + +## 17. `find_best_task` 的调整 + +当前： + +- `pkg/planner/core/find_best_task.go:2275` + +会把任意 `FtsQueryInfo != nil` 且带 sort property 的 path 直接判 invalid。 + +V1 建议改为： + +- 不再直接 invalid +- 但也不宣称该 path 满足 sort property +- 让 planner 正常走“无序 scan + root sort/root topN” + +这样： + +- `ORDER BY score` 至少可以执行 +- 同时不会错误地把 score 排序当成 index-order 能力 + +## 18. validation 的调整 + +当前 validation 过于保守，全部禁止了 `SELECT` / `ORDER BY`。 + +V1 建议调整成： + +- `Projection/Sort/TopN`：不再一刀切报错 +- `Join/Aggregation/Window/GroupBy/Having`：继续禁止 +- `Selection`：仍然要求 FTS expr 必须是可下推 access cond，不允许当普通 TiDB 本地表达式 + +也就是说： + +- shape 约束继续保留 +- score 绑定是否成功，不在 validation 中决定 +- 绑定失败在物理计划阶段报更具体的错误 + +## 19. 错误模型 + +建议在物理绑定阶段报下列明确错误： + +- `FTS score expression must also appear in a pushed-down FTS predicate` +- `multiple distinct FTS score expressions are not supported yet` +- `FTS score is only supported on single-read TiCI index path` +- `FTS score in JOIN/GROUP BY/HAVING/WINDOW is not supported` + +这样比当前统一报： + +- `cannot be used in SELECT fields` +- `ORDER BY is not supported` + +要精确得多。 + +## 20. 为什么这套设计能保证“同一个函数多次出现都映射到那个值” + +这里把关键逻辑单独说明。 + +### 20.1 重复出现点不会各自分配新列 + +score 列的注入由 `ensureFTSScoreInjected()` 统一负责： + +- 若 scan schema 已经有 `-2050` +- 直接复用 + +因此同一条 root plan 上： + +- 第一个消费者会触发注入 +- 后续消费者只会复用同一个 injected 列 + +### 20.2 绑定不是按位置，而是按 canonical hash + +所有 consumer expr 会先归一化，然后比对： + +- `CanonicalHashCode()` + +因此： + +- `SELECT` 中的 `fts_match()` +- `ORDER BY` 中的 `fts_match()` +- 同一个 projection 中多次出现的 `fts_match()` + +只要语义等价，都会命中同一个 hash。 + +### 20.3 rewrite 发生在每个父节点自己的表达式树里 + +`Projection`、`Sort`、`TopN` 分别对自己的表达式树做 rewrite： + +- 只要某个节点语义等价于绑定 expr +- 直接替换成 `scoreCol` + +因此“同一个函数多次出现”本质上只是一棵表达式树里命中了多次同一个规则，而不是需要维护“第几个出现点对第几个返回列”的 fragile mapping。 + +## 21. 为什么 V1 不直接支持 `IndexLookUpReader` + +`IndexLookUpReader` 的 schema 默认是 table plan schema： + +- `pkg/planner/core/operator/physicalop/physical_indexlookup_reader.go:221` + +而 index side 额外返回列会先落进： + +- `lookupTableTask.idxRows` +- `pkg/executor/distsql.go:82` + +若要让 score 穿过 lookup，必须再做以下事情： + +1. index request 把 score 一起返回 +2. score 保存在 `idxRows` +3. table worker 回表后，按 handle 把 score 拼回最终 row +4. `IndexLookUpReader.Schema()` 扩出 score 列 + +这会同时涉及： + +- planner +- index worker +- table worker +- row merge + +因此明显应分期处理，而不是和单读路径一起做。 + +## 22. V2 方向：双读与回表 + +后续如果要支持： + +```sql +select non_index_col, fts_match(...) +from t +where fts_match(...); +``` + +则需要进入 V2。 + +V2 的最小实现路径： + +- 让 score 作为 index-side 辅助列进入 `lookupTableTask.idxRows` +- 建立 `handle -> score` 关联 +- table side 回表后，把 score 和 table row 合并 +- 最终 `IndexLookUpReader` 输出扩展 schema + +这部分与当前 partition by-items / keep-order 的 index 辅助列处理模式相近，但不是本设计文档的 V1 范围。 + +## 23. 备选方案与取舍 + +### 23.1 备选方案 A：在逻辑层直接扩 `DataSource.Schema()` + +优点： + +- 上层所有节点天然能看到 score 列 + +缺点： + +- 侵入逻辑 rule 太多 +- 会影响 covering / prune columns / single scan 判定 +- 风险大 + +结论： + +- 不采用 + +### 23.2 备选方案 B：每个出现点都单独分配一个特殊列 + +优点： + +- 实现直观 + +缺点： + +- 同一个函数出现两次会返回两列，浪费 +- executor / schema 管理复杂 +- 很难保证等价表达式共享一个值 + +结论： + +- 不采用 + +### 23.3 备选方案 C：先做统一的 “index-returned computed column” 抽象 + +优点： + +- 抽象最干净 + +缺点： + +- 牵涉 vector / FTS / hybrid index 多条线 +- 一次性改动过大 + +结论： + +- 长期可考虑 +- 当前以 FTS score 特化实现为主 + +## 24. 测试建议 + +### 24.1 planner / explain 用例 + +建议增加： + +- `select f(), f() from t where f()` +- `select f() from t where f() order by f()` +- `select f()+1 from t where f()` +- `select f1(), f2() from t where f1() and f2()` 报错 +- `select f() from t where other_cond` 报错 + +其中 `f()` 代表 FTS score expr。 + +### 24.2 explain 验证 + +验证点： + +- 计划中出现 `_tidb_fts_score` +- 不再出现 TiDB 侧 eval 的 FTS builtin +- TiCI path 的 `FtsQueryInfo.QueryType` 为 `WithScore` + +### 24.3 执行验证 + +验证： + +- `SELECT/ORDER BY` 同时使用同一个 score expr 时结果正确 +- 重复出现的 score expr 数值一致 +- `ORDER BY score DESC LIMIT N` 结果稳定 + +## 25. 建议的实施顺序 + +### Phase 1：准备工作 + +1. 新增 `ExtraFTSScoreID = -2050` +2. 新增 `NewExtraFTSScoreColInfo()` +3. `PhysicalIndexScan.ToPB()` 识别 `-2050` + +### Phase 2：放开限制 + +1. 调整 `ftsFuncValidation` +2. 放开 `find_best_task.go` 中 FTS + sort property 的 blanket ban + +### Phase 3：单读路径绑定与重写 + +1. 在 `task.go` 增加 normalize / binding / inject / rewrite helpers +2. 在 `Projection/Sort/TopN` 的 `Attach2Task` 中接入 +3. 仅支持 `PhysicalIndexReader` + +### Phase 4：测试与收尾 + +1. explain / planner case +2. execution case +3. 错误模型 case + +### Phase 5：V2 + +1. `IndexLookUpReader` +2. score 透传 `lookupTableTask` +3. 可选的 TiCI TopK pushdown + +## 26. 最终结论 + +对于当前代码库，最合适的实现不是“简单传一个 `-2050` 列”，而是： + +- 在单读 TiCI FTS path 上注入 `-2050` score 特殊列 +- 用 canonical hash 做语义绑定 +- 把 `Projection/Sort/TopN` 中所有等价的 score expr 全部重写到同一个列 + +这样可以最小改动地解决两个核心问题： + +- score 必须在索引侧执行 +- 同一个函数可能出现多次，所有出现点都要正确映射到同一个值 + +同时，这套设计与现有 vector distance 的“特殊列回填”模式一致，易于复用和扩展，也为后续 BM25、多 score expr、回表路径和 TiCI TopK pushdown 留出了清晰演进路径。 From 5bec583d982f284623b668691029a45eef7ed5d5 Mon Sep 17 00:00:00 2001 From: Yiding Date: Wed, 22 Jul 2026 02:28:19 +0800 Subject: [PATCH 19/25] docs: fix preview RU write request ownership --- ...-07-22-preview-ru-resource-formula-plan.md | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md index 26337b6805fd4..16ecdc062fb86 100644 --- a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md +++ b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md @@ -21,6 +21,8 @@ This plan deliberately does not implement production code. It freezes the model, - [x] (2026-07-22 11:15Z) Incorporated the final ordering clarification: Sort owns `n*log(n)`, TopN owns `n*log(k)` with `k` derived from offset plus count, and expression evaluation remains Projection-owned. - [x] (2026-07-22 11:40Z) A fresh-context release reviewer re-read the updated user handoff, plan, and code and concluded: “当前版本无需必要修改”. - [x] (2026-07-22 11:45Z) Applied the Ready profile to this design-only diff: no Go/Bazel trigger was present and the staged Markdown patch passed `git diff --cached --check`. +- [x] (2026-07-22 13:05Z) A later fresh-context completion audit found that explicit-transaction DML write RPCs are statement-local and cannot all be deferred to COMMIT; revised ownership to charge each DML and COMMIT from its own frozen `RUV2Metrics` snapshot. +- [x] (2026-07-22 13:35Z) A new no-inherited-context reviewer inspected the corrected plan and current code and concluded exactly: “当前版本无需必要修改”. Because this iteration changed a substantive ownership rule, the convergence policy still requires another fresh-context iteration before design-loop completion. - [ ] Implement Milestone 1: introduce v4 work units and expression-count helpers without changing production RUv2. - [ ] Implement Milestone 2: construct operator units from current runtime details and add only the permitted HashJoin state-row detail. - [ ] Implement Milestone 3: migrate all preview outputs atomically and add regression coverage. @@ -55,6 +57,9 @@ This plan deliberately does not implement production code. It freezes the model, - Observation: zero-valued RUv2 getters do not prove that their payload was present, and the read-RPC counter is broader than cop response bytes. Evidence: absent/bypassed `RUV2Metrics` reads as zero; `ResourceManagerReadCnt` covers TiKV read RPC producers including unsupported point/ancillary paths, while `TiKVCoprocessorResponseBytes` covers cop responses. Runtime task coverage and a closed producer set are therefore required before zero or statement-wide totals are attributable. +- Observation: write RPC counters are statement-local even inside an explicit transaction. + Evidence: `session.executeStmtImpl` installs the current statement's `RUV2Metrics`, `ExecStmt.finalizeStatementRUV2Metrics` drains raw/commit details into it before preview construction, and `TestRUV2MetricsIsolatedPerStatementInExplicitTxn` proves successive statements use distinct instances. A pessimistic DML's write requests therefore cannot be deferred to the later COMMIT snapshot. + ## Decision Log - Decision: replace, rather than layer on top of, the v3 fixed/row/byte opclass matrix for new samples. @@ -101,13 +106,17 @@ This plan deliberately does not implement production code. It freezes the model, Rationale: this preserves one CPU-priced unit without adding independent mutation weights. `mutation_bytes_per_cpu_unit` is a versioned normalization constant, not an RU coefficient; it must be calibrated and validated as positive before a weighted total is published. Date/Author: 2026-07-22 / Codex. +- Decision: charge write requests to the statement whose frozen `RUV2Metrics` snapshot contains them; an explicit-transaction DML and the eventual COMMIT each own only their respective snapshots. + Rationale: TiDB installs a fresh `RUV2Metrics` object for every statement and finalizes raw RU details into that statement before preview construction. Pessimistic DML can issue write RPCs before COMMIT, so deferring all request work to COMMIT would permanently omit those DML-local requests. A cross-statement accumulator is unnecessary and would weaken the existing statement-local contract. + Date/Author: 2026-07-22 / Codex. + - Decision: do not guess numerical v4 weights or silently map the heterogeneous v3 opclass weights. Rationale: there is no evidence-backed one-to-one mapping. The implementation first publishes v4 base units with `uncalibrated_weights` and no total until an explicit v4 weight set and positive mutation normalization are installed. The preview flag is off by default, so this is safer than presenting arbitrary numbers as RU. Date/Author: 2026-07-22 / Codex. ## Outcomes & Retrospective -The design phase is stable and independently reviewed. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(offset+count)` work without repeating Projection expression evaluation, removes IndexJoin request double charging, limits new runtime data to one HashJoin state-row count, and defines a no-guess migration. A final reviewer with no inherited conversation context concluded that the current version needs no necessary modification. Implementation is intentionally pending and must happen in the separate worktree specified below. +The design keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(offset+count)` work without repeating Projection expression evaluation, removes IndexJoin request double charging, limits new runtime data to one HashJoin state-row count, and defines a no-guess migration. A later fresh-context audit also corrected explicit-transaction write-request ownership so pessimistic DML requests remain attached to their actual statement snapshot rather than being lost at COMMIT. A subsequent no-inherited-context reviewer found no further necessary modification. Implementation is intentionally pending and must happen in the separate worktree specified below; because this iteration changed a substantive ownership rule, one more fresh-context iteration is required by the convergence policy before the design phase is closed. ## Context and Orientation @@ -276,7 +285,7 @@ The v4 combined unit is: `mutation_bytes_per_cpu_unit` is a positive, finite, versioned normalization constant with units bytes per expression-equivalent CPU-work unit. It is stored alongside the v4 weights and included in output metadata. It is not independently multiplied by RU. If it is unset, zero, negative, NaN, or infinite, mutation base components remain visible but weighted v4 total is unavailable with `uncalibrated_weights`. -`write_request_count` is `RUV2Metrics.ResourceManagerWriteCnt()` and uses the same `request_weight` as reads. The frozen snapshot must exist and be non-bypassed. Autocommit DML normally emits mutation and write-request work together; a nonzero mutation with a zero/missing request payload is partial. Explicit-transaction DML emits statement-local mutation work only; the eventual COMMIT emits transaction-scoped write requests with empty `dml_kind`, without back-attributing them to earlier DML. A known empty transaction COMMIT emits observed zero request work; an absent lifecycle snapshot is missing, not zero. SQL `ROLLBACK` remains unsupported in v4 and emits neither a zero unit nor a total: current routing has no complete rollback-RPC attribution, and declaring it free would be unsafe. +`write_request_count` is `RUV2Metrics.ResourceManagerWriteCnt()` and uses the same `request_weight` as reads. The frozen snapshot must exist and be non-bypassed. Every DML statement, whether autocommit or inside an explicit transaction, emits the write-request count present in its own finalized snapshot alongside its statement-local mutation work. This is required for pessimistic transactions, whose DML statements can issue nonzero write RPCs before COMMIT. The eventual COMMIT emits only the write requests in its own fresh snapshot, with empty `dml_kind`; it neither absorbs nor back-attributes earlier DML requests. Thus each physical request is owned once by the statement whose RU details recorded it. A nonzero mutation with a missing request payload is partial, while an observed zero DML request count is valid only when the non-bypassed statement snapshot is present and finalized. A known empty transaction COMMIT likewise emits observed zero request work; an absent lifecycle snapshot is missing, not zero. SQL `ROLLBACK` remains unsupported in v4 and emits neither a zero unit nor a total: current routing has no complete rollback-RPC attribution, and declaring it free would be unsafe. Pipelined transactions retain valid mutation units but mark the write-request component `pipelined_tikv_payload_unsupported` until current details prove a complete logical flush request count. Deprecated batch DML accumulates available write-request snapshots across internal transaction switches; any missing switch makes the request component partial. Optimistic retry/replay keeps the mutation behavior above and marks unavailable request attribution partial. Thus no retry, pipeline, or batch path publishes a known-incomplete zero. @@ -288,7 +297,7 @@ The existing preview gates remain: the feature is default off; `EXPLAIN ANALYZE For side-effect-free SELECT, billing is statement-atomic. If any executed supported operator lacks a required input, the statement records status and reason but emits no billable v4 units and no `total_preview_ru`. This prevents a partial plan from looking cheap. Diagnostic status rows may still identify every missing operator. -For DML, read-tree, mutation, and transaction-request components keep independent status because explicit transactions separate their lifetimes. Complete components may retain coefficient-free units for calibration, but a statement-level weighted total is absent unless every component expected at that lifecycle point is complete and the weight set is calibrated. +For DML, read-tree, mutation, and statement-local write-request components keep independent status. COMMIT has its own write-request component because explicit transactions separate statement lifetimes. Complete components may retain coefficient-free units for calibration, but a statement-level weighted total is absent unless every component expected at that lifecycle point is complete and the weight set is calibrated. Observed zero is accepted only with presence evidence: an existing root stat, a cop stat with complete task coverage, a HashJoin state-row runtime stat, a mutation recorder snapshot, or a frozen RUv2 snapshot plus the reader consistency checks above. Missing, negative, overflowed, NaN, and infinite inputs have bounded reasons. New reasons are `missing_expression_count`, `missing_ordering_projection`, `invalid_topn_bound`, `missing_reader_transport_details`, `ambiguous_reader_transport_producers`, `missing_hash_state_rows`, `invalid_hash_state_rows`, and `uncalibrated_weights`. @@ -338,7 +347,7 @@ Use the existing flat-plan build/probe/left/right labels for join rows and the j Add `HashTableRuntimeStats` in `pkg/util/execdetails/runtime_stats.go`. Implement it for both HashJoin runtime-stat versions in `pkg/executor/join/hash_join_stats.go`, recording successful v1/v2 state admissions at the existing build completion points in `hash_join_v1.go` and `hash_join_v2.go`. Do not add a unique-key collector or any non-Join runtime field. -Change write construction in `pkg/planner/core/explain_ru.go` to derive `mutation_work` and use statement/COMMIT write RPC count. Retain raw mutation units as diagnostics. +Change write construction in `pkg/planner/core/explain_ru.go` to derive `mutation_work` and use the current statement's finalized write RPC count for both DML and COMMIT. Do not carry request counts across statements or defer explicit-transaction DML requests to COMMIT. Retain raw mutation units as diagnostics. Acceptance: every required formula input can be traced to the source table below; source searches show exactly one new Join-only state-row counter and no other runtime field. @@ -368,7 +377,7 @@ Acceptance: internal formula tests observe exact calibrated totals. EXPLAIN, met | HashJoin `hash_state_rows` | v1 hash-table `Len` plus NAAJ null-bucket entries; v2 row-table `validKeyCount` | completed build round, cumulative across rebuilds | **yes, Join only** | | Join `output_rows` | Join node own `BasicRuntimeStats.GetActRows` | executed root stat required | no | | `mutation_count/bytes` | `StatementContext` preview mutation recorder | current attempted-call semantics | no | -| `write_request_count` | `RUV2Metrics.ResourceManagerWriteCnt` | DML/COMMIT lifecycle snapshot | no | +| `write_request_count` | current statement's `RUV2Metrics.ResourceManagerWriteCnt` | finalized, present, non-bypassed snapshot; each DML and COMMIT owns only its own count | no | ## Concrete Steps @@ -400,7 +409,7 @@ Implement milestones in order. During WIP, run the smallest targeted tests. The ./tools/check/failpoint-go-test.sh pkg/planner/core -run 'TestExplainRU(PlanFormulaAndOperatorClasses|ComponentSnapshotStatusAndWeights)|TestReadBillingDemo' ./tools/check/failpoint-go-test.sh pkg/executor -run 'TestExplainAnalyzeFormatRU|TestReadBillingDemoMetricsHook|TestReadBillingDemoGeneralLogUnits|TestWriteSlowLog' ./tools/check/failpoint-go-test.sh pkg/executor/join -run 'Test.*HashJoin.*RuntimeStats' - ./tools/check/failpoint-go-test.sh pkg/session -run 'TestPreviewKVMutationRecorder|TestRUV2MetricsIsolatedPerStatementInExplicitTxn' + ./tools/check/failpoint-go-test.sh pkg/session -run 'TestPreviewKVMutationRecorder|TestRUV2Metrics(IsolatedPerStatementInExplicitTxn|WriteRequestsInPessimisticTxn)' ./tools/check/failpoint-go-test.sh pkg/util/stmtsummary -run 'TestReadBillingDemo(BaseUnitsToDatum|StructuredRowsToDatum|AggregationCaps|DMLKindAggregation|ReservedStatusMergeBypassesStatusCap)' ./tools/check/failpoint-go-test.sh pkg/util/stmtsummary/v2 -run 'TestStmtRecordReadBillingDemoStructuredStats|TestReadBillingDemo(MemReader|HistoryReader)' go test -tags=intest,deadlock ./pkg/metrics -run 'TestExplainRUMetrics|TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues' @@ -419,7 +428,7 @@ The implementation is accepted only when all of the following are observable. With private test weights set to simple values inside `pkg/planner/core`, table-driven formula tests show exact results for every operator row in the formula table, including zero rows, one row, multiple expressions, multi-key joins, all three IndexJoin-family key representations, V1 NAAJ null-bucket state, and Window frame expressions. Sort uses `log2(max(rows,2))`; TopN uses `log2(max(offset+count,2))`, with checked addition and cases where offset is nonzero. Neither ordering operator has an expression/key-count multiplier, and unmaterialized scalar ordering expressions fail closed rather than being charged there. -End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. +End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. An explicit pessimistic transaction case must prove that a DML-local nonzero `ResourceManagerWriteCnt` is emitted by that DML, the later COMMIT emits only its own fresh-snapshot count, and neither count is lost or duplicated. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. A multi-reader or IndexMerge case proves that statement `net_bytes` and read `request_count` appear once, while every scan retains its own `scan_bytes`. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. @@ -466,4 +475,4 @@ At milestone completion, the key internal interfaces should have these conceptua The exact private names may follow nearby conventions, but the semantics, data sources, one-Join-runtime-datum boundary, and de-duplication rules in this plan are mandatory. -Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, TopN owns `n*log(k)` with checked `k=offset+count`, and inner readers own IndexJoin request cost. HashJoin uses the sole permitted runtime addition to expose actual admitted hash-state rows instead of approximating them with all build rows. +Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, TopN owns `n*log(k)` with checked `k=offset+count`, and inner readers own IndexJoin request cost. HashJoin uses the sole permitted runtime addition to expose actual admitted hash-state rows instead of approximating them with all build rows. A later fresh-context audit corrected write-request ownership: explicit-transaction DML and COMMIT each charge only the write RPCs in their own finalized statement snapshot, preventing pessimistic DML requests from being lost. From 385d6e6c1327f3f3fd1429da0d83f715dcb57ca3 Mon Sep 17 00:00:00 2001 From: Yiding Date: Wed, 22 Jul 2026 02:32:41 +0800 Subject: [PATCH 20/25] docs: complete preview RU design review --- ...-07-22-preview-ru-resource-formula-plan.md | 5 +- .../fts_score_special_column_design.md | 989 ------------------ 2 files changed, 3 insertions(+), 991 deletions(-) delete mode 100644 docs/note/planner/fts_score_special_column_design.md diff --git a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md index 16ecdc062fb86..d83316fabb57e 100644 --- a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md +++ b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md @@ -23,6 +23,7 @@ This plan deliberately does not implement production code. It freezes the model, - [x] (2026-07-22 11:45Z) Applied the Ready profile to this design-only diff: no Go/Bazel trigger was present and the staged Markdown patch passed `git diff --cached --check`. - [x] (2026-07-22 13:05Z) A later fresh-context completion audit found that explicit-transaction DML write RPCs are statement-local and cannot all be deferred to COMMIT; revised ownership to charge each DML and COMMIT from its own frozen `RUV2Metrics` snapshot. - [x] (2026-07-22 13:35Z) A new no-inherited-context reviewer inspected the corrected plan and current code and concluded exactly: “当前版本无需必要修改”. Because this iteration changed a substantive ownership rule, the convergence policy still requires another fresh-context iteration before design-loop completion. +- [x] (2026-07-22 14:20Z) A further fresh-context reviewer independently audited the original formula contract, current exec-detail sources, operator ownership, degradation rules, migration, and corrected statement-local write-request lifecycle and concluded exactly: “当前版本无需必要修改”. No substantive design change was made, so the required convergence review is complete. - [ ] Implement Milestone 1: introduce v4 work units and expression-count helpers without changing production RUv2. - [ ] Implement Milestone 2: construct operator units from current runtime details and add only the permitted HashJoin state-row detail. - [ ] Implement Milestone 3: migrate all preview outputs atomically and add regression coverage. @@ -116,7 +117,7 @@ This plan deliberately does not implement production code. It freezes the model, ## Outcomes & Retrospective -The design keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(offset+count)` work without repeating Projection expression evaluation, removes IndexJoin request double charging, limits new runtime data to one HashJoin state-row count, and defines a no-guess migration. A later fresh-context audit also corrected explicit-transaction write-request ownership so pessimistic DML requests remain attached to their actual statement snapshot rather than being lost at COMMIT. A subsequent no-inherited-context reviewer found no further necessary modification. Implementation is intentionally pending and must happen in the separate worktree specified below; because this iteration changed a substantive ownership rule, one more fresh-context iteration is required by the convergence policy before the design phase is closed. +The design is stable after the required convergence review. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(offset+count)` work without repeating Projection expression evaluation, removes IndexJoin request double charging, limits new runtime data to one HashJoin state-row count, and defines a no-guess migration. A later fresh-context audit corrected explicit-transaction write-request ownership so pessimistic DML requests remain attached to their actual statement snapshot rather than being lost at COMMIT. After that substantive correction, two successive no-inherited-context reviewers found no further necessary modification; the final review changed only this progress record. Implementation remains intentionally pending and must happen in the separate worktree specified below. ## Context and Orientation @@ -475,4 +476,4 @@ At milestone completion, the key internal interfaces should have these conceptua The exact private names may follow nearby conventions, but the semantics, data sources, one-Join-runtime-datum boundary, and de-duplication rules in this plan are mandatory. -Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, TopN owns `n*log(k)` with checked `k=offset+count`, and inner readers own IndexJoin request cost. HashJoin uses the sole permitted runtime addition to expose actual admitted hash-state rows instead of approximating them with all build rows. A later fresh-context audit corrected write-request ownership: explicit-transaction DML and COMMIT each charge only the write RPCs in their own finalized statement snapshot, preventing pessimistic DML requests from being lost. +Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, TopN owns `n*log(k)` with checked `k=offset+count`, and inner readers own IndexJoin request cost. HashJoin uses the sole permitted runtime addition to expose actual admitted hash-state rows instead of approximating them with all build rows. A later fresh-context audit corrected write-request ownership: explicit-transaction DML and COMMIT each charge only the write RPCs in their own finalized statement snapshot, preventing pessimistic DML requests from being lost. A final fresh-context convergence review found no further necessary change to formulas, sources, ownership, degradation, units, or migration. diff --git a/docs/note/planner/fts_score_special_column_design.md b/docs/note/planner/fts_score_special_column_design.md deleted file mode 100644 index 098a0d7e5f136..0000000000000 --- a/docs/note/planner/fts_score_special_column_design.md +++ /dev/null @@ -1,989 +0,0 @@ -# FTS / BM25 / Vector Score 特殊列映射设计 - -## 1. 背景 - -在全文检索和向量检索场景中，常见查询形态如下： - -```sql -select fts_match(title, 'database') as score -from t -where fts_match(title, 'database') -order by fts_match(title, 'database') desc; -``` - -或者： - -```sql -select vec_l2_distance(vec, '[1,2,3]') -from t -where ... -order by vec_l2_distance(vec, '[1,2,3]'); -``` - -这些函数有一个共同点： - -- 真正的计算必须发生在索引侧，而不是 TiDB 本地。 -- 上层 `SELECT`、`ORDER BY`、后续可能的 `TopN`，都需要复用这个计算结果。 -- 同一个函数可能在同一棵计划树里出现多次，所有出现点都必须映射到同一个返回值，而不是重复计算，也不能按“出现位置”猜测绑定关系。 - -当前 TiDB 在 vector distance 路径上已经有一套“索引返回特殊列，planner 重写父节点表达式”的成熟模式；FTS/BM25 也应当沿用同样的范式。 - -本设计文档讨论： - -- 如何在当前代码结构下引入 `id = -2050` 的 score 特殊列。 -- 如何确保“同一个函数出现多次时，全部正确映射到同一个值”。 -- 如何分阶段落地，先做单读，再考虑回表和索引侧 TopK。 - -## 2. 当前实现概况 - -### 2.1 FTS builtin 的语义 - -当前 FTS builtin 的返回类型就是 `ETReal`： - -- `pkg/expression/builtin_fts.go:113` -- `pkg/expression/builtin_fts.go:187` - -这意味着： - -- FTS 表达式本来就不是纯布尔函数。 -- `WHERE fts_match(...)` 只是上层通过 `IS TRUE` 语义把 `real` 转成布尔条件。 -- 这也正是后续 score / BM25 的预留口。 - -`expr_to_pb.go` 中已经有对应注释： - -- `pkg/expression/expr_to_pb.go:276` - -注释明确写了：TiDB 的 FTS functions 返回 float，是为了潜在的 BM25 score 场景。 - -### 2.2 TiCI FTS path 当前只支持 filter，不支持 score - -`DataSource.analyzeTiCIIndex()` 在 `PredicatePushDown` 阶段尝试构造 TiCI FTS path： - -- `pkg/planner/core/operator/logicalop/logical_datasource.go:173` -- `pkg/planner/core/operator/logicalop/logical_datasource.go:656` - -最终在 `buildTiCIFTSPathAndCleanUp()` 里构建 `FtsQueryInfo`： - -- `pkg/planner/core/operator/logicalop/logical_datasource.go:902` -- `pkg/planner/core/operator/logicalop/logical_datasource.go:951` - -当前固定写死： - -```go -QueryType: tipb.FTSQueryType_FTSQueryTypeNoScore -``` - -因此当前 path 本质上是： - -- TiCI 负责做 FTS filter -- TiDB 不接收 score 列 - -### 2.3 当前 validation 直接禁止 SELECT / ORDER BY 中使用 FTS - -当前逻辑规则 `ftsFuncValidation` 直接禁止： - -- `SELECT fts_match(...)` -- `ORDER BY fts_match(...)` - -对应位置： - -- `pkg/planner/core/rule_ftsfunc_validation.go:44` -- `pkg/planner/core/rule_ftsfunc_validation.go:52` - -这也是为什么现在用户写： - -```sql -select fts_match() from ... -where fts_match() -order by fts_match() -``` - -根本进不到后续物理计划阶段。 - -### 2.4 当前物理路径选择也直接拒绝 “FTS path + sort property” - -即便放开 validation，当前物理路径选择仍会把这类计划判 invalid： - -- `pkg/planner/core/find_best_task.go:2275` - -代码含义是： - -- 只要 `candidate.path.FtsQueryInfo != nil` -- 且当前父节点请求了 sort property -- 直接放弃该 path - -这意味着 V1 方案至少要允许： - -- 无序 TiCI scan -- TiDB root sort / root TopN - -不然 `ORDER BY score` 仍然无法落地。 - -### 2.5 TiCI 当前 row layout 对尾部列顺序有强假设 - -TiCI index scan 当前的特殊 row layout 在： - -- `pkg/planner/core/operator/physicalop/physical_index_scan.go:453` - -其尾部逻辑为： - -- 可选 `ExtraPhysTblID` -- 必定追加 `ExtraVersion` - -executor 侧又依赖两个重要假设： - -- `pkg/executor/builder.go:4574` -- `pkg/executor/distsql.go:796` - -具体假设是： - -- 最后一列一定是 `_tidb_mvcc_version` -- 如果有 `ExtraPhysTblID`，它被认为在 version 前一个位置 - -因此 `-2050` score 列如果要插入 TiCI 返回 schema，必须避开这两个假设。 - -### 2.6 vector distance 已经有成熟的“特殊列映射”范式 - -vector distance 优化的核心逻辑位于： - -- `pkg/planner/core/task.go:883` -- `pkg/planner/core/task.go:1011` - -尤其是 `tryReturnDistanceFromIndex()`： - -- 发现 index 侧已有 distance -- 注入一个特殊列 `VirtualColVecSearchDistanceID` -- 把上层 `TopN/Projection` 中的函数重写成这个列 - -这正是 FTS/BM25 score 需要复用的基本范式。 - -## 3. 问题本质 - -这里真正要解决的问题不是“传一个 `-2050` 列”本身，而是： - -1. planner 必须识别哪些表达式需要复用 score。 -2. planner 必须判断哪些表达式在语义上是同一个函数。 -3. planner 必须保证这些语义等价表达式全部绑定到同一个特殊列。 -4. executor 必须稳定接收到该列，并且列顺序不能破坏现有 TiCI 假设。 - -如果只做“索引返回一个 `-2050` 列”但不做语义绑定，会出现以下问题： - -- `SELECT` 和 `ORDER BY` 中各自都有一个 `fts_match()`，TiDB 不知道它们应不应该共用同一个返回值。 -- 如果存在 rewrite，文本不同但语义等价的表达式可能无法匹配。 -- 如果靠出现顺序绑定，一旦 projection / sort / topN 形态变化，绑定就会漂移。 - -因此，绑定必须基于表达式语义，而不是文本或位置。 - -## 4. 设计目标 - -### 4.1 功能目标 - -- 支持 `SELECT score_expr` -- 支持 `ORDER BY score_expr` -- 支持同一个 score expr 在同一棵计划树里重复出现多次 -- 所有重复出现点都映射到同一个 `-2050` 返回列 -- 避免 TiDB 本地重新执行 FTS/BM25/vector score - -### 4.2 非目标 - -V1 不追求： - -- 同一个 `DataSource` 上多个不同 score expr 同时返回 -- `JOIN ON`、`GROUP BY`、`HAVING`、window 中使用 score -- 回表路径 `IndexLookUpReader` 上直接输出 score -- TiCI index side 的 score TopK pushdown - -### 4.3 兼容性目标 - -- 不污染 `DataSource` 的逻辑 schema -- 不破坏 `ExtraVersion` 在最后一列的约定 -- 不破坏现有 `ExtraPhysTblID` 假设 -- 不影响普通 TiCI FTS filter-only 路径 - -## 5. 总体方案 - -总体采用两层结构： - -- 逻辑层：放松限制，但不改逻辑 schema，只保留 shape 校验。 -- 物理层：在 `Attach2Task` 阶段完成 score 绑定、特殊列注入、父节点表达式重写。 - -更具体地说： - -1. 新增特殊列 `ExtraFTSScoreID = -2050` -2. 当物理父节点发现自己消费了 FTS score expr 时： - - 向下定位单读 TiCI path - - 判断这些 expr 是否都语义等价 - - 判断它们是否与 index access cond 中的 FTS expr 语义一致 - - 成功则把 `FtsQueryInfo.QueryType` 切成 `WithScore` - - 向 `PhysicalIndexScan` / `PhysicalIndexReader` 注入 score 列 - - 把父节点表达式中的所有等价 `fts_match()` 重写成 `Column{-2050}` - -这样： - -- 同一个函数出现多次，不再重复求值 -- 也不需要在 TiDB 本地 eval FTS builtin -- 所有映射都绑定到同一个 score 列 - -## 6. 为什么绑定放在物理阶段，而不是逻辑阶段 - -这是本方案的核心取舍。 - -### 6.1 逻辑阶段可见信息不足 - -`analyzeTiCIIndex()` 运行于 `PredicatePushDown`： - -- `pkg/planner/core/operator/logicalop/logical_datasource.go:173` - -当时 `DataSource` 只知道： - -- `PushedDownConds` -- 以及能否构造 TiCI FTS path - -它并不知道： - -- 上层 `Projection` 是否真的要输出 score -- 上层 `Sort/TopN` 是否要消费 score -- 最终是否走单读还是双读 - -### 6.2 逻辑 schema 改写侵入面过大 - -如果在逻辑阶段把 score 列塞进 `DataSource.Schema()`，会牵涉： - -- `PruneColumns` -- index covering -- single/double read 判定 -- 各类 rule 对 schema 的假设 - -这会显著放大改动范围。 - -### 6.3 物理阶段已经有现成范式 - -vector distance 在物理阶段完成： - -- 特殊列注入 -- 父节点表达式替换 - -而且已经证明这样做能稳定工作： - -- `pkg/planner/core/task.go:1011` - -因此 FTS/BM25 score 沿用相同范式是更自然的。 - -## 7. V1 范围与边界 - -V1 明确只支持： - -- 单个 `DataSource` -- 单个 distinct score expr -- `PhysicalIndexReader` / 单读 TiCI path -- `Projection`、`Sort`、`TopN` 消费 score - -V1 不支持： - -- `IndexLookUpReader` -- 多个不同 score expr -- 没有对应 access expr 的纯 score 查询 -- JOIN / AGG / HAVING / WINDOW / CTE - -### 7.1 为什么 V1 只做单读 - -`PhysicalIndexReader` 的数据流更简单： - -- index side 返回什么 -- reader 就可以直接输出什么 - -而 `PhysicalIndexLookUpReader` 的 schema 默认是 table plan schema： - -- `pkg/planner/core/operator/physicalop/physical_indexlookup_reader.go:221` - -它不会自动把 index side 额外列带到最终输出，需要显式把 score 从 index rows 穿过 lookup task 再拼回最终 row。复杂度高很多，适合第二阶段处理。 - -## 8. 特殊列建模 - -### 8.1 model 常量 - -建议在 `pkg/meta/model/table.go` 中新增： - -```go -const ExtraFTSScoreID int64 = -2050 -var ExtraFTSScoreName = ast.NewCIStr("_tidb_fts_score") -``` - -位置建议靠近： - -- `ExtraVersionID` -- `VirtualColVecSearchDistanceID` - -理由： - -- 它和 `ExtraVersionID` 一样，会进入 PB 和 executor 特殊列路径 -- 它不像 vector distance 那样纯粹局限于 table scan 场景 - -### 8.2 ColumnInfo helper - -建议在 `pkg/meta/model/column.go` 中新增： - -```go -func NewExtraFTSScoreColInfo() *ColumnInfo -``` - -字段建议： - -- `ID: ExtraFTSScoreID` -- `Name: ExtraFTSScoreName` -- `Type: mysql.TypeDouble` -- `Flag: 0` 或按需要设置 `NotNullFlag` -- binary charset / collation - -### 8.3 为什么类型建议用 DOUBLE - -当前 FTS builtin 的返回类型是 `ETReal`，更接近 `DOUBLE`，而不是强行缩成 `FLOAT`： - -- `pkg/expression/builtin_fts.go:113` -- `pkg/expression/builtin_fts.go:187` - -如果 score 列类型与 builtin 返回类型不一致，会产生潜在问题： - -- projection type 不一致 -- sort 比较类型不一致 -- PB 侧列类型和父节点表达式预期不一致 - -因此 V1 直接使用 `mysql.TypeDouble` 更稳。 - -## 9. 语义等价判定 - -### 9.1 判定工具 - -必须复用现有表达式语义等价工具： - -- `pkg/expression/scalar_function.go:591` -- `pkg/expression/scalar_function.go:600` - -也就是： - -- `CanonicalHashCode()` -- `ExpressionsSemanticEqual()` - -不能做的事情： - -- 不能用 `expr.String()` -- 不能用文本匹配 -- 不能按出现位置绑定 - -### 9.2 归一化步骤 - -对所有候选 score expr，需要做统一归一化，伪代码如下： - -```go -func normalizeFTSScoreExpr(expr expression.Expression) expression.Expression { - // 1. 如果是 MATCH ... AGAINST，先 rewrite 到内部 helper - // 2. 如果外层包了 IsTruthWithNull / IsTruthWithoutNull，把壳去掉 - // 3. 返回规范化后的 expr -} -``` - -原因： - -- `WHERE fts_match(...)` 往往会包 `IS TRUE` -- `SELECT fts_match(...)` 则通常不会 -- 如果不剥壳，这两个位置虽然语义同源，但 hash 可能不一致 - -归一化逻辑应尽量和： - -- `pkg/planner/core/operator/logicalop/logical_datasource.go:875` -- `pkg/expression/expr_to_pb.go:276` - -保持同一语义。 - -### 9.3 V1 的 distinct expr 限制 - -V1 要求： - -- 同一条 path 上所有 score consumer 归一化后，只能得到 1 个 distinct hash - -支持： - -```sql -select f(), f() -from t -where f() -order by f(); -``` - -不支持： - -```sql -select f1(), f2() -from t -where f1() and f2(); -``` - -后一类在 V1 直接报错： - -- `multiple distinct FTS score expressions are not supported yet` - -## 10. 物理阶段的绑定与重写 - -### 10.1 关键思想 - -score 的绑定不在逻辑层做，而在以下父节点的 `Attach2Task` 中做： - -- `PhysicalProjection` -- `PhysicalSort` -- `PhysicalTopN` - -对应位置： - -- `pkg/planner/core/task.go:1472` -- `pkg/planner/core/task.go:864` -- `pkg/planner/core/task.go:1294` - -做法是： - -1. 父节点扫描自己的表达式，看是否包含 FTS score consumer。 -2. 如果没有，走原逻辑。 -3. 如果有，强制把 child task 转成 root task。 -4. 在 root task 中向下找到 `PhysicalIndexReader + PhysicalIndexScan`。 -5. 检查是否为单读 TiCI FTS path。 -6. 成功则注入 `-2050`，并把父节点中所有等价函数都改成同一个 score 列。 - -### 10.2 为什么强制先转 root task - -V1 中，如果父节点消费了 score，我建议不要尝试把这个节点继续 pushdown 到 cop 侧，而是： - -- 直接转 root -- 在 root 上消费 score 列 - -理由： - -- 改动更小 -- 避免和现有 projection/topN pushdown 逻辑交叉 -- 避免出现 “cop 侧想 push 一个 FTS function，但我们其实只想要 score 列”的混合状态 - -因此 V1 的策略很简单： - -- 发现 score consumer -- 先 root 化 -- 再 rewrite 为列 - -## 11. 单读路径上的 plan shape - -`CopTask.ConvertToRootTask()` 在单读 index path 下，会构造： - -- `PhysicalIndexReader` - -对应位置： - -- `pkg/planner/core/operator/physicalop/task_base.go:485` -- `pkg/planner/core/operator/physicalop/task_base.go:544` - -也就是说，单读 TiCI path 的根形态通常是： - -```text -Projection / Sort / TopN - └─ IndexReader - └─ IndexScan(TiCI) -``` - -因此 V1 的核心 helper 可以明确只处理这条链： - -- 向下找到 `PhysicalIndexReader` -- 再找到 `PhysicalIndexScan` - -如果不是这条链，V1 直接返回“不支持”。 - -## 12. 建议新增的 helper - -### 12.1 找路径 helper - -建议新增： - -```go -func findSingleReadTiCIFTSReader(plan base.PhysicalPlan) (*physicalop.PhysicalIndexReader, *physicalop.PhysicalIndexScan, bool) -``` - -约束： - -- 只接受单读 `PhysicalIndexReader` -- 允许中间存在一层轻量 wrapper，但最终必须能定位到唯一 `PhysicalIndexScan` -- `scan.StoreType == kv.TiCI` -- `scan.FtsQueryInfo != nil` - -### 12.2 score consumer 收集 - -```go -func collectFTSScoreExprs(exprs []expression.Expression) []expression.Expression -``` - -用于： - -- `Projection.Exprs` -- `TopN.ByItems` -- `Sort.ByItems` - -### 12.3 归一化与 distinct 检查 - -```go -func buildFTSScoreBinding( - scoreExprs []expression.Expression, - accessConds []expression.Expression, - parserType model.FullTextParserType, -) (*FTSScoreBinding, error) -``` - -该函数应完成： - -- consumer expr 归一化 -- access cond expr 归一化 -- consumer distinct hash 统计 -- consumer 是否出现在 access cond hash 集合中的校验 - -### 12.4 score 列注入 - -```go -func ensureFTSScoreInjected( - reader *physicalop.PhysicalIndexReader, - scan *physicalop.PhysicalIndexScan, - binding *FTSScoreBinding, -) (*expression.Column, error) -``` - -职责： - -- 若 scan 已有 `-2050`，复用 -- 否则向 scan schema 注入 -- 同步向 reader schema / outputColumns 注入 -- 返回“供父节点 rewrite 使用的 reader schema 中的 score 列” - -### 12.5 父节点表达式重写 - -```go -func rewriteFTSScoreExpr( - expr expression.Expression, - binding *FTSScoreBinding, - scoreCol *expression.Column, -) expression.Expression -``` - -规则： - -- 若 `normalize(expr)` 与 `binding.ExprHash` 相同，替换为 `scoreCol.Clone()` -- 若是 scalar function，递归处理参数 -- 否则原样返回 - -## 13. `ensureFTSScoreInjected()` 的精确行为 - -这是整套方案最关键的实现点之一。 - -### 13.1 先检查 scan schema 中是否已存在 score 列 - -如果已存在： - -- 直接复用 -- 保证多个父节点共用同一个 injected score 列 - -这正是“同一个函数可能出现多次，他们要都能正确映射到那个值”的核心保证。 - -### 13.2 注入到 `PhysicalIndexScan.Schema()` - -当前 TiCI row layout 在： - -- `pkg/planner/core/operator/physicalop/physical_index_scan.go:459` - -V1 要求插入规则为： - -- 没有 `ExtraPhysTblID`：`[..., score, ExtraVersion]` -- 有 `ExtraPhysTblID`：`[..., score, ExtraPhysTblID, ExtraVersion]` - -不能做成： - -- `[..., ExtraPhysTblID, score, ExtraVersion]` -- `[..., ExtraPhysTblID, ExtraVersion, score]` - -原因： - -- [builder.go:4578](/DATA/disk4/yiding/gocode/tidb/pkg/executor/builder.go#L4578) 假定 `ExtraPhysTblID` 在 version 前一列 -- [distsql.go:800](/DATA/disk4/yiding/gocode/tidb/pkg/executor/distsql.go#L800) 假定最后一列是 version - -### 13.3 注入到 `PhysicalIndexReader.Schema()` 与 `OutputColumns` - -`PhysicalIndexReader` 默认 schema 不是 index scan schema，而是 `DataSourceSchema`： - -- `pkg/planner/core/operator/physicalop/physical_index_reader.go:77` - -因此仅仅修改 `scan.Schema()` 不够，还必须： - -- 在 `reader.Schema()` 中 append 一个 score 列 -- 在 `reader.OutputColumns` 中 append 对应列 - -不然 executor builder 仍然不会请求这个列： - -- `pkg/executor/builder.go:4379` - -### 13.4 `scoreCol.Index` 的处理 - -`IndexReader` 在执行前会对 `OutputColumns` 做 `ResolveIndices()`： - -- `pkg/planner/core/operator/physicalop/physical_index_reader.go:185` - -因此建议： - -- 注入时先构造列对象并挂入 schema -- `ResolveIndices()` 自然会把它解析到 scan schema 中的实际位置 -- 避免手工写死 index - -## 14. `ToPB()` 与 PB schema - -在 `PhysicalIndexScan.ToPB()` 中，目前只识别： - -- `ExtraHandleID` -- `ExtraPhysTblID` -- `ExtraVersionID` - -位置： - -- `pkg/planner/core/operator/physicalop/physical_index_scan.go:628` - -V1 需要新增： - -```go -else if col.ID == model.ExtraFTSScoreID { - columns = append(columns, model.NewExtraFTSScoreColInfo()) -} -``` - -否则： - -- `-2050` 会被误当成普通 table column 去查 `FindColumnInfoByID` -- 直接失败 - -## 15. `Projection` 的处理细节 - -### 15.1 触发条件 - -`PhysicalProjection` 中只要 `Exprs` 里存在 FTS score consumer，就走特殊路径。 - -### 15.2 处理流程 - -1. `t := tasks[0].Copy()` -2. `t = t.ConvertToRootTask(...)` -3. 在 root plan 中定位 `IndexReader + IndexScan` -4. 构建 `FTSScoreBinding` -5. `scan.FtsQueryInfo.QueryType = WithScore` -6. 注入 score 列 -7. 将 `p.Exprs` 中所有等价 score expr 替换为 `scoreCol` -8. 再正常 `attachPlan2Task(p, t)` - -### 15.3 替换后的收益 - -替换完成后： - -- `Projection` 不再包含 FTS builtin -- 只包含一个普通列 - -因此： - -- executor 不会本地 eval FTS function -- 多个 `SELECT f()` 会直接复用同一个 score 列 - -## 16. `Sort` / `TopN` 的处理细节 - -### 16.1 `Sort` - -`PhysicalSort` 的逻辑和 `Projection` 基本一致： - -- 发现 `ByItems` 中存在 score expr -- root 化 -- 绑定、注入、rewrite -- 最终 root sort 按 score 列排序 - -### 16.2 `TopN` - -`TopN` 比 `Sort` 更复杂，因为它当前有一套 pushdown 和 heavy-function 优化逻辑： - -- `pkg/planner/core/task.go:883` -- `pkg/planner/core/task.go:1294` - -V1 建议不要尝试复用 `getPushedDownTopN()` 去做 FTS score pushdown，而是： - -- 只要 `TopN.ByItems` 包含 score expr -- 直接转 root -- rewrite 成 score 列 -- root 上执行 TopN - -这样可以规避两个复杂问题： - -- TiCI score TopK pushdown 语义 -- 与现有 vector heavy-function 优化逻辑交织 - -### 16.3 为什么 V1 不做 TiCI score TopK pushdown - -当前 `TryToPassTiCITopN()`： - -- `pkg/planner/core/operator/physicalop/physical_index_scan.go:726` - -只支持 hybrid index sort columns 与普通列排序匹配，不支持函数排序。把 score TopK pushdown 一起做，会引入更多协议和执行语义约束。V1 不需要承担这部分复杂度。 - -## 17. `find_best_task` 的调整 - -当前： - -- `pkg/planner/core/find_best_task.go:2275` - -会把任意 `FtsQueryInfo != nil` 且带 sort property 的 path 直接判 invalid。 - -V1 建议改为： - -- 不再直接 invalid -- 但也不宣称该 path 满足 sort property -- 让 planner 正常走“无序 scan + root sort/root topN” - -这样： - -- `ORDER BY score` 至少可以执行 -- 同时不会错误地把 score 排序当成 index-order 能力 - -## 18. validation 的调整 - -当前 validation 过于保守，全部禁止了 `SELECT` / `ORDER BY`。 - -V1 建议调整成： - -- `Projection/Sort/TopN`：不再一刀切报错 -- `Join/Aggregation/Window/GroupBy/Having`：继续禁止 -- `Selection`：仍然要求 FTS expr 必须是可下推 access cond，不允许当普通 TiDB 本地表达式 - -也就是说： - -- shape 约束继续保留 -- score 绑定是否成功，不在 validation 中决定 -- 绑定失败在物理计划阶段报更具体的错误 - -## 19. 错误模型 - -建议在物理绑定阶段报下列明确错误： - -- `FTS score expression must also appear in a pushed-down FTS predicate` -- `multiple distinct FTS score expressions are not supported yet` -- `FTS score is only supported on single-read TiCI index path` -- `FTS score in JOIN/GROUP BY/HAVING/WINDOW is not supported` - -这样比当前统一报： - -- `cannot be used in SELECT fields` -- `ORDER BY is not supported` - -要精确得多。 - -## 20. 为什么这套设计能保证“同一个函数多次出现都映射到那个值” - -这里把关键逻辑单独说明。 - -### 20.1 重复出现点不会各自分配新列 - -score 列的注入由 `ensureFTSScoreInjected()` 统一负责： - -- 若 scan schema 已经有 `-2050` -- 直接复用 - -因此同一条 root plan 上： - -- 第一个消费者会触发注入 -- 后续消费者只会复用同一个 injected 列 - -### 20.2 绑定不是按位置，而是按 canonical hash - -所有 consumer expr 会先归一化，然后比对： - -- `CanonicalHashCode()` - -因此： - -- `SELECT` 中的 `fts_match()` -- `ORDER BY` 中的 `fts_match()` -- 同一个 projection 中多次出现的 `fts_match()` - -只要语义等价，都会命中同一个 hash。 - -### 20.3 rewrite 发生在每个父节点自己的表达式树里 - -`Projection`、`Sort`、`TopN` 分别对自己的表达式树做 rewrite： - -- 只要某个节点语义等价于绑定 expr -- 直接替换成 `scoreCol` - -因此“同一个函数多次出现”本质上只是一棵表达式树里命中了多次同一个规则，而不是需要维护“第几个出现点对第几个返回列”的 fragile mapping。 - -## 21. 为什么 V1 不直接支持 `IndexLookUpReader` - -`IndexLookUpReader` 的 schema 默认是 table plan schema： - -- `pkg/planner/core/operator/physicalop/physical_indexlookup_reader.go:221` - -而 index side 额外返回列会先落进： - -- `lookupTableTask.idxRows` -- `pkg/executor/distsql.go:82` - -若要让 score 穿过 lookup，必须再做以下事情： - -1. index request 把 score 一起返回 -2. score 保存在 `idxRows` -3. table worker 回表后，按 handle 把 score 拼回最终 row -4. `IndexLookUpReader.Schema()` 扩出 score 列 - -这会同时涉及： - -- planner -- index worker -- table worker -- row merge - -因此明显应分期处理，而不是和单读路径一起做。 - -## 22. V2 方向：双读与回表 - -后续如果要支持： - -```sql -select non_index_col, fts_match(...) -from t -where fts_match(...); -``` - -则需要进入 V2。 - -V2 的最小实现路径： - -- 让 score 作为 index-side 辅助列进入 `lookupTableTask.idxRows` -- 建立 `handle -> score` 关联 -- table side 回表后，把 score 和 table row 合并 -- 最终 `IndexLookUpReader` 输出扩展 schema - -这部分与当前 partition by-items / keep-order 的 index 辅助列处理模式相近，但不是本设计文档的 V1 范围。 - -## 23. 备选方案与取舍 - -### 23.1 备选方案 A：在逻辑层直接扩 `DataSource.Schema()` - -优点： - -- 上层所有节点天然能看到 score 列 - -缺点： - -- 侵入逻辑 rule 太多 -- 会影响 covering / prune columns / single scan 判定 -- 风险大 - -结论： - -- 不采用 - -### 23.2 备选方案 B：每个出现点都单独分配一个特殊列 - -优点： - -- 实现直观 - -缺点： - -- 同一个函数出现两次会返回两列，浪费 -- executor / schema 管理复杂 -- 很难保证等价表达式共享一个值 - -结论： - -- 不采用 - -### 23.3 备选方案 C：先做统一的 “index-returned computed column” 抽象 - -优点： - -- 抽象最干净 - -缺点： - -- 牵涉 vector / FTS / hybrid index 多条线 -- 一次性改动过大 - -结论： - -- 长期可考虑 -- 当前以 FTS score 特化实现为主 - -## 24. 测试建议 - -### 24.1 planner / explain 用例 - -建议增加： - -- `select f(), f() from t where f()` -- `select f() from t where f() order by f()` -- `select f()+1 from t where f()` -- `select f1(), f2() from t where f1() and f2()` 报错 -- `select f() from t where other_cond` 报错 - -其中 `f()` 代表 FTS score expr。 - -### 24.2 explain 验证 - -验证点： - -- 计划中出现 `_tidb_fts_score` -- 不再出现 TiDB 侧 eval 的 FTS builtin -- TiCI path 的 `FtsQueryInfo.QueryType` 为 `WithScore` - -### 24.3 执行验证 - -验证： - -- `SELECT/ORDER BY` 同时使用同一个 score expr 时结果正确 -- 重复出现的 score expr 数值一致 -- `ORDER BY score DESC LIMIT N` 结果稳定 - -## 25. 建议的实施顺序 - -### Phase 1：准备工作 - -1. 新增 `ExtraFTSScoreID = -2050` -2. 新增 `NewExtraFTSScoreColInfo()` -3. `PhysicalIndexScan.ToPB()` 识别 `-2050` - -### Phase 2：放开限制 - -1. 调整 `ftsFuncValidation` -2. 放开 `find_best_task.go` 中 FTS + sort property 的 blanket ban - -### Phase 3：单读路径绑定与重写 - -1. 在 `task.go` 增加 normalize / binding / inject / rewrite helpers -2. 在 `Projection/Sort/TopN` 的 `Attach2Task` 中接入 -3. 仅支持 `PhysicalIndexReader` - -### Phase 4：测试与收尾 - -1. explain / planner case -2. execution case -3. 错误模型 case - -### Phase 5：V2 - -1. `IndexLookUpReader` -2. score 透传 `lookupTableTask` -3. 可选的 TiCI TopK pushdown - -## 26. 最终结论 - -对于当前代码库，最合适的实现不是“简单传一个 `-2050` 列”，而是： - -- 在单读 TiCI FTS path 上注入 `-2050` score 特殊列 -- 用 canonical hash 做语义绑定 -- 把 `Projection/Sort/TopN` 中所有等价的 score expr 全部重写到同一个列 - -这样可以最小改动地解决两个核心问题： - -- score 必须在索引侧执行 -- 同一个函数可能出现多次，所有出现点都要正确映射到同一个值 - -同时，这套设计与现有 vector distance 的“特殊列回填”模式一致，易于复用和扩展，也为后续 BM25、多 score expr、回表路径和 TiCI TopK pushdown 留出了清晰演进路径。 From 0310c5f69d0cd60442b5f9c98e0302bcd459fd4f Mon Sep 17 00:00:00 2001 From: Yiding Date: Wed, 22 Jul 2026 16:16:38 +0800 Subject: [PATCH 21/25] planner: implement preview RU v4 resource formula --- .../references/distsql-case-map.md | 2 +- .../references/executor-case-map.md | 8 +- .../references/metrics-case-map.md | 2 +- .../references/planner-case-map.md | 2 +- .../references/session-case-map.md | 2 +- .../references/util-case-map.md | 4 +- .../2026-07-01-read-billing-demo-ru-model.md | 4 + .../2026-07-10-preview-ru-tidb-kv-mutation.md | 10 + ...-07-22-preview-ru-resource-formula-plan.md | 214 +++- pkg/distsql/select_result_test.go | 23 + pkg/executor/adapter.go | 20 +- pkg/executor/adapter_internal_test.go | 52 +- pkg/executor/adapter_test.go | 66 + pkg/executor/explain_test.go | 239 +++- pkg/executor/join/hash_join_stats.go | 45 +- pkg/executor/join/hash_join_v1.go | 2 + pkg/executor/join/hash_join_v2.go | 24 +- pkg/executor/join/hash_table_v1.go | 12 + pkg/executor/join/hash_table_v1_test.go | 6 + pkg/executor/join/join_stats_test.go | 24 + pkg/infoschema/tables.go | 6 +- pkg/metrics/explain_ru.go | 32 +- pkg/metrics/metrics_internal_test.go | 46 +- pkg/planner/core/BUILD.bazel | 1 + pkg/planner/core/common_plans_test.go | 820 +++++++++++- pkg/planner/core/explain_ru.go | 1109 ++++++++++------- pkg/session/tidb_test.go | 35 + pkg/util/execdetails/execdetails_test.go | 34 + pkg/util/execdetails/runtime_stats.go | 42 +- pkg/util/stmtsummary/read_billing.go | 6 + pkg/util/stmtsummary/read_billing_test.go | 92 +- pkg/util/stmtsummary/v2/record_test.go | 8 +- 32 files changed, 2375 insertions(+), 617 deletions(-) diff --git a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md index b7df4b7e2e64b..7b5f37c91c3a3 100644 --- a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md @@ -14,7 +14,7 @@ - `pkg/distsql/distsql_test.go` - Tests normal select. - `pkg/distsql/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/distsql/request_builder_test.go` - Tests table handles to KV ranges. -- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership, multi-response aggregation, independent expected-task coverage, and close-time stats-only handling that cannot replay stale summaries. +- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership/provenance counts, multi-response aggregation, independent expected-task coverage, and close-time stats-only handling that cannot replay stale summaries. ## pkg/distsql/context diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index d0356aadec4f6..19aabd6b48446 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -9,7 +9,7 @@ ## pkg/executor ### Tests -- `pkg/executor/adapter_internal_test.go` - executor: Tests private adapter runtime accounting, including preview RU General Log gates, reduced unit schema, aggregation, and deterministic ordering. +- `pkg/executor/adapter_internal_test.go` - executor: Tests private adapter runtime accounting, including preview RU General Log gates, mutation `cpu_work` plus raw-unit provenance, DML/source/side aggregation, serialization, and deterministic ordering. - `pkg/executor/adapter_test.go` - executor: Tests statement adapter behavior, including RU v2 finalization and self-describing preview RU General Log completion after lazy SELECT close, DML/early errors, compile-error suppression, cursor-detach gating, repeated finish calls, and EXPLAIN ANALYZE DML commit errors. - `pkg/executor/analyze_test.go` - executor: Tests analyze index extract top n. - `pkg/executor/analyze_utils_test.go` - executor: Tests get analyze panic err. @@ -30,7 +30,7 @@ - `pkg/executor/executor_failpoint_test.go` - executor: Tests TiDB last txn info commit mode. - `pkg/executor/executor_pkg_test.go` - executor: Tests build KV ranges for index join without CWC. - `pkg/executor/executor_required_rows_test.go` - executor: Tests limit required rows. -- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including preview RU v3 TiKV cop diagnostics, DML mutation/write operators, exact index and optimistic-retry mutation counts, join/sort/read-tree preservation, operator-level unavailable-TiKV status rows, restricted/feature-off isolation, RUv2 A/B equivalence, explicit-COMMIT ownership, and diagnostic compatibility. +- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including production-default uncalibrated v4 units and absent totals, distinct read/write request output units, six pushed TiKV operator classes (including an explicitly ordered StreamAgg) with native Unistore's incomplete scan-width fail-closed behavior, bounded scan/transport/point-lookup failures, all raw DML mutation diagnostics with gated mutation `cpu_work`, statement-summary/metrics output versions, restricted/feature-off isolation, and legacy v3 behavior retained as explicitly superseded coverage. - `pkg/executor/explain_unit_test.go` - executor: Tests explain analyze invoke next and close. - `pkg/executor/explainfor_test.go` - executor: Tests explain for. - `pkg/executor/grant_test.go` - executor: Tests grant global. @@ -189,12 +189,12 @@ - `pkg/executor/join/anti_semi_join_probe_test.go` - executor/join: Tests anti-semi join basic. - `pkg/executor/join/bench_test.go` - executor/join: Tests hash table build. - `pkg/executor/join/concurrent_map_test.go` - executor/join: Tests concurrent map. -- `pkg/executor/join/hash_table_v1_test.go` - executor/join: Tests hash row container. +- `pkg/executor/join/hash_table_v1_test.go` - executor/join: Tests hash row container, including V1 NAAJ null-bucket rows in HashJoin state accounting. - `pkg/executor/join/hash_table_v2_test.go` - executor/join: Tests hash table size. - `pkg/executor/join/inner_join_probe_test.go` - executor/join: Tests inner join probe basic. - `pkg/executor/join/inner_join_spill_test.go` - executor/join: Tests inner join spill basic. - `pkg/executor/join/join_row_table_test.go` - executor/join: Tests heap object can move. -- `pkg/executor/join/join_stats_test.go` - executor/join: Tests hash join runtime stats. +- `pkg/executor/join/join_stats_test.go` - executor/join: Tests HashJoin V1/V2 runtime stats, valid-key state-row accounting, overflow rejection, and clone/merge preservation. - `pkg/executor/join/join_table_meta_test.go` - executor/join: Tests join table meta key mode. - `pkg/executor/join/joiner_test.go` - executor/join: Tests required rows. - `pkg/executor/join/left_outer_anti_semi_join_probe_test.go` - executor/join: Tests left-outer anti-semi join probe basic. diff --git a/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md b/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md index 648fe5daf2299..175902cfe4f0d 100644 --- a/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/metrics-case-map.md @@ -10,7 +10,7 @@ ### Tests - `pkg/metrics/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/metrics/metrics_internal_test.go` - Tests retained labels and preview-RU bounded metric propagation, including measured zero base-unit samples. +- `pkg/metrics/metrics_internal_test.go` - Tests retained labels and preview-RU bounded metric propagation, including distinct read/write request units, dimension-qualified mutation `cpu_work`, raw mutation samples, and measured zero base-unit samples. - `pkg/metrics/metrics_test.go` - Tests metrics. ## pkg/metrics/common diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index bf8b89be6b1e8..c1bb64ff18f8b 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, weights, Sort/TopN algorithmic `order_work`, TiDB Join/Agg and guarded TiKV Agg diagnostic output shadows, TiKV component/child-row/scan-width input estimation and width barriers, DML mutation/write and explicit-COMMIT payload construction, pipelined payload suppression, and ancillary-work partial diagnostics. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, private v4 semantic weights and overflow rejection, independent read/write request units and weights, expression-slot counts for Agg/Join/Window families, Sort CPU work plus TopN input saturation/zero-count/checked bounds, UnionScan direct-child CPU work, reader transport attribution, real distsql ScanDetail ownership/provenance and fail-closed coverage, HashJoin state inputs, mutation normalization into dimension-qualified `cpu_work` with raw diagnostics, and statement-local DML/COMMIT request ownership. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/.agents/skills/tidb-test-guidelines/references/session-case-map.md b/.agents/skills/tidb-test-guidelines/references/session-case-map.md index 64df94f729bf7..7dc5396d53549 100644 --- a/.agents/skills/tidb-test-guidelines/references/session-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/session-case-map.md @@ -13,7 +13,7 @@ - `pkg/session/bootstrap_test.go` - session: Tests MySQL DB tables. - `pkg/session/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/session/session_test.go` - session: Tests get start mode. -- `pkg/session/tidb_test.go` - session: Tests domain-map basics, RUv2 isolation, all four preview KV mutation APIs including failed attempts, and pipelined COMMIT diagnostics. +- `pkg/session/tidb_test.go` - session: Tests domain-map basics, per-statement RUv2 isolation including pessimistic DML/COMMIT write-request snapshots, and all four preview KV mutation APIs including failed attempts. - `pkg/session/upgrade_test.go` - session: Tests upgrade to ver functions check. ## pkg/session/cursor diff --git a/.agents/skills/tidb-test-guidelines/references/util-case-map.md b/.agents/skills/tidb-test-guidelines/references/util-case-map.md index 6b1d1b380ae2b..1097da4ce1c53 100644 --- a/.agents/skills/tidb-test-guidelines/references/util-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/util-case-map.md @@ -215,7 +215,7 @@ ## pkg/util/execdetails ### Tests -- `pkg/util/execdetails/execdetails_test.go` - util/execdetails: Tests string. +- `pkg/util/execdetails/execdetails_test.go` - util/execdetails: Tests runtime-detail formatting, cop summary/task aggregation, and ScanDetail attachment provenance snapshots. - `pkg/util/execdetails/main_test.go` - Configures default goleak settings and registers testdata. ## pkg/util/expensivequery @@ -528,7 +528,7 @@ ### Tests - `pkg/util/stmtsummary/evicted_test.go` - util/stmtsummary: Tests map to evicted count datum. - `pkg/util/stmtsummary/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/util/stmtsummary/read_billing_test.go` - util/stmtsummary: Tests preview RU dimension aggregation, caps, and DML-kind separation. +- `pkg/util/stmtsummary/read_billing_test.go` - util/stmtsummary: Tests preview RU dimension aggregation, caps, distinct read/write request units, DML-kind separation for mutation `cpu_work` and raw units, and v4 detail retention with zero legacy convenience totals. - `pkg/util/stmtsummary/statement_summary_test.go` - util/stmtsummary: Tests set up. ## pkg/util/stmtsummary/v2 diff --git a/docs/design/2026-07-01-read-billing-demo-ru-model.md b/docs/design/2026-07-01-read-billing-demo-ru-model.md index 15a06e3d95bff..a163147645d23 100644 --- a/docs/design/2026-07-01-read-billing-demo-ru-model.md +++ b/docs/design/2026-07-01-read-billing-demo-ru-model.md @@ -1,5 +1,9 @@ # Read Billing Demo RU 模型与校准统计面设计 +> **当前模型说明（2026-07-22）：** 本文描述的是历史 v1-v3 模型，仅保留为迁移与校准背景。当前已实现的规范是 +> `docs/design/2026-07-22-preview-ru-resource-formula-plan.md` 中的 v4：使用语义化资源单位，三个旧 convenience totals +> 对 v4 恒为零，并且权重未校准时不发布 `total_preview_ru`。下文与此冲突的“当前”表述仅适用于 v3。 + - Author(s): TBD - Discussion PR: TBD - Tracking Issue: TBD diff --git a/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md b/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md index 8882457a2ec57..fdc60790dd8e1 100644 --- a/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md +++ b/docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md @@ -1,5 +1,15 @@ # Preview RU TiDB KV mutation operator +> **Current-model notice (2026-07-22):** the mutation lifecycle evidence in +> this document remains valid, but its v3 two-weight and commit-detail formulas +> are historical. The implemented v4 formula and statement-local DML/COMMIT +> request ownership are normative in +> `docs/design/2026-07-22-preview-ru-resource-formula-plan.md`. +> In that v4 contract, normalized mutation work is emitted as the shared +> `cpu_work` unit and identified by `site=tidb`, `op_class=kv_mutation`, +> `operator_kind=memdb_mutation`, `input_source=stmt_memdb_mutation_calls`, and +> `input_side=all`; it is not a separate weight-bearing unit. + ## Status and scope This document defines the write-side model implemented by the preview RU demo. diff --git a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md index d83316fabb57e..85b7f8b1b586a 100644 --- a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md +++ b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md @@ -8,9 +8,9 @@ Reference: `PLANS.md` at repository root. This plan must be maintained according The current preview RU demo uses an operator-specific matrix of fixed, row, byte, and ordering weights. That matrix is useful for collecting evidence, but it does not express the intended model directly: expression evaluation should consume one shared CPU weight, scans should consume one scan-byte weight, transport should consume network-byte and request weights, and stateful operators should add small hash-table or join-output terms. -After this plan is implemented, `EXPLAIN ANALYZE FORMAT='RU'`, the bounded metrics, the general-log detail, and the statement-summary detail tables will expose the same coefficient-free resource work units. A reader can reproduce every operator's preview RU with six named weights and can see exactly which current runtime detail supplied each input. The preview remains isolated from production RUv2 charging and resource-control reporting. +After this plan is implemented, `EXPLAIN ANALYZE FORMAT='RU'`, the bounded metrics, the general-log detail, and the statement-summary detail tables will expose the same coefficient-free resource work units. A reader can reproduce every operator's preview RU with seven named weights and can see exactly which current runtime detail supplied each input. The preview remains isolated from production RUv2 charging and resource-control reporting. -This plan deliberately does not implement production code. It freezes the model, field mappings, failure behavior, migration, and validation needed by the later implementation worktree. +This plan first froze the model, field mappings, failure behavior, migration, and validation, and now records the implementation and verification evidence from the implementation worktree. ## Progress @@ -24,10 +24,32 @@ This plan deliberately does not implement production code. It freezes the model, - [x] (2026-07-22 13:05Z) A later fresh-context completion audit found that explicit-transaction DML write RPCs are statement-local and cannot all be deferred to COMMIT; revised ownership to charge each DML and COMMIT from its own frozen `RUV2Metrics` snapshot. - [x] (2026-07-22 13:35Z) A new no-inherited-context reviewer inspected the corrected plan and current code and concluded exactly: “当前版本无需必要修改”. Because this iteration changed a substantive ownership rule, the convergence policy still requires another fresh-context iteration before design-loop completion. - [x] (2026-07-22 14:20Z) A further fresh-context reviewer independently audited the original formula contract, current exec-detail sources, operator ownership, degradation rules, migration, and corrected statement-local write-request lifecycle and concluded exactly: “当前版本无需必要修改”. No substantive design change was made, so the required convergence review is complete. -- [ ] Implement Milestone 1: introduce v4 work units and expression-count helpers without changing production RUv2. -- [ ] Implement Milestone 2: construct operator units from current runtime details and add only the permitted HashJoin state-row detail. -- [ ] Implement Milestone 3: migrate all preview outputs atomically and add regression coverage. -- [ ] After implementation, run the implementation Ready validation profile and record concise evidence here. +- [x] (2026-07-22 19:10Z) Implemented Milestone 1: introduced the private six-weight v4 container, semantic work units, centralized expression-slot counts, checked ordering arithmetic, and exact formula tests without changing production RUv2. +- [x] (2026-07-22 19:35Z) Implemented Milestone 2: constructed root/cop/read-transport/write units from existing frozen details and added only the narrow HashJoin `HashTableRows()` runtime interface with V1/V2 state accounting. +- [x] (2026-07-22 20:05Z) Implemented Milestone 3: migrated EXPLAIN, metrics, statement summary, General Log, version labels, legacy-total behavior, documentation, and focused regression tests atomically. +- [x] (2026-07-22 20:55Z) Completed the implementation Ready profile: `make bazel_prepare`, all six prescribed failpoint-wrapped package test sets, the tagged metrics test, `make lint`, and `git diff --check` passed; the HashJoin set additionally ran `TestHashRowContainer` for V1 NAAJ state coverage. +- [x] (2026-07-22 22:10Z) A completion audit found and fixed a v4 migration regression: the write constructor had stopped consulting the existing statement mutation snapshot's `Pipelined` flag. Pipelined DML and COMMIT now retain mutation evidence but fail the write-request component closed with `pipelined_tikv_payload_unsupported`; focused DML and COMMIT regressions cover the restored gate. +- [x] (2026-07-22 04:25+08:00) The final completion audit found that the ordering gate rejected residual scalar functions but did not prove that column `ByItems` belonged to the direct child schema. Sort/TopN now validate the executed flat child, including Projection expression/schema alignment, and a regression was observed failing before the fix and passing afterward. +- [x] (2026-07-22 04:30+08:00) Recompleted the Ready gate after the ordering fix: the full planner/core and executor target sets passed with failpoint cleanup, the earlier same-state join/session/statement-summary/metrics runs remained applicable, `make lint` passed after the code edit, and `git diff --check` plus API/range review found no remaining mismatch. +- [x] (2026-07-22 05:35+08:00) A requirement-by-requirement completion audit found that the private weight container omitted its specified immutable version, so a test-only calibrated set could publish a total under the production uncalibrated label. Added version ownership and validation to the weight container, propagated its active version to every output, and observed the focused regression fail before the fix and pass afterward. +- [x] (2026-07-22 05:45+08:00) Recompleted the full Ready gate from the current worktree: `make bazel_prepare` produced no generated diff; all six failpoint-wrapped target groups, the HashJoin NAAJ coverage, and tagged metrics tests passed; `make lint`, `git diff --check`, API-boundary searches, and the baseline-to-HEAD scope review passed. Failpoint refcount returned to zero after every run. +- [x] (2026-07-22 07:15+08:00) A subsequent output-version audit found that background Prometheus statement, operator-status, base-unit, and row-width series carried only `model_version`. Added the bounded active `weight_version` label to all four families and froze one active version per result recording; the new metric-contract regression failed before the change and passed afterward. +- [x] (2026-07-22 07:20+08:00) Recompleted the Ready gate after the Prometheus provenance fix: `make bazel_prepare` produced no generated diff; all six failpoint-wrapped target groups, HashJoin NAAJ coverage, and tagged metrics tests passed with failpoint refcount returning to zero; `make lint`, `git diff --check`, call-site search, version-label review, and the baseline scope review passed. +- [x] (2026-07-22 15:42+08:00) Applied the final write-unit contract: normalized mutation work now reuses `cpu_work`, is emitted only under a valid calibrated weight snapshot, and remains distinguishable through `site=tidb`, `op_class=kv_mutation`, `operator_kind=memdb_mutation`, `input_source=stmt_memdb_mutation_calls`, and `input_side=all`. The six raw mutation diagnostics remain visible when uncalibrated. +- [x] (2026-07-22 15:53+08:00) Completed the revised write-unit Ready gate: the new regression failed against the old independent-unit behavior and passed after restoration; all six failpoint-wrapped target groups, tagged metrics tests, `make lint`, residual-name search, and `git diff --check` passed with failpoint refcount returning to zero. +- [x] (2026-07-22 16:09+08:00) Split the shared request contract into `read_request_count`/`read_request_weight` and `write_request_count`/`write_request_weight`, preserving the existing statement-scope ownership and fail-closed rules. Updated EXPLAIN, General Log, Prometheus, statement-summary, formula, DML, and COMMIT regressions plus the affected test case maps. +- [x] (2026-07-22 16:14+08:00) Completed the request-split Ready gate: both reader and writer regressions failed when their constructors were temporarily restored to the old shared `request_count` output, then passed with the split implementation. All six failpoint-wrapped target groups, tagged metrics tests, `make lint`, residual-name search, and `git diff --check` passed with failpoint refcount returning to zero. The Bazel gate found no trigger, so `make bazel_prepare` was not required. +- [x] (2026-07-22 18:45+08:00) Corrected TopN ordering work to saturate its legal `offset+count` heap bound at actual input rows and to emit zero work when `count=0`. The focused regression failed against the unsaturated formula and passed after the fix; overflow remains fail-closed for every positive count. +- [x] (2026-07-22 18:47+08:00) Completed the TopN correction Ready gate: the full preview-RU planner target, EXPLAIN/metrics/general-log executor targets, `make lint`, and `git diff --check` passed with failpoint refcount returning to zero. The Bazel gate found no trigger, so `make bazel_prepare` was not required. +- [x] (2026-07-23 23:35+08:00) Reproduced the real-TiKV non-Scan publication failure in code: distsql attaches response ScanDetail to the component's last plan ID while the scan leaf receives only an execution summary, and the v4 estimator counted both the real holder and the leaf's default-zero detail as holders. Three rounds of ownership, fail-closed, and validation review converged on explicit TiDB-side ScanDetail attachment provenance without changing the distsql ownership or TiKV protocol. +- [x] (2026-07-24 00:05+08:00) Added a regression that exactly uses `RecordOneCopTask` for the scan leaf and `RecordCopStats` for the parent holder. It failed before the fix with `ambiguous_cop_scan_width` and passed after `RuntimeStatsColl` began recording ScanDetail attachments and the estimator validated holder/task coverage from one consistent snapshot. +- [x] (2026-07-24 00:20+08:00) Completed the provenance regression matrix: explicit empty detail, repeated same-holder attachments, attachment-without-holder-summary, independent scan-summary and attachment coverage failures, no holder, partial tuple, multi-holder ambiguity, ExecutorId remap, and statement-atomic failure are all covered. Independent implementation and test reviewers both concluded LGTM with no P0/P1/P2. +- [x] (2026-07-24 00:33+08:00) Completed cleanup-safe real-TiKV validation against PD/TiKV `v9.0.0-beta.2.pre-363-g8cabcf1` and the current-worktree TiDB binary. Selection, Projection, Limit, TopN, HashAgg, and ordered StreamAgg all succeeded in `FORMAT='RU'`; the Selection probe emitted `tikv/filter_eval cpu_work=4` and `tikv/kv_range_scan scan_bytes=212.5` identically through General Log, Prometheus, and statement summary. The exact TiUP tag was cleaned and the PD endpoint was unreachable afterward. +- [x] (2026-07-24 00:36+08:00) Completed the final Ready gate after the stable ordered-StreamAgg fixture revision: targeted execdetails, distsql, planner/core, and executor tests passed; `make lint` and `git diff --check` passed; failpoint refcount was zero. The Bazel gate found no added/moved Go file, import change, new top-level test, Bazel edit, or module edit, so `make bazel_prepare` was not required. +- [x] (2026-07-24 01:20+08:00) Added statement-scoped RPC-only preview units for closed-set read-only PointGet and BatchPointGet plans. The implementation emits exactly one synthetic point-lookup operator, retains physical lookup rows as non-billable diagnostics, and keeps locking, DML, mixed cop-reader, and unknown producer shapes fail-closed. +- [x] (2026-07-24 01:35+08:00) Observed the focused EXPLAIN regression fail against the prior `ambiguous_reader_transport_producers` behavior and pass after the implementation. Planner, EXPLAIN, plan-digest, General Log, Prometheus, and statement-summary regressions passed; `make bazel_prepare`, `make lint`, and `git diff --check` passed. A cleanup-safe real-TiKV run confirmed PointGet and BatchPointGet each publish only one statement-scoped `read_request_count=1` unit and no uncalibrated total. +- [x] (2026-07-24 02:05+08:00) Bound UnionScan's intentionally simple CPU formula to its direct child's actual runtime rows: `cpu_work=input_rows`. The focused regression failed while `overlay_reader` was non-billable and passed after the formula was enabled, with no expression-count multiplier or new runtime datum. +- [x] (2026-07-24 02:25+08:00) Completed UnionScan output and Ready verification. A cleanup-safe real-TiKV transaction produced UnionScan output rows 4 over direct-child rows 3 and published exactly `tidb/overlay_reader/unionscan cpu_work=3` from `runtime_child_act_rows`; the focused preview-RU planner suite, `make lint`, and `git diff --check` passed with failpoint refcount zero. The Bazel gate found no import, file, top-level-test, Bazel, or module trigger, so `make bazel_prepare` was not required. ## Surprises & Discoveries @@ -61,18 +83,54 @@ This plan deliberately does not implement production code. It freezes the model, - Observation: write RPC counters are statement-local even inside an explicit transaction. Evidence: `session.executeStmtImpl` installs the current statement's `RUV2Metrics`, `ExecStmt.finalizeStatementRUV2Metrics` drains raw/commit details into it before preview construction, and `TestRUV2MetricsIsolatedPerStatementInExplicitTxn` proves successive statements use distinct instances. A pessimistic DML's write requests therefore cannot be deferred to the later COMMIT snapshot. +- Observation: MockStore does not synthesize resource-manager write RPC detail for pessimistic KV traffic. + Evidence: the new session lifecycle test observes zero from a real MockStore DML before injection. It therefore injects finalized values into the actual per-statement `RUV2Metrics` objects to prove DML/COMMIT snapshot separation, while private core construction tests independently prove that values 7 and 2 become distinct `write_request_count` units. + +- Observation: pipelined completeness is already preserved in the statement-local mutation snapshot and on the COMMIT statement before transaction invalidation. + Evidence: `stmtctx.PreviewKVMutationSnapshot.Pipelined`, `LazyTxn.previewKVMutationRecorder`, and `session.markPreviewKVMutationTxnPipelined` provide the lifecycle evidence. The v4 write constructor must consume that flag before publishing `ResourceManagerWriteCnt`; a present counter alone does not prove complete pipelined logical-flush attribution. + +- Observation: a non-scalar Sort/TopN `ByItems` expression alone does not prove that inline expression evaluation was materialized by the executed child. + Evidence: `InjectProjBelowSort` rewrites scalar ordering expressions to newly allocated Projection columns, but the preview constructor previously accepted any column without checking the direct child's schema. Column membership and Projection expression/schema alignment are therefore required presence evidence before publishing ordering work. + +- Observation: a constant output-side weight version does not prove that the coefficient set used to calculate a total has that version. + Evidence: the initial v4 implementation kept `readBillingDemoWeightVersion` outside `readBillingDemoWeights`; private calibrated formula fixtures could therefore make `total_preview_ru` available while EXPLAIN, statement summary, and metrics still labeled the result `v3-resource-formula-uncalibrated`. + +- Observation: `model_version` alone is insufficient provenance for background Prometheus units whose semantics can change with the weight container. + Evidence: mutation-derived `cpu_work` depends on the versioned `MutationBytesPerCPUUnit`, while the four `tidb_read_billing_demo_*` families previously omitted `weight_version`; a rolling transition could therefore merge distinct v4 contracts into the same series. + +- Observation: distsql response ScanDetail ownership differs from executor-summary ownership. + Evidence: `pkg/distsql/select_result.go::updateCopRuntimeStats` records each valid execution summary under its exact `copPlanIDs` entry but attaches the response-level ScanDetail only through `RecordCopStats` on the last plan ID. `pkg/distsql/select_result_test.go::TestUpdateCopRuntimeStats` proves that the scan leaf has tasks and zero detail while its parent holds the merged detail. + +- Observation: a zero-valued `CopRuntimeStats.GetScanDetail()` does not prove an observed empty scan. + Evidence: both a summary-only scan leaf and an actually attached all-zero ScanDetail read back as the zero struct. The previous `tasks>0 && detail==zero` sentinel therefore double-counted the leaf beside a real parent holder and could also turn a missing or partial detail into a false observed zero. + +- Observation: native EmbedUnistore is not a complete scan-width oracle for this regression. + Evidence: its cop response supplies processed versions but not the complete total/processed-size tuple required by `readBillingDemoRangeScanInput`. A test-only response enrichment may exercise output wiring, but the corrected Ready evidence must include real TiKV. + +- Observation: a bare `stream_agg()` hint did not produce StreamAgg consistently across native EmbedUnistore and real TiKV. + Evidence: the stable validation shape also orders the result and uses `order_index(table, idx_b)`, making the ordered index property explicit; ordinary EXPLAIN and `FORMAT='RU'` then both show `StreamAgg cop[tikv]`. + +- Observation: TiKV client-go already classifies every successfully completed TiKV `CmdGet` and `CmdBatchGet` RPC as one raw read RPC, and TiDB drains that counter into the statement's frozen `ResourceManagerReadCnt`. + Evidence: `client-go/internal/client/completedTiKVRUV2RPCCount` returns `(1, 0)` for non-write TiKV requests and its regression names both Get and BatchGet; `config.UpdateTiKVRUV2FromExecDetailsV2` adds that count to `kvrpcpb.RUV2.ReadRpcCount`; `ExecStmt.finalizeStatementRUV2Metrics` drains it through `SyncRUV2MetricsFromRUDetails`. + +- Observation: PointGet and BatchPointGet have no attributable response-byte counter in the frozen preview details, while their statement read-RPC counter may cover more than one plan node. + Evidence: `RUV2Metrics.TiKVCoprocessorResponseBytes` covers coprocessor responses rather than Get/BatchGet payloads, and `ResourceManagerReadCnt` is statement-scoped rather than keyed by physical plan ID. Per-node publication would therefore invent byte work and duplicate RPC work for multi-lookup statements. + +- Observation: UnionScan merges snapshot rows from its direct child with transaction mem-buffer rows, but the frozen preview details expose only the direct child's exact actual-row count. + Evidence: `UnionScanExec.getOneRow` merges `getSnapshotRow` and `getAddedRow`; `RuntimeStatsColl` records the direct child and UnionScan output rows but no separate count of mem-buffer rows considered by the merge. Output rows cannot reconstruct input work because deletes, replacements, and predicates can suppress rows. + ## Decision Log - Decision: replace, rather than layer on top of, the v3 fixed/row/byte opclass matrix for new samples. Rationale: keeping both billable models would double count and make `cpu_weight` cease to mean one expression-slot evaluation. Historical v3 samples remain queryable under their model version. Date/Author: 2026-07-22 / Codex. -- Decision: use six semantic RU weights: `cpu_weight`, `scan_weight`, `net_weight`, `request_weight`, `hash_table_weight`, and `join_weight`. - Rationale: these are exactly the independent resource terms in the requested formulas. Operator identity remains a diagnostic dimension, not a weight key. +- Decision: use seven semantic RU weights: `cpu_weight`, `scan_weight`, `net_weight`, `read_request_weight`, `write_request_weight`, `hash_table_weight`, and `join_weight`. + Rationale: read and write requests have independent calibration coefficients and externally distinct units. Operator identity remains a diagnostic dimension, not a weight key. Date/Author: 2026-07-22 / Codex. -- Decision: Sort CPU work is `rows * log2(max(rows, 2))`; TopN CPU work is `rows * log2(max(k, 2))`, where `k = offset + count`. - Rationale: this is the final requested ownership split. Inline Projection alone charges scalar expression evaluation; the ordering node charges aggregate algorithmic work, with no expression/key-count multiplier. TopN uses its heap bound rather than full input cardinality, and offset is part of that bound. The implementation rejects `offset + count` overflow instead of wrapping; zero/one rows and zero/one `k` use base two so the work is finite and deterministic. +- Decision: Sort CPU work is `rows * log2(max(rows, 2))`; for positive `count`, TopN CPU work is `rows * log2(max(min(rows, k), 2))`, where `k = offset + count`; `count=0` produces zero TopN work. + Rationale: this is the final requested ownership split. Inline Projection alone charges scalar expression evaluation; the ordering node charges aggregate algorithmic work, with no expression/key-count multiplier. TopN retains at most the rows that actually exist, so a configured heap bound larger than input cardinality cannot increase work. Offset remains part of the legal heap bound. The implementation rejects `offset + count` overflow for positive counts instead of wrapping; zero/one rows use base two so positive-count work is finite and deterministic, while `count=0` matches the executor's no-child-read fast path. Date/Author: 2026-07-22 / Codex. - Decision: count top-level executable expression slots, not recursive AST nodes. @@ -83,6 +141,14 @@ This plan deliberately does not implement production code. It freezes the model, Rationale: the existing byte and RPC details are statement-scoped and all listed reader kinds share the same weights. Algebraically, charging the totals once equals summing per-reader formulas. Proportional allocation by logical output bytes is rejected because it invents attribution and is badly biased for IndexLookup/IndexMerge internal legs. Date/Author: 2026-07-22 / Codex. +- Decision: publish PointGet and BatchPointGet read RPCs once per statement under `id=point_lookup@statement`, `site=tikv`, and `op_class=kv_point_lookup`, with only `read_request_count`. + Rationale: Get/BatchGet RPC counts are exact but statement-scoped, so one synthetic operator preserves the requested lookup dimensions without multiplying the total across multiple plan nodes. Pure PointGet and pure BatchPointGet use their respective operator kind; a closed statement containing both uses `mixed_point_lookup`. No CPU, scan, or network work is inferred. Locking lookups, DML, and statements mixing point lookup with cop readers remain fail-closed because the current counter cannot split ancillary or heterogeneous read RPCs. + Date/Author: 2026-07-24 / Codex. + +- Decision: make root UnionScan weight-bearing with exactly `cpu_work = direct_child_actual_rows`. + Rationale: this is the requested simple formula and reuses the same stable `runtime_child_act_rows` source as other root unary operators. UnionScan emits no `expression_count` and does not infer mem-buffer input rows from output rows. A future model that charges the second input would require an explicit executor datum and a separate design revision. + Date/Author: 2026-07-24 / Codex. + - Decision: interpret the requested HashJoin `distinct_rows` term as runtime `hash_state_rows`, the cumulative rows actually admitted into hash lookup state; interpret the HashAgg term as `group_rows = own output rows`. Rationale: duplicate join keys still allocate/probe row-backed entries, while null/filter-rejected build rows do not enter that structure. Exact admitted rows are therefore more faithful than all build-child rows, and counting unique keys would require a second hash set solely for billing. This is the one permitted Join-only exec-details addition. Date/Author: 2026-07-22 / Codex. @@ -103,21 +169,49 @@ This plan deliberately does not implement production code. It freezes the model, Rationale: historical rows already contain values and versions, while private per-statement results do not survive process upgrade. Keeping a second calculation path would create an unnecessary mixed-model state; an eventual calibrated set must receive another immutable version. Date/Author: 2026-07-22 / Codex. -- Decision: combine write mutation count and bytes into `mutation_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit`, then apply `cpu_weight` once. - Rationale: this preserves one CPU-priced unit without adding independent mutation weights. `mutation_bytes_per_cpu_unit` is a versioned normalization constant, not an RU coefficient; it must be calibrated and validated as positive before a weighted total is published. +- Decision: combine write mutation count and bytes into `cpu_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit`, then apply `cpu_weight` once. Distinguish mutation CPU from expression CPU through the operator dimensions rather than a separate unit name. + Rationale: normalization produces expression-equivalent CPU work, so reusing the shared semantic unit preserves one CPU coefficient. `mutation_bytes_per_cpu_unit` is a versioned normalization constant, not an RU coefficient; the complete weight snapshot must be calibrated and valid before mutation-derived `cpu_work` is emitted. Raw mutation count/byte diagnostics remain available without calibration. Date/Author: 2026-07-22 / Codex. - Decision: charge write requests to the statement whose frozen `RUV2Metrics` snapshot contains them; an explicit-transaction DML and the eventual COMMIT each own only their respective snapshots. Rationale: TiDB installs a fresh `RUV2Metrics` object for every statement and finalizes raw RU details into that statement before preview construction. Pessimistic DML can issue write RPCs before COMMIT, so deferring all request work to COMMIT would permanently omit those DML-local requests. A cross-statement accumulator is unnecessary and would weaken the existing statement-local contract. Date/Author: 2026-07-22 / Codex. +- Decision: expose read and write requests as `read_request_count` and `write_request_count`, with independent `read_request_weight` and `write_request_weight` coefficients. + Rationale: the counters already have distinct authoritative sources and workload behavior, and independent unit names prevent downstream consumers from collapsing them before calibration. This changes only preview-v4 semantics; statement-scope ownership, fail-closed gates, and production RUv2 remain unchanged. + Date/Author: 2026-07-22 / Codex. + - Decision: do not guess numerical v4 weights or silently map the heterogeneous v3 opclass weights. Rationale: there is no evidence-backed one-to-one mapping. The implementation first publishes v4 base units with `uncalibrated_weights` and no total until an explicit v4 weight set and positive mutation normalization are installed. The preview flag is off by default, so this is safer than presenting arbitrary numbers as RU. Date/Author: 2026-07-22 / Codex. +- Decision: validate Sort/TopN ordering columns against the direct executed flat child schema, and additionally require a Projection child's schema width to match its expression list. + Rationale: this proves the ownership handoff to the child Projection without charging expression CPU at the ordering node, while preserving ordinary column and constant ordering. Relying only on optimizer invariants would violate the explicit fail-closed contract for missing or misaligned Projection schema. + Date/Author: 2026-07-22 / Codex. + +- Decision: make the private weight container own its immutable version and reject calibration under the shipped uncalibrated label. + Rationale: coefficients and their version are one atomic contract. Deriving all output labels from the active container prevents a future calibrated set from publishing a total under stale metadata, while the production default remains explicitly `v3-resource-formula-uncalibrated` and cannot publish totals. + Date/Author: 2026-07-22 / Codex. + +- Decision: add the active weight version to every background preview Prometheus family, not only to base-unit samples. + Rationale: keeping statement status, operator status, units, and row-width observations on the same bounded version key makes the Prometheus output an atomic contract and prevents partial cross-version aggregation. The version is immutable per binary configuration, so the extra label remains bounded. + Date/Author: 2026-07-22 / Codex. + +- Decision: record TiDB-internal ScanDetail attachment counts and use them, rather than detail values, to identify the unique component holder. + Rationale: `RecordCopStats(scan != nil)` already knows whether a response attached ScanDetail, including a legitimate all-zero detail, while `RecordOneCopTask` is summary-only. Counting attachments under the original plan ID preserves the existing distsql/EXPLAIN ownership contract, distinguishes empty from absent evidence, and makes multiple holders fail closed without relying on flat-tree traversal order. This is internal provenance metadata, not a new TiKV protocol field or formula input. + Date/Author: 2026-07-23 / Codex. + +- Decision: validate scan executor summaries and ScanDetail attachments as independent coverage channels. + Rationale: the scan leaf must have `observed summaries == expected responses > 0`, while the unique holder must have `attachment records == holder expected == scan expected`. The holder's own execution summary is not required for scan bytes because `RecordCopStats` can receive complete ScanDetail with a nil summary. Only after both channels are complete may an all-zero tuple mean an observed empty scan. + Date/Author: 2026-07-23 / Codex. + +- Decision: keep the broader non-Scan direct-child expected-task hardening outside this focused ownership correction. + Rationale: the existing relative `maxSummaryTasks` gate can miss the case where every executor omits the same response summary, but changing every unary/Agg/DML path is not required to fix the confirmed double-holder failure and needs its own calibrated-release regression matrix. + Date/Author: 2026-07-23 / Codex. + ## Outcomes & Retrospective -The design is stable after the required convergence review. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(offset+count)` work without repeating Projection expression evaluation, removes IndexJoin request double charging, limits new runtime data to one HashJoin state-row count, and defines a no-guess migration. A later fresh-context audit corrected explicit-transaction write-request ownership so pessimistic DML requests remain attached to their actual statement snapshot rather than being lost at COMMIT. After that substantive correction, two successive no-inherited-context reviewers found no further necessary modification; the final review changed only this progress record. Implementation remains intentionally pending and must happen in the separate worktree specified below. +The design is stable after the required convergence review. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(min(n, offset+count))` work without repeating Projection expression evaluation, treats zero-count TopN as zero work, validates ordering-column ownership against the executed child schema, removes IndexJoin request double charging, and defines a no-guess migration. The implementation now publishes exactly seven weight-bearing v4 semantic units, reuses dimension-qualified `cpu_work` for normalized mutation preparation, retains the raw mutation diagnostics, and keeps production weights intentionally uncalibrated. It atomically binds any future calibrated coefficients to their immutable output version across EXPLAIN, statement summary, general log, and all background Prometheus families, removes the executable v3 formula from production, gives read and write requests independent unit names and weights while preserving statement-scoped ownership, and preserves the pipelined write-request fail-closed gate. Runtime additions remain narrow: HashJoin exposes actual admitted state rows as a formula datum, while cop stats record only TiDB-side ScanDetail attachment provenance needed to distinguish an observed empty scan from absent detail. MockStore cannot produce real resource-manager write RPC details, so the session lifecycle test injects finalized observations into the actual fresh per-statement snapshots; private construction tests separately verify their exact 7/2 unit values. The ownership correction now passes the complete focused regression matrix and real-TiKV output verification: six non-Scan cop operator classes publish their units, and one Selection sample agrees across EXPLAIN, General Log, Prometheus, and statement summary without publishing an uncalibrated total. The subsequent point-lookup extension consumes the already-frozen TiKV read-RPC counter without introducing a new runtime datum: pure read-only PointGet and BatchPointGet statements now publish one exact RPC-only synthetic operator, while every shape whose statement counter cannot be attributed uniquely remains fail-closed. UnionScan likewise reuses existing evidence and the existing CPU weight: it now publishes direct-child actual rows as `cpu_work` without charging its conditions or inventing a mem-buffer input estimate. ## Context and Orientation @@ -137,13 +231,14 @@ For one statement, weighted preview RU is the sum of the following unit families cpu_work * cpu_weight + scan_bytes * scan_weight + net_bytes * net_weight - + request_count * request_weight + + read_request_count * read_request_weight + + write_request_count * write_request_weight + hash_state_rows * hash_table_weight + join_output_rows * join_weight -All arithmetic is float64 with explicit negative, NaN, infinity, and overflow rejection. Integer counters are converted only after validating that they are non-negative. Each pre-aggregation operator result retains `site`, `op_class`, `operator_kind`, `operator_id`, `source`, and, for joins, `input_side`. Physical results use the executed plan's Explain ID. Statement-scoped synthetic results use reserved, non-plan IDs: `reader_transport@statement`, `mutation@statement`, and `txn_write@statement`. Statement-summary detail intentionally aggregates away `operator_id`; its remaining bounded dimensions and version still preserve formula provenance. +All arithmetic is float64 with explicit negative, NaN, infinity, and overflow rejection. Integer counters are converted only after validating that they are non-negative. Each pre-aggregation operator result retains `site`, `op_class`, `operator_kind`, `operator_id`, `source`, and, for joins, `input_side`. Physical results use the executed plan's Explain ID. Statement-scoped synthetic results use reserved, non-plan IDs: `reader_transport@statement`, `point_lookup@statement`, `mutation@statement`, and `txn_write@statement`. Statement-summary detail intentionally aggregates away `operator_id`; its remaining bounded dimensions and version still preserve formula provenance. -There is no billable fixed-event term in v4. Setup costs that correlate with remote fanout use `request_count`; other constant setup costs stay outside this intentionally simple model. +There is no billable fixed-event term in v4. Setup costs that correlate with remote fanout use the direction-specific request unit; other constant setup costs stay outside this intentionally simple model. ### Operator formulas @@ -152,9 +247,11 @@ There is no billable fixed-event term in v4. Setup costs that correlate with rem | Selection | `rows * n_expr * cpu_weight` | direct child actual rows; selection expression slots | | Projection | `rows * n_expr * cpu_weight` | direct child actual rows; projected expression slots | | Sort | `rows * log2(max(rows,2)) * cpu_weight` | direct child actual rows; scalar expression evaluation belongs to inline Projection | -| TopN | `rows * log2(max(k,2)) * cpu_weight` | direct child actual rows; checked `k=offset+count`; scalar expression evaluation belongs to inline Projection | +| TopN | `count == 0 ? 0 : rows * log2(max(min(rows,k),2)) * cpu_weight` | direct child actual rows; checked `k=offset+count` for positive count; scalar expression evaluation belongs to inline Projection | | TableScan / IndexScan | `scan_bytes * scan_weight` | attributable TiKV `ScanDetail` | -| TableReader / IndexReader / IndexLookup / IndexMerge transport | `net_bytes * net_weight + request_count * request_weight` | statement `RUV2Metrics`, emitted once | +| TableReader / IndexReader / IndexLookup / IndexMerge transport | `net_bytes * net_weight + read_request_count * read_request_weight` | statement `RUV2Metrics`, emitted once | +| PointGet / BatchPointGet | `read_request_count * read_request_weight` | statement `RUV2Metrics`, emitted once for a closed point-lookup producer set | +| UnionScan | `rows * cpu_weight` | direct child actual rows; no expression-count multiplier | | StreamAgg | `rows * n_expr * cpu_weight` | direct child actual rows; group and aggregate slots | | HashAgg | `rows * n_expr * cpu_weight + group_rows * hash_table_weight` | StreamAgg inputs plus own actual output rows | | MergeJoin | `(left_rows + right_rows) * n_expr * cpu_weight + output_rows * join_weight` | both child rows, join slots, own output rows | @@ -162,7 +259,7 @@ There is no billable fixed-event term in v4. Setup costs that correlate with rem | IndexJoin family | `(left_rows + right_rows) * n_expr * cpu_weight + output_rows * join_weight` | both child rows, join slots, own output rows; inner reader already owns requests | | Limit | `rows * cpu_weight` | direct child actual rows | | Window | `rows * n_expr * cpu_weight` | direct child rows and the refined Window slot count below | -| Write mutation and commit | `mutation_work * cpu_weight + write_request_count * request_weight` | existing mutation recorder and statement/commit RUv2 details | +| Write mutation and commit | `cpu_work * cpu_weight + write_request_count * write_request_weight` | existing mutation recorder and statement/commit RUv2 details; mutation provenance is carried by operator dimensions | The table is normative. Diagnostic `input_rows`, logical chunk bytes, output bytes, mutation component counters, and operator status may still be emitted, but they have no weight and never enter `preview_ru`. @@ -172,7 +269,7 @@ The table is normative. Diagnostic `input_rows`, logical chunk bytes, output byt Selection uses `len(PhysicalSelection.Conditions)`. Projection uses `len(PhysicalProjection.Exprs)`, including inline ordering expressions materialized for Sort/TopN and column pass-through expressions because the executor still materializes an output column. -Sort and TopN do not emit an expression-count unit and do not multiply algorithmic work by `len(ByItems)` or `len(PartitionBy)`. Ordinary column, constant, and multi-key ordering are covered by the single aggregate sorting term. Before publishing that term, inspect `ByItems`: a remaining `ScalarFunction` proves that expression evaluation was not materialized into a child Projection, so the operator fails closed with `missing_ordering_projection` rather than charging expression CPU at Sort/TopN. If `ByItems` references columns produced by an aligned child `PhysicalProjection`, that Projection's normal formula owns all of its `Exprs` once. A missing/misaligned Projection schema also fails closed. For TopN, compute `k=offset+count` with checked unsigned addition and convert it to float only after validation; do not use `count` alone. +Sort and TopN do not emit an expression-count unit and do not multiply algorithmic work by `len(ByItems)` or `len(PartitionBy)`. Ordinary column, constant, and multi-key ordering are covered by the single aggregate sorting term. Before publishing that term, inspect `ByItems`: a remaining `ScalarFunction` proves that expression evaluation was not materialized into a child Projection, so the operator fails closed with `missing_ordering_projection` rather than charging expression CPU at Sort/TopN. If `ByItems` references columns produced by an aligned child `PhysicalProjection`, that Projection's normal formula owns all of its `Exprs` once. A missing/misaligned Projection schema also fails closed. For positive-count TopN, compute `k=offset+count` with checked unsigned addition, saturate it to `effective_k=min(uint64(rows), k)`, and only then convert it to float; do not use `count` alone. A zero-count TopN emits zero ordering work regardless of offset because the executor returns without reading its child. StreamAgg and HashAgg use `len(GroupByItems) + len(AggFuncs)`. One aggregate function descriptor is one slot regardless of partial/final/complete mode and regardless of argument count. This intentionally avoids a two-phase special model. `COUNT(*)` therefore has one slot rather than zero. Ordered aggregate arguments remain inside their aggregate function slot for v4. @@ -229,7 +326,7 @@ TableScan and IndexScan use the current v3 scan-byte proxy: otherwise: scan_bytes = TotalKeys * ProcessedKeysSize / ProcessedKeys -The all-zero case is an observed zero only when the cop plan has at least one observed task and coverage is complete. Otherwise it is missing. In the nonzero case, all three values must be positive and the result must be finite. This proxy preserves scan work for MVCC/skipped keys while using only current `ScanDetail`; it is labeled `scan_detail_processed_key_avg_estimate`, not presented as encoded response bytes. +The all-zero case is an observed zero only when the scan leaf has `observed execution summaries == expected responses > 0` and the unique ScanDetail holder has `attachment records == holder expected == scan expected`. An attachment record is one `RecordCopStats` call with a non-nil ScanDetail under the original plan ID; a summary-only `RecordOneCopTask` is not an attachment. The holder's own execution summary is independent and is not required for scan bytes. Otherwise the value is missing or incomplete. In the nonzero case, all three values must be positive and the result must be finite. This proxy preserves scan work for MVCC/skipped keys while using only current `ScanDetail`; it is labeled `scan_detail_processed_key_avg_estimate`, not presented as encoded response bytes. Each scan detail must be attributable to exactly one scan component. Multi-scan IndexMerge is supported by evaluating each partial scan component separately; it must not share one detail across siblings. Ambiguous or absent attribution fails the affected statement according to the atomicity rules below. @@ -237,13 +334,13 @@ Each scan detail must be attributable to exactly one scan component. Multi-scan The four named reader families share one statement-level transport formula. The constructor identifies all executed TableReader, IndexReader, IndexLookup, and IndexMerge nodes, then emits exactly one `id=reader_transport@statement`, `site=tidb`, `op_class=reader_transport` operator with a bounded `operator_kind` set: `table_reader`, `index_reader`, `index_lookup`, `index_merge`, or `mixed_reader`. -`net_bytes` is `RUV2Metrics.TiKVCoprocessorResponseBytes()`. `request_count` is `RUV2Metrics.ResourceManagerReadCnt()`. Both are snapshots from the same frozen statement details used by existing preview outputs. The unit source is `ruv2_metrics`; logical `BasicRuntimeStats.GetOutputBytes` remains a diagnostic and is never substituted for transport bytes. +`net_bytes` is `RUV2Metrics.TiKVCoprocessorResponseBytes()`. `read_request_count` is `RUV2Metrics.ResourceManagerReadCnt()`. Both are snapshots from the same frozen statement details used by existing preview outputs. The unit source is `ruv2_metrics`; logical `BasicRuntimeStats.GetOutputBytes` remains a diagnostic and is never substituted for transport bytes. The statement-wide counters are publishable only when the executed flat plan proves a closed producer set: every possible TiKV read-RPC producer for the statement belongs to those four supported cop-reader families. The initial implementation uses this exact algorithm: 1. Only a read-only `SELECT` can pass the closed-set gate. If a DML flat plan contains any supported reader, its reader-transport component is always `unknown_input/ambiguous_reader_transport_producers`; v4 has no DML allowlist because current details cannot exclude uniqueness, locking, FK, transaction, or other ancillary reads. 2. Walk the complete executed `FlatPhysicalPlan`, including IndexJoin inner plans. Classify `*physicalop.PhysicalTableReader` with `StoreType == kv.TiKV`, `*physicalop.PhysicalIndexReader`, `*physicalop.PhysicalIndexLookUpReader`, and `*physicalop.PhysicalIndexMergeReader` as supported producers. -3. Reject `*physicalop.PointGetPlan`, `*physicalop.BatchPointGetPlan`, any `PhysicalTableReader` whose `StoreType != kv.TiKV`, `*physicalop.PhysicalExchangeReceiver`, and `*physicalop.PhysicalExchangeSender`. Also reject any node that the existing preview classifier marks as a reader/store-access class but that is not one of the four supported types. This catches TiFlash/MPP and future external reader types without treating a new producer as free. +3. Reject any `*physicalop.PointGetPlan` or `*physicalop.BatchPointGetPlan` from the cop-reader transport component; a separate point-lookup component below may publish it only when the complete statement contains point lookup producers and no cop reader or other open read producer. Also reject any `PhysicalTableReader` whose `StoreType != kv.TiKV`, `*physicalop.PhysicalExchangeReceiver`, and `*physicalop.PhysicalExchangeSender`. Any node that the existing preview classifier marks as a reader/store-access class but that is not one of the four supported cop-reader types also opens the set. This catches TiFlash/MPP and future external reader types without treating a new producer as free. 4. Other already-supported CPU, join, aggregation, wrapper, scan-descendant, UnionScan, MemTable, and TableDual nodes are not independent TiKV read-RPC producers and do not open the set. Any structurally unknown plan node continues to fail through the existing unsupported-operator gate, so transport is never published alongside an unknown tree. A supported reader mixed with any rejected producer is not partially charged from the total counter: SELECT fails atomically; DML marks only its reader-transport component unknown. This is conservative because the current statement counter cannot subtract unsupported RPCs. @@ -251,15 +348,23 @@ A supported reader mixed with any rejected producer is not partially charged fro Presence and zero handling are normative: - A nil, bypassed, or otherwise unavailable RUv2 snapshot is missing, even though public getters return zero. -- Inspect `GetTasks()` and `GetExpectedCopTasks()` for every supported reader/cop descendant. If any observed or expected cop task exists, `net_bytes == 0 && request_count == 0` is missing rather than free. -- `net_bytes > 0 && request_count == 0` is invalid. `request_count > 0 && net_bytes == 0` is valid for an empty cop response. -- `net_bytes == 0 && request_count == 0` is an observed zero only when a present, non-bypassed frozen RUv2 snapshot exists, no supported descendant has an observed or expected cop task, every supported reader root has observed zero output rows, and the producer set is closed. This represents an empty range/no-request execution. +- Inspect `GetTasks()` and `GetExpectedCopTasks()` for every supported reader/cop descendant. If any observed or expected cop task exists, `net_bytes == 0 && read_request_count == 0` is missing rather than free. +- `net_bytes > 0 && read_request_count == 0` is invalid. `read_request_count > 0 && net_bytes == 0` is valid for an empty cop response. +- `net_bytes == 0 && read_request_count == 0` is an observed zero only when a present, non-bypassed frozen RUv2 snapshot exists, no supported descendant has an observed or expected cop task, every supported reader root has observed zero output rows, and the producer set is closed. This represents an empty range/no-request execution. If no supported reader executed, no reader-transport operator is emitted. Unsupported producers retain explicit bounded status rows until an attributable mapping is designed. The bounded transport reasons are `missing_reader_transport_details` for presence/coverage failure and `ambiguous_reader_transport_producers` for an open producer set. +### Point lookup RPC transport + +Read-only, non-locking PointGet and BatchPointGet plans publish only the exact statement `RUV2Metrics.ResourceManagerReadCnt()` value. The constructor emits it once as `id=point_lookup@statement`, `site=tikv`, `op_class=kv_point_lookup`, `input_source=ruv2_metrics`, `input_side=all`, and unit `read_request_count`. Its bounded `operator_kind` is `point_get`, `batch_point_get`, or `mixed_point_lookup` when both physical kinds occur in one otherwise closed statement. The physical PointGet/BatchPointGet plan rows remain non-billable diagnostics; they do not each copy the statement counter. + +A present, non-bypassed frozen RUv2 snapshot is required. Zero is valid because a point lookup can be satisfied without a TiKV RPC, for example from transaction-local state; nonzero plan output therefore does not imply a missing counter. A negative counter is invalid. The component emits no `cpu_work`, `scan_bytes`, or `net_bytes`, because current details expose neither point-lookup executor CPU nor attributable Get/BatchGet response bytes. + +The point-lookup producer set is closed only for a read-only statement whose complete flat plan contains one or more non-locking PointGet/BatchPointGet nodes and no cop-reader producer, TiFlash/MPP producer, locking point lookup, or unknown store-access producer. DML remains fail-closed because uniqueness checks, locking, foreign keys, or transaction work can add read RPCs. A statement mixing point lookup and cop readers also remains fail-closed rather than allocating the statement total between components. + ### IndexJoin request de-duplication -IndexJoin, IndexHashJoin, and IndexMergeJoin do not emit a Join-local `request_count`. Their dynamic inner executors use TableReader, IndexReader, or IndexLookup paths, so the resulting physical read RPCs are already present in the statement-level reader-transport `ResourceManagerReadCnt`. PhysicalIndexMergeReader remains a supported standalone transport producer, but `dataReaderBuilder.BuildExecutorForIndexJoin` does not construct it as an IndexJoin inner path. +IndexJoin, IndexHashJoin, and IndexMergeJoin do not emit a Join-local `read_request_count`. Their dynamic inner executors use TableReader, IndexReader, or IndexLookup paths, so the resulting physical read RPCs are already present in the statement-level reader-transport `ResourceManagerReadCnt`. PhysicalIndexMergeReader remains a supported standalone transport producer, but `dataReaderBuilder.BuildExecutorForIndexJoin` does not construct it as an IndexJoin inner path. This is the explicit de-duplication refinement to the initial simple IndexJoin formula: @@ -280,13 +385,15 @@ The existing statement-local mutation recorder remains authoritative. Its comple `docs/design/2026-07-10-preview-ru-tidb-kv-mutation.md` remains background evidence, but the bullets above are the normative v4 contract. -The v4 combined unit is: +The v4 mutation normalization derives the shared CPU unit: - mutation_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit + cpu_work = mutation_count + mutation_bytes / mutation_bytes_per_cpu_unit `mutation_bytes_per_cpu_unit` is a positive, finite, versioned normalization constant with units bytes per expression-equivalent CPU-work unit. It is stored alongside the v4 weights and included in output metadata. It is not independently multiplied by RU. If it is unset, zero, negative, NaN, or infinite, mutation base components remain visible but weighted v4 total is unavailable with `uncalibrated_weights`. -`write_request_count` is `RUV2Metrics.ResourceManagerWriteCnt()` and uses the same `request_weight` as reads. The frozen snapshot must exist and be non-bypassed. Every DML statement, whether autocommit or inside an explicit transaction, emits the write-request count present in its own finalized snapshot alongside its statement-local mutation work. This is required for pessimistic transactions, whose DML statements can issue nonzero write RPCs before COMMIT. The eventual COMMIT emits only the write requests in its own fresh snapshot, with empty `dml_kind`; it neither absorbs nor back-attributes earlier DML requests. Thus each physical request is owned once by the statement whose RU details recorded it. A nonzero mutation with a missing request payload is partial, while an observed zero DML request count is valid only when the non-bypassed statement snapshot is present and finalized. A known empty transaction COMMIT likewise emits observed zero request work; an absent lifecycle snapshot is missing, not zero. SQL `ROLLBACK` remains unsupported in v4 and emits neither a zero unit nor a total: current routing has no complete rollback-RPC attribution, and declaring it free would be unsafe. +The externally weight-bearing semantic units are fixed to `cpu_work`, `scan_bytes`, `net_bytes`, `read_request_count`, `write_request_count`, `hash_state_rows`, and `join_output_rows`. Mutation-derived `cpu_work` uses `site=tidb`, `op_class=kv_mutation`, `operator_kind=memdb_mutation`, `input_source=stmt_memdb_mutation_calls`, and `input_side=all`; consumers must use these dimensions to distinguish it from expression CPU work. + +`write_request_count` is `RUV2Metrics.ResourceManagerWriteCnt()` and uses its independent `write_request_weight`; read transport uses `read_request_weight`. The frozen snapshot must exist and be non-bypassed. Every DML statement, whether autocommit or inside an explicit transaction, emits the write-request count present in its own finalized snapshot alongside its statement-local mutation work. This is required for pessimistic transactions, whose DML statements can issue nonzero write RPCs before COMMIT. The eventual COMMIT emits only the write requests in its own fresh snapshot, with empty `dml_kind`; it neither absorbs nor back-attributes earlier DML requests. Thus each physical request is owned once by the statement whose RU details recorded it. A nonzero mutation with a missing request payload is partial, while an observed zero DML request count is valid only when the non-bypassed statement snapshot is present and finalized. A known empty transaction COMMIT likewise emits observed zero request work; an absent lifecycle snapshot is missing, not zero. SQL `ROLLBACK` remains unsupported in v4 and emits neither a zero unit nor a total: current routing has no complete rollback-RPC attribution, and declaring it free would be unsafe. Pipelined transactions retain valid mutation units but mark the write-request component `pipelined_tikv_payload_unsupported` until current details prove a complete logical flush request count. Deprecated batch DML accumulates available write-request snapshots across internal transaction switches; any missing switch makes the request component partial. Optimistic retry/replay keeps the mutation behavior above and marks unavailable request attribution partial. Thus no retry, pipeline, or batch path publishes a known-incomplete zero. @@ -313,14 +420,15 @@ The v4 weight container is preview-only: CPUPerWorkUnit float64 ScanPerByte float64 NetworkPerByte float64 - Request float64 + ReadRequest float64 + WriteRequest float64 HashTablePerRow float64 JoinPerOutputRow float64 MutationBytesPerCPUUnit float64 Calibrated bool } -The six RU fields have units stated by their names. `MutationBytesPerCPUUnit` is a normalization, not RU. Validation requires a nonempty new version, finite non-negative RU weights, a positive finite mutation normalization, and `Calibrated=true` before any weighted total is published. Formula tests in `pkg/planner/core` use the private container directly with small deterministic values. No exported setter, session/global variable, or failpoint is added. Package-external executor tests see the production default and assert `uncalibrated_weights`, coefficient-free units, and no `total_preview_ru` until a later calibration change supplies production values. +The seven RU fields have units stated by their names. `MutationBytesPerCPUUnit` is a normalization, not RU. Validation requires a nonempty new version, finite non-negative RU weights, a positive finite mutation normalization, and `Calibrated=true` before any weighted total is published. Formula tests in `pkg/planner/core` use the private container directly with small deterministic values. No exported setter, session/global variable, or failpoint is added. Package-external executor tests see the production default and assert `uncalibrated_weights`, coefficient-free units, and no `total_preview_ru` until a later calibration change supplies production values. Set the exact constants `model_version='v4'` and `weight_version='v3-resource-formula-uncalibrated'`. Do not reuse the old model `v3` or weight `v2` labels. The weight-version string intentionally describes the shipped state; a later calibrated weight set must use another immutable version rather than changing values behind this label. Existing statement-summary detail already carries model and weight versions, so old rows remain self-describing and are not rewritten. Queries that compare workload windows must group by both versions. @@ -334,9 +442,9 @@ All outputs must switch together: EXPLAIN unit rows, `total_preview_ru`, Prometh ### Milestone 1: represent v4 work without changing collection -In `pkg/planner/core/explain_ru.go`, add bounded v4 unit names (`cpu_work`, `expression_count`, `scan_bytes`, `net_bytes`, `request_count`, `hash_state_rows`, `join_output_rows`, and `mutation_work`), the validated weight container, and one formula application function. Replace opclass-specific billable lookup for new results with semantic-unit lookup. Keep diagnostic legacy units zero-weight. +In `pkg/planner/core/explain_ru.go`, add bounded v4 unit names (`cpu_work`, `expression_count`, `scan_bytes`, `net_bytes`, `read_request_count`, `write_request_count`, `hash_state_rows`, and `join_output_rows`), the validated weight container, and one formula application function. Replace opclass-specific billable lookup for new results with semantic-unit lookup. Keep diagnostic legacy units zero-weight. -Add physical-plan helpers that return expression-slot count for every supported concrete type. Unit-test exact counts for simple and compound Selection, Projection, Agg, Join, and Window plans, including the distinct ordinary IndexJoin, IndexHashJoin, and IndexMergeJoin key representations. For Sort/TopN, test that a root scalar expression backed by inline Projection is evaluated only at Projection, ordinary-column/multi-key plans still receive exactly one aggregate sorting term, and a TiKV pushed TopN with an unmaterialized scalar `ByItems` fails `missing_ordering_projection`. Test checked `offset+count`, a nonzero offset, overflow rejection, and zero/one boundaries. At this milestone, custom unit fixtures in internal `package core` tests use the private calibrated weight container to prove exact algebra and invalid-number rejection before runtime constructors change. +Add physical-plan helpers that return expression-slot count for every supported concrete type. Unit-test exact counts for simple and compound Selection, Projection, Agg, Join, and Window plans, including the distinct ordinary IndexJoin, IndexHashJoin, and IndexMergeJoin key representations. For Sort/TopN, test that a root scalar expression backed by inline Projection is evaluated only at Projection, ordinary-column/multi-key plans still receive exactly one aggregate sorting term, and a TiKV pushed TopN with an unmaterialized scalar `ByItems` fails `missing_ordering_projection`. Test checked `offset+count`, a nonzero offset, saturation when `k` exceeds actual input rows, a legal bound near `MaxUint64`, positive-count overflow rejection, zero-count fast-path work, and zero/one-row boundaries. At this milestone, custom unit fixtures in internal `package core` tests use the private calibrated weight container to prove exact algebra and invalid-number rejection before runtime constructors change. Acceptance: formula tests with injected weights reproduce hand-calculated totals; no production RUv2 API or configuration changes. @@ -346,9 +454,9 @@ Refactor root and cop constructors in `pkg/planner/core/explain_ru.go` around th Use the existing flat-plan build/probe/left/right labels for join rows and the join node's own rows for output. Do not expose or add an IndexJoin lookup-task counter: its dynamic inner readers are already included by the statement-scope reader transport unit. -Add `HashTableRuntimeStats` in `pkg/util/execdetails/runtime_stats.go`. Implement it for both HashJoin runtime-stat versions in `pkg/executor/join/hash_join_stats.go`, recording successful v1/v2 state admissions at the existing build completion points in `hash_join_v1.go` and `hash_join_v2.go`. Do not add a unique-key collector or any non-Join runtime field. +Add `HashTableRuntimeStats` in `pkg/util/execdetails/runtime_stats.go`. Implement it for both HashJoin runtime-stat versions in `pkg/executor/join/hash_join_stats.go`, recording successful v1/v2 state admissions at the existing build completion points in `hash_join_v1.go` and `hash_join_v2.go`. Do not add a unique-key collector. The later ScanDetail ownership correction may add only an attachment-presence counter and consistent coverage snapshot to `RuntimeStatsColl`; that metadata does not enter a formula and must not alter distsql ownership. -Change write construction in `pkg/planner/core/explain_ru.go` to derive `mutation_work` and use the current statement's finalized write RPC count for both DML and COMMIT. Do not carry request counts across statements or defer explicit-transaction DML requests to COMMIT. Retain raw mutation units as diagnostics. +Change write construction in `pkg/planner/core/explain_ru.go` to derive mutation `cpu_work` only from a valid calibrated normalization/weight snapshot and use the current statement's finalized write RPC count for both DML and COMMIT. Do not carry request counts across statements or defer explicit-transaction DML requests to COMMIT. Retain raw mutation units as diagnostics even when the derived unit is unavailable. Acceptance: every required formula input can be traced to the source table below; source searches show exactly one new Join-only state-row counter and no other runtime field. @@ -371,9 +479,9 @@ Acceptance: internal formula tests observe exact calibrated totals. EXPLAIN, met | root `rows` | `RuntimeStatsColl` direct child `BasicRuntimeStats.GetActRows` | exact child plan ID must exist | no | | cop `rows` | direct child `CopRuntimeStats.GetActRows/GetTasks` | exact plan ID and coverage checks | no | | `n_expr` | concrete `physicalop` plan fields | centralized type switch, structural validation | no; immutable plan metadata | -| `scan_bytes` | `CopRuntimeStats.GetScanDetail` | unique scan component and complete tasks | no | +| `scan_bytes` | `CopRuntimeStats.GetScanDetail` plus TiDB attachment provenance | unique holder by non-nil `RecordCopStats` attachment; scan summaries and holder attachments each cover all expected responses | **yes, TiDB-only attachment count; no protocol field or formula term** | | `net_bytes` | `RUV2Metrics.TiKVCoprocessorResponseBytes` | once per statement; non-bypassed presence, descendant task gate, closed read producer set | no | -| reader `request_count` | `RUV2Metrics.ResourceManagerReadCnt` | once per statement only when every read-RPC producer is attributable to supported cop readers | no | +| reader `read_request_count` | `RUV2Metrics.ResourceManagerReadCnt` | once per statement only when every read-RPC producer is attributable to supported cop readers | no | | HashAgg `group_rows` | Agg node own runtime rows | TiKV additionally needs expected/observed coverage | no | | HashJoin `hash_state_rows` | v1 hash-table `Len` plus NAAJ null-bucket entries; v2 row-table `validKeyCount` | completed build round, cumulative across rebuilds | **yes, Join only** | | Join `output_rows` | Join node own `BasicRuntimeStats.GetActRows` | executed root stat required | no | @@ -421,21 +529,21 @@ At Ready, run the minimum targeted set again after any required `make bazel_prep make lint -Do not run `make bazel_lint_changed` or RealTiKV tests unless a discovered dependency, CI reproduction, or explicit request expands the scope. This model consumes already-collected TiKV details and does not require a live TiKV cluster for its formula unit tests; end-to-end mockstore tests remain necessary for rendered output. +Do not run `make bazel_lint_changed`. Formula unit tests do not require a live TiKV cluster, but the ScanDetail ownership correction must complete one scoped real-TiKV Ready verification because the defect was observed only with the complete real response tuple and native EmbedUnistore cannot supply that evidence. ## Validation and Acceptance The implementation is accepted only when all of the following are observable. -With private test weights set to simple values inside `pkg/planner/core`, table-driven formula tests show exact results for every operator row in the formula table, including zero rows, one row, multiple expressions, multi-key joins, all three IndexJoin-family key representations, V1 NAAJ null-bucket state, and Window frame expressions. Sort uses `log2(max(rows,2))`; TopN uses `log2(max(offset+count,2))`, with checked addition and cases where offset is nonzero. Neither ordering operator has an expression/key-count multiplier, and unmaterialized scalar ordering expressions fail closed rather than being charged there. +With private test weights set to simple values inside `pkg/planner/core`, table-driven formula tests show exact results for every operator row in the formula table, including zero rows, one row, multiple expressions, multi-key joins, all three IndexJoin-family key representations, V1 NAAJ null-bucket state, and Window frame expressions. Sort uses `log2(max(rows,2))`; positive-count TopN uses `log2(max(min(rows,offset+count),2))` with checked addition, while zero-count TopN emits zero work. Cases cover nonzero offset, `k>rows`, a legal bound near `MaxUint64`, and overflow rejection. Neither ordering operator has an expression/key-count multiplier, and unmaterialized scalar ordering expressions fail closed rather than being charged there. -End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. An explicit pessimistic transaction case must prove that a DML-local nonzero `ResourceManagerWriteCnt` is emitted by that DML, the later COMMIT emits only its own fresh-snapshot count, and neither count is lost or duplicated. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. +End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, RPC-only PointGet/BatchPointGet, UnionScan, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. An explicit pessimistic transaction case must prove that a DML-local nonzero `ResourceManagerWriteCnt` is emitted by that DML, the later COMMIT emits only its own fresh-snapshot count, and neither count is lost or duplicated. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. -A multi-reader or IndexMerge case proves that statement `net_bytes` and read `request_count` appear once, while every scan retains its own `scan_bytes`. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. +A multi-reader or IndexMerge case proves that statement `net_bytes` and `read_request_count` appear once, while every scan retains its own `scan_bytes`. PointGet and BatchPointGet cases prove that a pure read-only plan publishes the statement `read_request_count` exactly once under the bounded lookup kind, publishes no inferred CPU/scan/network unit, accepts a present zero counter, and rejects locking, DML, bypassed/missing detail, and mixed point/cop-reader producer sets. A UnionScan case proves that `tidb/overlay_reader/unionscan` publishes `cpu_work` equal to its direct child's exact actual rows, carries `runtime_child_act_rows`, and has no expression-count multiplier. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. DML and COMMIT cases prove that `write_request_count` remains statement-local and uses a coefficient independent from read requests. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. Missing root stats, incomplete cop summaries, ambiguous scan details, missing reader transport, invalid expression structure, invalid mutation normalization, negative inputs, overflow, NaN, and infinity all fail closed with bounded reasons. SELECT produces no partial billable total. DML preserves complete independent units but does not claim a complete statement total. -Search and API review prove that HashJoin state rows are the only runtime field added. `config.RUV2`, TiKV request charging, `ReportRUV2Consumption`, and resource-control behavior are unchanged. +Search and API review prove that runtime additions are limited to HashJoin state rows and the TiDB-only ScanDetail attachment count. The latter is provenance only and never enters a formula. `config.RUV2`, TiKV request charging, `ReportRUV2Consumption`, and resource-control behavior are unchanged. Statement-summary detail, Prometheus metrics hooks, EXPLAIN rows, and general-log details are built from the same frozen result and agree on unit values. General-log records retain DML kind, input source, and input side. Historical v3 rows remain distinguishable by version; legacy three-column convenience totals keep v3 behavior and remain zero for v4, whose memory/history-reader details remain queryable. @@ -445,7 +553,7 @@ All formula construction is read-only over frozen plan/runtime snapshots and mus `make bazel_prepare`, formatting, and targeted tests are safe to rerun. The failpoint test wrapper always disables failpoints during cleanup. If a milestone leaves mixed output versions, revert only that milestone's focused changes or finish all output consumers before running behavioral tests; never commit a mixed v3/v4 renderer state. -If the runtime source cannot prove a required input, add a bounded status reason and keep the formula unavailable. Do not recover by parsing runtime strings or by introducing an estimate not recorded in this plan. Any newly discovered need for another runtime field requires revisiting this design before coding it; the current v4 plan consumes the optional Join-only allowance solely for HashJoin state rows. +If the runtime source cannot prove a required input, add a bounded status reason and keep the formula unavailable. Do not recover by parsing runtime strings or by introducing an estimate not recorded in this plan. Any newly discovered formula datum still requires revisiting this design. The ScanDetail attachment count is a provenance exception recorded here after real-TiKV evidence showed that a zero value alone cannot distinguish missing detail from an observed empty scan. ## Artifacts and Notes @@ -454,13 +562,19 @@ Current evidence commands used while drafting this plan included: rg -n 'type (BasicRuntimeStats|CopRuntimeStats|RuntimeStatsColl)' pkg/util/execdetails/runtime_stats.go rg -n 'TiKVCoprocessorResponseBytes|ResourceManagerReadCnt|ResourceManagerWriteCnt|Bypass' pkg/util/execdetails/ruv2_metrics.go rg -n 'innerWorker.task|type indexLookUpJoinRuntimeStats' pkg/executor/join - rg -n 'type Physical(Selection|Projection|TopN|Sort|HashAgg|StreamAgg|HashJoin|MergeJoin|IndexJoin|Window)' pkg/planner/core/operator/physicalop + rg -n 'type Physical(Selection|Projection|TopN|Sort|HashAgg|StreamAgg|HashJoin|MergeJoin|IndexJoin|Window|UnionScan)' pkg/planner/core/operator/physicalop The evidence establishes availability and location, not completion of the later implementation. +The final ScanDetail-ownership evidence additionally includes a cleanup-safe real-TiKV run with one current-worktree TiDB, one PD, and one TiKV. It observed `cpu_work=4` for `site=tikv, op_class=filter_eval, operator_kind=selection` and `scan_bytes=212.5` for the corresponding table scan in General Log, Prometheus, and statement summary. The shipped `v3-resource-formula-uncalibrated` version left total preview RU absent as required. + +The point-lookup evidence additionally includes a cleanup-safe real-TiKV run using the current-worktree TiDB binary and TiKV isolation. `SELECT ... WHERE primary_key = 1` emitted `id=point_lookup@statement, site=tikv, op_class=kv_point_lookup, operator_kind=point_get, read_request_count=1`; the two-key `IN` form emitted the same dimensions with `operator_kind=batch_point_get` and `read_request_count=1`. Neither row contained CPU, scan, or network work, and the uncalibrated summary contained no total preview RU. The exact TiUP tag and data directory were removed and all dedicated ports were closed afterward. + +The UnionScan evidence includes a cleanup-safe real-TiKV transaction over three committed rows plus one uncommitted inserted row. Ordinary `EXPLAIN ANALYZE` reported `UnionScan actRows=4` and its direct `TableReader actRows=3`; `FORMAT='RU'` emitted `site=tidb, op_class=overlay_reader, operator_kind=unionscan, input_rows=3` and `cpu_work=3`, both sourced from `runtime_child_act_rows`, with no expression-count unit and no uncalibrated total. The exact `preview-ru-unionscan-v4` TiUP tag and data directory were removed and ports 25000, 23379, 41160, and 24930 were closed afterward. + ## Interfaces and Dependencies -Keep all v4 model types private to `pkg/planner/core` except the narrow `execdetails.HashTableRuntimeStats` read interface. Do not export the preview weight container, concrete executor-private Join stats, or add public session/global variables in the initial implementation. +Keep all v4 model types private to `pkg/planner/core` except the narrow `execdetails.HashTableRuntimeStats` read interface and the `RuntimeStatsColl` coverage-snapshot method needed by planner/core. Do not export the preview weight container, concrete executor-private Join stats, or add public session/global variables. The implementation depends only on existing TiDB packages: `physicalop` for immutable plan expressions, `execdetails` for runtime rows/scan/RUv2 details, the statement mutation recorder for write inputs, and existing statement-summary/metrics/log renderers. It adds no third-party dependency and no protocol field. @@ -476,4 +590,10 @@ At milestone completion, the key internal interfaces should have these conceptua The exact private names may follow nearby conventions, but the semantics, data sources, one-Join-runtime-datum boundary, and de-duplication rules in this plan are mandatory. -Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, TopN owns `n*log(k)` with checked `k=offset+count`, and inner readers own IndexJoin request cost. HashJoin uses the sole permitted runtime addition to expose actual admitted hash-state rows instead of approximating them with all build rows. A later fresh-context audit corrected write-request ownership: explicit-transaction DML and COMMIT each charge only the write RPCs in their own finalized statement snapshot, preventing pessimistic DML requests from being lost. A final fresh-context convergence review found no further necessary change to formulas, sources, ownership, degradation, units, or migration. +Revision note (2026-07-22): first complete design draft created from current branch evidence, then revised for explicit de-duplication and the final ordering contract: inline Projection alone owns scalar Sort/TopN evaluation, Sort owns `n*log(n)`, positive-count TopN owns `n*log(min(n,k))` with checked `k=offset+count`, zero-count TopN owns zero work, and inner readers own IndexJoin request cost. HashJoin exposes actual admitted hash-state rows instead of approximating them with all build rows. A later fresh-context audit corrected write-request ownership: explicit-transaction DML and COMMIT each charge only the write RPCs in their own finalized statement snapshot, preventing pessimistic DML requests from being lost. + +Revision note (2026-07-23): real TiKV proved that non-Scan execution summaries were present but all units were suppressed by false ScanDetail ambiguity. The plan now records non-nil ScanDetail attachment counts under the original plan ID, validates scan-summary and attachment coverage independently, preserves true empty scans, and requires a scoped real-TiKV Ready verification. This is a minimal evidence/provenance revision; formulas, weights, protocol, distsql ownership, output dimensions, and production RUv2 remain unchanged. + +Revision note (2026-07-24): PointGet and BatchPointGet now consume only the existing statement-scoped TiKV read-RPC counter. The plan adds one synthetic point-lookup publisher with a closed producer set and retains fail-closed handling wherever the counter may include locking, DML, cop-reader, MPP, or unknown work. No CPU or byte estimate, runtime field, protocol change, or new weight-bearing unit was introduced. + +Revision note (2026-07-24): UnionScan now reuses the existing `cpu_work` semantic unit with `input_source=runtime_child_act_rows` and value equal to the direct child's actual rows. This intentionally simple first formula does not multiply by UnionScan conditions and does not estimate its transaction mem-buffer input. diff --git a/pkg/distsql/select_result_test.go b/pkg/distsql/select_result_test.go index 44dacb5731992..5726983ddf3e6 100644 --- a/pkg/distsql/select_result_test.go +++ b/pkg/distsql/select_result_test.go @@ -133,6 +133,14 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int64(100), parentStats.GetScanDetail().ProcessedKeysSize) require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, scanDetailRecords, scanObservedTasks, scanExpectedTasks := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(scanPlanID) + _, parentDetailRecords, parentObservedTasks, parentExpectedTasks := ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Zero(t, scanDetailRecords) + require.Equal(t, int32(1), scanObservedTasks) + require.Equal(t, int32(1), scanExpectedTasks) + require.Equal(t, int32(1), parentDetailRecords) + require.Equal(t, int32(1), parentObservedTasks) + require.Equal(t, int32(1), parentExpectedTasks) update(3, 1, &tikvutil.ScanDetail{TotalKeys: 2, ProcessedKeys: 2, ProcessedKeysSize: 40}) require.Equal(t, int64(7), scanStats.GetActRows()) @@ -143,6 +151,14 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int64(140), parentStats.GetScanDetail().ProcessedKeysSize) require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) require.Equal(t, int32(2), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, scanDetailRecords, scanObservedTasks, scanExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(scanPlanID) + _, parentDetailRecords, parentObservedTasks, parentExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Zero(t, scanDetailRecords) + require.Equal(t, int32(2), scanObservedTasks) + require.Equal(t, int32(2), scanExpectedTasks) + require.Equal(t, int32(2), parentDetailRecords) + require.Equal(t, int32(2), parentObservedTasks) + require.Equal(t, int32(2), parentExpectedTasks) ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) sr.ctx = ctx.GetDistSQLCtx() @@ -157,6 +173,10 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int64(5), parentStats.GetScanDetail().ProcessedKeys) require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) require.Equal(t, int32(1), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, parentDetailRecords, parentObservedTasks, parentExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Equal(t, int32(1), parentDetailRecords) + require.Zero(t, parentObservedTasks) + require.Equal(t, int32(1), parentExpectedTasks) ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) sr.ctx = ctx.GetDistSQLCtx() @@ -180,6 +200,9 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.Equal(t, int32(2), parentStats.GetTasks()) require.Equal(t, int32(3), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(scanPlanID)) require.Equal(t, int32(3), ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetExpectedCopTasks(parentPlanID)) + _, parentDetailRecords, _, parentExpectedTasks = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopScanDetailAndCoverage(parentPlanID) + require.Equal(t, int32(2), parentDetailRecords) + require.Equal(t, int32(3), parentExpectedTasks) ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(nil) sr.ctx = ctx.GetDistSQLCtx() diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go index a5aef4afc9941..245a49297735f 100644 --- a/pkg/executor/adapter.go +++ b/pkg/executor/adapter.go @@ -1797,7 +1797,10 @@ type readBillingDemoGeneralLogUnit struct { site string opClass string operatorKind string + dmlKind string unit string + inputSource string + inputSide string value float64 } @@ -1805,7 +1808,10 @@ func (u readBillingDemoGeneralLogUnit) MarshalLogObject(enc zapcore.ObjectEncode enc.AddString("site", u.site) enc.AddString("op_class", u.opClass) enc.AddString("operator_kind", u.operatorKind) + enc.AddString("dml_kind", u.dmlKind) enc.AddString("unit", u.unit) + enc.AddString("input_source", u.inputSource) + enc.AddString("input_side", u.inputSide) enc.AddFloat64("value", u.value) return nil } @@ -1828,7 +1834,10 @@ func buildReadBillingDemoGeneralLogUnits(stats stmtsummary.ReadBillingDemoStatem site: sample.Site, opClass: sample.OpClass, operatorKind: sample.OperatorKind, + dmlKind: sample.DMLKind, unit: sample.Unit, + inputSource: sample.InputSource, + inputSide: sample.InputSide, } totals[key] += sample.Value } @@ -1848,7 +1857,16 @@ func buildReadBillingDemoGeneralLogUnits(stats stmtsummary.ReadBillingDemoStatem if cmp := strings.Compare(left.operatorKind, right.operatorKind); cmp != 0 { return cmp } - return strings.Compare(left.unit, right.unit) + if cmp := strings.Compare(left.dmlKind, right.dmlKind); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.unit, right.unit); cmp != 0 { + return cmp + } + if cmp := strings.Compare(left.inputSource, right.inputSource); cmp != 0 { + return cmp + } + return strings.Compare(left.inputSide, right.inputSide) }) return units } diff --git a/pkg/executor/adapter_internal_test.go b/pkg/executor/adapter_internal_test.go index 015ca5e59a27e..07e2d68e39d0e 100644 --- a/pkg/executor/adapter_internal_test.go +++ b/pkg/executor/adapter_internal_test.go @@ -118,8 +118,8 @@ func TestReadBillingDemoGeneralLogUnits(t *testing.T) { sessVars.StmtCtx.OriginalSQL = "select 12345, 'secret_literal'" expectedNormalizedSQL, expectedDigest := parser.NormalizeDigest(sessVars.StmtCtx.OriginalSQL) stats := stmtsummary.ReadBillingDemoStatementStats{ - ModelVersion: "v3", - WeightVersion: "v2", + ModelVersion: "v4", + WeightVersion: "test-v4-calibrated", BaseUnits: []stmtsummary.ReadBillingDemoBaseUnitSample{ { Site: "tikv", OpClass: "join_hash", OperatorKind: "hashjoin", Unit: "input_rows", Value: 2, @@ -131,6 +131,15 @@ func TestReadBillingDemoGeneralLogUnits(t *testing.T) { }, {Site: "tidb", OpClass: "agg_hash", OperatorKind: "hashagg", Unit: "output_rows", Value: 1}, {Site: "tidb", OpClass: "agg_hash", OperatorKind: "hashagg", Unit: "input_bytes", Value: 8}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "cpu_work", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 3.5}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "encoded_mutation_count", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 2}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "encoded_mutation_bytes", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 15}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "set_count", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 2}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "delete_count", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 0}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "key_bytes", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 5}, + {Site: "tidb", OpClass: "kv_mutation", OperatorKind: "memdb_mutation", DMLKind: "insert", Unit: "value_bytes", InputSource: "stmt_memdb_mutation_calls", InputSide: "all", Value: 10}, + {Site: "tidb", OpClass: "reader_transport", OperatorKind: "mixed_reader", Unit: "read_request_count", InputSource: "ruv2_metrics", InputSide: "all", Value: 4}, + {Site: "tikv", OpClass: "kv_write", OperatorKind: "txn_write", DMLKind: "update", Unit: "write_request_count", InputSource: "ruv2_metrics", InputSide: "all", Value: 2}, }, } @@ -161,8 +170,8 @@ func TestReadBillingDemoGeneralLogUnits(t *testing.T) { fields := entries[0].ContextMap() require.Len(t, fields, 6) require.Equal(t, uint64(42), fields["conn"]) - require.Equal(t, "v3", fields["model_version"]) - require.Equal(t, "v2", fields["weight_version"]) + require.Equal(t, "v4", fields["model_version"]) + require.Equal(t, "test-v4-calibrated", fields["weight_version"]) require.Equal(t, expectedNormalizedSQL, fields["normalized_sql"]) require.Equal(t, expectedDigest.String(), fields["sql_digest"]) require.NotContains(t, fields["normalized_sql"], "12345") @@ -170,30 +179,49 @@ func TestReadBillingDemoGeneralLogUnits(t *testing.T) { require.NotContains(t, fields, "sql") rawUnits, ok := fields["units"].([]any) require.True(t, ok) - require.Len(t, rawUnits, 3) + require.Len(t, rawUnits, 13) units := make([]map[string]any, 0, len(rawUnits)) for _, rawUnit := range rawUnits { unit, ok := rawUnit.(map[string]any) require.True(t, ok) - require.Len(t, unit, 5) - for _, field := range []string{"site", "op_class", "operator_kind", "unit", "value"} { + require.Len(t, unit, 8) + for _, field := range []string{"site", "op_class", "operator_kind", "dml_kind", "unit", "input_source", "input_side", "value"} { require.Contains(t, unit, field) } - for _, internalField := range []string{"dml_kind", "input_source", "input_side", "row_width_source", "row_width", "operator_id"} { + for _, internalField := range []string{"row_width_source", "row_width", "operator_id"} { require.NotContains(t, unit, internalField) } units = append(units, unit) } require.Equal(t, map[string]any{ - "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "unit": "input_bytes", "value": float64(8), + "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "dml_kind": "", "unit": "input_bytes", "input_source": "", "input_side": "", "value": float64(8), }, units[0]) require.Equal(t, map[string]any{ - "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "unit": "output_rows", "value": float64(1), + "site": "tidb", "op_class": "agg_hash", "operator_kind": "hashagg", "dml_kind": "", "unit": "output_rows", "input_source": "", "input_side": "", "value": float64(1), }, units[1]) + expectedMutationValues := map[string]float64{ + "cpu_work": 3.5, "delete_count": 0, "encoded_mutation_bytes": 15, "encoded_mutation_count": 2, + "key_bytes": 5, "set_count": 2, "value_bytes": 10, + } + for i, unitName := range []string{"cpu_work", "delete_count", "encoded_mutation_bytes", "encoded_mutation_count", "key_bytes", "set_count", "value_bytes"} { + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "kv_mutation", "operator_kind": "memdb_mutation", "dml_kind": "insert", + "unit": unitName, "input_source": "stmt_memdb_mutation_calls", "input_side": "all", "value": expectedMutationValues[unitName], + }, units[i+2]) + } + require.Equal(t, map[string]any{ + "site": "tidb", "op_class": "reader_transport", "operator_kind": "mixed_reader", "dml_kind": "", "unit": "read_request_count", "input_source": "ruv2_metrics", "input_side": "all", "value": float64(4), + }, units[9]) + require.Equal(t, map[string]any{ + "site": "tikv", "op_class": "join_hash", "operator_kind": "hashjoin", "dml_kind": "insert", "unit": "input_rows", "input_source": "child", "input_side": "left", "value": float64(2), + }, units[10]) + require.Equal(t, map[string]any{ + "site": "tikv", "op_class": "join_hash", "operator_kind": "hashjoin", "dml_kind": "update", "unit": "input_rows", "input_source": "runtime", "input_side": "right", "value": float64(3), + }, units[11]) require.Equal(t, map[string]any{ - "site": "tikv", "op_class": "join_hash", "operator_kind": "hashjoin", "unit": "input_rows", "value": float64(5), - }, units[2]) + "site": "tikv", "op_class": "kv_write", "operator_kind": "txn_write", "dml_kind": "update", "unit": "write_request_count", "input_source": "ruv2_metrics", "input_side": "all", "value": float64(2), + }, units[12]) // A valid completed snapshot with no unit samples stays structured and does // not invent one statement status from the snapshot's multi-status model. diff --git a/pkg/executor/adapter_test.go b/pkg/executor/adapter_test.go index 24dcf21706a5d..b3b25647dc0bc 100644 --- a/pkg/executor/adapter_test.go +++ b/pkg/executor/adapter_test.go @@ -707,6 +707,72 @@ func TestFinishExecuteStmtSyncsTiDBRUV2FromRUDetails(t *testing.T) { require.NotContains(t, completionFields["normalized_sql"], "secret_literal") }) + t.Run("preview RU general log publishes point lookup rpc only", func(t *testing.T) { + core, recorded := observer.New(zap.InfoLevel) + oldLogger := logutil.GeneralLogger + logutil.GeneralLogger = zap.New(core) + oldGeneralLog := vardef.ProcessGeneralLog.Swap(false) + t.Cleanup(func() { + logutil.GeneralLogger = oldLogger + vardef.ProcessGeneralLog.Store(oldGeneralLog) + }) + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table point_get_rpc_ru_units (id int primary key, v int)") + tk.MustExec("insert into point_get_rpc_ru_units values (1, 10), (2, 20)") + tk.MustExec("set tidb_enable_read_billing_demo = on") + vardef.ProcessGeneralLog.Store(true) + + testCases := []struct { + name string + querySQL string + operatorKind string + requests int64 + rows []string + }{ + {"point get", "select v from point_get_rpc_ru_units where id = 1", "point_get", 3, []string{"10"}}, + {"batch point get", "select v from point_get_rpc_ru_units where id in (1, 2) order by v", "batch_point_get", 4, []string{"10", "20"}}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recorded.TakeAll() + ctx := execdetails.ContextWithInitializedExecDetails(context.Background()) + execdetails.RUV2MetricsFromContext(ctx).AddResourceManagerReadCnt(tc.requests) + tk.MustQueryWithContext(ctx, tc.querySQL).Check(testkit.Rows(tc.rows...)) + + entries := recorded.TakeAll() + var completionEntries []observer.LoggedEntry + for _, entry := range entries { + if entry.Message == "GENERAL_LOG_RU_UNITS" { + completionEntries = append(completionEntries, entry) + } + } + require.Len(t, completionEntries, 1) + requireReadBillingDemoGeneralLogIdentity(t, completionEntries[0], tc.querySQL) + rawUnits, ok := completionEntries[0].ContextMap()["units"].([]any) + require.True(t, ok) + pointUnits := 0 + for _, rawUnit := range rawUnits { + unit, ok := rawUnit.(map[string]any) + require.True(t, ok) + if unit["op_class"] != "kv_point_lookup" { + continue + } + pointUnits++ + require.Equal(t, "tikv", unit["site"]) + require.Equal(t, tc.operatorKind, unit["operator_kind"]) + require.Equal(t, "read_request_count", unit["unit"]) + require.Equal(t, "ruv2_metrics", unit["input_source"]) + require.Equal(t, "all", unit["input_side"]) + require.Equal(t, float64(tc.requests), unit["value"]) + } + require.Equal(t, 1, pointUnits) + }) + } + }) + t.Run("preview RU general log DML completion is self describing", func(t *testing.T) { core, recorded := observer.New(zap.InfoLevel) oldLogger := logutil.GeneralLogger diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 5febd0339198c..074c387d9759a 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/pkg/session" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/sqlexec" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -545,11 +546,32 @@ func TestExplainAnalyzeFormatRUOutput(t *testing.T) { _, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' select * from explain_ru_t where a > 0") require.Error(t, err) require.Contains(t, err.Error(), "status=unknown_input") - require.Contains(t, err.Error(), "reason=missing_scan_width_evidence") - require.Contains(t, err.Error(), "operator=tikv/kv_range_scan") - - rows = tk.MustQuery("explain analyze format='ru' select * from explain_ru_t where a = 1").Rows() - requireExplainRUWeightedOperatorClass(t, rows, "tikv/kv_point_lookup") + require.True(t, + strings.Contains(err.Error(), "reason=missing_scan_width_evidence operator=tikv/kv_range_scan/tablerangescan") || + strings.Contains(err.Error(), "reason=missing_reader_transport_details operator=tidb/reader_transport/table_reader"), + err.Error(), + ) + + for _, tc := range []struct { + sql string + operatorKind string + }{ + {"explain analyze format='ru' select * from explain_ru_t where a = 1", "point_get"}, + {"explain analyze format='ru' select * from explain_ru_t where a in (1, 2)", "batch_point_get"}, + } { + rows, err = queryExplainRURowsOrErrWithContext(execdetails.ContextWithInitializedExecDetails(context.Background()), t, tk, tc.sql) + require.NoError(t, err, tc.sql) + require.NotEmpty(t, rows, tc.sql) + requireExplainRUOperatorClass(t, rows, "tikv/kv_point_lookup") + require.GreaterOrEqual(t, explainRUCountUnitValue(t, rows, "tikv/kv_point_lookup", "read_request_count"), 0.0) + for _, row := range rows { + if len(row) != 17 || row[0] != "plan" || row[3] != "tikv/kv_point_lookup" { + continue + } + require.Equal(t, tc.operatorKind, row[2], tc.sql) + require.NotContains(t, []any{"cpu_work", "scan_bytes", "net_bytes"}, row[11], tc.sql) + } + } } func TestExplainAnalyzeFormatRUTiKVCopOperatorClasses(t *testing.T) { @@ -563,48 +585,68 @@ func TestExplainAnalyzeFormatRUTiKVCopOperatorClasses(t *testing.T) { cases := []struct { sql string nonScanOpClass string + planOperator string }{ { sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) where b > 10", nonScanOpClass: "tikv/filter_eval", + planOperator: "Selection", }, { sql: "explain analyze format='ru' select a from explain_ru_cop where b > 10", nonScanOpClass: "tikv/projection_eval", + planOperator: "Projection", }, { sql: "explain analyze format='ru' select * from explain_ru_cop limit 2", nonScanOpClass: "tikv/row_limit", + planOperator: "Limit", }, { sql: "explain analyze format='ru' select * from explain_ru_cop ignore index(idx_b) order by c limit 2", nonScanOpClass: "tikv/bounded_topn", + planOperator: "TopN", }, { sql: "explain analyze format='ru' select /*+ agg_to_cop(), hash_agg() */ b, count(*) from explain_ru_cop group by b", nonScanOpClass: "tikv/agg_hash", + planOperator: "HashAgg", }, { - sql: "explain analyze format='ru' select /*+ agg_to_cop(), stream_agg() */ b, count(*) from explain_ru_cop group by b", + sql: "explain analyze format='ru' select /*+ stream_agg(), agg_to_cop(), order_index(explain_ru_cop, idx_b) */ b, count(*) from explain_ru_cop group by b order by b", nonScanOpClass: "tikv/agg_stream", + planOperator: "StreamAgg", }, } for _, tc := range cases { - rows, err := queryExplainRURowsOrErr(t, tk, tc.sql) - if err != nil { - require.Contains(t, err.Error(), "status=unknown_input", tc.sql) - require.Contains(t, err.Error(), "reason=missing_scan_width_evidence", tc.sql) - require.Contains(t, err.Error(), "operator="+tc.nonScanOpClass, tc.sql) - continue + ordinarySQL := strings.Replace(tc.sql, "explain analyze format='ru'", "explain analyze", 1) + seenPushedOperator := false + for _, row := range tk.MustQuery(ordinarySQL).Rows() { + rowText := fmt.Sprint(row) + if strings.Contains(rowText, tc.planOperator) && strings.Contains(rowText, "cop[tikv]") { + seenPushedOperator = true + break + } } - requireExplainRUWeightedOperatorClass(t, rows, "tikv/kv_range_scan") - requireNoExplainRUOperatorClass(t, rows, tc.nonScanOpClass) + require.True(t, seenPushedOperator, "%s did not produce %s in cop[tikv]", tc.sql, tc.planOperator) + + _, err := queryExplainRURowsOrErr(t, tk, tc.sql) + require.Error(t, err, tc.nonScanOpClass) + require.Contains(t, err.Error(), "status=unknown_input", tc.sql) + // Native EmbedUnistore returns execution summaries for these pushed + // operators but does not populate the complete scan-width tuple. + require.Contains(t, err.Error(), "reason=missing_scan_width_evidence", tc.sql) + require.Contains(t, err.Error(), "operator=tikv/kv_range_scan/", tc.sql) } } func queryExplainRURowsOrErr(t *testing.T, tk *testkit.TestKit, sql string) ([][]any, error) { + return queryExplainRURowsOrErrWithContext(context.Background(), t, tk, sql) +} + +func queryExplainRURowsOrErrWithContext(ctx context.Context, t *testing.T, tk *testkit.TestKit, sql string) ([][]any, error) { t.Helper() - rs, err := tk.Exec(sql) + rs, err := tk.ExecWithContext(ctx, sql) if err != nil { if rs != nil { require.NoError(t, rs.Close()) @@ -653,10 +695,10 @@ func requireExplainRUPlanRow(t *testing.T, rows [][]any) { require.NotEmpty(t, row[8]) require.NotEmpty(t, row[11]) require.NotEmpty(t, row[12]) - require.NotEmpty(t, row[13]) - require.NotEmpty(t, row[14]) + require.Empty(t, row[13]) + require.Empty(t, row[14]) require.NotEmpty(t, row[15]) - require.Contains(t, fmt.Sprint(row[16]), "weight_version=v2") + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v3-resource-formula-uncalibrated") return } require.Fail(t, "missing FORMAT='RU' plan row") @@ -679,9 +721,10 @@ func requireExplainRUWeightedOperatorClass(t *testing.T, rows [][]any, operatorC if row[0] != "plan" || row[3] != operatorClass { continue } - require.NotEmpty(t, row[13], "missing weight for %s row %v", operatorClass, row) - require.NotEmpty(t, row[14], "missing preview RU for %s row %v", operatorClass, row) - require.Contains(t, fmt.Sprint(row[16]), "weight_version=v2") + require.NotEmpty(t, row[12], "missing semantic unit value for %s row %v", operatorClass, row) + require.Empty(t, row[13], "uncalibrated v4 must not publish a weight for %s row %v", operatorClass, row) + require.Empty(t, row[14], "uncalibrated v4 must not publish preview RU for %s row %v", operatorClass, row) + require.Contains(t, fmt.Sprint(row[16]), "weight_version=v3-resource-formula-uncalibrated") return } require.Failf(t, "missing weighted FORMAT='RU' operator class", "operatorClass=%s rows=%v", operatorClass, rows) @@ -807,7 +850,14 @@ func explainRUCountUnitValue(t *testing.T, rows [][]any, operatorClass, unit str if len(row) != 17 || row[0] != "plan" || row[3] != operatorClass || row[11] != unit { continue } - value, err := strconv.ParseFloat(fmt.Sprint(row[12]), 64) + valueColumn := 12 + switch unit { + case "cpu_work": + valueColumn = 9 + case "scan_bytes", "net_bytes", "encoded_mutation_bytes", "key_bytes", "value_bytes": + valueColumn = 10 + } + value, err := strconv.ParseFloat(fmt.Sprint(row[valueColumn]), 64) require.NoError(t, err) return value } @@ -840,13 +890,67 @@ func TestExplainAnalyzeFormatRUPlanDigest(t *testing.T) { digestRows := tk.MustQuery("select plan_digest from information_schema.statements_summary_history where digest_text like 'select `b` from `explain_ru_digest`%' and plan_digest != ''").Rows() require.NotEmpty(t, digestRows) planDigest := fmt.Sprint(digestRows[0][0]) - rows := tk.MustQuery(fmt.Sprintf("explain analyze format='ru' '%s'", planDigest)).Rows() + rows, err := queryExplainRURowsOrErr(t, tk, fmt.Sprintf("explain analyze format='ru' '%s'", planDigest)) + require.NoError(t, err) + requireExplainRUOperatorClass(t, rows, "tikv/kv_point_lookup") + require.GreaterOrEqual(t, explainRUCountUnitValue(t, rows, "tikv/kv_point_lookup", "read_request_count"), 0.0) +} + +func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists explain_ru_write_v4") + tk.MustExec("create table explain_ru_write_v4(a int primary key, b varchar(20))") + + rows, err := queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' insert into explain_ru_write_v4 values (1, 'one')") + require.NoError(t, err) require.NotEmpty(t, rows) require.Equal(t, "summary", rows[0][0]) - requireExplainRUPlanRow(t, rows) + require.Empty(t, rows[0][14], rows) + require.Contains(t, fmt.Sprint(rows[0][16]), "weight_version=v3-resource-formula-uncalibrated") + require.Contains(t, fmt.Sprint(rows[0][16]), "uncalibrated_weights") + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_bytes"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "set_count"), rows) + require.Zero(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "delete_count"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "key_bytes"), rows) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "value_bytes"), rows) + requireExplainRUOperatorUnitAbsent(t, rows, "tidb/kv_mutation", "cpu_work") + requireExplainRUUnitAbsent(t, rows, "write_request_count") + require.Contains(t, fmt.Sprint(rows[0][16]), "partial_missing_write_rpc_count") + tk.MustQuery("select * from explain_ru_write_v4").Check(testkit.Rows("1 one")) + + tk.MustExec("begin pessimistic") + rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write_v4 set b = 'two' where a = 1") + require.NoError(t, err) + require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) + requireExplainRUOperatorUnitAbsent(t, rows, "tidb/kv_mutation", "cpu_work") + requireExplainRUUnitAbsent(t, rows, "write_request_count") + tk.MustExec("commit") + tk.MustQuery("select * from explain_ru_write_v4").Check(testkit.Rows("1 two")) } -func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { +func requireExplainRUUnitAbsent(t *testing.T, rows [][]any, unit string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" { + require.NotEqual(t, unit, row[11], rows) + } + } +} + +func requireExplainRUOperatorUnitAbsent(t *testing.T, rows [][]any, operatorClass, unit string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" && row[3] == operatorClass { + require.NotEqual(t, unit, row[11], rows) + } + } +} + +func TestExplainAnalyzeFormatRUWriteDMLV3Legacy(t *testing.T) { + t.Skip("v3 per-unit weighting and commit-detail ownership are superseded by the v4 resource formula") store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -1072,20 +1176,87 @@ func TestReadBillingDemoMetricsHook(t *testing.T) { tk.MustExec("set global tidb_enable_stmt_summary = 0") tk.MustExec("set global tidb_enable_stmt_summary = 1") + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v4", "v3-resource-formula-uncalibrated") + cpuWork := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "cpu_work", "runtime_child_act_rows", "all", "v4", "v3-resource-formula-uncalibrated") + pointGetRequests := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_point_lookup", "point_get", "read_request_count", "ruv2_metrics", "all", "v4", "v3-resource-formula-uncalibrated") + batchPointGetRequests := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_point_lookup", "batch_point_get", "read_request_count", "ruv2_metrics", "all", "v4", "v3-resource-formula-uncalibrated") + mutationCPUWork := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "cpu_work", "stmt_memdb_mutation_calls", "all", "v4", "v3-resource-formula-uncalibrated") + mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v4", "v3-resource-formula-uncalibrated") + + tk.MustExec("set tidb_enable_read_billing_demo=off") + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Zero(t, readExecutorCounterValue(t, success)) + + tk.MustExec("set tidb_enable_read_billing_demo=on") + tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) + require.Equal(t, 1.0, readExecutorCounterValue(t, success)) + require.Greater(t, readExecutorCounterValue(t, cpuWork), 0.0) + tk.MustQuery(`select exec_count, sum_read_billing_demo_fixed_events, sum_read_billing_demo_input_rows, sum_read_billing_demo_input_bytes from information_schema.statements_summary where digest_text = 'select ? + ?'`).Check(testkit.Rows("2 0 0 0")) + tk.MustQuery(`select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text = 'select ? + ?' and unit = 'cpu_work'`).Check(testkit.Rows("tidb projection_eval projection cpu_work runtime_child_act_rows all v4 v3-resource-formula-uncalibrated 1 1")) + tk.MustQuery(`select site, op_class, operator_kind, status, reason, count from information_schema.statements_summary_read_billing_demo_status where digest_text = 'select ? + ?' and site = 'statement'`).Check(testkit.Rows("statement statement statement success none 1")) + + tk.MustExec("drop table if exists read_billing_demo_v4") + tk.MustExec("create table read_billing_demo_v4(a int primary key)") + beforeMutationCPUWork := readExecutorCounterValue(t, mutationCPUWork) + beforeMutation := readExecutorCounterValue(t, mutationCount) + tk.MustExec("insert into read_billing_demo_v4 values (1), (2)") + require.Equal(t, beforeMutationCPUWork, readExecutorCounterValue(t, mutationCPUWork)) + require.Greater(t, readExecutorCounterValue(t, mutationCount), beforeMutation) + tk.MustQuery("select site, op_class, operator_kind, dml_kind, unit, input_source, input_side, model_version, weight_version, value > 0 from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'insert into `read_billing_demo_v4`%' and unit in ('encoded_mutation_bytes', 'encoded_mutation_count', 'set_count', 'delete_count', 'key_bytes', 'value_bytes') order by unit").Check(testkit.Rows( + "tidb kv_mutation memdb_mutation insert delete_count stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 0", + "tidb kv_mutation memdb_mutation insert encoded_mutation_bytes stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert encoded_mutation_count stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert key_bytes stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert set_count stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + "tidb kv_mutation memdb_mutation insert value_bytes stmt_memdb_mutation_calls all v4 v3-resource-formula-uncalibrated 1", + )) + + pointCtx := execdetails.ContextWithInitializedExecDetails(context.Background()) + execdetails.RUV2MetricsFromContext(pointCtx).AddResourceManagerReadCnt(3) + beforePointGetRequests := readExecutorCounterValue(t, pointGetRequests) + tk.MustQueryWithContext(pointCtx, "select * from read_billing_demo_v4 where a = 1").Check(testkit.Rows("1")) + require.Equal(t, beforePointGetRequests+3, readExecutorCounterValue(t, pointGetRequests)) + + batchCtx := execdetails.ContextWithInitializedExecDetails(context.Background()) + execdetails.RUV2MetricsFromContext(batchCtx).AddResourceManagerReadCnt(4) + beforeBatchPointGetRequests := readExecutorCounterValue(t, batchPointGetRequests) + tk.MustQueryWithContext(batchCtx, "select * from read_billing_demo_v4 where a in (1, 2)").Sort().Check(testkit.Rows("1", "2")) + require.Equal(t, beforeBatchPointGetRequests+4, readExecutorCounterValue(t, batchPointGetRequests)) + + tk.MustQuery("select site, op_class, operator_kind, unit, input_source, input_side, model_version, weight_version, sample_count, value from information_schema.statements_summary_read_billing_demo_base_units where digest_text like 'select * from `read_billing_demo_v4`%' and op_class = 'kv_point_lookup' order by operator_kind").Check(testkit.Rows( + "tikv kv_point_lookup batch_point_get read_request_count ruv2_metrics all v4 v3-resource-formula-uncalibrated 1 4", + "tikv kv_point_lookup point_get read_request_count ruv2_metrics all v4 v3-resource-formula-uncalibrated 1 3", + )) +} + +func TestReadBillingDemoV3MetricsHookLegacy(t *testing.T) { + t.Skip("v3 formula expectations are superseded by TestReadBillingDemoMetricsHook") + metrics.InitExplainRUMetrics() + + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) + tk.MustExec("use test") + + originEnableStmtSummary := fmt.Sprint(tk.MustQuery("select @@global.tidb_enable_stmt_summary").Rows()[0][0]) + defer tk.MustExec(fmt.Sprintf("set global tidb_enable_stmt_summary = %s", originEnableStmtSummary)) + tk.MustExec("set global tidb_enable_stmt_summary = 0") + tk.MustExec("set global tidb_enable_stmt_summary = 1") + tk.MustQuery("select @@tidb_enable_read_billing_demo").Check(testkit.Rows("0")) tk.MustExec("drop table if exists read_billing_demo") tk.MustExec("create table read_billing_demo(a int primary key)") tk.MustExec("insert into read_billing_demo values (1), (2)") - success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v3") - unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v3") - unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v3") - errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v3") - projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v3") - mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v3") - mutationBytes := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v3") - writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_keys", "commit_detail", "all", "v3") - writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_byte", "commit_detail", "all", "v3") + success := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("success", "v3", "v2") + unsupported := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unsupported", "v3", "v2") + unknownInput := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("unknown_input", "v3", "v2") + errorStatus := metrics.ReadBillingDemoStatementsCounter.WithLabelValues("error", "v3", "v2") + projectionFixedEvents := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "fixed_events", "runtime_chunk_bytes", "all", "v3", "v2") + mutationCount := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v3", "v2") + mutationBytes := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v3", "v2") + writeKeys := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_keys", "commit_detail", "all", "v3", "v2") + writeByte := metrics.ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_prewrite", "write_byte", "commit_detail", "all", "v3", "v2") tk.MustQuery("select 1 + 1").Check(testkit.Rows("2")) require.Equal(t, 0.0, readExecutorCounterValue(t, success)) diff --git a/pkg/executor/join/hash_join_stats.go b/pkg/executor/join/hash_join_stats.go index 10d82b323c246..d538b828f252f 100644 --- a/pkg/executor/join/hash_join_stats.go +++ b/pkg/executor/join/hash_join_stats.go @@ -17,6 +17,7 @@ package join import ( "bytes" "fmt" + "math" "strconv" "sync/atomic" "time" @@ -25,6 +26,37 @@ import ( "github.com/pingcap/tidb/pkg/util/execdetails" ) +func addHashTableRows(addr *int64, rows uint64) { + if rows > math.MaxInt64 { + atomic.StoreInt64(addr, -1) + return + } + delta := int64(rows) + for { + current := atomic.LoadInt64(addr) + if current < 0 { + return + } + if delta > math.MaxInt64-current { + if atomic.CompareAndSwapInt64(addr, current, -1) { + return + } + continue + } + if atomic.CompareAndSwapInt64(addr, current, current+delta) { + return + } + } +} + +func mergeHashTableRows(addr *int64, rows int64) { + if rows < 0 { + atomic.StoreInt64(addr, -1) + return + } + addHashTableRows(addr, uint64(rows)) +} + func writeSpilledPartitionNumStatsToString(buf *bytes.Buffer, partitionNum int, spilledPartitionNumPerRound []int) { buf.WriteString("[") fmt.Fprintf(buf, "%d/%d", spilledPartitionNumPerRound[0], partitionNum) @@ -53,8 +85,11 @@ type hashJoinRuntimeStats struct { probe int64 concurrent int maxFetchAndProbe int64 + hashTableRows int64 } +func (e *hashJoinRuntimeStats) HashTableRows() int64 { return atomic.LoadInt64(&e.hashTableRows) } + // Tp implements the RuntimeStats interface. func (*hashJoinRuntimeStats) Tp() int { return execdetails.TpHashJoinRuntimeStats @@ -101,6 +136,7 @@ func (e *hashJoinRuntimeStats) Clone() execdetails.RuntimeStats { probe: e.probe, concurrent: e.concurrent, maxFetchAndProbe: e.maxFetchAndProbe, + hashTableRows: e.HashTableRows(), } } @@ -114,6 +150,7 @@ func (e *hashJoinRuntimeStats) Merge(rs execdetails.RuntimeStats) { e.hashStat.probeCollision += tmp.hashStat.probeCollision e.fetchAndProbe += tmp.fetchAndProbe e.probe += tmp.probe + mergeHashTableRows(&e.hashTableRows, tmp.HashTableRows()) if e.maxFetchAndProbe < tmp.maxFetchAndProbe { e.maxFetchAndProbe = tmp.maxFetchAndProbe } @@ -163,9 +200,12 @@ type hashJoinRuntimeStatsV2 struct { spill spillStats - isHashJoinGA bool + isHashJoinGA bool + hashTableRows int64 } +func (e *hashJoinRuntimeStatsV2) HashTableRows() int64 { return atomic.LoadInt64(&e.hashTableRows) } + func setMaxValue(addr *int64, currentValue int64) { for { value := atomic.LoadInt64(addr) @@ -179,6 +219,7 @@ func setMaxValue(addr *int64, currentValue int64) { } func (e *hashJoinRuntimeStatsV2) reset() { + atomic.StoreInt64(&e.hashTableRows, 0) e.probeCollision = 0 e.fetchAndBuildHashTable = 0 e.partitionData = 0 @@ -308,6 +349,7 @@ func (e *hashJoinRuntimeStatsV2) Clone() execdetails.RuntimeStats { maxBuildHashTableForCurrentRound: e.maxBuildHashTableForCurrentRound, maxProbeForCurrentRound: e.maxProbeForCurrentRound, maxWorkerFetchAndProbeForCurrentRound: e.maxWorkerFetchAndProbeForCurrentRound, + hashTableRows: e.HashTableRows(), } } @@ -326,6 +368,7 @@ func (e *hashJoinRuntimeStatsV2) Merge(rs execdetails.RuntimeStats) { e.maxPartitionData = tmp.maxPartitionData } e.probeCollision += tmp.probeCollision + mergeHashTableRows(&e.hashTableRows, tmp.HashTableRows()) e.fetchAndProbe += tmp.fetchAndProbe e.probe += tmp.probe if e.maxWorkerFetchAndProbe < tmp.maxWorkerFetchAndProbe { diff --git a/pkg/executor/join/hash_join_v1.go b/pkg/executor/join/hash_join_v1.go index 6976ea6a834eb..6e90320e5345a 100644 --- a/pkg/executor/join/hash_join_v1.go +++ b/pkg/executor/join/hash_join_v1.go @@ -1082,6 +1082,8 @@ func (e *HashJoinV1Exec) fetchAndBuildHashTable(ctx context.Context) { if err == nil { if err = <-fetchBuildSideRowsOk; err != nil { e.buildFinished <- err + } else if e.stats != nil && e.RowContainer != nil { + addHashTableRows(&e.stats.hashTableRows, e.RowContainer.hashStateRows()) } } } diff --git a/pkg/executor/join/hash_join_v2.go b/pkg/executor/join/hash_join_v2.go index 51a278bbcce73..1bb6eb0c9b717 100644 --- a/pkg/executor/join/hash_join_v2.go +++ b/pkg/executor/join/hash_join_v2.go @@ -93,6 +93,23 @@ func (htc *hashTableContext) getAllMemoryUsageInHashTable() int64 { return totalMemoryUsage } +func (htc *hashTableContext) hashStateRows() uint64 { + var rows uint64 + if htc == nil || htc.hashTable == nil { + return 0 + } + for _, table := range htc.hashTable.tables { + if table != nil && table.rowData != nil { + validRows := table.rowData.validKeyCount() + if math.MaxUint64-rows < validRows { + return math.MaxUint64 + } + rows += validRows + } + } + return rows +} + func (htc *hashTableContext) clearHashTable() { partNum := len(htc.hashTable.tables) for i := range partNum { @@ -1324,7 +1341,12 @@ func (e *HashJoinV2Exec) fetchAndBuildHashTableImpl(ctx context.Context) { buildTaskCh := e.createBuildTasks(totalSegmentCnt, wg, errCh, doneCh) e.buildHashTable(buildTaskCh, wg, errCh, doneCh) - waitJobDone(wg, errCh) + if !waitJobDone(wg, errCh) { + return + } + if e.stats != nil { + addHashTableRows(&e.stats.hashTableRows, e.hashTableContext.hashStateRows()) + } } func (e *HashJoinV2Exec) fetchBuildSideRows(ctx context.Context, fetcherAndWorkerSyncer *sync.WaitGroup, wg *sync.WaitGroup, errCh chan error, doneCh chan struct{}) chan *chunk.Chunk { diff --git a/pkg/executor/join/hash_table_v1.go b/pkg/executor/join/hash_table_v1.go index 5e676e7972e8c..cba4df90d41fa 100644 --- a/pkg/executor/join/hash_table_v1.go +++ b/pkg/executor/join/hash_table_v1.go @@ -532,6 +532,18 @@ func (c *hashRowContainer) Len() uint64 { return c.hashTable.Len() } +func (c *hashRowContainer) hashStateRows() uint64 { + rows := c.Len() + if c.hashNANullBucket != nil { + nullRows := uint64(len(c.hashNANullBucket.entries)) + if ^uint64(0)-rows < nullRows { + return ^uint64(0) + } + rows += nullRows + } + return rows +} + func (c *hashRowContainer) Close() error { failpoint.Inject("issue60926", nil) diff --git a/pkg/executor/join/hash_table_v1_test.go b/pkg/executor/join/hash_table_v1_test.go index 775cb87a02be1..d769c8efaa27d 100644 --- a/pkg/executor/join/hash_table_v1_test.go +++ b/pkg/executor/join/hash_table_v1_test.go @@ -84,6 +84,12 @@ func (h hashCollision) Size() int { panic("not implement func (h hashCollision) BlockSize() int { panic("not implemented") } func TestHashRowContainer(t *testing.T) { + stateContainer := &hashRowContainer{ + hashTable: &unsafeHashTable{length: 2}, + hashNANullBucket: &hashNANullBucket{entries: []*naEntry{{}, {}, {}}}, + } + require.Equal(t, uint64(5), stateContainer.hashStateRows()) + hashFunc := fnv.New64 rowContainer, copiedRC := testHashRowContainer(t, hashFunc, false) require.Equal(t, int64(0), rowContainer.stat.probeCollision) diff --git a/pkg/executor/join/join_stats_test.go b/pkg/executor/join/join_stats_test.go index cba47f81e19fb..63927ff79fe12 100644 --- a/pkg/executor/join/join_stats_test.go +++ b/pkg/executor/join/join_stats_test.go @@ -15,6 +15,8 @@ package join import ( + "math" + "sync/atomic" "testing" "time" @@ -22,6 +24,16 @@ import ( ) func TestHashJoinRuntimeStats(t *testing.T) { + overflowRows := int64(1) + addHashTableRows(&overflowRows, math.MaxInt64) + require.Equal(t, int64(-1), overflowRows) + addHashTableRows(&overflowRows, 1) + require.Equal(t, int64(-1), overflowRows) + + rowData := &rowTable{segments: []*rowTableSegment{{validJoinKeyPos: []int{0, 2}}, {validJoinKeyPos: []int{1}}}} + hashCtx := &hashTableContext{hashTable: &hashTableV2{tables: []*subTable{{rowData: rowData}}}} + require.Equal(t, uint64(3), hashCtx.hashStateRows()) + stats := &hashJoinRuntimeStats{ fetchAndBuildHashTable: 2 * time.Second, hashStat: hashStatistic{ @@ -37,6 +49,18 @@ func TestHashJoinRuntimeStats(t *testing.T) { require.Equal(t, stats.Clone().String(), stats.String()) stats.Merge(stats.Clone()) require.Equal(t, "build_hash_table:{total:4s, fetch:3.8s, build:200ms}, probe:{concurrency:4, total:10s, max:2s, probe:8s, fetch and wait:2s, probe_collision:2}", stats.String()) + atomic.StoreInt64(&stats.hashTableRows, 7) + cloned := stats.Clone().(*hashJoinRuntimeStats) + require.Equal(t, int64(7), cloned.HashTableRows()) + stats.Merge(cloned) + require.Equal(t, int64(14), stats.HashTableRows()) + + statsV2 := &hashJoinRuntimeStatsV2{} + atomic.StoreInt64(&statsV2.hashTableRows, 11) + clonedV2 := statsV2.Clone().(*hashJoinRuntimeStatsV2) + require.Equal(t, int64(11), clonedV2.HashTableRows()) + statsV2.Merge(clonedV2) + require.Equal(t, int64(22), statsV2.HashTableRows()) } func TestIndexJoinRuntimeStats(t *testing.T) { diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index 1cfb3c4ef73e9..a439a67057de9 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -1445,9 +1445,9 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.AvgQueuedRcTimeStr, tp: mysql.TypeLonglong, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of waiting for available request-units"}, {name: stmtsummary.MaxRequestUnitV2Str, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Max request-unit v2 cost of these statements"}, {name: stmtsummary.AvgRequestUnitV2Str, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Average request-unit v2 cost of these statements"}, - {name: stmtsummary.SumReadBillingDemoFixedEventsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Total read billing demo fixed-event base units"}, - {name: stmtsummary.SumReadBillingDemoInputRowsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Total read billing demo input-row base units"}, - {name: stmtsummary.SumReadBillingDemoInputBytesStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Total read billing demo input-byte base units"}, + {name: stmtsummary.SumReadBillingDemoFixedEventsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Legacy v3 read billing demo fixed-event units; zero for v4"}, + {name: stmtsummary.SumReadBillingDemoInputRowsStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Legacy v3 read billing demo input-row units; zero for v4"}, + {name: stmtsummary.SumReadBillingDemoInputBytesStr, tp: mysql.TypeDouble, flag: mysql.NotNullFlag | mysql.UnsignedFlag, size: 22, comment: "Legacy v3 read billing demo input-byte units; zero for v4"}, {name: stmtsummary.ResourceGroupName, tp: mysql.TypeVarchar, size: 64, comment: "Bind resource group name"}, {name: stmtsummary.PlanCacheUnqualifiedStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag, comment: "The number of times that these statements are not supported by the plan cache"}, {name: stmtsummary.PlanCacheUnqualifiedLastReasonStr, tp: mysql.TypeBlob, size: types.UnspecifiedLength, comment: "The last reason why the statement is not supported by the plan cache"}, diff --git a/pkg/metrics/explain_ru.go b/pkg/metrics/explain_ru.go index a3dc234e5f6f9..553e811cc3dc7 100644 --- a/pkg/metrics/explain_ru.go +++ b/pkg/metrics/explain_ru.go @@ -110,7 +110,7 @@ func InitExplainRUMetrics() { Subsystem: "read_billing_demo", Name: "statements_total", Help: "Counter of read billing demo statement status.", - }, []string{"status", readBillingDemoLabelModelVersion}, + }, []string{"status", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, ) ReadBillingDemoOperatorStatusCounter = metricscommon.NewCounterVec( prometheus.CounterOpts{ @@ -118,7 +118,7 @@ func InitExplainRUMetrics() { Subsystem: "read_billing_demo", Name: "operator_status_total", Help: "Counter of read billing demo operator status.", - }, []string{"site", "op_class", "operator_kind", "status", "reason", readBillingDemoLabelModelVersion}, + }, []string{"site", "op_class", "operator_kind", "status", "reason", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, ) ReadBillingDemoBaseUnitsCounter = metricscommon.NewCounterVec( prometheus.CounterOpts{ @@ -126,7 +126,7 @@ func InitExplainRUMetrics() { Subsystem: "read_billing_demo", Name: "base_units_total", Help: "Counter of coefficient-free read billing demo base units.", - }, []string{"site", "op_class", "operator_kind", "unit", "input_source", "input_side", readBillingDemoLabelModelVersion}, + }, []string{"site", "op_class", "operator_kind", "unit", "input_source", "input_side", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, ) ReadBillingDemoRowWidthHistogram = metricscommon.NewHistogramVec( prometheus.HistogramOpts{ @@ -135,7 +135,7 @@ func InitExplainRUMetrics() { Name: "row_width_bytes", Help: "Histogram of row-width factors used by read billing demo.", Buckets: prometheus.ExponentialBuckets(1, 2, 12), - }, []string{"site", "op_class", "operator_kind", "row_width_source", readBillingDemoLabelModelVersion}, + }, []string{"site", "op_class", "operator_kind", "row_width_source", readBillingDemoLabelModelVersion, explainRULabelWeightVersion}, ) } @@ -182,37 +182,37 @@ func ObserveExplainRURow(section, component, operator, source, rowWidthSource, w } // RecordReadBillingDemoStatement records a bounded read billing demo statement status. -func RecordReadBillingDemoStatement(status, modelVersion string) { - if ReadBillingDemoStatementsCounter == nil || status == "" || modelVersion == "" { +func RecordReadBillingDemoStatement(status, modelVersion, weightVersion string) { + if ReadBillingDemoStatementsCounter == nil || status == "" || modelVersion == "" || weightVersion == "" { return } - ReadBillingDemoStatementsCounter.WithLabelValues(status, modelVersion).Inc() + ReadBillingDemoStatementsCounter.WithLabelValues(status, modelVersion, weightVersion).Inc() } // RecordReadBillingDemoOperatorStatus records a bounded read billing demo operator status. -func RecordReadBillingDemoOperatorStatus(site, opClass, operatorKind, status, reason, modelVersion string) { +func RecordReadBillingDemoOperatorStatus(site, opClass, operatorKind, status, reason, modelVersion, weightVersion string) { if ReadBillingDemoOperatorStatusCounter == nil || - site == "" || opClass == "" || operatorKind == "" || status == "" || reason == "" || modelVersion == "" { + site == "" || opClass == "" || operatorKind == "" || status == "" || reason == "" || modelVersion == "" || weightVersion == "" { return } - ReadBillingDemoOperatorStatusCounter.WithLabelValues(site, opClass, operatorKind, status, reason, modelVersion).Inc() + ReadBillingDemoOperatorStatusCounter.WithLabelValues(site, opClass, operatorKind, status, reason, modelVersion, weightVersion).Inc() } // AddReadBillingDemoBaseUnits records coefficient-free read billing demo base units. -func AddReadBillingDemoBaseUnits(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion string, value float64) { +func AddReadBillingDemoBaseUnits(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion, weightVersion string, value float64) { if ReadBillingDemoBaseUnitsCounter == nil || - site == "" || opClass == "" || operatorKind == "" || unit == "" || inputSource == "" || inputSide == "" || modelVersion == "" || + site == "" || opClass == "" || operatorKind == "" || unit == "" || inputSource == "" || inputSide == "" || modelVersion == "" || weightVersion == "" || value < 0 { return } - ReadBillingDemoBaseUnitsCounter.WithLabelValues(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion).Add(value) + ReadBillingDemoBaseUnitsCounter.WithLabelValues(site, opClass, operatorKind, unit, inputSource, inputSide, modelVersion, weightVersion).Add(value) } // ObserveReadBillingDemoRowWidth records row-width factors used by read billing demo. -func ObserveReadBillingDemoRowWidth(site, opClass, operatorKind, rowWidthSource, modelVersion string, rowWidth float64) { +func ObserveReadBillingDemoRowWidth(site, opClass, operatorKind, rowWidthSource, modelVersion, weightVersion string, rowWidth float64) { if ReadBillingDemoRowWidthHistogram == nil || - site == "" || opClass == "" || operatorKind == "" || rowWidthSource == "" || modelVersion == "" || rowWidth <= 0 { + site == "" || opClass == "" || operatorKind == "" || rowWidthSource == "" || modelVersion == "" || weightVersion == "" || rowWidth <= 0 { return } - ReadBillingDemoRowWidthHistogram.WithLabelValues(site, opClass, operatorKind, rowWidthSource, modelVersion).Observe(rowWidth) + ReadBillingDemoRowWidthHistogram.WithLabelValues(site, opClass, operatorKind, rowWidthSource, modelVersion, weightVersion).Observe(rowWidth) } diff --git a/pkg/metrics/metrics_internal_test.go b/pkg/metrics/metrics_internal_test.go index 7c4c73e02b5b8..f2ce1ee4d3808 100644 --- a/pkg/metrics/metrics_internal_test.go +++ b/pkg/metrics/metrics_internal_test.go @@ -102,20 +102,30 @@ func TestExplainRUMetrics(t *testing.T) { ObserveExplainRURenderDuration("success", 0.01) RecordExplainRUComponentSnapshot("ok") ObserveExplainRURow("plan", "", "projection", "read_billing_model", "runtime_chunk_avg", "v2", 1.25, 3.5, 24, 8) - RecordReadBillingDemoStatement("success", "v2") - RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2") - AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", 3) - AddReadBillingDemoBaseUnits("tikv", "agg_hash", "hashagg", "output_rows", "runtime_operator_act_rows", "all", "v3", 0) - ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", 8) + RecordReadBillingDemoStatement("success", "v2", "weights-v2") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2", "weights-v2") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "weights-v2", 3) + AddReadBillingDemoBaseUnits("tikv", "agg_hash", "hashagg", "output_rows", "runtime_operator_act_rows", "all", "v3", "weights-v3", 0) + AddReadBillingDemoBaseUnits("tidb", "kv_mutation", "memdb_mutation", "cpu_work", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated", 3.5) + AddReadBillingDemoBaseUnits("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated", 2) + AddReadBillingDemoBaseUnits("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated", 15) + AddReadBillingDemoBaseUnits("tidb", "reader_transport", "mixed_reader", "read_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated", 4) + AddReadBillingDemoBaseUnits("tikv", "kv_write", "txn_write", "write_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated", 2) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", "weights-v2", 8) require.Equal(t, 1.0, readCounterValue(t, ExplainRUStatementsCounter.WithLabelValues("success"))) require.Equal(t, 1.0, readCounterValue(t, ExplainRUComponentSnapshotCounter.WithLabelValues("ok"))) require.Equal(t, 1.25, readCounterValue(t, ExplainRUPreviewRUCounter.WithLabelValues("plan", "", "projection", "read_billing_model", "v2"))) require.Equal(t, 3.5, readCounterValue(t, ExplainRUWorkRowsCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) require.Equal(t, 24.0, readCounterValue(t, ExplainRUWorkBytesCounter.WithLabelValues("plan", "", "projection", "read_billing_model"))) - require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2"))) - require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v2"))) - require.Equal(t, 3.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoStatementsCounter.WithLabelValues("success", "v2", "weights-v2"))) + require.Equal(t, 1.0, readCounterValue(t, ReadBillingDemoOperatorStatusCounter.WithLabelValues("tidb", "projection_eval", "projection", "ok", "none", "v2", "weights-v2"))) + require.Equal(t, 3.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "weights-v2"))) + require.Equal(t, 3.5, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "cpu_work", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 2.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_count", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 15.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "kv_mutation", "memdb_mutation", "encoded_mutation_bytes", "stmt_memdb_mutation_calls", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 4.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tidb", "reader_transport", "mixed_reader", "read_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated"))) + require.Equal(t, 2.0, readCounterValue(t, ReadBillingDemoBaseUnitsCounter.WithLabelValues("tikv", "kv_write", "txn_write", "write_request_count", "ruv2_metrics", "all", "v4", "test-v4-calibrated"))) registry := prometheus.NewRegistry() require.NoError(t, registry.Register(ExplainRUPreviewRUCounter)) @@ -150,13 +160,13 @@ func TestExplainRUMetrics(t *testing.T) { requireMetricFamilyHasLabels(t, componentSnapshotFamily, "component_snapshot_status") readBillingStatementFamily := findMetricFamily(families, "tidb_read_billing_demo_statements_total") require.NotNil(t, readBillingStatementFamily) - requireMetricFamilyHasLabels(t, readBillingStatementFamily, "status", "model_version") + requireMetricFamilyHasLabels(t, readBillingStatementFamily, "status", "model_version", "weight_version") readBillingOperatorFamily := findMetricFamily(families, "tidb_read_billing_demo_operator_status_total") require.NotNil(t, readBillingOperatorFamily) - requireMetricFamilyHasLabels(t, readBillingOperatorFamily, "site", "op_class", "operator_kind", "status", "reason", "model_version") + requireMetricFamilyHasLabels(t, readBillingOperatorFamily, "site", "op_class", "operator_kind", "status", "reason", "model_version", "weight_version") readBillingBaseUnitFamily := findMetricFamily(families, "tidb_read_billing_demo_base_units_total") require.NotNil(t, readBillingBaseUnitFamily) - requireMetricFamilyHasLabels(t, readBillingBaseUnitFamily, "site", "op_class", "operator_kind", "unit", "input_source", "input_side", "model_version") + requireMetricFamilyHasLabels(t, readBillingBaseUnitFamily, "site", "op_class", "operator_kind", "unit", "input_source", "input_side", "model_version", "weight_version") var zeroOutputRows *dto.Metric for _, metric := range readBillingBaseUnitFamily.GetMetric() { if metricHasLabelValue(metric, "unit", "output_rows") { @@ -168,7 +178,7 @@ func TestExplainRUMetrics(t *testing.T) { require.Zero(t, zeroOutputRows.GetCounter().GetValue()) readBillingRowWidthFamily := findMetricFamily(families, "tidb_read_billing_demo_row_width_bytes") require.NotNil(t, readBillingRowWidthFamily) - requireMetricFamilyHasLabels(t, readBillingRowWidthFamily, "site", "op_class", "operator_kind", "row_width_source", "model_version") + requireMetricFamilyHasLabels(t, readBillingRowWidthFamily, "site", "op_class", "operator_kind", "row_width_source", "model_version", "weight_version") } func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { @@ -180,10 +190,14 @@ func TestExplainRUMetricsIgnoreEmptyLabelsAndMissingValues(t *testing.T) { ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", "", 1, -1, -1, -1) ObserveExplainRURow("summary", "total_preview_ru", "", "summary_total", "", "v2", 0, -1, -1, -1) ObserveExplainRURow("plan", "", "", "read_billing_model", "runtime_chunk_avg", "v2", -1, -1, -1, 32) - RecordReadBillingDemoStatement("", "v2") - RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v2") - AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", -1) - ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", 0) + RecordReadBillingDemoStatement("", "v2", "weights-v2") + RecordReadBillingDemoStatement("success", "v2", "") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "", "none", "v2", "weights-v2") + RecordReadBillingDemoOperatorStatus("tidb", "projection_eval", "projection", "ok", "none", "v2", "") + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "weights-v2", -1) + AddReadBillingDemoBaseUnits("tidb", "projection_eval", "projection", "input_rows", "runtime_chunk_bytes", "all", "v2", "", 1) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", "weights-v2", 0) + ObserveReadBillingDemoRowWidth("tidb", "projection_eval", "projection", "runtime_chunk_avg", "v2", "", 8) require.Equal(t, 0, countCollectedMetrics(ExplainRUStatementsCounter)) require.Equal(t, 0, countCollectedMetrics(ExplainRURenderDurationHistogram)) diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 2e04a1615b1cb..b62e14732b951 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -333,6 +333,7 @@ go_test( "//pkg/util/plancodec", "//pkg/util/ranger", "//pkg/util/sem/v2:sem", + "//pkg/util/stmtsummary", "@com_github_docker_go_units//:go-units", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index a23dd4e827670..ec1bf6bed225b 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -15,14 +15,19 @@ package core import ( + "math" "testing" "time" "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/expression/aggregation" "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/core/operator/logicalop" "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" "github.com/pingcap/tidb/pkg/planner/property" plannerutil "github.com/pingcap/tidb/pkg/planner/util" @@ -30,6 +35,7 @@ import ( "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/mock" + "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" tikvutil "github.com/tikv/client-go/v2/util" @@ -238,8 +244,767 @@ func TestExplainRURowFormatting(t *testing.T) { }, row.toStrings()) } +const ( + readBillingDemoWriteKeyWeight = 0.6 + readBillingDemoWriteByteWeight = 0.00002 +) + +func buildWriteBillingDemoResultFromDetails(dmlKind string, _ *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { + return readBillingDemoResult{ + status: readBillingDemoStatusSuccess, + reason: readBillingDemoReasonNone, + operators: buildTiKVWriteBillingDemoOperators(dmlKind, ruv2Metrics, false), + } +} + +func readBillingDemoWriteDiagnosticStatus(dmlKind, reason string) readBillingDemoOperatorResult { + return readBillingDemoOperatorResult{ + id: "txn_write@statement", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassKVWrite, + operatorKind: readBillingDemoOperatorTxnWrite, + dmlKind: dmlKind, + scope: readBillingDemoScopeTxnPrewritePayload, + status: readBillingDemoStatusPartial, + reason: reason, + } +} + +type legacyReadBillingDemoWeights struct { + fixedEvent, row, byte, orderWork float64 + mutationCount, mutationByte float64 + writeKey, writeByte float64 + writeRPC, region float64 +} + +func (legacyReadBillingDemoWeights) valid() bool { return true } + +func (weights legacyReadBillingDemoWeights) unitWeight(unit string) (float64, bool) { + switch unit { + case readBillingDemoUnitFixedEvents: + return weights.fixedEvent, true + case readBillingDemoUnitInputRows: + return weights.row, true + case readBillingDemoUnitInputBytes: + return weights.byte, true + case readBillingDemoUnitOrderWork: + return weights.orderWork, true + case readBillingDemoUnitEncodedMutationCount: + return weights.mutationCount, true + case readBillingDemoUnitEncodedMutationBytes: + return weights.mutationByte, true + case readBillingDemoUnitWriteKeys: + return weights.writeKey, true + case readBillingDemoUnitWriteByte: + return weights.writeByte, true + case readBillingDemoUnitPrewriteRegionNum: + return weights.region, true + case readBillingDemoUnitTiKVWriteRPCCount: + return weights.writeRPC, true + default: + return 0, false + } +} + +func readBillingDemoResolveWeights(site, opClass, version string) (legacyReadBillingDemoWeights, bool) { + if version != readBillingDemoWeightVersion || opClass == readBillingDemoOpClassPointLookup && site != readBillingDemoSiteTiKV { + return legacyReadBillingDemoWeights{}, false + } + w := legacyReadBillingDemoWeights{ + fixedEvent: 0.1, row: 0.2, byte: 0.3, orderWork: 0.4, + } + if opClass == readBillingDemoOpClassTopN { + w.row = 0 + } + if opClass == readBillingDemoOpClassKVWrite { + w.writeKey = readBillingDemoWriteKeyWeight + w.writeByte = readBillingDemoWriteByteWeight + } + return w, true +} + +func TestReadBillingDemoV4FormulaContract(t *testing.T) { + weights := readBillingDemoWeights{ + Version: "test-v4-calibrated", + CPUWeight: 2, ScanWeight: 3, NetWeight: 5, ReadRequestWeight: 7, WriteRequestWeight: 17, + HashTableWeight: 11, JoinWeight: 13, MutationBytesPerCPUUnit: 10, Calibrated: true, + } + for _, tc := range []struct { + unit string + value float64 + weight float64 + }{ + {readBillingDemoUnitCPUWork, 4, 2}, {readBillingDemoUnitScanBytes, 6, 3}, + {readBillingDemoUnitNetBytes, 8, 5}, {readBillingDemoUnitReadRequestCount, 10, 7}, + {readBillingDemoUnitWriteRequestCount, 2, 17}, + {readBillingDemoUnitHashStateRows, 12, 11}, {readBillingDemoUnitJoinOutputRows, 14, 13}, + } { + weight, ru, ok := readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: tc.unit, value: tc.value}, weights) + require.True(t, ok) + require.Equal(t, tc.weight, weight) + require.Equal(t, tc.value*tc.weight, ru) + } + for _, invalid := range []float64{-1, math.NaN(), math.Inf(1)} { + _, _, ok := readBillingDemoUnitPreviewRU(readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, value: invalid}, weights) + require.False(t, ok) + } + for _, invalidWeights := range []readBillingDemoWeights{ + {}, + {MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: readBillingDemoWeightVersion, MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", CPUWeight: -1, MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", CPUWeight: math.NaN(), MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", CPUWeight: math.Inf(1), MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", ReadRequestWeight: -1, MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", WriteRequestWeight: math.NaN(), MutationBytesPerCPUUnit: 1, Calibrated: true}, + {Version: "test", MutationBytesPerCPUUnit: math.NaN(), Calibrated: true}, + {Version: "test", MutationBytesPerCPUUnit: math.Inf(1), Calibrated: true}, + } { + require.False(t, readBillingDemoWeightsValid(invalidWeights)) + } + + oldWeights := readBillingDemoV4Weights + readBillingDemoV4Weights = weights + t.Cleanup(func() { readBillingDemoV4Weights = oldWeights }) + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone, operators: []readBillingDemoOperatorResult{{ + id: "formula", site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassProjection, + operatorKind: "projection", status: readBillingDemoStatusOperatorOK, + units: []readBillingDemoUnit{ + {unit: readBillingDemoUnitCPUWork, value: 4}, {unit: readBillingDemoUnitScanBytes, value: 6}, + {unit: readBillingDemoUnitNetBytes, value: 8}, {unit: readBillingDemoUnitReadRequestCount, value: 10}, + {unit: readBillingDemoUnitWriteRequestCount, value: 2}, + {unit: readBillingDemoUnitHashStateRows, value: 12}, {unit: readBillingDemoUnitJoinOutputRows, value: 14}, + }, + }}} + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.True(t, rows[0].hasPreviewRU) + require.Equal(t, 484.0, rows[0].previewRU) + require.Contains(t, rows[0].note, "weight_version=test-v4-calibrated") + require.Equal(t, "test-v4-calibrated", buildReadBillingDemoStatementStats(result).WeightVersion) + overflowWeights := weights + overflowWeights.CPUWeight = 1 + readBillingDemoV4Weights = overflowWeights + overflowResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, operators: []readBillingDemoOperatorResult{{ + status: readBillingDemoStatusOperatorOK, opClass: readBillingDemoOpClassProjection, + units: []readBillingDemoUnit{{unit: readBillingDemoUnitCPUWork, value: math.MaxFloat64}, {unit: readBillingDemoUnitCPUWork, value: math.MaxFloat64}}, + }}} + overflowRows := explainRUBuildReadBillingRows(overflowResult, explainRUComponentSnapshotOK) + require.False(t, overflowRows[0].hasPreviewRU) + readBillingDemoV4Weights = weights + + readBillingDemoV4Weights = readBillingDemoWeights{} + rows = explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.False(t, rows[0].hasPreviewRU) + require.Contains(t, rows[0].note, readBillingDemoReasonUncalibratedWeights) + stats := buildReadBillingDemoStatementStats(result) + require.Equal(t, "v4", stats.ModelVersion) + require.Equal(t, "v3-resource-formula-uncalibrated", stats.WeightVersion) + require.Equal(t, stmtsummary.ReadBillingDemoBaseUnitSummary{}, stats.Totals) + + t.Run("reader transport is emitted once and fails closed", func(t *testing.T) { + ctx := mock.NewContext() + reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + flat := &FlatPhysicalPlan{Main: FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: scan, ChildrenEndIdx: 1, StoreType: kv.TiKV}, + }} + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + basic := runtimeStats.GetBasicRuntimeStats(reader.ID(), true) + basic.Record(time.Millisecond, 0) + metrics := execdetails.NewRUV2Metrics() + metrics.AddResourceManagerReadCnt(3) + metrics.AddTiKVCoprocessorResponseBytes(128) + op, present := readBillingDemoReaderTransport(flat, runtimeStats, metrics, false) + require.True(t, present) + require.Equal(t, readBillingDemoStatusOperatorOK, op.status) + require.Equal(t, 128.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitNetBytes, readBillingDemoInputSideAll)) + require.Equal(t, 3.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + + runtimeStats.RecordExpectedCopTasks([]int{scan.ID()}) + op, _ = readBillingDemoReaderTransport(flat, runtimeStats, execdetails.NewRUV2Metrics(), false) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, op.reason) + op, _ = readBillingDemoReaderTransport(flat, runtimeStats, metrics, true) + require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, op.reason) + }) + + t.Run("point lookup transport is rpc only and emitted once", func(t *testing.T) { + ctx := mock.NewContext() + stats := &property.StatsInfo{RowCount: 1} + tblInfo := &model.TableInfo{} + pointPlan := physicalop.PointGetPlan{TblInfo: tblInfo} + pointPlan.SetSchema(expression.NewSchema()) + point := pointPlan.Init(ctx, stats, 0) + batch := (&physicalop.BatchPointGetPlan{TblInfo: tblInfo}).Init(ctx, stats, expression.NewSchema(), nil, 0) + metrics := execdetails.NewRUV2Metrics() + metrics.AddResourceManagerReadCnt(3) + + testCases := []struct { + name string + flat *FlatPhysicalPlan + kind string + }{ + { + name: "point get", + flat: &FlatPhysicalPlan{Main: FlatPlanTree{{Origin: point, IsRoot: true, StoreType: kv.TiDB}}}, + kind: "point_get", + }, + { + name: "batch point get", + flat: &FlatPhysicalPlan{Main: FlatPlanTree{{Origin: batch, IsRoot: true, StoreType: kv.TiDB}}}, + kind: "batch_point_get", + }, + { + name: "mixed point lookup", + flat: &FlatPhysicalPlan{ + Main: FlatPlanTree{{Origin: point, IsRoot: true, StoreType: kv.TiDB}}, + ScalarSubQueries: []FlatPlanTree{{{Origin: batch, IsRoot: true, StoreType: kv.TiDB}}}, + }, + kind: "mixed_point_lookup", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + op, present := readBillingDemoPointLookupTransport(tc.flat, metrics, false) + require.True(t, present) + require.Equal(t, "point_lookup@statement", op.id) + require.Equal(t, readBillingDemoStatusOperatorOK, op.status) + require.Equal(t, readBillingDemoSiteTiKV, op.site) + require.Equal(t, readBillingDemoOpClassPointLookup, op.opClass) + require.Equal(t, tc.kind, op.operatorKind) + require.Len(t, op.units, 1) + require.Equal(t, 3.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.True(t, readBillingDemoOperatorBillable(op)) + }) + } + + zeroOp, present := readBillingDemoPointLookupTransport(testCases[0].flat, execdetails.NewRUV2Metrics(), false) + require.True(t, present) + require.Equal(t, readBillingDemoStatusOperatorOK, zeroOp.status) + require.Zero(t, readBillingDemoUnitValue(zeroOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + + missingOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, false) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, missingOp.reason) + bypassedMetrics := execdetails.NewRUV2Metrics() + bypassedMetrics.SetBypass(true) + bypassedOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, bypassedMetrics, false) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, bypassedOp.reason) + dmlOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, metrics, true) + require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, dmlOp.reason) + + point.Lock = true + lockOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, metrics, false) + require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, lockOp.reason) + point.Lock = false + reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + mixedReaderFlat := &FlatPhysicalPlan{ + Main: testCases[0].flat.Main, + ScalarSubQueries: []FlatPlanTree{{{Origin: reader, IsRoot: true, StoreType: kv.TiDB}}}, + } + mixedReaderOp, _ := readBillingDemoPointLookupTransport(mixedReaderFlat, metrics, false) + require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, mixedReaderOp.reason) + + physicalOp, supported, reason := readBillingDemoClassifyOperator(testCases[0].flat.Main[0]) + physicalOp.id = point.ExplainID().String() + require.True(t, supported) + require.Empty(t, reason) + require.False(t, readBillingDemoOperatorBillable(physicalOp)) + }) +} + +func TestReadBillingDemoV4ExpressionCountsAndOrdering(t *testing.T) { + ctx := mock.NewContext() + stats := &property.StatsInfo{RowCount: 1} + col := &expression.Column{Index: 0, RetType: types.NewFieldType(mysql.TypeLonglong)} + selection := physicalop.PhysicalSelection{Conditions: []expression.Expression{col, col}}.Init(ctx, stats, 0) + projection := physicalop.PhysicalProjection{Exprs: []expression.Expression{col, col, col}}.Init(ctx, stats, 0) + hashAgg := (&physicalop.BasePhysicalAgg{GroupByItems: []expression.Expression{col}, AggFuncs: []*aggregation.AggFuncDesc{nil, nil}}).InitForHash(ctx, stats, 0, expression.NewSchema(col)) + window := physicalop.PhysicalWindow{WindowFuncDescs: []*aggregation.WindowFuncDesc{nil}, PartitionBy: []property.SortItem{{Col: col}}, OrderBy: []property.SortItem{{Col: col}}, Frame: &logicalop.WindowFrame{Start: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col}}, End: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col, col}}}}.Init(ctx, stats, 0) + for _, tc := range []struct { + plan base.Plan + want int64 + }{{selection, 2}, {projection, 3}, {hashAgg, 3}, {window, 6}} { + got, ok := readBillingDemoExpressionCount(tc.plan) + require.True(t, ok) + require.Equal(t, tc.want, got) + } + + baseJoin := physicalop.BasePhysicalJoin{ + LeftConditions: expression.CNFExprs{col}, + RightConditions: expression.CNFExprs{col}, + OtherConditions: expression.CNFExprs{col}, + LeftJoinKeys: []*expression.Column{col, col}, + RightJoinKeys: []*expression.Column{col, col}, + OuterJoinKeys: []*expression.Column{col, col}, + InnerJoinKeys: []*expression.Column{col, col}, + } + compareFilters := &physicalop.ColWithCmpFuncManager{OpType: []string{"gt", "lt"}} + joins := []struct { + name string + plan base.Plan + want int64 + }{ + {name: "hash join", plan: &physicalop.PhysicalHashJoin{BasePhysicalJoin: baseJoin, EqualConditions: []*expression.ScalarFunction{{}, {}}, NAEqualConditions: []*expression.ScalarFunction{{}}}, want: 6}, + {name: "merge join", plan: &physicalop.PhysicalMergeJoin{BasePhysicalJoin: baseJoin, CompareFuncs: []expression.CompareFunc{nil, nil}}, want: 5}, + {name: "index join", plan: &physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, CompareFilters: compareFilters}, want: 7}, + {name: "index hash join", plan: &physicalop.PhysicalIndexHashJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, OuterHashKeys: []*expression.Column{col, col, col}, InnerHashKeys: []*expression.Column{col, col, col}, CompareFilters: compareFilters}}, want: 8}, + {name: "index merge join", plan: &physicalop.PhysicalIndexMergeJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, CompareFilters: compareFilters}, CompareFuncs: []expression.CompareFunc{nil, nil}, OuterCompareFuncs: []expression.CompareFunc{nil}, NeedOuterSort: true}, want: 8}, + } + for _, tc := range joins { + t.Run(tc.name, func(t *testing.T) { + got, ok := readBillingDemoExpressionCount(tc.plan) + require.True(t, ok) + require.Equal(t, tc.want, got) + }) + } + invalidHashJoin := &physicalop.PhysicalHashJoin{BasePhysicalJoin: baseJoin} + invalidHashJoin.RightJoinKeys = invalidHashJoin.RightJoinKeys[:1] + _, ok := readBillingDemoExpressionCount(invalidHashJoin) + require.False(t, ok) + invalidIndexJoin := &physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin} + invalidIndexJoin.InnerJoinKeys = invalidIndexJoin.InnerJoinKeys[:1] + _, ok = readBillingDemoExpressionCount(invalidIndexJoin) + require.False(t, ok) + invalidIndexHashJoin := &physicalop.PhysicalIndexHashJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin, OuterHashKeys: []*expression.Column{col}}} + _, ok = readBillingDemoExpressionCount(invalidIndexHashJoin) + require.False(t, ok) + invalidIndexMergeJoin := &physicalop.PhysicalIndexMergeJoin{PhysicalIndexJoin: physicalop.PhysicalIndexJoin{BasePhysicalJoin: baseJoin}, OuterCompareFuncs: []expression.CompareFunc{nil}} + _, ok = readBillingDemoExpressionCount(invalidIndexMergeJoin) + require.False(t, ok) + + topN := physicalop.PhysicalTopN{Offset: 3, Count: 5, ByItems: []*plannerutil.ByItems{{Expr: col}, {Expr: col}}}.Init(ctx, stats, 0) + unit, ok := readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN, 4) + require.True(t, ok) + require.Equal(t, 8.0, unit.value) + for _, tc := range []struct { + name string + rows int64 + offset, count uint64 + want float64 + }{ + {name: "zero rows", rows: 0, count: 1, want: 0}, + {name: "one row", rows: 1, count: 1, want: 1}, + {name: "bound saturates at input rows", rows: 4, offset: 3, count: 5, want: 8}, + {name: "huge bound saturates at one input row", rows: 1, offset: math.MaxUint64 - 1, count: 1, want: 1}, + {name: "zero count ignores offset", rows: 4, offset: math.MaxUint64, count: 0, want: 0}, + } { + topN.Offset, topN.Count = tc.offset, tc.count + unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN, tc.rows) + require.True(t, ok, tc.name) + require.Equal(t, tc.want, unit.value, tc.name) + } + topN.Offset, topN.Count = math.MaxUint64, 1 + _, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN, 1) + require.False(t, ok) + require.Equal(t, readBillingDemoReasonInvalidTopNBound, readBillingDemoOrderingFailureReason(&FlatOperator{Origin: topN, IsRoot: true}, readBillingDemoOpClassTopN)) + topN.Offset, topN.Count = 0, 1 + topN.ByItems = []*plannerutil.ByItems{{Expr: &expression.ScalarFunction{}}} + require.False(t, readBillingDemoOrderingMaterialized(&FlatOperator{Origin: topN, IsRoot: false}, nil)) + sort := physicalop.PhysicalSort{ByItems: []*plannerutil.ByItems{{Expr: col}}}.Init(ctx, stats, 0) + unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: sort, IsRoot: true}, readBillingDemoOpClassSort, 3) + require.True(t, ok) + require.InDelta(t, 3*math.Log2(3), unit.value, 1e-12) + + formulaWeights := readBillingDemoWeights{ + Version: "test-v4-calibrated", + CPUWeight: 2, ScanWeight: 3, NetWeight: 5, ReadRequestWeight: 7, WriteRequestWeight: 17, + HashTableWeight: 11, JoinWeight: 13, MutationBytesPerCPUUnit: 10, Calibrated: true, + } + weightedTotal := func(t *testing.T, units []readBillingDemoUnit) float64 { + t.Helper() + var total float64 + for _, unit := range units { + _, ru, ok := readBillingDemoUnitPreviewRU(unit, formulaWeights) + if _, semantic := readBillingDemoUnitWeight(formulaWeights, unit.unit); !semantic { + continue + } + require.True(t, ok, "unit=%+v", unit) + total += ru + } + return total + } + recordRoot := func(runtimeStats *execdetails.RuntimeStatsColl, planID int, rows int) { + rootStats := runtimeStats.GetBasicRuntimeStats(planID, true) + rootStats.Record(time.Millisecond, rows) + rootStats.RecordBytes(0, int64(rows*8)) + } + schema := expression.NewSchema(col) + + t.Run("root unary formulas use exact semantic terms", func(t *testing.T) { + for _, tc := range []struct { + name string + opClass string + buildPlan func() base.Plan + wantRU float64 + }{ + {name: "selection", opClass: readBillingDemoOpClassFilter, buildPlan: func() base.Plan { + return physicalop.PhysicalSelection{Conditions: []expression.Expression{col, col}}.Init(ctx, stats, 0) + }, wantRU: 16}, + {name: "projection", opClass: readBillingDemoOpClassProjection, buildPlan: func() base.Plan { + return physicalop.PhysicalProjection{Exprs: []expression.Expression{col, col, col}}.Init(ctx, stats, 0) + }, wantRU: 24}, + {name: "stream agg", opClass: readBillingDemoOpClassStreamAgg, buildPlan: func() base.Plan { + return (&physicalop.BasePhysicalAgg{GroupByItems: []expression.Expression{col}, AggFuncs: []*aggregation.AggFuncDesc{nil, nil}}).InitForStream(ctx, stats, 0, schema) + }, wantRU: 24}, + {name: "hash agg", opClass: readBillingDemoOpClassHashAgg, buildPlan: func() base.Plan { + return (&physicalop.BasePhysicalAgg{GroupByItems: []expression.Expression{col}, AggFuncs: []*aggregation.AggFuncDesc{nil, nil}}).InitForHash(ctx, stats, 0, schema) + }, wantRU: 46}, + {name: "limit", opClass: readBillingDemoOpClassLimit, buildPlan: func() base.Plan { + return physicalop.PhysicalLimit{}.Init(ctx, stats, 0) + }, wantRU: 8}, + {name: "union scan", opClass: readBillingDemoOpClassOverlayReader, buildPlan: func() base.Plan { + return physicalop.PhysicalUnionScan{}.Init(ctx, stats, 0) + }, wantRU: 8}, + {name: "window", opClass: readBillingDemoOpClassWindow, buildPlan: func() base.Plan { + return physicalop.PhysicalWindow{WindowFuncDescs: []*aggregation.WindowFuncDesc{nil}, PartitionBy: []property.SortItem{{Col: col}}, OrderBy: []property.SortItem{{Col: col}}, Frame: &logicalop.WindowFrame{Start: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col}}, End: &logicalop.FrameBound{CalcFuncs: []expression.Expression{col, col}}}}.Init(ctx, stats, 0) + }, wantRU: 48}, + {name: "sort", opClass: readBillingDemoOpClassSort, buildPlan: func() base.Plan { + return physicalop.PhysicalSort{ByItems: []*plannerutil.ByItems{{Expr: col}, {Expr: col}}}.Init(ctx, stats, 0) + }, wantRU: 16}, + {name: "topn with offset", opClass: readBillingDemoOpClassTopN, buildPlan: func() base.Plan { + return physicalop.PhysicalTopN{Offset: 3, Count: 5, ByItems: []*plannerutil.ByItems{{Expr: col}, {Expr: col}}}.Init(ctx, stats, 0) + }, wantRU: 16}, + } { + t.Run(tc.name, func(t *testing.T) { + plan := tc.buildPlan() + child := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + child.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: plan, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRoot(runtimeStats, plan.ID(), 2) + recordRoot(runtimeStats, child.ID(), 4) + operator := readBillingDemoOperatorResult{id: plan.ExplainID().String(), opClass: tc.opClass} + require.True(t, readBillingDemoOperatorBillable(operator)) + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], operator) + require.True(t, ok, reason) + require.Empty(t, reason) + require.Equal(t, tc.wantRU, weightedTotal(t, units)) + if tc.opClass == readBillingDemoOpClassOverlayReader { + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) + require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceRuntimeChildActRows, readBillingDemoUnitSource(units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, -1.0, readBillingDemoUnitValue(units, readBillingDemoUnitExpressionCount, readBillingDemoInputSideAll)) + } + }) + } + }) + + t.Run("ordering rejects columns outside the child schema", func(t *testing.T) { + plan := physicalop.PhysicalSort{ByItems: []*plannerutil.ByItems{{Expr: col}}}.Init(ctx, stats, 0) + child := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + child.SetSchema(expression.NewSchema()) + tree := FlatPlanTree{ + {Origin: plan, ChildrenIdx: []int{1}, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + {Origin: child, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRoot(runtimeStats, plan.ID(), 2) + recordRoot(runtimeStats, child.ID(), 4) + _, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], readBillingDemoOperatorResult{opClass: readBillingDemoOpClassSort}) + require.False(t, ok) + require.Equal(t, readBillingDemoReasonMissingOrderingProjection, reason) + }) + + t.Run("root join formulas use both inputs and subtype expressions", func(t *testing.T) { + joinBase := physicalop.BasePhysicalJoin{ + OtherConditions: expression.CNFExprs{col}, + LeftJoinKeys: []*expression.Column{col}, RightJoinKeys: []*expression.Column{col}, + OuterJoinKeys: []*expression.Column{col}, InnerJoinKeys: []*expression.Column{col}, + } + for _, tc := range []struct { + name string + opClass string + buildPlan func() base.Plan + hashRows int64 + wantRU float64 + }{ + {name: "merge join", opClass: readBillingDemoOpClassMergeJoin, buildPlan: func() base.Plan { + return physicalop.PhysicalMergeJoin{BasePhysicalJoin: joinBase, CompareFuncs: []expression.CompareFunc{nil}}.Init(ctx, stats, 0) + }, wantRU: 66}, + {name: "hash join", opClass: readBillingDemoOpClassHashJoin, buildPlan: func() base.Plan { + return physicalop.PhysicalHashJoin{BasePhysicalJoin: joinBase, EqualConditions: []*expression.ScalarFunction{{}}}.Init(ctx, stats, 0) + }, hashRows: 3, wantRU: 99}, + {name: "index join", opClass: readBillingDemoOpClassLookupJoin, buildPlan: func() base.Plan { + return physicalop.PhysicalIndexJoin{BasePhysicalJoin: joinBase}.Init(ctx, stats, 0) + }, wantRU: 66}, + {name: "index hash join", opClass: readBillingDemoOpClassLookupJoin, buildPlan: func() base.Plan { + indexJoin := physicalop.PhysicalIndexJoin{BasePhysicalJoin: joinBase, OuterHashKeys: []*expression.Column{col}, InnerHashKeys: []*expression.Column{col}}.Init(ctx, stats, 0) + return physicalop.PhysicalIndexHashJoin{PhysicalIndexJoin: *indexJoin}.Init(ctx) + }, wantRU: 66}, + {name: "index merge join", opClass: readBillingDemoOpClassLookupJoin, buildPlan: func() base.Plan { + indexJoin := physicalop.PhysicalIndexJoin{BasePhysicalJoin: joinBase}.Init(ctx, stats, 0) + return physicalop.PhysicalIndexMergeJoin{PhysicalIndexJoin: *indexJoin, CompareFuncs: []expression.CompareFunc{nil}}.Init(ctx) + }, wantRU: 66}, + } { + t.Run(tc.name, func(t *testing.T) { + plan := tc.buildPlan() + left := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + right := physicalop.PhysicalProjection{Exprs: []expression.Expression{col}}.Init(ctx, stats, 0) + left.SetSchema(schema) + right.SetSchema(schema) + tree := FlatPlanTree{ + {Origin: plan, ChildrenIdx: []int{1, 2}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: left, ChildrenEndIdx: 1, IsRoot: true, StoreType: kv.TiDB, Label: BuildSide}, + {Origin: right, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB, Label: ProbeSide}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + recordRoot(runtimeStats, plan.ID(), 2) + recordRoot(runtimeStats, left.ID(), 4) + recordRoot(runtimeStats, right.ID(), 6) + if tc.opClass == readBillingDemoOpClassHashJoin { + runtimeStats.RegisterStats(plan.ID(), &readBillingDemoHashStatsForTest{rows: tc.hashRows}) + } + units, reason, ok := readBillingDemoRootUnits(runtimeStats, tree, 0, tree[0], readBillingDemoOperatorResult{opClass: tc.opClass}) + require.True(t, ok, reason) + require.Empty(t, reason) + require.Equal(t, tc.wantRU, weightedTotal(t, units)) + }) + } + }) + + t.Run("cop selection uses exact child rows without logical byte width", func(t *testing.T) { + reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + scan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tree := FlatPlanTree{ + {Origin: reader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: selection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + {Origin: scan, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + summary := func(rows uint64) *tipb.ExecutorExecutionSummary { + return &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &rows, + NumIterations: &one, + } + } + runtimeStats.RecordExpectedCopTasks([]int{scan.ID(), selection.ID()}) + runtimeStats.RecordOneCopTask(scan.ID(), kv.TiKV, summary(4)) + runtimeStats.RecordCopStats(selection.ID(), kv.TiKV, &tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, tikvutil.TimeDetail{}, summary(2)) + estimator := newReadBillingDemoCopEstimator(tree, runtimeStats) + outcome := readBillingDemoCopUnits(estimator, 1, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassFilter, operatorKind: "selection"}) + require.True(t, outcome.success, "%+v", outcome.failure) + require.Equal(t, 2.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitExpressionCount, readBillingDemoInputSideAll)) + require.Equal(t, 8.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, -1.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + + scanOutcome := readBillingDemoCopUnits(estimator, 2, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, scanOutcome.success, "%+v", scanOutcome.failure) + require.Equal(t, 40.0, readBillingDemoUnitValue(scanOutcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + + selection.SetChildren(scan) + reader.TablePlan = selection + recordRoot(runtimeStats, reader.ID(), 2) + ctx.GetSessionVars().StmtCtx.RuntimeStatsColl = runtimeStats + metrics := execdetails.NewRUV2Metrics() + metrics.AddResourceManagerReadCnt(1) + metrics.AddTiKVCoprocessorResponseBytes(40) + result := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, metrics) + require.Equal(t, readBillingDemoStatusSuccess, result.status) + seenScanBytes := false + seenSelectionCPU := false + for _, sample := range buildReadBillingDemoStatementStats(result).BaseUnits { + switch { + case sample.Site == readBillingDemoSiteTiKV && sample.OpClass == readBillingDemoOpClassRangeScan && sample.Unit == readBillingDemoUnitScanBytes: + seenScanBytes = true + require.Equal(t, 40.0, sample.Value) + case sample.Site == readBillingDemoSiteTiKV && sample.OpClass == readBillingDemoOpClassFilter && sample.Unit == readBillingDemoUnitCPUWork: + seenSelectionCPU = true + require.Equal(t, 8.0, sample.Value) + } + } + require.True(t, seenScanBytes) + require.True(t, seenSelectionCPU) + + runtimeStats.RecordExpectedCopTasks([]int{scan.ID(), selection.ID()}) + failedResult := buildReadBillingDemoResult(ctx, reader, &ast.SelectStmt{}, nil, metrics) + require.Equal(t, readBillingDemoStatusUnknownInput, failedResult.status) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, failedResult.reason) + require.Empty(t, buildReadBillingDemoStatementStats(failedResult).BaseUnits) + }) + + t.Run("cop scan detail provenance stays fail closed", func(t *testing.T) { + buildEstimator := func(detail *tikvutil.ScanDetail, responses, scanSummaries, detailRecords int, holderSummaries bool) (*readBillingDemoCopEstimator, int) { + localReader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + localSelection := physicalop.PhysicalSelection{Conditions: []expression.Expression{col}}.Init(ctx, stats, 0) + localScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tree := FlatPlanTree{ + {Origin: localReader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: localSelection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + {Origin: localScan, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + zero := uint64(0) + summary := &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &zero, + NumIterations: &one, + } + for range responses { + runtimeStats.RecordExpectedCopTasks([]int{localScan.ID(), localSelection.ID()}) + } + for range scanSummaries { + runtimeStats.RecordOneCopTask(localScan.ID(), kv.TiKV, summary) + } + for range detailRecords { + var holderSummary *tipb.ExecutorExecutionSummary + if holderSummaries { + holderSummary = summary + } + runtimeStats.RecordCopStats(localSelection.ID(), kv.TiKV, detail, tikvutil.TimeDetail{}, holderSummary) + } + return newReadBillingDemoCopEstimator(tree, runtimeStats), 2 + } + + estimator, scanIdx := buildEstimator(&tikvutil.ScanDetail{}, 1, 1, 1, true) + outcome := readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, outcome.success, "%+v", outcome.failure) + require.Zero(t, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 2, 2, 2, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, outcome.success, "%+v", outcome.failure) + require.Equal(t, 80.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 1, 1, 1, false) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.True(t, outcome.success, "%+v", outcome.failure) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4}, 1, 1, 1, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonMissingScanWidthEvidence, outcome.failure.reason) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 2, 1, 2, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, outcome.failure.reason) + + estimator, scanIdx = buildEstimator(&tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40}, 2, 2, 1, true) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonIncompleteCopRuntimeRows, outcome.failure.reason) + + estimator, scanIdx = buildEstimator(nil, 1, 1, 0, false) + outcome = readBillingDemoCopUnits(estimator, scanIdx, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonMissingScanWidthEvidence, outcome.failure.reason) + + localReader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) + localSelection := physicalop.PhysicalSelection{Conditions: []expression.Expression{col}}.Init(ctx, stats, 0) + localScan := physicalop.PhysicalIndexScan{}.Init(ctx, 0) + tree := FlatPlanTree{ + {Origin: localReader, ChildrenIdx: []int{1}, ChildrenEndIdx: 2, IsRoot: true, StoreType: kv.TiDB}, + {Origin: localSelection, ChildrenIdx: []int{2}, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + {Origin: localScan, ChildrenEndIdx: 2, StoreType: kv.TiKV}, + } + runtimeStats := execdetails.NewRuntimeStatsColl(nil) + one := uint64(1) + zero := uint64(0) + summary := &tipb.ExecutorExecutionSummary{TimeProcessedNs: &one, NumProducedRows: &zero, NumIterations: &one} + detail := &tikvutil.ScanDetail{TotalKeys: 4, ProcessedKeys: 4, ProcessedKeysSize: 40} + runtimeStats.RecordExpectedCopTasks([]int{localScan.ID(), localSelection.ID()}) + runtimeStats.RecordCopStats(localScan.ID(), kv.TiKV, detail, tikvutil.TimeDetail{}, summary) + runtimeStats.RecordCopStats(localSelection.ID(), kv.TiKV, detail, tikvutil.TimeDetail{}, summary) + outcome = readBillingDemoCopUnits(newReadBillingDemoCopEstimator(tree, runtimeStats), 2, readBillingDemoOperatorResult{site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassRangeScan, operatorKind: "indexscan"}) + require.False(t, outcome.success) + require.Equal(t, readBillingDemoReasonAmbiguousCopScanWidth, outcome.failure.reason) + }) +} + +func TestReadBillingDemoV4WriteRequests(t *testing.T) { + oldWeights := readBillingDemoV4Weights + readBillingDemoV4Weights = readBillingDemoWeights{ + Version: "test-v4-calibrated", CPUWeight: 2, MutationBytesPerCPUUnit: 10, Calibrated: true, + } + t.Cleanup(func() { readBillingDemoV4Weights = oldWeights }) + ctx := mock.NewContext() + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordSet(5, 7) + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordDelete(3) + result := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&result, ctx, "update") + mutation := result.operators[0] + require.Equal(t, readBillingDemoOpClassKVMutation, mutation.opClass) + require.Equal(t, readBillingDemoOperatorMemDBMutation, mutation.operatorKind) + require.Equal(t, 3.5, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitEncodedMutationCount, readBillingDemoInputSideAll)) + require.Equal(t, 15.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitEncodedMutationBytes, readBillingDemoInputSideAll)) + require.Equal(t, 1.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitSetCount, readBillingDemoInputSideAll)) + require.Equal(t, 1.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitDeleteCount, readBillingDemoInputSideAll)) + require.Equal(t, 8.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitKeyBytes, readBillingDemoInputSideAll)) + require.Equal(t, 7.0, readBillingDemoUnitValue(mutation.units, readBillingDemoUnitValueBytes, readBillingDemoInputSideAll)) + + rows := explainRUBuildReadBillingRows(result, explainRUComponentSnapshotOK) + require.True(t, rows[0].hasPreviewRU) + require.Equal(t, 7.0, rows[0].previewRU) + var mutationCPUWorkRows int + for _, row := range rows { + if row.operatorClass == "tidb/kv_mutation" && row.component == readBillingDemoOperatorMemDBMutation && row.unit == readBillingDemoUnitCPUWork { + mutationCPUWorkRows++ + require.Equal(t, 3.5, row.workRows) + require.Equal(t, readBillingDemoInputSourceStmtMemDBMutation, row.source) + } + } + require.Equal(t, 1, mutationCPUWorkRows) + + stats := buildReadBillingDemoStatementStats(result) + var mutationCPUWorkSamples int + for _, sample := range stats.BaseUnits { + if sample.Unit == readBillingDemoUnitCPUWork { + mutationCPUWorkSamples++ + require.Equal(t, readBillingDemoSiteTiDB, sample.Site) + require.Equal(t, readBillingDemoOpClassKVMutation, sample.OpClass) + require.Equal(t, readBillingDemoOperatorMemDBMutation, sample.OperatorKind) + require.Equal(t, readBillingDemoInputSourceStmtMemDBMutation, sample.InputSource) + require.Equal(t, readBillingDemoInputSideAll, sample.InputSide) + require.Equal(t, 3.5, sample.Value) + } + } + require.Equal(t, 1, mutationCPUWorkSamples) + + readBillingDemoV4Weights = readBillingDemoWeights{MutationBytesPerCPUUnit: 10} + uncalibratedResult := readBillingDemoResult{status: readBillingDemoStatusSuccess, reason: readBillingDemoReasonNone} + appendReadBillingDemoMutation(&uncalibratedResult, ctx, "update") + require.Len(t, uncalibratedResult.operators[0].units, 6) + require.Equal(t, -1.0, readBillingDemoUnitValue(uncalibratedResult.operators[0].units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, 2.0, readBillingDemoUnitValue(uncalibratedResult.operators[0].units, readBillingDemoUnitEncodedMutationCount, readBillingDemoInputSideAll)) + require.Equal(t, 15.0, readBillingDemoUnitValue(uncalibratedResult.operators[0].units, readBillingDemoUnitEncodedMutationBytes, readBillingDemoInputSideAll)) + require.Len(t, uncalibratedResult.operators, 2) + require.Equal(t, readBillingDemoReasonUncalibratedMutation, uncalibratedResult.operators[1].reason) + + dmlMetrics := execdetails.NewRUV2Metrics() + dmlMetrics.AddResourceManagerWriteCnt(7) + dml := buildTiKVWriteBillingDemoOperators("update", dmlMetrics, false) + require.Equal(t, 7.0, readBillingDemoUnitValue(dml[0].units, readBillingDemoUnitWriteRequestCount, readBillingDemoInputSideAll)) + commitMetrics := execdetails.NewRUV2Metrics() + commitMetrics.AddResourceManagerWriteCnt(2) + commit := buildTiKVWriteBillingDemoOperators("", commitMetrics, false) + require.Equal(t, 2.0, readBillingDemoUnitValue(commit[0].units, readBillingDemoUnitWriteRequestCount, readBillingDemoInputSideAll)) + require.Empty(t, commit[0].dmlKind) + + pipelinedDML := buildTiKVWriteBillingDemoOperators("update", dmlMetrics, true) + require.Len(t, pipelinedDML, 1) + require.Equal(t, readBillingDemoStatusPartial, pipelinedDML[0].status) + require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedDML[0].reason) + require.Empty(t, pipelinedDML[0].units) + + ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.MarkPipelined() + pipelinedCommit := buildTxnCommitBillingDemoResult(ctx, commitMetrics, nil) + require.Len(t, pipelinedCommit.operators, 1) + require.Equal(t, readBillingDemoStatusPartial, pipelinedCommit.operators[0].status) + require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedCommit.operators[0].reason) + require.Empty(t, pipelinedCommit.operators[0].units) +} + func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { - require.Equal(t, "v2", readBillingDemoWeightVersion) + t.Skip("v3 opclass-weight expectations are superseded by TestReadBillingDemoV4FormulaContract") + require.Equal(t, "v3-resource-formula-uncalibrated", readBillingDemoWeightVersion) tidbWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) require.True(t, ok) tikvWeights, ok := readBillingDemoResolveWeights(readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion) @@ -468,7 +1233,7 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { require.Equal(t, 1, orderRows) stats := buildReadBillingDemoStatementStats(result) - require.Equal(t, "v2", stats.WeightVersion) + require.Equal(t, "v3-resource-formula-uncalibrated", stats.WeightVersion) var orderSamples int for _, sample := range stats.BaseUnits { if sample.Unit == readBillingDemoUnitOrderWork { @@ -582,11 +1347,10 @@ func TestExplainRUPlanFormulaAndOperatorClasses(t *testing.T) { require.True(t, ok) require.Greater(t, unit.value, float64(maxInt64)) - hugeTopN := physicalop.PhysicalTopN{Offset: ^uint64(0), Count: ^uint64(0)}.Init(ctx, stats, 0) + hugeTopN := physicalop.PhysicalTopN{Offset: math.MaxUint64 - 1, Count: 1}.Init(ctx, stats, 0) unit, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: hugeTopN, IsRoot: true, StoreType: kv.TiDB}, readBillingDemoOpClassTopN, 1) require.True(t, ok) - require.Greater(t, unit.value, 64.0) - require.LessOrEqual(t, unit.value, 65.0) + require.Equal(t, 1.0, unit.value) malformedCopTopN := physicalop.PhysicalTopN{Offset: 1, Count: 3}.Init(ctx, stats, 0) _, ok = readBillingDemoOrderingWorkUnit(&FlatOperator{Origin: malformedCopTopN, IsRoot: false, StoreType: kv.TiKV}, readBillingDemoOpClassTopN, 8) @@ -641,6 +1405,7 @@ func TestExplainRUComponentSnapshotStatusAndWeights(t *testing.T) { } func TestReadBillingDemoWriteDMLResult(t *testing.T) { + t.Skip("v3 commit-detail formula expectations are superseded by TestReadBillingDemoV4WriteRequests") ctx := mock.NewContext() ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder = &stmtctx.PreviewKVMutationRecorder{} ctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder.RecordSet(5, 7) @@ -752,7 +1517,7 @@ func TestReadBillingDemoWriteDMLResult(t *testing.T) { // Pipelined transactions expose a non-nil CommitDetails without logical // WriteKeys/WriteSize. The incomplete payload must not become billable zero // units merely because the detail object exists. - pipelinedResult := buildTiKVWriteBillingDemoOperators("update", &tikvutil.CommitDetails{}, ruv2Metrics, true) + pipelinedResult := buildTiKVWriteBillingDemoOperators("update", ruv2Metrics, true) require.Len(t, pipelinedResult, 1) require.Equal(t, readBillingDemoStatusPartial, pipelinedResult[0].status) require.Equal(t, readBillingDemoReasonPipelinedWritePartial, pipelinedResult[0].reason) @@ -853,7 +1618,14 @@ func TestReadBillingDemoRangeScanUsesProcessedKeyAverage(t *testing.T) { buildUnits := func(scanDetail *tikvutil.ScanDetail) readBillingDemoCopUnitOutcome { runtimeStats := execdetails.NewRuntimeStatsColl(nil) - runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, scanDetail, tikvutil.TimeDetail{}, nil) + one := uint64(1) + zero := uint64(0) + runtimeStats.RecordExpectedCopTasks([]int{scan.ID()}) + runtimeStats.RecordCopStats(scan.ID(), kv.TiKV, scanDetail, tikvutil.TimeDetail{}, &tipb.ExecutorExecutionSummary{ + TimeProcessedNs: &one, + NumProducedRows: &zero, + NumIterations: &one, + }) return readBillingDemoCopUnits( newReadBillingDemoCopEstimator(tree, runtimeStats), 1, @@ -862,21 +1634,22 @@ func TestReadBillingDemoRangeScanUsesProcessedKeyAverage(t *testing.T) { } outcome := buildUnits(&tikvutil.ScanDetail{TotalKeys: 10, ProcessedKeys: 5, ProcessedKeysSize: 100}) - require.True(t, outcome.success) + require.True(t, outcome.success, "%+v", outcome.failure) units := outcome.units require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) require.Equal(t, 10.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) require.Equal(t, 200.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) + require.Equal(t, 200.0, readBillingDemoUnitValue(units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) require.Equal(t, readBillingDemoInputSourceScanDetail, readBillingDemoUnitSource(units, readBillingDemoUnitInputRows, readBillingDemoInputSideAll)) require.Equal(t, explainRUWidthSourceScanDetailProcessedAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideAll)) outcome = buildUnits(&tikvutil.ScanDetail{}) - require.False(t, outcome.success) - require.Nil(t, outcome.units) - require.Equal(t, readBillingDemoReasonMissingScanWidthEvidence, outcome.failure.reason) + require.True(t, outcome.success) + require.Equal(t, 0.0, readBillingDemoUnitValue(outcome.units, readBillingDemoUnitScanBytes, readBillingDemoInputSideAll)) } func TestReadBillingDemoCopInputEstimator(t *testing.T) { + t.Skip("v3 byte-width estimator expectations do not apply to the v4 resource formula") ctx := mock.NewContext() col := &expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)} schema := expression.NewSchema(col) @@ -938,7 +1711,7 @@ func TestReadBillingDemoCopInputEstimator(t *testing.T) { }{ {name: "selection", node: &FlatOperator{Origin: selection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, {name: "limit", node: &FlatOperator{Origin: limit, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: -1}, - {name: "topn", node: &FlatOperator{Origin: topN, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: 12}, + {name: "topn", node: &FlatOperator{Origin: topN, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthKnown, expectedOrderWork: 8}, {name: "projection", node: &FlatOperator{Origin: projection, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1}, {name: "hashagg", node: &FlatOperator{Origin: hashAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1, expectOutputUnits: true}, {name: "streamagg", node: &FlatOperator{Origin: streamAgg, IsRoot: false, StoreType: kv.TiKV}, widthState: readBillingDemoCopWidthBarrier, expectedOrderWork: -1, expectOutputUnits: true}, @@ -1635,6 +2408,9 @@ func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { schema := expression.NewSchema(col) stats := &property.StatsInfo{RowCount: 10} join := (&physicalop.PhysicalHashJoin{}).Init(ctx, stats, 0) + join.EqualConditions = []*expression.ScalarFunction{{}} + join.LeftJoinKeys = []*expression.Column{col} + join.RightJoinKeys = []*expression.Column{col} left := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) right := physicalop.PhysicalProjection{}.Init(ctx, stats, 0) join.SetSchema(schema) @@ -1655,6 +2431,7 @@ func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { recordRootRows(join.ID(), 6) recordRootRows(left.ID(), 4) recordRootRows(right.ID(), 6) + runtimeStats.RegisterStats(join.ID(), &readBillingDemoHashStatsForTest{rows: 3}) units, reason, ok := readBillingDemoRootUnits( runtimeStats, @@ -1668,16 +2445,27 @@ func TestReadBillingDemoHashJoinUnitsUseBuildProbeSides(t *testing.T) { require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitFixedEvents, readBillingDemoInputSideAll)) require.Equal(t, 4.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideBuild)) require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputRows, readBillingDemoInputSideProbe)) - require.Equal(t, 40.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideBuild)) - require.Equal(t, 60.0, readBillingDemoUnitValue(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideProbe)) + require.Equal(t, 1.0, readBillingDemoUnitValue(units, readBillingDemoUnitExpressionCount, readBillingDemoInputSideAll)) + require.Equal(t, 10.0, readBillingDemoUnitValue(units, readBillingDemoUnitCPUWork, readBillingDemoInputSideAll)) + require.Equal(t, 3.0, readBillingDemoUnitValue(units, readBillingDemoUnitHashStateRows, readBillingDemoInputSideBuild)) + require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitJoinOutputRows, readBillingDemoInputSideAll)) require.Equal(t, 6.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputRows, readBillingDemoInputSideAll)) require.Equal(t, 60.0, readBillingDemoUnitValue(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) - require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideBuild)) - require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitInputBytes, readBillingDemoInputSideProbe)) + require.Equal(t, readBillingDemoInputSourceHashJoinRuntime, readBillingDemoUnitSource(units, readBillingDemoUnitHashStateRows, readBillingDemoInputSideBuild)) require.Equal(t, readBillingDemoInputSourceRuntimeChunkBytes, readBillingDemoUnitSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) require.Equal(t, explainRUWidthSourceRuntimeChunkAvg, readBillingDemoUnitWidthSource(units, readBillingDemoUnitOutputBytes, readBillingDemoInputSideAll)) } +type readBillingDemoHashStatsForTest struct{ rows int64 } + +func (*readBillingDemoHashStatsForTest) String() string { return "" } +func (s *readBillingDemoHashStatsForTest) Clone() execdetails.RuntimeStats { + return &readBillingDemoHashStatsForTest{rows: s.rows} +} +func (*readBillingDemoHashStatsForTest) Merge(execdetails.RuntimeStats) {} +func (*readBillingDemoHashStatsForTest) Tp() int { return 1_000_000 } +func (s *readBillingDemoHashStatsForTest) HashTableRows() int64 { return s.rows } + func readBillingDemoUnitValue(units []readBillingDemoUnit, unitName, side string) float64 { for _, unit := range units { if unit.unit == unitName && unit.side == side { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 78d609b641a0e..5db692f878233 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -18,10 +18,10 @@ import ( "math" "strconv" "strings" - "sync/atomic" "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/parser/ast" @@ -60,8 +60,8 @@ const ( explainRUWidthSourceScanDetailProcessedAvg = "scan_detail_processed_key_avg" explainRUWidthSourceScanDetailProcessedEstimate = "scan_detail_processed_key_avg_estimate" explainRUWidthSourceNotApplicable = "not_applicable" - readBillingDemoModelVersion = "v3" - readBillingDemoWeightVersion = "v2" + readBillingDemoModelVersion = "v4" + readBillingDemoWeightVersion = "v3-resource-formula-uncalibrated" readBillingDemoStatusSuccess = "success" readBillingDemoStatusUnsupported = "unsupported" readBillingDemoStatusUnknownInput = "unknown_input" @@ -103,6 +103,15 @@ const ( readBillingDemoReasonIncompleteCopRuntimeRows = "incomplete_cop_runtime_rows" readBillingDemoReasonDependentCopInputUnavailable = "dependent_cop_input_unavailable" readBillingDemoReasonInvalidOrderingWork = "invalid_ordering_work" + readBillingDemoReasonMissingOrderingProjection = "missing_ordering_projection" + readBillingDemoReasonMissingExpressionCount = "missing_expression_count" + readBillingDemoReasonInvalidTopNBound = "invalid_topn_bound" + readBillingDemoReasonMissingHashStateRows = "missing_hash_state_rows" + readBillingDemoReasonInvalidHashStateRows = "invalid_hash_state_rows" + readBillingDemoReasonMissingReaderTransport = "missing_reader_transport_details" + readBillingDemoReasonAmbiguousReaderTransport = "ambiguous_reader_transport_producers" + readBillingDemoReasonUncalibratedWeights = "uncalibrated_weights" + readBillingDemoReasonUnsupportedStatement = "unsupported_statement" readBillingDemoSiteStatement = "statement" readBillingDemoSiteTiDB = "tidb" readBillingDemoSiteTiKV = "tikv" @@ -126,11 +135,13 @@ const ( readBillingDemoOpClassRangeScan = "kv_range_scan" readBillingDemoOpClassKVMutation = "kv_mutation" readBillingDemoOpClassKVWrite = "kv_write" + readBillingDemoOpClassReaderTransport = "reader_transport" readBillingDemoOpClassWrapper = "wrapper" readBillingDemoOpClassSynthetic = "synthetic_source" readBillingDemoOperatorStatement = "statement" readBillingDemoOperatorMemDBMutation = "memdb_mutation" readBillingDemoOperatorTxnPrewrite = "txn_prewrite" + readBillingDemoOperatorTxnWrite = "txn_write" readBillingDemoUnitFixedEvents = "fixed_events" readBillingDemoUnitInputRows = "input_rows" readBillingDemoUnitInputBytes = "input_bytes" @@ -147,6 +158,14 @@ const ( readBillingDemoUnitWriteByte = "write_byte" readBillingDemoUnitPrewriteRegionNum = "prewrite_region_num" readBillingDemoUnitTiKVWriteRPCCount = "tikv_write_rpc_count" + readBillingDemoUnitCPUWork = "cpu_work" + readBillingDemoUnitExpressionCount = "expression_count" + readBillingDemoUnitScanBytes = "scan_bytes" + readBillingDemoUnitNetBytes = "net_bytes" + readBillingDemoUnitReadRequestCount = "read_request_count" + readBillingDemoUnitWriteRequestCount = "write_request_count" + readBillingDemoUnitHashStateRows = "hash_state_rows" + readBillingDemoUnitJoinOutputRows = "join_output_rows" readBillingDemoInputSourceRuntimeChunkBytes = "runtime_chunk_bytes" readBillingDemoInputSourceScanDetail = "scan_detail" readBillingDemoInputSourceRuntimeChildActRows = "runtime_child_act_rows" @@ -156,6 +175,8 @@ const ( readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" readBillingDemoInputSourceCommitDetail = "commit_detail" readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" + readBillingDemoInputSourcePhysicalPlan = "physical_plan" + readBillingDemoInputSourceHashJoinRuntime = "hash_join_runtime_stats" readBillingDemoInputSideAll = "all" readBillingDemoInputSideBuild = "build" readBillingDemoInputSideProbe = "probe" @@ -163,21 +184,6 @@ const ( readBillingDemoInputSideRight = "right" readBillingDemoScopeStatementAttempted = "statement_attempted" readBillingDemoScopeTxnPrewritePayload = "txn_prewrite_payload" - - // The first write-side preview formula intentionally has no fixed/RPC/region - // terms: write keys use the current scaled RUv2 write-key coefficient, and - // write bytes use the existing TiKV range-scan byte coefficient as a - // calibration seed because RUv2 only shadows commit WriteSize today. - readBillingDemoWriteRUScale = 2.01 - readBillingDemoRUV2WriteKeysWeight = 0.330760861554226 - readBillingDemoWriteKeyWeight = readBillingDemoWriteRUScale * readBillingDemoRUV2WriteKeysWeight - readBillingDemoWriteByteWeight = 0.000020 - // Mutation weights are independent preview calibration slots. They stay at - // zero until mutation-only TiDB CPU calibration supplies a defensible seed; - // they must never alias RUv2 or TiKV write coefficients. - readBillingDemoMutationCountWeight = 0.0 - readBillingDemoMutationByteWeight = 0.0 - readBillingDemoDiagnosticZeroWeight = 0.0 ) type explainRUComponentSnapshotStatus string @@ -269,11 +275,15 @@ type readBillingDemoCopInputEstimate struct { } type readBillingDemoCopComponentEvidence struct { - scanCount int - scanIdx int - detailHolderCount int - scanDetail tikvutil.ScanDetail - maxSummaryTasks int32 + scanCount int + scanIdx int + scanObservedTasks int32 + scanExpectedTasks int32 + detailHolderCount int + scanDetail tikvutil.ScanDetail + scanDetailRecords int32 + scanDetailExpectedTasks int32 + maxSummaryTasks int32 } type readBillingDemoCopEstimator struct { @@ -308,68 +318,28 @@ type readBillingDemoAppendOutcome struct { cause readBillingDemoCopFailure } -type readBillingDemoOperatorWeights struct { - fixedEvent float64 - row float64 - byte float64 - orderWork float64 - mutationCount float64 - mutationByte float64 - setCount float64 - deleteCount float64 - keyByte float64 - valueByte float64 - writeKey float64 - writeByte float64 - writeRPC float64 - region float64 -} - -type readBillingDemoWeightKey struct { - site string - opClass string - version string -} - -var readBillingDemoWeights = map[readBillingDemoWeightKey]readBillingDemoOperatorWeights{ - {readBillingDemoSiteTiKV, readBillingDemoOpClassRangeScan, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000020}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassKVWrite, readBillingDemoWeightVersion}: {writeKey: readBillingDemoWriteKeyWeight, writeByte: readBillingDemoWriteByteWeight, writeRPC: readBillingDemoDiagnosticZeroWeight, region: readBillingDemoDiagnosticZeroWeight}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000040, byte: 0.000006}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000006}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000008, byte: 0.000002}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, byte: 0.000012, orderWork: 0.000075}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.080, row: 0.000100, byte: 0.000014}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.060, row: 0.000065, byte: 0.000010}, - {readBillingDemoSiteTiKV, readBillingDemoOpClassPointLookup, readBillingDemoWeightVersion}: {fixedEvent: 0.045, row: 0.000030, byte: 0.000012}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassKVMutation, readBillingDemoWeightVersion}: { - mutationCount: readBillingDemoMutationCountWeight, - mutationByte: readBillingDemoMutationByteWeight, - setCount: readBillingDemoDiagnosticZeroWeight, - deleteCount: readBillingDemoDiagnosticZeroWeight, - keyByte: readBillingDemoDiagnosticZeroWeight, - valueByte: readBillingDemoDiagnosticZeroWeight, - }, - {readBillingDemoSiteTiDB, readBillingDemoOpClassFilter, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000030, byte: 0.000005}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassProjection, readBillingDemoWeightVersion}: {fixedEvent: 0.020, row: 0.000020, byte: 0.000004}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassLimit, readBillingDemoWeightVersion}: {fixedEvent: 0.010, row: 0.000006, byte: 0.000001}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassTopN, readBillingDemoWeightVersion}: {fixedEvent: 0.060, byte: 0.000010, orderWork: 0.000060}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassSort, readBillingDemoWeightVersion}: {fixedEvent: 0.080, byte: 0.000012, orderWork: 0.000070}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassWindow, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000070, byte: 0.000010}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassHashAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000085, byte: 0.000012}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassStreamAgg, readBillingDemoWeightVersion}: {fixedEvent: 0.050, row: 0.000055, byte: 0.000008}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassHashJoin, readBillingDemoWeightVersion}: {fixedEvent: 0.110, row: 0.000115, byte: 0.000020}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassMergeJoin, readBillingDemoWeightVersion}: {fixedEvent: 0.090, row: 0.000075, byte: 0.000012}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassLookupJoin, readBillingDemoWeightVersion}: {fixedEvent: 0.120, row: 0.000120, byte: 0.000020}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassReaderReceive, readBillingDemoWeightVersion}: {fixedEvent: 0.040, row: 0.000025, byte: 0.000014}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassLookupReader, readBillingDemoWeightVersion}: {fixedEvent: 0.070, row: 0.000045, byte: 0.000016}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassOverlayReader, readBillingDemoWeightVersion}: {fixedEvent: 0.050, row: 0.000035, byte: 0.000012}, - {readBillingDemoSiteTiDB, readBillingDemoOpClassMetadataReader, readBillingDemoWeightVersion}: { - fixedEvent: 0.020, - row: 0.000008, - byte: 0.000002, - }, +type readBillingDemoWeights struct { + Version string + CPUWeight float64 + ScanWeight float64 + NetWeight float64 + ReadRequestWeight float64 + WriteRequestWeight float64 + HashTableWeight float64 + JoinWeight float64 + MutationBytesPerCPUUnit float64 + Calibrated bool +} + +type readBillingDemoWeightProvider interface { + valid() bool + unitWeight(string) (float64, bool) } +// Production intentionally starts without guessed coefficients. Formula tests +// inject a calibrated private value directly. +var readBillingDemoV4Weights = readBillingDemoWeights{Version: readBillingDemoWeightVersion} + type explainRURow struct { section string id string @@ -424,6 +394,9 @@ func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast. if _, ok := stmt.(*ast.CommitStmt); ok { return buildTxnCommitBillingDemoResult(sctx, ruv2Metrics, execErr) } + if _, ok := stmt.(*ast.RollbackStmt); ok { + return readBillingDemoFailure(readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedStatement) + } if dmlKind, ok := explainRUWriteDMLKind(stmt); ok { return buildWriteBillingDemoResult(sctx, plan, dmlKind, ruv2Metrics, execErr) } @@ -463,6 +436,21 @@ func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast. return readBillingDemoFailedOperator(status, op) } } + if ruv2Metrics == nil && sctx.GetSessionVars() != nil { + ruv2Metrics = sctx.GetSessionVars().RUV2Metrics + } + if op, present := readBillingDemoReaderTransport(flat, runtimeStats, ruv2Metrics, false); present { + if op.status != readBillingDemoStatusOperatorOK { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, op) + } + result.operators = append(result.operators, op) + } + if op, present := readBillingDemoPointLookupTransport(flat, ruv2Metrics, false); present { + if op.status != readBillingDemoStatusOperatorOK { + return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, op) + } + result.operators = append(result.operators, op) + } return result } @@ -476,25 +464,20 @@ func buildTxnCommitBillingDemoResult(sctx base.PlanContext, ruv2Metrics *execdet result.reason = readBillingDemoReasonStatementError } - var commitDetail *tikvutil.CommitDetails var pipelined bool if sctx != nil && sctx.GetSessionVars() != nil { vars := sctx.GetSessionVars() if ruv2Metrics == nil { ruv2Metrics = vars.RUV2Metrics } - if vars.StmtCtx != nil { - commitDetail = vars.StmtCtx.GetExecDetails().CommitDetail - if recorder := vars.StmtCtx.PreviewKVMutationRecorder; recorder != nil { - pipelined = recorder.Snapshot().Pipelined - } + if vars.StmtCtx != nil && vars.StmtCtx.PreviewKVMutationRecorder != nil { + pipelined = vars.StmtCtx.PreviewKVMutationRecorder.Snapshot().Pipelined } } - // A final COMMIT owns the transaction-scoped TiKV payload. Keep dml_kind - // empty rather than guessing how the payload should be split among prior - // statements in the explicit transaction. - result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators("", commitDetail, ruv2Metrics, pipelined)...) + // COMMIT owns only the write requests in its fresh statement snapshot. Keep + // dml_kind empty because earlier DML statements own their own snapshots. + result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators("", ruv2Metrics, pipelined)...) return result } @@ -508,27 +491,23 @@ func buildWriteBillingDemoResult(sctx base.PlanContext, plan base.Plan, dmlKind result.reason = readBillingDemoReasonStatementError } - var commitDetail *tikvutil.CommitDetails - var mutationPipelined bool + var pipelined bool if sctx != nil && sctx.GetSessionVars() != nil { if ruv2Metrics == nil { ruv2Metrics = sctx.GetSessionVars().RUV2Metrics } - if sctx.GetSessionVars().StmtCtx != nil { - commitDetail = sctx.GetSessionVars().StmtCtx.GetExecDetails().CommitDetail - if recorder := sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder; recorder != nil { - mutationPipelined = recorder.Snapshot().Pipelined - } + if stmtCtx := sctx.GetSessionVars().StmtCtx; stmtCtx != nil && stmtCtx.PreviewKVMutationRecorder != nil { + pipelined = stmtCtx.PreviewKVMutationRecorder.Snapshot().Pipelined } } - appendReadBillingDemoDMLPlan(&result, sctx, plan) + appendReadBillingDemoDMLPlan(&result, sctx, plan, ruv2Metrics) appendReadBillingDemoMutation(&result, sctx, dmlKind) - result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators(dmlKind, commitDetail, ruv2Metrics, mutationPipelined)...) + result.operators = append(result.operators, buildTiKVWriteBillingDemoOperators(dmlKind, ruv2Metrics, pipelined)...) return result } -func appendReadBillingDemoDMLPlan(result *readBillingDemoResult, sctx base.PlanContext, plan base.Plan) { +func appendReadBillingDemoDMLPlan(result *readBillingDemoResult, sctx base.PlanContext, plan base.Plan, ruv2Metrics *execdetails.RUV2Metrics) { if plan == nil || sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx == nil { result.operators = append(result.operators, readBillingDemoPlanDiagnostic(readBillingDemoReasonMissingPlan)) return @@ -553,6 +532,12 @@ func appendReadBillingDemoDMLPlan(result *readBillingDemoResult, sctx base.PlanC for _, tree := range flat.ScalarSubQueries { appendTree(tree) } + if op, present := readBillingDemoReaderTransport(flat, runtimeStats, ruv2Metrics, true); present { + result.operators = append(result.operators, op) + } + if op, present := readBillingDemoPointLookupTransport(flat, ruv2Metrics, true); present { + result.operators = append(result.operators, op) + } } func readBillingDemoPlanDiagnostic(reason string) readBillingDemoOperatorResult { @@ -569,13 +554,13 @@ func readBillingDemoPlanDiagnostic(reason string) readBillingDemoOperatorResult func appendReadBillingDemoMutation(result *readBillingDemoResult, sctx base.PlanContext, dmlKind string) { operator := readBillingDemoOperatorResult{ - id: "stmt_memdb_mutation", + id: "mutation@statement", site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassKVMutation, operatorKind: readBillingDemoOperatorMemDBMutation, dmlKind: dmlKind, scope: readBillingDemoScopeStatementAttempted, - uncalibrated: true, + uncalibrated: !readBillingDemoWeightsValid(readBillingDemoV4Weights), } if sctx == nil || sctx.GetSessionVars() == nil || sctx.GetSessionVars().StmtCtx == nil || sctx.GetSessionVars().StmtCtx.PreviewKVMutationRecorder == nil { @@ -599,13 +584,17 @@ func appendReadBillingDemoMutation(result *readBillingDemoResult, sctx base.Plan readBillingDemoUnit{unit: readBillingDemoUnitKeyBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.KeyBytes), widthSource: explainRUWidthSourceNotApplicable}, readBillingDemoUnit{unit: readBillingDemoUnitValueBytes, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: float64(snapshot.ValueBytes), widthSource: explainRUWidthSourceNotApplicable}, ) + if readBillingDemoWeightsValid(readBillingDemoV4Weights) { + normalization := readBillingDemoV4Weights.MutationBytesPerCPUUnit + work := float64(snapshot.EncodedMutationCount) + float64(snapshot.EncodedMutationBytes)/normalization + if work >= 0 && !math.IsNaN(work) && !math.IsInf(work, 0) { + operator.units = append(operator.units, readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceStmtMemDBMutation, side: readBillingDemoInputSideAll, value: work, widthSource: explainRUWidthSourceNotApplicable}) + } + } result.operators = append(result.operators, operator) - result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonUncalibratedMutation)) - // The foreground gate does not currently distinguish simple DML from - // schemas that require unmodeled row preparation, constraint checks, or FK - // orchestration. Keep the encoded mutation units available, but report that - // wider DML CPU as partial for every supported DML kind, including DELETE. - result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonDMLAncillaryPartial)) + if !readBillingDemoWeightsValid(readBillingDemoV4Weights) { + result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonUncalibratedMutation)) + } vars := sctx.GetSessionVars() if vars.InTxn() && vars.TxnCtx != nil && vars.TxnCtx.CouldRetry { result.operators = append(result.operators, readBillingDemoMutationDiagnostic(dmlKind, readBillingDemoReasonOptimisticReplayPartial)) @@ -614,7 +603,7 @@ func appendReadBillingDemoMutation(result *readBillingDemoResult, sctx base.Plan func readBillingDemoMutationDiagnostic(dmlKind, reason string) readBillingDemoOperatorResult { return readBillingDemoOperatorResult{ - id: "stmt_memdb_mutation", + id: "mutation@statement", site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassKVMutation, operatorKind: readBillingDemoOperatorMemDBMutation, @@ -625,112 +614,36 @@ func readBillingDemoMutationDiagnostic(dmlKind, reason string) readBillingDemoOp } } -func buildWriteBillingDemoResultFromDetails(dmlKind string, commitDetail *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics) readBillingDemoResult { - return readBillingDemoResult{ - status: readBillingDemoStatusSuccess, - reason: readBillingDemoReasonNone, - operators: buildTiKVWriteBillingDemoOperators(dmlKind, commitDetail, ruv2Metrics, false), - } -} - -func buildTiKVWriteBillingDemoOperators(dmlKind string, commitDetail *tikvutil.CommitDetails, ruv2Metrics *execdetails.RUV2Metrics, pipelined bool) []readBillingDemoOperatorResult { +func buildTiKVWriteBillingDemoOperators(dmlKind string, ruv2Metrics *execdetails.RUV2Metrics, pipelined bool) []readBillingDemoOperatorResult { operator := readBillingDemoOperatorResult{ - id: "commit_txn", + id: "txn_write@statement", site: readBillingDemoSiteTiKV, opClass: readBillingDemoOpClassKVWrite, - operatorKind: readBillingDemoOperatorTxnPrewrite, + operatorKind: readBillingDemoOperatorTxnWrite, dmlKind: dmlKind, scope: readBillingDemoScopeTxnPrewritePayload, emitStatusRow: true, } - // Pipelined transactions allocate CommitDetails, but the logical flush - // payload is not accumulated into WriteKeys/WriteSize. Do not publish those - // empty fields as a complete zero payload merely because the detail exists. if pipelined { operator.status = readBillingDemoStatusPartial operator.reason = readBillingDemoReasonPipelinedWritePartial return []readBillingDemoOperatorResult{operator} } - if commitDetail == nil { + if ruv2Metrics == nil || ruv2Metrics.Bypass() { operator.status = readBillingDemoStatusPartial - operator.reason = readBillingDemoReasonMissingCommitDetail + operator.reason = readBillingDemoReasonMissingWriteRPCCount + return []readBillingDemoOperatorResult{operator} + } + writeRPCCount := ruv2Metrics.ResourceManagerWriteCnt() + if writeRPCCount < 0 { + operator.status = readBillingDemoStatusPartial + operator.reason = readBillingDemoReasonMissingWriteRPCCount return []readBillingDemoOperatorResult{operator} } - operator.status = readBillingDemoStatusOperatorOK operator.reason = readBillingDemoReasonNone - writeKeys := int64(commitDetail.WriteKeys) - writeBytes := int64(commitDetail.WriteSize) - if writeKeys == 0 && writeBytes == 0 { - operator.reason = readBillingDemoReasonZeroMutation - } - operator.units = append(operator.units, - readBillingDemoUnit{ - unit: readBillingDemoUnitWriteKeys, - source: readBillingDemoInputSourceCommitDetail, - side: readBillingDemoInputSideAll, - value: float64(writeKeys), - widthSource: explainRUWidthSourceNotApplicable, - }, - readBillingDemoUnit{ - unit: readBillingDemoUnitWriteByte, - source: readBillingDemoInputSourceCommitDetail, - side: readBillingDemoInputSideAll, - value: float64(writeBytes), - widthSource: explainRUWidthSourceNotApplicable, - }, - ) - operators := []readBillingDemoOperatorResult{operator} - if writeKeys == 0 && writeBytes > 0 { - operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingWriteKeys)) - } - if writeBytes == 0 && writeKeys > 0 { - operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingWriteByte)) - } - prewriteRegionNum := int64(atomic.LoadInt32(&commitDetail.PrewriteRegionNum)) - if prewriteRegionNum > 0 { - operator.units = append(operator.units, readBillingDemoUnit{ - unit: readBillingDemoUnitPrewriteRegionNum, - source: readBillingDemoInputSourceCommitDetail, - side: readBillingDemoInputSideAll, - value: float64(prewriteRegionNum), - widthSource: explainRUWidthSourceNotApplicable, - }) - } - writeRPCCount := int64(0) - if ruv2Metrics != nil && !ruv2Metrics.Bypass() { - writeRPCCount = ruv2Metrics.ResourceManagerWriteCnt() - } - if writeRPCCount > 0 { - operator.units = append(operator.units, readBillingDemoUnit{ - unit: readBillingDemoUnitTiKVWriteRPCCount, - source: readBillingDemoInputSourceRUV2Metrics, - side: readBillingDemoInputSideAll, - value: float64(writeRPCCount), - widthSource: explainRUWidthSourceNotApplicable, - }) - } - if prewriteRegionNum == 0 { - operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingPrewriteRegion)) - } - if writeRPCCount == 0 { - operators = append(operators, readBillingDemoWriteDiagnosticStatus(dmlKind, readBillingDemoReasonMissingWriteRPCCount)) - } - operators[0] = operator - return operators -} - -func readBillingDemoWriteDiagnosticStatus(dmlKind, reason string) readBillingDemoOperatorResult { - return readBillingDemoOperatorResult{ - id: "commit_txn", - site: readBillingDemoSiteTiKV, - opClass: readBillingDemoOpClassKVWrite, - operatorKind: readBillingDemoOperatorTxnPrewrite, - dmlKind: dmlKind, - scope: readBillingDemoScopeTxnPrewritePayload, - status: readBillingDemoStatusPartial, - reason: reason, - } + operator.units = []readBillingDemoUnit{{unit: readBillingDemoUnitWriteRequestCount, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(writeRPCCount), widthSource: explainRUWidthSourceNotApplicable}} + return []readBillingDemoOperatorResult{operator} } func readBillingDemoPlanContext(plan base.Plan) base.PlanContext { @@ -740,52 +653,226 @@ func readBillingDemoPlanContext(plan base.Plan) base.PlanContext { return plan.SCtx() } -func readBillingDemoResolveWeights(site, opClass, version string) (readBillingDemoOperatorWeights, bool) { - weights, ok := readBillingDemoWeights[readBillingDemoWeightKey{site: site, opClass: opClass, version: version}] - return weights, ok +func readBillingDemoAllTrees(flat *FlatPhysicalPlan) []FlatPlanTree { + if flat == nil { + return nil + } + trees := make([]FlatPlanTree, 0, 1+len(flat.CTEs)+len(flat.ScalarSubQueries)) + trees = append(trees, flat.Main) + trees = append(trees, flat.CTEs...) + trees = append(trees, flat.ScalarSubQueries...) + return trees +} + +func readBillingDemoReaderTransport(flat *FlatPhysicalPlan, runtimeStats *execdetails.RuntimeStatsColl, ruv2Metrics *execdetails.RUV2Metrics, dml bool) (readBillingDemoOperatorResult, bool) { + op := readBillingDemoOperatorResult{ + id: "reader_transport@statement", + site: readBillingDemoSiteTiDB, + opClass: readBillingDemoOpClassReaderTransport, + operatorKind: "mixed_reader", + emitStatusRow: true, + } + kinds := make(map[string]struct{}) + openProducerSet := dml + hasTasks := false + allReaderRowsZero := true + for _, tree := range readBillingDemoAllTrees(flat) { + for _, node := range tree { + if node == nil || node.Origin == nil { + continue + } + kind := "" + switch plan := node.Origin.(type) { + case *physicalop.PhysicalTableReader: + if plan.StoreType == kv.TiKV { + kind = "table_reader" + } else { + openProducerSet = true + } + case *physicalop.PhysicalIndexReader: + kind = "index_reader" + case *physicalop.PhysicalIndexLookUpReader: + kind = "index_lookup" + case *physicalop.PhysicalIndexMergeReader: + kind = "index_merge" + case *physicalop.PointGetPlan, *physicalop.BatchPointGetPlan, *physicalop.PhysicalExchangeReceiver, *physicalop.PhysicalExchangeSender: + openProducerSet = true + } + if kind != "" { + kinds[kind] = struct{}{} + rows, ok := readBillingDemoPlanActRows(runtimeStats, node.Origin.ID()) + if !ok || rows != 0 { + allReaderRowsZero = false + } + } + if !node.IsRoot && node.StoreType == kv.TiKV && runtimeStats != nil { + stats := runtimeStats.GetCopStats(node.Origin.ID()) + if (stats != nil && stats.GetTasks() > 0) || runtimeStats.GetExpectedCopTasks(node.Origin.ID()) > 0 { + hasTasks = true + } + } + } + } + if len(kinds) == 0 { + return readBillingDemoOperatorResult{}, false + } + if len(kinds) == 1 { + for kind := range kinds { + op.operatorKind = kind + } + } + if openProducerSet { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonAmbiguousReaderTransport + return op, true + } + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + netBytes := ruv2Metrics.TiKVCoprocessorResponseBytes() + requests := ruv2Metrics.ResourceManagerReadCnt() + if netBytes < 0 || requests < 0 || (netBytes > 0 && requests == 0) || (netBytes == 0 && requests == 0 && (hasTasks || !allReaderRowsZero)) { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{ + {unit: readBillingDemoUnitNetBytes, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(netBytes), widthSource: explainRUWidthSourceNotApplicable}, + {unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable}, + } + return op, true +} + +func readBillingDemoPointLookupTransport(flat *FlatPhysicalPlan, ruv2Metrics *execdetails.RUV2Metrics, dml bool) (readBillingDemoOperatorResult, bool) { + op := readBillingDemoOperatorResult{ + id: "point_lookup@statement", + site: readBillingDemoSiteTiKV, + opClass: readBillingDemoOpClassPointLookup, + operatorKind: "mixed_point_lookup", + emitStatusRow: true, + } + kinds := make(map[string]struct{}) + openProducerSet := dml + for _, tree := range readBillingDemoAllTrees(flat) { + for _, node := range tree { + if node == nil || node.Origin == nil { + continue + } + switch plan := node.Origin.(type) { + case *physicalop.PointGetPlan: + kinds["point_get"] = struct{}{} + openProducerSet = openProducerSet || plan.Lock + case *physicalop.BatchPointGetPlan: + kinds["batch_point_get"] = struct{}{} + openProducerSet = openProducerSet || plan.Lock + case *physicalop.PhysicalTableReader: + openProducerSet = true + case *physicalop.PhysicalIndexReader, *physicalop.PhysicalIndexLookUpReader, + *physicalop.PhysicalIndexMergeReader, *physicalop.PhysicalExchangeReceiver, + *physicalop.PhysicalExchangeSender: + openProducerSet = true + } + } + } + if len(kinds) == 0 { + return readBillingDemoOperatorResult{}, false + } + if len(kinds) == 1 { + for kind := range kinds { + op.operatorKind = kind + } + } + if openProducerSet { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonAmbiguousReaderTransport + return op, true + } + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + requests := ruv2Metrics.ResourceManagerReadCnt() + if requests < 0 { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{{ + unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceRUV2Metrics, + side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable, + }} + return op, true +} + +func readBillingDemoWeightsValid(weights readBillingDemoWeights) bool { + if weights.Version == "" || weights.Version == readBillingDemoWeightVersion || !weights.Calibrated || + weights.MutationBytesPerCPUUnit <= 0 || math.IsNaN(weights.MutationBytesPerCPUUnit) || math.IsInf(weights.MutationBytesPerCPUUnit, 0) { + return false + } + for _, weight := range []float64{weights.CPUWeight, weights.ScanWeight, weights.NetWeight, weights.ReadRequestWeight, weights.WriteRequestWeight, weights.HashTableWeight, weights.JoinWeight} { + if weight < 0 || math.IsNaN(weight) || math.IsInf(weight, 0) { + return false + } + } + return true +} + +func readBillingDemoActiveWeightVersion() string { + if readBillingDemoV4Weights.Version != "" { + return readBillingDemoV4Weights.Version + } + return readBillingDemoWeightVersion +} + +func (weights readBillingDemoWeights) valid() bool { + return readBillingDemoWeightsValid(weights) +} + +func (weights readBillingDemoWeights) unitWeight(unit string) (float64, bool) { + return readBillingDemoUnitWeight(weights, unit) } -func readBillingDemoUnitWeight(weights readBillingDemoOperatorWeights, unit string) (float64, bool) { +func readBillingDemoUnitWeight(weights readBillingDemoWeights, unit string) (float64, bool) { switch unit { - case readBillingDemoUnitFixedEvents: - return weights.fixedEvent, true - case readBillingDemoUnitInputRows: - return weights.row, true - case readBillingDemoUnitInputBytes: - return weights.byte, true - case readBillingDemoUnitOrderWork: - return weights.orderWork, true - case readBillingDemoUnitEncodedMutationCount: - return weights.mutationCount, true - case readBillingDemoUnitEncodedMutationBytes: - return weights.mutationByte, true - case readBillingDemoUnitSetCount: - return weights.setCount, true - case readBillingDemoUnitDeleteCount: - return weights.deleteCount, true - case readBillingDemoUnitKeyBytes: - return weights.keyByte, true - case readBillingDemoUnitValueBytes: - return weights.valueByte, true - case readBillingDemoUnitWriteKeys: - return weights.writeKey, true - case readBillingDemoUnitWriteByte: - return weights.writeByte, true - case readBillingDemoUnitPrewriteRegionNum: - return weights.region, true - case readBillingDemoUnitTiKVWriteRPCCount: - return weights.writeRPC, true + case readBillingDemoUnitCPUWork: + return weights.CPUWeight, true + case readBillingDemoUnitScanBytes: + return weights.ScanWeight, true + case readBillingDemoUnitNetBytes: + return weights.NetWeight, true + case readBillingDemoUnitReadRequestCount: + return weights.ReadRequestWeight, true + case readBillingDemoUnitWriteRequestCount: + return weights.WriteRequestWeight, true + case readBillingDemoUnitHashStateRows: + return weights.HashTableWeight, true + case readBillingDemoUnitJoinOutputRows: + return weights.JoinWeight, true default: return 0, false } } -func readBillingDemoUnitPreviewRU(unit readBillingDemoUnit, weights readBillingDemoOperatorWeights) (float64, float64, bool) { - weight, ok := readBillingDemoUnitWeight(weights, unit.unit) +func readBillingDemoUnitPreviewRU(unit readBillingDemoUnit, weights readBillingDemoWeightProvider) (float64, float64, bool) { + if !weights.valid() || unit.value < 0 || math.IsNaN(unit.value) || math.IsInf(unit.value, 0) { + return 0, 0, false + } + weight, ok := weights.unitWeight(unit.unit) if !ok { return 0, 0, false } - return weight, unit.value * weight, true + ru := unit.value * weight + if ru < 0 || math.IsNaN(ru) || math.IsInf(ru, 0) { + return 0, 0, false + } + return weight, ru, true } func readBillingDemoFailure(status, reason string) readBillingDemoResult { @@ -815,29 +902,15 @@ func readBillingDemoFailedOperator(status string, op readBillingDemoOperatorResu } func summarizeReadBillingDemoBaseUnits(result readBillingDemoResult) stmtsummary.ReadBillingDemoBaseUnitSummary { - var summary stmtsummary.ReadBillingDemoBaseUnitSummary - for _, op := range result.operators { - if op.status != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { - continue - } - for _, unit := range op.units { - switch unit.unit { - case readBillingDemoUnitFixedEvents: - summary.SumReadBillingDemoFixedEvents += unit.value - case readBillingDemoUnitInputRows: - summary.SumReadBillingDemoInputRows += unit.value - case readBillingDemoUnitInputBytes: - summary.SumReadBillingDemoInputBytes += unit.value - } - } - } - return summary + // The three convenience totals are a v3 schema. V4 detail is preserved in + // the versioned base-unit table and must not be projected into those fields. + return stmtsummary.ReadBillingDemoBaseUnitSummary{} } func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummary.ReadBillingDemoStatementStats { stats := stmtsummary.ReadBillingDemoStatementStats{ ModelVersion: readBillingDemoModelVersion, - WeightVersion: readBillingDemoWeightVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), } status := result.status if status == "" { @@ -849,7 +922,7 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar } stats.Statuses = append(stats.Statuses, stmtsummary.ReadBillingDemoStatusSample{ ModelVersion: readBillingDemoModelVersion, - WeightVersion: readBillingDemoWeightVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), Site: readBillingDemoSiteStatement, OpClass: readBillingDemoOpClassStatement, OperatorKind: readBillingDemoOperatorStatement, @@ -875,7 +948,7 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar opReason == reason) { stats.Statuses = append(stats.Statuses, stmtsummary.ReadBillingDemoStatusSample{ ModelVersion: readBillingDemoModelVersion, - WeightVersion: readBillingDemoWeightVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), Site: op.site, OpClass: op.opClass, OperatorKind: op.operatorKind, @@ -889,7 +962,7 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar for _, unit := range op.units { sample := stmtsummary.ReadBillingDemoBaseUnitSample{ ModelVersion: readBillingDemoModelVersion, - WeightVersion: readBillingDemoWeightVersion, + WeightVersion: readBillingDemoActiveWeightVersion(), Site: op.site, OpClass: op.opClass, OperatorKind: op.operatorKind, @@ -902,14 +975,6 @@ func buildReadBillingDemoStatementStats(result readBillingDemoResult) stmtsummar RowWidth: unit.rowWidth, } stats.BaseUnits = append(stats.BaseUnits, sample) - switch unit.unit { - case readBillingDemoUnitFixedEvents: - stats.Totals.SumReadBillingDemoFixedEvents += unit.value - case readBillingDemoUnitInputRows: - stats.Totals.SumReadBillingDemoInputRows += unit.value - case readBillingDemoUnitInputBytes: - stats.Totals.SumReadBillingDemoInputBytes += unit.value - } } } return stats @@ -1081,17 +1146,19 @@ func newReadBillingDemoCopEstimator(tree FlatPlanTree, runtimeStats *execdetails if runtimeStats == nil { continue } - stats := runtimeStats.GetCopStats(node.Origin.ID()) - if stats == nil { - continue + detail, detailRecords, observedTasks, expectedTasks := runtimeStats.GetCopScanDetailAndCoverage(node.Origin.ID()) + if observedTasks > component.maxSummaryTasks { + component.maxSummaryTasks = observedTasks } - if tasks := stats.GetTasks(); tasks > component.maxSummaryTasks { - component.maxSummaryTasks = tasks + if supported && operator.opClass == readBillingDemoOpClassRangeScan { + component.scanObservedTasks = observedTasks + component.scanExpectedTasks = expectedTasks } - detail := stats.GetScanDetail() - if detail.ProcessedKeys > 0 && detail.ProcessedKeysSize > 0 { + if detailRecords > 0 { component.detailHolderCount++ component.scanDetail = detail + component.scanDetailRecords = detailRecords + component.scanDetailExpectedTasks = expectedTasks } } estimator.auxiliaryEntries = len(estimator.parentIdx) + len(estimator.componentID) + len(estimator.components) + len(estimator.nodeFailures) + len(estimator.treeFailures) @@ -1542,7 +1609,19 @@ func (op readBillingDemoOperatorResult) withReason(reason string) readBillingDem } func readBillingDemoOperatorBillable(op readBillingDemoOperatorResult) bool { - return op.opClass != readBillingDemoOpClassWrapper && op.opClass != readBillingDemoOpClassSynthetic + switch op.opClass { + case readBillingDemoOpClassFilter, readBillingDemoOpClassProjection, readBillingDemoOpClassLimit, + readBillingDemoOpClassTopN, readBillingDemoOpClassSort, readBillingDemoOpClassWindow, + readBillingDemoOpClassHashAgg, readBillingDemoOpClassStreamAgg, readBillingDemoOpClassHashJoin, + readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin, readBillingDemoOpClassRangeScan, + readBillingDemoOpClassReaderTransport, readBillingDemoOpClassOverlayReader, + readBillingDemoOpClassKVMutation, readBillingDemoOpClassKVWrite: + return true + case readBillingDemoOpClassPointLookup: + return op.id == "point_lookup@statement" + default: + return false + } } func readBillingDemoOperatorActRows(runtimeStats *execdetails.RuntimeStatsColl, op *FlatOperator) (int64, bool) { @@ -1595,7 +1674,7 @@ func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorR case plancodec.TypeExchangeReceiver, plancodec.TypeExchangeSender: return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedMPP case plancodec.TypeIndexMerge: - return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupReader, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedIndexMerge + return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassLookupReader, operatorKind: operatorKind}, true, "" case plancodec.TypeLock: return readBillingDemoOperatorResult{site: readBillingDemoSiteTiDB, opClass: readBillingDemoOpClassReaderReceive, operatorKind: operatorKind}, false, readBillingDemoReasonUnsupportedLock case plancodec.TypePointGet, plancodec.TypeBatchPointGet: @@ -1639,45 +1718,227 @@ func readBillingDemoClassifyOperator(op *FlatOperator) (readBillingDemoOperatorR } } -func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { - outputRows, outputBytes, hasOutput := readBillingDemoRootOutputRowsAndBytes(runtimeStats, op.Origin.ID()) - if !hasOutput { - if _, rowsOK := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()); !rowsOK { - return nil, readBillingDemoReasonMissingRuntimeRows, false +func readBillingDemoJoinConditionCount(join *physicalop.BasePhysicalJoin) int64 { + return int64(len(join.LeftConditions) + len(join.RightConditions) + len(join.OtherConditions)) +} + +func readBillingDemoCompareFilterCount(filters *physicalop.ColWithCmpFuncManager) int64 { + if filters == nil { + return 0 + } + return int64(len(filters.OpType)) +} + +func readBillingDemoExpressionCount(plan base.Plan) (int64, bool) { + switch p := plan.(type) { + case *physicalop.PhysicalSelection: + return int64(len(p.Conditions)), true + case *physicalop.PhysicalProjection: + return int64(len(p.Exprs)), true + case *physicalop.PhysicalHashAgg: + return int64(len(p.GroupByItems) + len(p.AggFuncs)), true + case *physicalop.PhysicalStreamAgg: + return int64(len(p.GroupByItems) + len(p.AggFuncs)), true + case *physicalop.PhysicalHashJoin: + if len(p.LeftJoinKeys) != len(p.RightJoinKeys) || len(p.LeftNAJoinKeys) != len(p.RightNAJoinKeys) { + return 0, false + } + return int64(len(p.EqualConditions)+len(p.NAEqualConditions)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin), true + case *physicalop.PhysicalMergeJoin: + if len(p.LeftJoinKeys) != len(p.RightJoinKeys) { + return 0, false + } + return int64(len(p.CompareFuncs)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin), true + case *physicalop.PhysicalIndexHashJoin: + if len(p.OuterHashKeys) != len(p.InnerHashKeys) { + return 0, false + } + return int64(len(p.OuterHashKeys)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin) + readBillingDemoCompareFilterCount(p.CompareFilters), true + case *physicalop.PhysicalIndexMergeJoin: + if p.NeedOuterSort != (len(p.OuterCompareFuncs) > 0) { + return 0, false + } + return int64(len(p.CompareFuncs)+len(p.OuterCompareFuncs)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin) + readBillingDemoCompareFilterCount(p.CompareFilters), true + case *physicalop.PhysicalIndexJoin: + if len(p.OuterJoinKeys) != len(p.InnerJoinKeys) { + return 0, false + } + return int64(len(p.OuterJoinKeys)) + readBillingDemoJoinConditionCount(&p.BasePhysicalJoin) + readBillingDemoCompareFilterCount(p.CompareFilters), true + case *physicalop.PhysicalWindow: + count := len(p.WindowFuncDescs) + len(p.PartitionBy) + len(p.OrderBy) + if p.Frame != nil { + if p.Frame.Start != nil { + count += len(p.Frame.Start.CalcFuncs) + } + if p.Frame.End != nil { + count += len(p.Frame.End.CalcFuncs) + } + } + return int64(count), true + default: + return 0, false + } +} + +func readBillingDemoOrderingMaterialized(op, child *FlatOperator) bool { + if op == nil || op.Origin == nil || child == nil || child.Origin == nil { + return false + } + childSchema := child.Origin.Schema() + if projection, ok := child.Origin.(*physicalop.PhysicalProjection); ok && + (childSchema == nil || childSchema.Len() != len(projection.Exprs)) { + return false + } + checkExpr := func(expr expression.Expression) bool { + if expr == nil { + return false + } + _, scalar := expr.(*expression.ScalarFunction) + if scalar { + return false + } + if col, ok := expr.(*expression.Column); ok { + return childSchema != nil && childSchema.ColumnIndex(col) >= 0 } - return nil, readBillingDemoReasonMissingRuntimeBytes, false + return true + } + switch p := op.Origin.(type) { + case *physicalop.PhysicalSort: + for _, item := range p.ByItems { + if item == nil || !checkExpr(item.Expr) { + return false + } + } + return true + case *physicalop.PhysicalTopN: + for _, item := range p.ByItems { + if item == nil || !checkExpr(item.Expr) { + return false + } + } + return true + default: + return false + } +} + +func readBillingDemoCheckedWork(rows int64, multiplier float64) (float64, bool) { + if rows < 0 || multiplier < 0 || math.IsNaN(multiplier) || math.IsInf(multiplier, 0) { + return 0, false + } + work := float64(rows) * multiplier + return work, work >= 0 && !math.IsNaN(work) && !math.IsInf(work, 0) +} + +func readBillingDemoHashStateRows(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, bool) { + if runtimeStats == nil || !runtimeStats.ExistsRootStats(planID) { + return 0, false + } + _, groups := runtimeStats.GetRootStats(planID).MergeStats() + for _, group := range groups { + if stats, ok := group.(execdetails.HashTableRuntimeStats); ok { + return stats.HashTableRows(), true + } + } + return 0, false +} + +func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, op *FlatOperator, operator readBillingDemoOperatorResult) ([]readBillingDemoUnit, string, bool) { + outputRows, hasOutputRows := readBillingDemoPlanActRows(runtimeStats, op.Origin.ID()) + if !hasOutputRows || outputRows < 0 { + return nil, readBillingDemoReasonMissingRuntimeRows, false } units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChunkBytes)} - switch operator.opClass { - case readBillingDemoOpClassHashJoin: - var reason string - var ok bool - units, reason, ok = appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, true) - if !ok { - return nil, reason, false + appendExpressionCPU := func(rows int64) (string, bool) { + exprCount, ok := readBillingDemoExpressionCount(op.Origin) + if !ok || exprCount < 0 { + return readBillingDemoReasonMissingExpressionCount, false } - case readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: - var reason string - var ok bool - units, reason, ok = appendReadBillingDemoJoinUnits(units, runtimeStats, tree, idx, false) + work, ok := readBillingDemoCheckedWork(rows, float64(exprCount)) if !ok { + return readBillingDemoReasonMissingExpressionCount, false + } + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitExpressionCount, source: readBillingDemoInputSourcePhysicalPlan, side: readBillingDemoInputSideAll, value: float64(exprCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: work, widthSource: explainRUWidthSourceNotApplicable}, + ) + return "", true + } + switch operator.opClass { + case readBillingDemoOpClassHashJoin, readBillingDemoOpClassMergeJoin, readBillingDemoOpClassLookupJoin: + if idx < 0 || idx >= len(tree) || len(tree[idx].ChildrenIdx) != 2 { + return nil, readBillingDemoReasonMissingExpressionCount, false + } + var inputRows int64 + for childOrder, childIdx := range tree[idx].ChildrenIdx { + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + rows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + if !ok || rows < 0 || (rows > 0 && inputRows > math.MaxInt64-rows) { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + inputRows += rows + side := readBillingDemoInputSideLeft + if childOrder == 1 { + side = readBillingDemoInputSideRight + } + if operator.opClass == readBillingDemoOpClassHashJoin { + if tree[childIdx].Label == BuildSide { + side = readBillingDemoInputSideBuild + } else if tree[childIdx].Label == ProbeSide { + side = readBillingDemoInputSideProbe + } + } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChildActRows, side: side, value: float64(rows), widthSource: explainRUWidthSourceNotApplicable}) + } + if reason, ok := appendExpressionCPU(inputRows); !ok { return nil, reason, false } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitJoinOutputRows, source: readBillingDemoInputSourceRuntimeOperatorActRows, side: readBillingDemoInputSideAll, value: float64(outputRows), widthSource: explainRUWidthSourceNotApplicable}) + if operator.opClass == readBillingDemoOpClassHashJoin { + stateRows, ok := readBillingDemoHashStateRows(runtimeStats, op.Origin.ID()) + if !ok { + return nil, readBillingDemoReasonMissingHashStateRows, false + } + if stateRows < 0 { + return nil, readBillingDemoReasonInvalidHashStateRows, false + } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitHashStateRows, source: readBillingDemoInputSourceHashJoinRuntime, side: readBillingDemoInputSideBuild, value: float64(stateRows), widthSource: explainRUWidthSourceNotApplicable}) + } default: - inputRows, inputBytes, reason, ok := readBillingDemoDirectLocalInputRowsAndBytes(runtimeStats, tree, idx, operator.opClass) - if !ok { - return nil, reason, false + if idx < 0 || idx >= len(tree) || len(tree[idx].ChildrenIdx) != 1 { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + childIdx := tree[idx].ChildrenIdx[0] + if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil { + return nil, readBillingDemoReasonMissingRuntimeRows, false + } + inputRows, ok := readBillingDemoPlanActRows(runtimeStats, tree[childIdx].Origin.ID()) + if !ok || inputRows < 0 { + return nil, readBillingDemoReasonMissingRuntimeRows, false } - units = append(units, readBillingDemoRuntimeChunkInputUnits(inputRows, inputBytes, readBillingDemoInputSideAll)...) + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(inputRows), widthSource: explainRUWidthSourceNotApplicable}) orderWork, ok := readBillingDemoOrderingWorkUnit(op, operator.opClass, inputRows) if !ok { - return nil, readBillingDemoReasonInvalidOrderingWork, false + return nil, readBillingDemoOrderingFailureReason(op, operator.opClass), false } if orderWork.unit != "" { + if !readBillingDemoOrderingMaterialized(op, tree[childIdx]) { + return nil, readBillingDemoReasonMissingOrderingProjection, false + } + orderWork.unit = readBillingDemoUnitCPUWork units = append(units, orderWork) + } else if operator.opClass == readBillingDemoOpClassLimit || operator.opClass == readBillingDemoOpClassOverlayReader { + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(inputRows), widthSource: explainRUWidthSourceNotApplicable}) + } else if reason, ok := appendExpressionCPU(inputRows); !ok { + return nil, reason, false + } + if operator.opClass == readBillingDemoOpClassHashAgg { + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitHashStateRows, source: readBillingDemoInputSourceRuntimeOperatorActRows, side: readBillingDemoInputSideAll, value: float64(outputRows), widthSource: explainRUWidthSourceNotApplicable}) } } - if readBillingDemoOperatorHasOutputShadows(operator.opClass) { + if _, outputBytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, op.Origin.ID()); ok && readBillingDemoOperatorHasOutputShadows(operator.opClass) { units = append(units, readBillingDemoRuntimeChunkOutputUnits(outputRows, outputBytes)...) } return units, "", true @@ -1693,14 +1954,6 @@ func readBillingDemoFixedEventUnit(inputSource string) readBillingDemoUnit { } } -func readBillingDemoRuntimeChunkInputUnits(rows, bytes int64, side string) []readBillingDemoUnit { - rowWidth := readBillingDemoAverageRowWidth(rows, float64(bytes)) - return []readBillingDemoUnit{ - {unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChunkBytes, side: side, value: float64(rows), rowWidth: rowWidth, widthSource: explainRUWidthSourceRuntimeChunkAvg}, - {unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceRuntimeChunkBytes, side: side, value: float64(bytes), rowWidth: rowWidth, widthSource: explainRUWidthSourceRuntimeChunkAvg}, - } -} - func readBillingDemoRuntimeChunkOutputUnits(rows, bytes int64) []readBillingDemoUnit { if rows < 0 || bytes < 0 { return nil @@ -1748,18 +2001,20 @@ func readBillingDemoOrderingWorkUnit(op *FlatOperator, opClass string, inputRows if !ok { return readBillingDemoUnit{}, false } - if op.IsRoot { - // Add after conversion so an extreme OFFSET + COUNT cannot wrap uint64. - logWidth = max(float64(topN.Offset)+float64(topN.Count), 2) - } else { - // Pushdown folds the original OFFSET + COUNT into Count. TiKV's TopN - // protobuf has only Limit, so a non-zero cop Offset is not executable - // evidence for the heap bound used here. - if topN.Offset != 0 { - return readBillingDemoUnit{}, false - } - logWidth = max(float64(topN.Count), 2) + if topN.Count == 0 { + return readBillingDemoUnit{ + unit: readBillingDemoUnitOrderWork, + source: readBillingDemoInputSourceRuntimeOrderingWork, + side: readBillingDemoInputSideAll, + value: 0, + widthSource: explainRUWidthSourceNotApplicable, + }, true + } + if topN.Count > math.MaxUint64-topN.Offset { + return readBillingDemoUnit{}, false } + effectiveK := min(uint64(inputRows), topN.Offset+topN.Count) + logWidth = max(float64(effectiveK), 2) } work := float64(inputRows) * math.Log2(logWidth) if work < 0 || math.IsNaN(work) || math.IsInf(work, 0) { @@ -1774,72 +2029,13 @@ func readBillingDemoOrderingWorkUnit(op *FlatOperator, opClass string, inputRows }, true } -func readBillingDemoUseOutputRowsAsInput(opClass string) bool { - switch opClass { - case readBillingDemoOpClassReaderReceive, readBillingDemoOpClassLookupReader, readBillingDemoOpClassMetadataReader, readBillingDemoOpClassPointLookup: - return true - default: - return false - } -} - -func appendReadBillingDemoJoinUnits(units []readBillingDemoUnit, runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, useBuildProbe bool) ([]readBillingDemoUnit, string, bool) { - for childOrder, childIdx := range tree[idx].ChildrenIdx { - if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { - continue +func readBillingDemoOrderingFailureReason(op *FlatOperator, opClass string) string { + if opClass == readBillingDemoOpClassTopN && op != nil { + if topN, ok := op.Origin.(*physicalop.PhysicalTopN); ok && topN.Count > math.MaxUint64-topN.Offset { + return readBillingDemoReasonInvalidTopNBound } - rows, bytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, tree[childIdx].Origin.ID()) - if !ok { - return nil, readBillingDemoReasonMissingInputBytes, false - } - side := readBillingDemoInputSideAll - if useBuildProbe { - switch tree[childIdx].Label { - case BuildSide: - side = readBillingDemoInputSideBuild - case ProbeSide: - side = readBillingDemoInputSideProbe - default: - if childOrder == 0 { - side = readBillingDemoInputSideBuild - } else { - side = readBillingDemoInputSideProbe - } - } - } else if childOrder == 0 { - side = readBillingDemoInputSideLeft - } else { - side = readBillingDemoInputSideRight - } - units = append(units, readBillingDemoRuntimeChunkInputUnits(rows, bytes, side)...) - } - return units, "", true -} - -func readBillingDemoDirectLocalInputRowsAndBytes(runtimeStats *execdetails.RuntimeStatsColl, tree FlatPlanTree, idx int, opClass string) (int64, int64, string, bool) { - if idx < 0 || idx >= len(tree) || tree[idx] == nil { - return 0, 0, "", true - } - if len(tree[idx].ChildrenIdx) == 0 || readBillingDemoUseOutputRowsAsInput(opClass) { - rows, bytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, tree[idx].Origin.ID()) - if !ok { - return 0, 0, readBillingDemoReasonMissingRuntimeBytes, false - } - return rows, bytes, "", true - } - var rows, inputBytes int64 - for _, childIdx := range tree[idx].ChildrenIdx { - if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { - continue - } - childRows, childBytes, ok := readBillingDemoRootOutputRowsAndBytes(runtimeStats, tree[childIdx].Origin.ID()) - if !ok { - return 0, 0, readBillingDemoReasonMissingInputBytes, false - } - rows += childRows - inputBytes += childBytes } - return rows, inputBytes, "", true + return readBillingDemoReasonInvalidOrderingWork } func readBillingDemoPlanActRows(runtimeStats *execdetails.RuntimeStatsColl, planID int) (int64, bool) { @@ -1926,54 +2122,92 @@ func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, op if idx < 0 || idx >= len(estimator.tree) || len(estimator.tree[idx].ChildrenIdx) != 0 { return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnsupported, readBillingDemoReasonUnsupportedCopStructure)} } - width := estimator.componentOutputWidth(idx) - if width.failure.present { - return readBillingDemoCopUnitOutcome{failure: width.failure} - } - switch width.widthState { - case readBillingDemoCopWidthAmbiguous: + component := estimator.components[estimator.componentID[idx]] + if component.scanCount != 1 || component.detailHolderCount > 1 { return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonAmbiguousCopScanWidth)} - case readBillingDemoCopWidthMissing: - return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} - case readBillingDemoCopWidthKnown: - default: + } + if component.detailHolderCount != 1 || component.scanObservedTasks <= 0 || component.scanExpectedTasks <= 0 { return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} } - component := estimator.components[estimator.componentID[idx]] + if component.scanObservedTasks != component.scanExpectedTasks || + component.scanDetailExpectedTasks != component.scanExpectedTasks || + component.scanDetailRecords != component.scanDetailExpectedTasks { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} + } scanDetail := component.scanDetail scanInputRows, scanInputBytes, ok := readBillingDemoRangeScanInput(scanDetail.TotalKeys, scanDetail.ProcessedKeys, scanDetail.ProcessedKeysSize) if !ok { return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} } + rowWidth := readBillingDemoAverageRowWidth(scanInputRows, scanInputBytes) units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceScanDetail)} units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: width.avgRowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: width.avgRowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + readBillingDemoUnit{unit: readBillingDemoUnitScanBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedEstimate}, ) return readBillingDemoCopUnitOutcome{success: true, units: units} } - - input := estimator.inputEstimate(idx) - if input.failure.present { - return readBillingDemoCopUnitOutcome{failure: input.failure} + childIdx, failure, ok := estimator.directCopChild(idx) + if !ok { + return readBillingDemoCopUnitOutcome{failure: failure} + } + rowsEvidence := readBillingDemoExactCopRowsEvidence(estimator.runtimeStats, estimator.tree[childIdx].Origin.ID()) + if rowsEvidence.state == readBillingDemoCopRowsMissing { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingCopChildRuntimeRows)} + } + if rowsEvidence.state == readBillingDemoCopRowsInvalid { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(childIdx, readBillingDemoCopFailureIntrinsicCause, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidCopRuntimeRows)} + } + component := estimator.components[estimator.componentID[idx]] + if component.maxSummaryTasks > 0 && rowsEvidence.tasks < component.maxSummaryTasks { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} } units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChildActRows)} units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: input.inputSource, side: readBillingDemoInputSideAll, value: float64(input.rows), rowWidth: input.avgRowWidth, widthSource: input.widthSource}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: input.inputSource, side: readBillingDemoInputSideAll, value: input.inputBytes, rowWidth: input.avgRowWidth, widthSource: input.widthSource}, + readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(rowsEvidence.rows), widthSource: explainRUWidthSourceNotApplicable}, ) - orderWork, ok := readBillingDemoOrderingWorkUnit(estimator.tree[idx], operator.opClass, input.rows) + orderWork, ok := readBillingDemoOrderingWorkUnit(estimator.tree[idx], operator.opClass, rowsEvidence.rows) if !ok { - return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonInvalidOrderingWork)} + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoOrderingFailureReason(estimator.tree[idx], operator.opClass))} } if orderWork.unit != "" { + if !readBillingDemoOrderingMaterialized(estimator.tree[idx], estimator.tree[childIdx]) { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingOrderingProjection)} + } + orderWork.unit = readBillingDemoUnitCPUWork units = append(units, orderWork) + } else if operator.opClass == readBillingDemoOpClassLimit { + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(rowsEvidence.rows), widthSource: explainRUWidthSourceNotApplicable}) + } else { + exprCount, ok := readBillingDemoExpressionCount(estimator.tree[idx].Origin) + if !ok || exprCount < 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingExpressionCount)} + } + work, ok := readBillingDemoCheckedWork(rowsEvidence.rows, float64(exprCount)) + if !ok { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingExpressionCount)} + } + units = append(units, + readBillingDemoUnit{unit: readBillingDemoUnitExpressionCount, source: readBillingDemoInputSourcePhysicalPlan, side: readBillingDemoInputSideAll, value: float64(exprCount), widthSource: explainRUWidthSourceNotApplicable}, + readBillingDemoUnit{unit: readBillingDemoUnitCPUWork, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: work, widthSource: explainRUWidthSourceNotApplicable}, + ) + } + aggUnits := estimator.aggOutputShadowUnits(idx, operator.opClass) + if operator.opClass == readBillingDemoOpClassHashAgg { + if len(aggUnits) == 0 { + return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} + } + units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitHashStateRows, source: readBillingDemoInputSourceRuntimeOperatorActRows, side: readBillingDemoInputSideAll, value: aggUnits[0].value, widthSource: explainRUWidthSourceNotApplicable}) } - units = append(units, estimator.aggOutputShadowUnits(idx, operator.opClass)...) + units = append(units, aggUnits...) return readBillingDemoCopUnitOutcome{success: true, units: units} } func readBillingDemoRangeScanInput(totalKeys, processedKeys, processedKeysSize int64) (int64, float64, bool) { + if totalKeys == 0 && processedKeys == 0 && processedKeysSize == 0 { + return 0, 0, true + } if totalKeys <= 0 || processedKeys <= 0 || processedKeysSize <= 0 { return 0, 0, false } @@ -1989,7 +2223,8 @@ func recordReadBillingDemoResult(result readBillingDemoResult) { if status == "" { status = readBillingDemoStatusUnknownInput } - metrics.RecordReadBillingDemoStatement(status, readBillingDemoModelVersion) + weightVersion := readBillingDemoActiveWeightVersion() + metrics.RecordReadBillingDemoStatement(status, readBillingDemoModelVersion, weightVersion) for _, op := range result.operators { opStatus := op.status if opStatus == "" { @@ -2002,13 +2237,13 @@ func recordReadBillingDemoResult(result readBillingDemoResult) { if reason == "" { reason = readBillingDemoReasonNone } - metrics.RecordReadBillingDemoOperatorStatus(op.site, op.opClass, op.operatorKind, opStatus, reason, readBillingDemoModelVersion) + metrics.RecordReadBillingDemoOperatorStatus(op.site, op.opClass, op.operatorKind, opStatus, reason, readBillingDemoModelVersion, weightVersion) if opStatus != readBillingDemoStatusOperatorOK || !readBillingDemoOperatorBillable(op) { continue } for _, unit := range op.units { - metrics.AddReadBillingDemoBaseUnits(op.site, op.opClass, op.operatorKind, unit.unit, unit.source, unit.side, readBillingDemoModelVersion, unit.value) - metrics.ObserveReadBillingDemoRowWidth(op.site, op.opClass, op.operatorKind, unit.widthSource, readBillingDemoModelVersion, unit.rowWidth) + metrics.AddReadBillingDemoBaseUnits(op.site, op.opClass, op.operatorKind, unit.unit, unit.source, unit.side, readBillingDemoModelVersion, weightVersion, unit.value) + metrics.ObserveReadBillingDemoRowWidth(op.site, op.opClass, op.operatorKind, unit.widthSource, readBillingDemoModelVersion, weightVersion, unit.rowWidth) } } } @@ -2242,15 +2477,19 @@ func (e *Explain) renderRUExplain() (err error) { func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus explainRUComponentSnapshotStatus) []explainRURow { rows := []explainRURow{{ - section: explainRUSectionSummary, - component: "total_preview_ru", - hasPreviewRU: true, - source: explainRUSourceSummaryTotal, - note: explainRUReadBillingSummaryNote(snapshotStatus, result), + section: explainRUSectionSummary, + component: "total_preview_ru", + source: explainRUSourceSummaryTotal, + note: explainRUReadBillingSummaryNote(snapshotStatus, result), }} totalPreviewRU := 0.0 + weightsReady := readBillingDemoWeightsValid(readBillingDemoV4Weights) + completeTotal := weightsReady && result.status == readBillingDemoStatusSuccess for _, op := range result.operators { if op.status != readBillingDemoStatusOperatorOK { + if readBillingDemoOperatorBillable(op) { + completeTotal = false + } if op.emitStatusRow { rows = append(rows, explainRUReadBillingStatusRow(op)) } @@ -2259,24 +2498,31 @@ func explainRUBuildReadBillingRows(result readBillingDemoResult, snapshotStatus if !readBillingDemoOperatorBillable(op) { continue } - weights, hasWeights := readBillingDemoResolveWeights(op.site, op.opClass, readBillingDemoWeightVersion) for _, unit := range op.units { row := explainRUReadBillingUnitRow(op, unit) - if hasWeights { - if weight, previewRU, ok := readBillingDemoUnitPreviewRU(unit, weights); ok { + if _, semantic := readBillingDemoUnitWeight(readBillingDemoV4Weights, unit.unit); semantic { + if weight, previewRU, ok := readBillingDemoUnitPreviewRU(unit, readBillingDemoV4Weights); ok { row.weight = weight row.hasWeight = true row.previewRU = previewRU row.hasPreviewRU = true - totalPreviewRU += previewRU + nextTotal := totalPreviewRU + previewRU + if nextTotal < 0 || math.IsNaN(nextTotal) || math.IsInf(nextTotal, 0) { + completeTotal = false + } else { + totalPreviewRU = nextTotal + } + } else { + completeTotal = false } - } else { - row.note = appendExplainRUNote(row.note, "missing_weight") } rows = append(rows, row) } } - rows[0].previewRU = totalPreviewRU + if completeTotal { + rows[0].previewRU = totalPreviewRU + rows[0].hasPreviewRU = true + } return rows } @@ -2286,7 +2532,7 @@ func explainRUReadBillingStatusRow(op readBillingDemoOperatorResult) explainRURo id: op.id, component: op.operatorKind, operatorClass: op.site + "/" + op.opClass, - note: "weight_version=" + readBillingDemoWeightVersion, + note: "weight_version=" + readBillingDemoActiveWeightVersion(), } row.note = appendExplainRUNote(row.note, "status="+op.status) if op.reason != "" { @@ -2308,7 +2554,10 @@ func explainRUReadBillingStatusRow(op readBillingDemoOperatorResult) explainRURo } func explainRUReadBillingSummaryNote(snapshotStatus explainRUComponentSnapshotStatus, result readBillingDemoResult) string { - note := "weight_version=" + readBillingDemoWeightVersion + note := "weight_version=" + readBillingDemoActiveWeightVersion() + if !readBillingDemoWeightsValid(readBillingDemoV4Weights) { + note = appendExplainRUNote(note, readBillingDemoReasonUncalibratedWeights) + } if snapshotStatus != explainRUComponentSnapshotOK { note = appendExplainRUNote(note, "component_snapshot_"+string(snapshotStatus)) } @@ -2334,7 +2583,7 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill rowWidthSource: unit.widthSource, unit: unit.unit, source: unit.source, - note: "input_side=" + unit.side + ",weight_version=" + readBillingDemoWeightVersion, + note: "input_side=" + unit.side + ",weight_version=" + readBillingDemoActiveWeightVersion(), } if op.scope != "" { row.note = appendExplainRUNote(row.note, "scope="+op.scope) @@ -2384,6 +2633,16 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill case readBillingDemoUnitOrderWork: row.workRows = unit.value row.hasWorkRows = true + case readBillingDemoUnitCPUWork: + row.workRows = unit.value + row.hasWorkRows = true + case readBillingDemoUnitScanBytes, readBillingDemoUnitNetBytes: + row.workBytes = unit.value + row.hasWorkBytes = true + case readBillingDemoUnitExpressionCount, readBillingDemoUnitReadRequestCount, readBillingDemoUnitWriteRequestCount, + readBillingDemoUnitHashStateRows, readBillingDemoUnitJoinOutputRows: + row.count = int64(unit.value) + row.hasCount = true case readBillingDemoUnitEncodedMutationCount, readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitWriteKeys, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount: row.count = int64(unit.value) @@ -2398,9 +2657,11 @@ func explainRUReadBillingUnitRow(op readBillingDemoOperatorResult, unit readBill func readBillingDemoUnitDiagnosticOnly(unit string) bool { switch unit { - case readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitKeyBytes, + case readBillingDemoUnitFixedEvents, readBillingDemoUnitInputRows, readBillingDemoUnitInputBytes, + readBillingDemoUnitOrderWork, readBillingDemoUnitExpressionCount, readBillingDemoUnitEncodedMutationCount, + readBillingDemoUnitEncodedMutationBytes, readBillingDemoUnitSetCount, readBillingDemoUnitDeleteCount, readBillingDemoUnitKeyBytes, readBillingDemoUnitValueBytes, readBillingDemoUnitPrewriteRegionNum, readBillingDemoUnitTiKVWriteRPCCount, - readBillingDemoUnitOutputRows, readBillingDemoUnitOutputBytes: + readBillingDemoUnitWriteKeys, readBillingDemoUnitWriteByte, readBillingDemoUnitOutputRows, readBillingDemoUnitOutputBytes: return true default: return false @@ -2506,7 +2767,7 @@ func explainRUObserveRow(row explainRURow) { rowWidth = row.rowWidth } component, operator := explainRUMetricComponentOperator(row) - metrics.ObserveExplainRURow(row.section, component, operator, row.source, row.rowWidthSource, readBillingDemoWeightVersion, previewRU, workRows, workBytes, rowWidth) + metrics.ObserveExplainRURow(row.section, component, operator, row.source, row.rowWidthSource, readBillingDemoActiveWeightVersion(), previewRU, workRows, workBytes, rowWidth) } func explainRUMetricComponentOperator(row explainRURow) (component, operator string) { diff --git a/pkg/session/tidb_test.go b/pkg/session/tidb_test.go index 0939d76aa7b92..d738f5634bbdc 100644 --- a/pkg/session/tidb_test.go +++ b/pkg/session/tidb_test.go @@ -397,6 +397,41 @@ func TestRUV2MetricsIsolatedPerStatementInExplicitTxn(t *testing.T) { }) } +func TestRUV2MetricsWriteRequestsInPessimisticTxn(t *testing.T) { + store, dom := CreateStoreAndBootstrap(t) + defer func() { require.NoError(t, store.Close()) }() + defer dom.Close() + + se, err := createSession(store) + require.NoError(t, err) + MustExec(t, se, "use test") + MustExec(t, se, "drop table if exists ruv2_pessimistic_requests") + MustExec(t, se, "create table ruv2_pessimistic_requests (id int primary key, v int)") + MustExec(t, se, "insert into ruv2_pessimistic_requests values (1, 1)") + MustExec(t, se, "set @@session.tidb_txn_mode = 'pessimistic'") + + MustExec(t, se, "begin pessimistic") + MustExec(t, se, "update ruv2_pessimistic_requests set v = v + 1 where id = 1") + dmlMetrics := se.sessionVars.RUV2Metrics + require.NotNil(t, dmlMetrics) + // MockStore does not produce resource-manager RPC detail. Inject the + // finalized DML observation into the same statement snapshot to prove that + // the following COMMIT cannot inherit, drain, or overwrite it. + dmlMetrics.AddResourceManagerWriteCnt(7) + dmlWriteRequests := dmlMetrics.ResourceManagerWriteCnt() + require.Equal(t, int64(7), dmlWriteRequests) + + MustExec(t, se, "commit") + commitMetrics := se.sessionVars.RUV2Metrics + require.NotNil(t, commitMetrics) + require.NotSame(t, dmlMetrics, commitMetrics) + require.Zero(t, commitMetrics.ResourceManagerWriteCnt(), "COMMIT must start with a fresh statement snapshot") + commitMetrics.AddResourceManagerWriteCnt(2) + commitWriteRequests := commitMetrics.ResourceManagerWriteCnt() + require.Equal(t, int64(2), commitWriteRequests) + require.Equal(t, dmlWriteRequests, dmlMetrics.ResourceManagerWriteCnt(), "COMMIT must not drain or mutate the completed DML snapshot") +} + func TestSchemaCacheSizeVar(t *testing.T) { store, err := mockstore.NewMockStore(mockstore.WithStoreType(mockstore.EmbedUnistore)) require.NoError(t, err) diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index 9762bae67e186..3ae49866b2bf9 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -358,10 +358,17 @@ func TestCopRuntimeStats(t *testing.T) { tableScanID := 1 aggID := 2 tableReaderID := 3 + stats.RecordExpectedCopTasks([]int{tableScanID, aggID}) + stats.RecordExpectedCopTasks([]int{tableScanID, aggID}) stats.RecordOneCopTask(tableScanID, kv.TiKV, mockExecutorExecutionSummary(1, 1, 1)) stats.RecordOneCopTask(tableScanID, kv.TiKV, mockExecutorExecutionSummary(2, 2, 2)) stats.RecordOneCopTask(aggID, kv.TiKV, mockExecutorExecutionSummary(3, 3, 3)) stats.RecordOneCopTask(aggID, kv.TiKV, mockExecutorExecutionSummary(4, 4, 4)) + detail, detailRecords, observedTasks, expectedTasks := stats.GetCopScanDetailAndCoverage(tableScanID) + require.Zero(t, detail) + require.Zero(t, detailRecords) + require.Equal(t, int32(2), observedTasks) + require.Equal(t, int32(2), expectedTasks) scanDetail := &util.ScanDetail{ TotalKeys: 15, ProcessedKeys: 10, @@ -373,6 +380,17 @@ func TestCopRuntimeStats(t *testing.T) { RocksdbBlockReadByte: 100, } stats.RecordCopStats(tableScanID, kv.TiKV, scanDetail, util.TimeDetail{}, nil) + stats.RecordCopStats(tableScanID, kv.TiKV, &util.ScanDetail{}, util.TimeDetail{}, nil) + stats.RecordCopStats(aggID, kv.TiKV, nil, util.TimeDetail{}, nil) + detail, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(tableScanID) + require.Equal(t, *scanDetail, detail) + require.Equal(t, int32(2), detailRecords) + require.Equal(t, int32(2), observedTasks) + require.Equal(t, int32(2), expectedTasks) + _, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(aggID) + require.Zero(t, detailRecords) + require.Equal(t, int32(2), observedTasks) + require.Equal(t, int32(2), expectedTasks) require.True(t, stats.ExistsCopStats(tableScanID)) cop := stats.GetCopStats(tableScanID) @@ -400,6 +418,22 @@ func TestCopRuntimeStats(t *testing.T) { require.Equal(t, "", zeroScanDetail.String()) require.Equal(t, "", zeroTimeDetail.String()) require.Equal(t, "", zeroCopStats.String()) + + originalPlanID := 4 + remappedPlanID := 5 + executorID := "selection_" + strconv.Itoa(remappedPlanID) + remappedSummary := mockExecutorExecutionSummary(1, 1, 1) + remappedSummary.ExecutorId = &executorID + stats.RecordExpectedCopTasks([]int{originalPlanID}) + stats.RecordCopStats(originalPlanID, kv.TiFlash, &zeroScanDetail, util.TimeDetail{}, remappedSummary) + _, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(originalPlanID) + require.Equal(t, int32(1), detailRecords) + require.Zero(t, observedTasks) + require.Equal(t, int32(1), expectedTasks) + _, detailRecords, observedTasks, expectedTasks = stats.GetCopScanDetailAndCoverage(remappedPlanID) + require.Zero(t, detailRecords) + require.Equal(t, int32(1), observedTasks) + require.Zero(t, expectedTasks) } func TestRUV2MetricsSnapshotCalculateRUValues(t *testing.T) { diff --git a/pkg/util/execdetails/runtime_stats.go b/pkg/util/execdetails/runtime_stats.go index 020125a82473d..affb8f6720130 100644 --- a/pkg/util/execdetails/runtime_stats.go +++ b/pkg/util/execdetails/runtime_stats.go @@ -80,6 +80,14 @@ type RuntimeStats interface { Tp() int } +// HashTableRuntimeStats exposes the number of rows admitted into a hash join's +// lookup state. It is deliberately narrow so preview resource accounting does +// not depend on executor-private runtime-stat implementations. +type HashTableRuntimeStats interface { + RuntimeStats + HashTableRows() int64 +} + type basicCopRuntimeStats struct { loop int32 rows int64 @@ -215,10 +223,11 @@ type CopRuntimeStats struct { // have many region leaders, several coprocessor tasks can be sent to the // same tikv-server instance. We have to use a list to maintain all tasks // executed on each instance. - stats basicCopRuntimeStats - scanDetail util.ScanDetail - timeDetail util.TimeDetail - storeType kv.StoreType + stats basicCopRuntimeStats + scanDetail util.ScanDetail + scanDetailRecords int32 + timeDetail util.TimeDetail + storeType kv.StoreType } // GetActRows return total rows of CopRuntimeStats. @@ -656,6 +665,21 @@ func (e *RuntimeStatsColl) GetExpectedCopTasks(planID int) int32 { return e.expectedCopTasks[planID] } +// GetCopScanDetailAndCoverage returns one consistent snapshot of the scan +// detail attachment and task coverage recorded for planID. detailRecords +// counts RecordCopStats calls with a non-nil scan detail; summary-only +// RecordOneCopTask calls do not contribute to it. +func (e *RuntimeStatsColl) GetCopScanDetailAndCoverage(planID int) (detail util.ScanDetail, detailRecords, observedTasks, expectedTasks int32) { + e.mu.Lock() + defer e.mu.Unlock() + expectedTasks = e.expectedCopTasks[planID] + copStats, ok := e.copStats[planID] + if !ok { + return detail, 0, 0, expectedTasks + } + return copStats.scanDetail, copStats.scanDetailRecords, copStats.GetTasks(), expectedTasks +} + func getPlanIDFromExecutionSummary(summary *tipb.ExecutorExecutionSummary) (int, bool) { if summary.GetExecutorId() != "" { strs := strings.Split(summary.GetExecutorId(), "_") @@ -676,16 +700,14 @@ func (e *RuntimeStatsColl) RecordCopStats(planID int, storeType kv.StoreType, sc timeDetail: time, storeType: storeType, } - if scan != nil { - copStats.scanDetail = *scan - } e.copStats[planID] = copStats } else { - if scan != nil { - copStats.scanDetail.Merge(scan) - } copStats.timeDetail.Merge(&time) } + if scan != nil { + copStats.scanDetailRecords++ + copStats.scanDetail.Merge(scan) + } if summary != nil { // for TiFlash cop response, ExecutorExecutionSummary contains executor id, so if there is a valid executor id in // summary, use it overwrite the planID diff --git a/pkg/util/stmtsummary/read_billing.go b/pkg/util/stmtsummary/read_billing.go index 6ac5980078dc8..503386affc16f 100644 --- a/pkg/util/stmtsummary/read_billing.go +++ b/pkg/util/stmtsummary/read_billing.go @@ -283,6 +283,9 @@ func (a *ReadBillingDemoStatusAgg) addEntry(entry ReadBillingDemoStatusAggEntry) } func (s *ReadBillingDemoBaseUnitSummary) addSample(sample ReadBillingDemoBaseUnitSample) { + if sample.ModelVersion != "v3" { + return + } switch sample.Unit { case "fixed_events": s.SumReadBillingDemoFixedEvents += sample.Value @@ -294,6 +297,9 @@ func (s *ReadBillingDemoBaseUnitSummary) addSample(sample ReadBillingDemoBaseUni } func (s *ReadBillingDemoBaseUnitSummary) addEntry(entry ReadBillingDemoBaseUnitAggEntry) { + if entry.ModelVersion != "v3" { + return + } switch entry.Unit { case "fixed_events": s.SumReadBillingDemoFixedEvents += entry.Value diff --git a/pkg/util/stmtsummary/read_billing_test.go b/pkg/util/stmtsummary/read_billing_test.go index 72118e9bb4072..3265a8b39cf4f 100644 --- a/pkg/util/stmtsummary/read_billing_test.go +++ b/pkg/util/stmtsummary/read_billing_test.go @@ -23,12 +23,12 @@ import ( func TestReadBillingDemoAggregationCaps(t *testing.T) { stats := ReadBillingDemoStatementStats{ - ModelVersion: "v1", + ModelVersion: "v3", WeightVersion: "v1", } for i := 0; i < MaxReadBillingDemoBaseUnitKeysPerRecord+2; i++ { stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ - ModelVersion: "v1", + ModelVersion: "v3", WeightVersion: "v1", Site: "tidb", OpClass: fmt.Sprintf("op_%03d", i), @@ -47,6 +47,26 @@ func TestReadBillingDemoAggregationCaps(t *testing.T) { require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, ReadBillingDemoStatusEntriesFromMap(statusAggs), readBillingDemoReasonAggregation).Count) require.Equal(t, float64(MaxReadBillingDemoBaseUnitKeysPerRecord*(MaxReadBillingDemoBaseUnitKeysPerRecord+1)/2), acceptedSummary.SumReadBillingDemoInputRows) + v4Stats := ReadBillingDemoStatementStats{ + ModelVersion: "v4", + WeightVersion: "v3-resource-formula-uncalibrated", + BaseUnits: []ReadBillingDemoBaseUnitSample{{ + ModelVersion: "v4", + WeightVersion: "v3-resource-formula-uncalibrated", + Site: "tidb", + OpClass: "projection_eval", + OperatorKind: "projection", + Unit: "input_rows", + InputSource: "runtime_child_act_rows", + InputSide: "all", + Value: 10, + }}, + } + _, _, v4Summary := AddReadBillingDemoStatementStatsToMaps(nil, nil, &v4Stats) + require.Zero(t, v4Summary.SumReadBillingDemoFixedEvents) + require.Zero(t, v4Summary.SumReadBillingDemoInputRows) + require.Zero(t, v4Summary.SumReadBillingDemoInputBytes) + baseEntries, statusEntries, acceptedSummary := AddReadBillingDemoStatementStatsToEntries(nil, nil, &stats) require.Len(t, baseEntries, MaxReadBillingDemoBaseUnitKeysPerRecord) require.Equal(t, uint64(2), requireReadBillingDemoStatusReason(t, statusEntries, readBillingDemoReasonAggregation).Count) @@ -84,33 +104,61 @@ func TestReadBillingDemoAggregationCaps(t *testing.T) { } func TestReadBillingDemoDMLKindAggregation(t *testing.T) { - stats := ReadBillingDemoStatementStats{ModelVersion: "v2", WeightVersion: "v1"} + stats := ReadBillingDemoStatementStats{ModelVersion: "v4", WeightVersion: "test-v4-calibrated"} for _, dmlKind := range []string{"insert", "update"} { - stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ - ModelVersion: "v2", - WeightVersion: "v1", - Site: "tidb", - OpClass: "kv_mutation", - OperatorKind: "memdb_mutation", - DMLKind: dmlKind, - Unit: "encoded_mutation_count", - InputSource: "stmt_memdb_mutation_calls", - InputSide: "all", - RowWidthSource: "not_applicable", - Value: 1, - }) + for _, unit := range []string{"cpu_work", "encoded_mutation_count"} { + stats.BaseUnits = append(stats.BaseUnits, ReadBillingDemoBaseUnitSample{ + ModelVersion: "v4", + WeightVersion: "test-v4-calibrated", + Site: "tidb", + OpClass: "kv_mutation", + OperatorKind: "memdb_mutation", + DMLKind: dmlKind, + Unit: unit, + InputSource: "stmt_memdb_mutation_calls", + InputSide: "all", + RowWidthSource: "not_applicable", + Value: 1, + }) + } } + stats.BaseUnits = append(stats.BaseUnits, + ReadBillingDemoBaseUnitSample{ + ModelVersion: "v4", WeightVersion: "test-v4-calibrated", Site: "tidb", OpClass: "reader_transport", OperatorKind: "mixed_reader", + Unit: "read_request_count", InputSource: "ruv2_metrics", InputSide: "all", RowWidthSource: "not_applicable", Value: 4, + }, + ReadBillingDemoBaseUnitSample{ + ModelVersion: "v4", WeightVersion: "test-v4-calibrated", Site: "tikv", OpClass: "kv_write", OperatorKind: "txn_write", DMLKind: "update", + Unit: "write_request_count", InputSource: "ruv2_metrics", InputSide: "all", RowWidthSource: "not_applicable", Value: 2, + }, + ) aggs, _, _ := AddReadBillingDemoStatementStatsToMaps(nil, nil, &stats) - require.Len(t, aggs, 2) + require.Len(t, aggs, 6) entries := ReadBillingDemoBaseUnitEntriesFromMap(aggs) - require.Equal(t, "insert", entries[0].DMLKind) - require.Equal(t, "update", entries[1].DMLKind) + seenUnits := make(map[string]int) for _, entry := range entries { - require.Equal(t, "kv_mutation", entry.OpClass) - require.Equal(t, "memdb_mutation", entry.OperatorKind) - require.Equal(t, "encoded_mutation_count", entry.Unit) + require.Equal(t, "all", entry.InputSide) + switch entry.Unit { + case "cpu_work", "encoded_mutation_count": + require.Equal(t, "kv_mutation", entry.OpClass) + require.Equal(t, "memdb_mutation", entry.OperatorKind) + require.Equal(t, "stmt_memdb_mutation_calls", entry.InputSource) + require.Contains(t, []string{"insert", "update"}, entry.DMLKind) + case "read_request_count": + require.Equal(t, "reader_transport", entry.OpClass) + require.Equal(t, "mixed_reader", entry.OperatorKind) + require.Equal(t, "ruv2_metrics", entry.InputSource) + case "write_request_count": + require.Equal(t, "kv_write", entry.OpClass) + require.Equal(t, "txn_write", entry.OperatorKind) + require.Equal(t, "ruv2_metrics", entry.InputSource) + default: + require.Failf(t, "unexpected unit", "entry=%+v", entry) + } + seenUnits[entry.Unit]++ } + require.Equal(t, map[string]int{"cpu_work": 2, "encoded_mutation_count": 2, "read_request_count": 1, "write_request_count": 1}, seenUnits) } func TestReadBillingDemoReservedStatusMergeBypassesStatusCap(t *testing.T) { diff --git a/pkg/util/stmtsummary/v2/record_test.go b/pkg/util/stmtsummary/v2/record_test.go index c73d99bb6a080..6b6ca96ca79c6 100644 --- a/pkg/util/stmtsummary/v2/record_test.go +++ b/pkg/util/stmtsummary/v2/record_test.go @@ -125,10 +125,10 @@ func TestStmtRecord(t *testing.T) { func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { info := GenerateStmtExecInfo4Test("digest_read_billing") info.ReadBillingDemoStats = stmtsummarybase.ReadBillingDemoStatementStats{ - ModelVersion: "v1", + ModelVersion: "v3", WeightVersion: "v1", Statuses: []stmtsummarybase.ReadBillingDemoStatusSample{{ - ModelVersion: "v1", + ModelVersion: "v3", WeightVersion: "v1", Site: "statement", OpClass: "statement", @@ -138,7 +138,7 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { }}, BaseUnits: []stmtsummarybase.ReadBillingDemoBaseUnitSample{ { - ModelVersion: "v1", + ModelVersion: "v3", WeightVersion: "v1", Site: "tidb", OpClass: "projection_eval", @@ -152,7 +152,7 @@ func TestStmtRecordReadBillingDemoStructuredStats(t *testing.T) { RowWidth: 16, }, { - ModelVersion: "v1", + ModelVersion: "v3", WeightVersion: "v1", Site: "tidb", OpClass: "projection_eval", From 10fbbd36de797379f7060a7e2b39d71428532ee0 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Fri, 24 Jul 2026 11:32:56 +0800 Subject: [PATCH 22/25] add some comments --- pkg/executor/join/hash_join_v2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/executor/join/hash_join_v2.go b/pkg/executor/join/hash_join_v2.go index 1bb6eb0c9b717..7ab2946bf1307 100644 --- a/pkg/executor/join/hash_join_v2.go +++ b/pkg/executor/join/hash_join_v2.go @@ -93,6 +93,7 @@ func (htc *hashTableContext) getAllMemoryUsageInHashTable() int64 { return totalMemoryUsage } +// hashStateRows returns the total number of valid rows in the hash table. func (htc *hashTableContext) hashStateRows() uint64 { var rows uint64 if htc == nil || htc.hashTable == nil { From 1d5da3935ab9317e2f4e267d2a75e11733c97539 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Fri, 24 Jul 2026 11:56:25 +0800 Subject: [PATCH 23/25] remove fixed_events, input_rows/bytes --- pkg/planner/core/explain_ru.go | 39 +++++----------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index 5db692f878233..bfd2f66975027 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -1848,7 +1848,7 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F if !hasOutputRows || outputRows < 0 { return nil, readBillingDemoReasonMissingRuntimeRows, false } - units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChunkBytes)} + var units []readBillingDemoUnit appendExpressionCPU := func(rows int64) (string, bool) { exprCount, ok := readBillingDemoExpressionCount(op.Origin) if !ok || exprCount < 0 { @@ -1870,7 +1870,7 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F return nil, readBillingDemoReasonMissingExpressionCount, false } var inputRows int64 - for childOrder, childIdx := range tree[idx].ChildrenIdx { + for _, childIdx := range tree[idx].ChildrenIdx { if childIdx < 0 || childIdx >= len(tree) || tree[childIdx] == nil || !tree[childIdx].IsRoot { return nil, readBillingDemoReasonMissingRuntimeRows, false } @@ -1879,18 +1879,6 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F return nil, readBillingDemoReasonMissingRuntimeRows, false } inputRows += rows - side := readBillingDemoInputSideLeft - if childOrder == 1 { - side = readBillingDemoInputSideRight - } - if operator.opClass == readBillingDemoOpClassHashJoin { - if tree[childIdx].Label == BuildSide { - side = readBillingDemoInputSideBuild - } else if tree[childIdx].Label == ProbeSide { - side = readBillingDemoInputSideProbe - } - } - units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChildActRows, side: side, value: float64(rows), widthSource: explainRUWidthSourceNotApplicable}) } if reason, ok := appendExpressionCPU(inputRows); !ok { return nil, reason, false @@ -1918,7 +1906,6 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F if !ok || inputRows < 0 { return nil, readBillingDemoReasonMissingRuntimeRows, false } - units = append(units, readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(inputRows), widthSource: explainRUWidthSourceNotApplicable}) orderWork, ok := readBillingDemoOrderingWorkUnit(op, operator.opClass, inputRows) if !ok { return nil, readBillingDemoOrderingFailureReason(op, operator.opClass), false @@ -1944,16 +1931,6 @@ func readBillingDemoRootUnits(runtimeStats *execdetails.RuntimeStatsColl, tree F return units, "", true } -func readBillingDemoFixedEventUnit(inputSource string) readBillingDemoUnit { - return readBillingDemoUnit{ - unit: readBillingDemoUnitFixedEvents, - source: inputSource, - side: readBillingDemoInputSideAll, - value: 1, - widthSource: explainRUWidthSourceNotApplicable, - } -} - func readBillingDemoRuntimeChunkOutputUnits(rows, bytes int64) []readBillingDemoUnit { if rows < 0 || bytes < 0 { return nil @@ -2140,12 +2117,9 @@ func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, op return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonMissingScanWidthEvidence)} } rowWidth := readBillingDemoAverageRowWidth(scanInputRows, scanInputBytes) - units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceScanDetail)} - units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: float64(scanInputRows), rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, - readBillingDemoUnit{unit: readBillingDemoUnitInputBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedAvg}, + units := []readBillingDemoUnit{ readBillingDemoUnit{unit: readBillingDemoUnitScanBytes, source: readBillingDemoInputSourceScanDetail, side: readBillingDemoInputSideAll, value: scanInputBytes, rowWidth: rowWidth, widthSource: explainRUWidthSourceScanDetailProcessedEstimate}, - ) + } return readBillingDemoCopUnitOutcome{success: true, units: units} } childIdx, failure, ok := estimator.directCopChild(idx) @@ -2163,10 +2137,7 @@ func readBillingDemoCopUnits(estimator *readBillingDemoCopEstimator, idx int, op if component.maxSummaryTasks > 0 && rowsEvidence.tasks < component.maxSummaryTasks { return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoReasonIncompleteCopRuntimeRows)} } - units := []readBillingDemoUnit{readBillingDemoFixedEventUnit(readBillingDemoInputSourceRuntimeChildActRows)} - units = append(units, - readBillingDemoUnit{unit: readBillingDemoUnitInputRows, source: readBillingDemoInputSourceRuntimeChildActRows, side: readBillingDemoInputSideAll, value: float64(rowsEvidence.rows), widthSource: explainRUWidthSourceNotApplicable}, - ) + var units []readBillingDemoUnit orderWork, ok := readBillingDemoOrderingWorkUnit(estimator.tree[idx], operator.opClass, rowsEvidence.rows) if !ok { return readBillingDemoCopUnitOutcome{failure: readBillingDemoCopFailureAt(idx, readBillingDemoCopFailureCurrent, readBillingDemoStatusUnknownInput, readBillingDemoOrderingFailureReason(estimator.tree[idx], operator.opClass))} From a0c6f71ccdb2eb99c6e5b31b8110800b3019b0d6 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 24 Jul 2026 12:29:27 +0800 Subject: [PATCH 24/25] planner: report DML PointGet read requests --- .../references/executor-case-map.md | 2 +- .../references/planner-case-map.md | 2 +- ...-07-22-preview-ru-resource-formula-plan.md | 22 ++++-- pkg/executor/explain_test.go | 19 ++++- pkg/planner/core/BUILD.bazel | 2 + pkg/planner/core/common_plans_test.go | 79 +++++++++++++++++-- pkg/planner/core/explain_ru.go | 65 ++++++++++++++- 7 files changed, 167 insertions(+), 24 deletions(-) diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index 19aabd6b48446..1ecf4aa555076 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -30,7 +30,7 @@ - `pkg/executor/executor_failpoint_test.go` - executor: Tests TiDB last txn info commit mode. - `pkg/executor/executor_pkg_test.go` - executor: Tests build KV ranges for index join without CWC. - `pkg/executor/executor_required_rows_test.go` - executor: Tests limit required rows. -- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including production-default uncalibrated v4 units and absent totals, distinct read/write request output units, six pushed TiKV operator classes (including an explicitly ordered StreamAgg) with native Unistore's incomplete scan-width fail-closed behavior, bounded scan/transport/point-lookup failures, all raw DML mutation diagnostics with gated mutation `cpu_work`, statement-summary/metrics output versions, restricted/feature-off isolation, and legacy v3 behavior retained as explicitly superseded coverage. +- `pkg/executor/explain_test.go` - executor: Tests EXPLAIN ANALYZE formats, including production-default uncalibrated v4 units and absent totals, distinct read/write request output units, DML PointGet read-request publication, six pushed TiKV operator classes (including an explicitly ordered StreamAgg) with native Unistore's incomplete scan-width fail-closed behavior, bounded scan/transport/point-lookup failures, all raw DML mutation diagnostics with gated mutation `cpu_work`, statement-summary/metrics output versions, restricted/feature-off isolation, and legacy v3 behavior retained as explicitly superseded coverage. - `pkg/executor/explain_unit_test.go` - executor: Tests explain analyze invoke next and close. - `pkg/executor/explainfor_test.go` - executor: Tests explain for. - `pkg/executor/grant_test.go` - executor: Tests grant global. diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index c1bb64ff18f8b..96af105cbc566 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, private v4 semantic weights and overflow rejection, independent read/write request units and weights, expression-slot counts for Agg/Join/Window families, Sort CPU work plus TopN input saturation/zero-count/checked bounds, UnionScan direct-child CPU work, reader transport attribution, real distsql ScanDetail ownership/provenance and fail-closed coverage, HashJoin state inputs, mutation normalization into dimension-qualified `cpu_work` with raw diagnostics, and statement-local DML/COMMIT request ownership. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, private v4 semantic weights and overflow rejection, independent read/write request units and weights, expression-slot counts for Agg/Join/Window families, Sort CPU work plus TopN input saturation/zero-count/checked bounds, UnionScan direct-child CPU work, reader transport attribution, plan-local DML PointGet RPC ownership and isolation from statement counters, real distsql ScanDetail ownership/provenance and fail-closed coverage, HashJoin state inputs, mutation normalization into dimension-qualified `cpu_work` with raw diagnostics, and statement-local DML/COMMIT request ownership. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md index 85b7f8b1b586a..6642c2c16be96 100644 --- a/docs/design/2026-07-22-preview-ru-resource-formula-plan.md +++ b/docs/design/2026-07-22-preview-ru-resource-formula-plan.md @@ -50,14 +50,17 @@ This plan first froze the model, field mappings, failure behavior, migration, an - [x] (2026-07-24 01:35+08:00) Observed the focused EXPLAIN regression fail against the prior `ambiguous_reader_transport_producers` behavior and pass after the implementation. Planner, EXPLAIN, plan-digest, General Log, Prometheus, and statement-summary regressions passed; `make bazel_prepare`, `make lint`, and `git diff --check` passed. A cleanup-safe real-TiKV run confirmed PointGet and BatchPointGet each publish only one statement-scoped `read_request_count=1` unit and no uncalibrated total. - [x] (2026-07-24 02:05+08:00) Bound UnionScan's intentionally simple CPU formula to its direct child's actual runtime rows: `cpu_work=input_rows`. The focused regression failed while `overlay_reader` was non-billable and passed after the formula was enabled, with no expression-count multiplier or new runtime datum. - [x] (2026-07-24 02:25+08:00) Completed UnionScan output and Ready verification. A cleanup-safe real-TiKV transaction produced UnionScan output rows 4 over direct-child rows 3 and published exactly `tidb/overlay_reader/unionscan cpu_work=3` from `runtime_child_act_rows`; the focused preview-RU planner suite, `make lint`, and `git diff --check` passed with failpoint refcount zero. The Bazel gate found no import, file, top-level-test, Bazel, or module trigger, so `make bazel_prepare` was not required. +- [x] (2026-07-24 12:20+08:00) Enabled DML PointGet and BatchPointGet request publication from their plan-local `SnapshotRuntimeStats.GetCmdRPCCount`. The focused regression failed against the blanket DML rejection and passed after the change; the DML statement producer set remains open, so pessimistic-lock and ancillary requests cannot be copied from the statement RUv2 counter into PointGet. +- [x] (2026-07-24 12:26+08:00) A cleanup-safe real-TiKV run with the current-worktree TiDB binary confirmed `source=snapshot_runtime_stats`: the autocommit UPDATE published PointGet `read_request_count=1`, while the explicit pessimistic UPDATE published PointGet `read_request_count=0` and independently retained `write_request_count=1`. The unique TiUP tag was removed, the PD endpoint was unreachable afterward, and the pre-existing playground tags were preserved. +- [x] (2026-07-24 12:27+08:00) Completed the Ready gate: `make bazel_prepare` generated only the required planner/core TiKV RPC dependencies; the focused planner and EXPLAIN failpoint-wrapped regressions passed with refcount returning to zero; `make server`, `make lint`, and `git diff --check` passed. ## Surprises & Discoveries - Observation: root executor byte accounting is logical live chunk bytes, not encoded network bytes. Evidence: `pkg/util/execdetails/runtime_stats.go` defines `BasicRuntimeStats.inputBytes/outputBytes`, and `pkg/executor/internal/exec/executor.go` records child/output chunk bytes around `Next`. -- Observation: exact TiKV response bytes and read/write RPC counters already exist, but only at statement scope. - Evidence: `pkg/util/execdetails/ruv2_metrics.go` exposes `TiKVCoprocessorResponseBytes`, `ResourceManagerReadCnt`, and `ResourceManagerWriteCnt`; none is keyed by reader plan ID. +- Observation: exact TiKV response bytes and chargeable read/write RPC counters exist at statement scope, while PointGet and BatchPointGet additionally retain plan-local RPC counts in their snapshot runtime stats. + Evidence: `pkg/util/execdetails/ruv2_metrics.go` exposes `TiKVCoprocessorResponseBytes`, `ResourceManagerReadCnt`, and `ResourceManagerWriteCnt`; `pkg/executor/point_get.go` and `pkg/executor/batch_point_get.go` register `SnapshotRuntimeStats` under their plan IDs and query `GetCmdRPCCount` for `CmdGet` and `CmdBatchGet`. - Observation: `selectResultRuntimeStats` already counts cop responses internally, but proportional or per-reader network-byte attribution is not available from current exec details. Evidence: `pkg/distsql/select_result.go` keeps `copRespTime` and `reqStat` inside the unexported `selectResultRuntimeStats`, while response bytes are drained into statement-level `RUV2Metrics`. @@ -142,7 +145,7 @@ This plan first froze the model, field mappings, failure behavior, migration, an Date/Author: 2026-07-22 / Codex. - Decision: publish PointGet and BatchPointGet read RPCs once per statement under `id=point_lookup@statement`, `site=tikv`, and `op_class=kv_point_lookup`, with only `read_request_count`. - Rationale: Get/BatchGet RPC counts are exact but statement-scoped, so one synthetic operator preserves the requested lookup dimensions without multiplying the total across multiple plan nodes. Pure PointGet and pure BatchPointGet use their respective operator kind; a closed statement containing both uses `mixed_point_lookup`. No CPU, scan, or network work is inferred. Locking lookups, DML, and statements mixing point lookup with cop readers remain fail-closed because the current counter cannot split ancillary or heterogeneous read RPCs. + Rationale: pure PointGet and pure BatchPointGet use their respective operator kind; a statement containing both uses `mixed_point_lookup`. Read-only lookups use the closed producer-set gate and the chargeable statement counter. DML keeps the statement producer set open and instead sums only plan-local `CmdGet`/`CmdBatchGet` counts, which excludes pessimistic-lock writes and unrelated ancillary reads by construction. No CPU, scan, or network work is inferred; missing plan-local runtime evidence remains fail-closed. Date/Author: 2026-07-24 / Codex. - Decision: make root UnionScan weight-bearing with exactly `cpu_work = direct_child_actual_rows`. @@ -211,7 +214,7 @@ This plan first froze the model, field mappings, failure behavior, migration, an ## Outcomes & Retrospective -The design is stable after the required convergence review. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(min(n, offset+count))` work without repeating Projection expression evaluation, treats zero-count TopN as zero work, validates ordering-column ownership against the executed child schema, removes IndexJoin request double charging, and defines a no-guess migration. The implementation now publishes exactly seven weight-bearing v4 semantic units, reuses dimension-qualified `cpu_work` for normalized mutation preparation, retains the raw mutation diagnostics, and keeps production weights intentionally uncalibrated. It atomically binds any future calibrated coefficients to their immutable output version across EXPLAIN, statement summary, general log, and all background Prometheus families, removes the executable v3 formula from production, gives read and write requests independent unit names and weights while preserving statement-scoped ownership, and preserves the pipelined write-request fail-closed gate. Runtime additions remain narrow: HashJoin exposes actual admitted state rows as a formula datum, while cop stats record only TiDB-side ScanDetail attachment provenance needed to distinguish an observed empty scan from absent detail. MockStore cannot produce real resource-manager write RPC details, so the session lifecycle test injects finalized observations into the actual fresh per-statement snapshots; private construction tests separately verify their exact 7/2 unit values. The ownership correction now passes the complete focused regression matrix and real-TiKV output verification: six non-Scan cop operator classes publish their units, and one Selection sample agrees across EXPLAIN, General Log, Prometheus, and statement summary without publishing an uncalibrated total. The subsequent point-lookup extension consumes the already-frozen TiKV read-RPC counter without introducing a new runtime datum: pure read-only PointGet and BatchPointGet statements now publish one exact RPC-only synthetic operator, while every shape whose statement counter cannot be attributed uniquely remains fail-closed. UnionScan likewise reuses existing evidence and the existing CPU weight: it now publishes direct-child actual rows as `cpu_work` without charging its conditions or inventing a mem-buffer input estimate. +The design is stable after the required convergence review. It keeps the requested simple formula family, identifies every current data source, assigns Sort `n*log(n)` and TopN `n*log(min(n, offset+count))` work without repeating Projection expression evaluation, treats zero-count TopN as zero work, validates ordering-column ownership against the executed child schema, removes IndexJoin request double charging, and defines a no-guess migration. The implementation now publishes exactly seven weight-bearing v4 semantic units, reuses dimension-qualified `cpu_work` for normalized mutation preparation, retains the raw mutation diagnostics, and keeps production weights intentionally uncalibrated. It atomically binds any future calibrated coefficients to their immutable output version across EXPLAIN, statement summary, general log, and all background Prometheus families, removes the executable v3 formula from production, gives read and write requests independent unit names and weights while preserving statement-scoped ownership, and preserves the pipelined write-request fail-closed gate. Runtime additions remain narrow: HashJoin exposes actual admitted state rows as a formula datum, while cop stats record only TiDB-side ScanDetail attachment provenance needed to distinguish an observed empty scan from absent detail. MockStore cannot produce real resource-manager write RPC details, so the session lifecycle test injects finalized observations into the actual fresh per-statement snapshots; private construction tests separately verify their exact 7/2 unit values. The ownership correction now passes the complete focused regression matrix and real-TiKV output verification: six non-Scan cop operator classes publish their units, and one Selection sample agrees across EXPLAIN, General Log, Prometheus, and statement summary without publishing an uncalibrated total. The subsequent point-lookup extension adds no runtime datum: pure read-only PointGet and BatchPointGet statements publish one exact RPC-only synthetic operator from the chargeable statement counter, while DML lookups publish only their existing plan-local snapshot RPC counts and leave the broader statement producer set open. UnionScan likewise reuses existing evidence and the existing CPU weight: it now publishes direct-child actual rows as `cpu_work` without charging its conditions or inventing a mem-buffer input estimate. ## Context and Orientation @@ -356,11 +359,11 @@ If no supported reader executed, no reader-transport operator is emitted. Unsupp ### Point lookup RPC transport -Read-only, non-locking PointGet and BatchPointGet plans publish only the exact statement `RUV2Metrics.ResourceManagerReadCnt()` value. The constructor emits it once as `id=point_lookup@statement`, `site=tikv`, `op_class=kv_point_lookup`, `input_source=ruv2_metrics`, `input_side=all`, and unit `read_request_count`. Its bounded `operator_kind` is `point_get`, `batch_point_get`, or `mixed_point_lookup` when both physical kinds occur in one otherwise closed statement. The physical PointGet/BatchPointGet plan rows remain non-billable diagnostics; they do not each copy the statement counter. +PointGet and BatchPointGet plans emit request work once as `id=point_lookup@statement`, `site=tikv`, `op_class=kv_point_lookup`, `input_side=all`, and unit `read_request_count`. Its bounded `operator_kind` is `point_get`, `batch_point_get`, or `mixed_point_lookup` when both physical kinds occur. The physical PointGet/BatchPointGet plan rows remain non-billable diagnostics. -A present, non-bypassed frozen RUv2 snapshot is required. Zero is valid because a point lookup can be satisfied without a TiKV RPC, for example from transaction-local state; nonzero plan output therefore does not imply a missing counter. A negative counter is invalid. The component emits no `cpu_work`, `scan_bytes`, or `net_bytes`, because current details expose neither point-lookup executor CPU nor attributable Get/BatchGet response bytes. +For read-only statements, a present, non-bypassed frozen RUv2 snapshot is required and `input_source=ruv2_metrics`. For DML, every PointGet/BatchPointGet plan ID must have snapshot runtime stats and `input_source=snapshot_runtime_stats`; the component sums only `CmdGet` or `CmdBatchGet` as appropriate. Zero is valid because a point lookup can be satisfied without a Get/BatchGet RPC, for example through a pessimistic lock request that returns the value. A negative count or overflow is invalid. The component emits no `cpu_work`, `scan_bytes`, or `net_bytes`, because current details expose neither point-lookup executor CPU nor attributable Get/BatchGet response bytes. -The point-lookup producer set is closed only for a read-only statement whose complete flat plan contains one or more non-locking PointGet/BatchPointGet nodes and no cop-reader producer, TiFlash/MPP producer, locking point lookup, or unknown store-access producer. DML remains fail-closed because uniqueness checks, locking, foreign keys, or transaction work can add read RPCs. A statement mixing point lookup and cop readers also remains fail-closed rather than allocating the statement total between components. +For a read-only statement, the point-lookup producer set is closed only when its complete flat plan contains one or more non-locking PointGet/BatchPointGet nodes and no cop-reader producer, TiFlash/MPP producer, or unknown store-access producer. The DML statement producer set is always open: uniqueness checks, foreign-key reads, pessimistic locks, or other transaction work may add requests, so its statement RUv2 read count is never assigned to PointGet. Plan-local snapshot stats still allow PointGet to publish its own exact Get/BatchGet RPC count even when other DML producers exist. ### IndexJoin request de-duplication @@ -482,6 +485,7 @@ Acceptance: internal formula tests observe exact calibrated totals. EXPLAIN, met | `scan_bytes` | `CopRuntimeStats.GetScanDetail` plus TiDB attachment provenance | unique holder by non-nil `RecordCopStats` attachment; scan summaries and holder attachments each cover all expected responses | **yes, TiDB-only attachment count; no protocol field or formula term** | | `net_bytes` | `RUV2Metrics.TiKVCoprocessorResponseBytes` | once per statement; non-bypassed presence, descendant task gate, closed read producer set | no | | reader `read_request_count` | `RUV2Metrics.ResourceManagerReadCnt` | once per statement only when every read-RPC producer is attributable to supported cop readers | no | +| point-lookup `read_request_count` | read-only: `RUV2Metrics.ResourceManagerReadCnt`; DML: plan-local `SnapshotRuntimeStats.GetCmdRPCCount` | once per statement; read-only requires a closed producer set, DML sums exact `CmdGet`/`CmdBatchGet` counts across PointGet/BatchPointGet plan IDs while the statement producer set stays open | no | | HashAgg `group_rows` | Agg node own runtime rows | TiKV additionally needs expected/observed coverage | no | | HashJoin `hash_state_rows` | v1 hash-table `Len` plus NAAJ null-bucket entries; v2 row-table `validKeyCount` | completed build round, cumulative across rebuilds | **yes, Join only** | | Join `output_rows` | Join node own `BasicRuntimeStats.GetActRows` | executed root stat required | no | @@ -539,7 +543,7 @@ With private test weights set to simple values inside `pkg/planner/core`, table- End-to-end `EXPLAIN ANALYZE FORMAT='RU'` cases cover Selection/Projection, Sort/TopN, Table/Index scans, each reader family including IndexMerge, RPC-only PointGet/BatchPointGet, UnionScan, Stream/HashAgg, Merge/Hash/IndexJoin, Limit, Window, autocommit write, explicit DML plus COMMIT, unsupported ROLLBACK, and zero-mutation/zero-row cases. An explicit pessimistic transaction case must prove that a DML-local nonzero `ResourceManagerWriteCnt` is emitted by that DML, the later COMMIT emits only its own fresh-snapshot count, and neither count is lost or duplicated. Each attributable case exposes its coefficient-free units, source, and model/weight versions. Because these tests are package-external and production defaults are not calibrated, they assert `uncalibrated_weights` and absence of `total_preview_ru`; exact weighted totals belong to private core formula tests. -A multi-reader or IndexMerge case proves that statement `net_bytes` and `read_request_count` appear once, while every scan retains its own `scan_bytes`. PointGet and BatchPointGet cases prove that a pure read-only plan publishes the statement `read_request_count` exactly once under the bounded lookup kind, publishes no inferred CPU/scan/network unit, accepts a present zero counter, and rejects locking, DML, bypassed/missing detail, and mixed point/cop-reader producer sets. A UnionScan case proves that `tidb/overlay_reader/unionscan` publishes `cpu_work` equal to its direct child's exact actual rows, carries `runtime_child_act_rows`, and has no expression-count multiplier. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. DML and COMMIT cases prove that `write_request_count` remains statement-local and uses a coefficient independent from read requests. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. +A multi-reader or IndexMerge case proves that statement `net_bytes` and `read_request_count` appear once, while every scan retains its own `scan_bytes`. PointGet and BatchPointGet cases prove that a pure read-only plan publishes the statement `read_request_count` exactly once under the bounded lookup kind, publishes no inferred CPU/scan/network unit, accepts a present zero counter, and rejects locking reads, bypassed/missing detail, and mixed point/cop-reader producer sets. A DML PointGet case proves that plan-local `CmdGet`/`CmdBatchGet` counts publish independently of a missing or larger statement counter, while missing exec details stays unknown. A UnionScan case proves that `tidb/overlay_reader/unionscan` publishes `cpu_work` equal to its direct child's exact actual rows, carries `runtime_child_act_rows`, and has no expression-count multiplier. An IndexJoin case proves that inner lookup requests appear only in reader transport and no Join-local request unit is emitted. DML and COMMIT cases prove that `write_request_count` remains statement-local and uses a coefficient independent from read requests. Sort/TopN cases prove that root scalar expressions materialized by inline Projection are evaluated only there, ordinary-column/multi-key ordering receives one aggregate complexity term without a key multiplier, TopN offset changes `k`, and an unmaterialized pushed scalar TopN fails closed. Reader-gate cases prove that zero rows with observed/expected cop tasks plus a zero RUv2 payload is missing, `requests > 0 && bytes == 0` is valid, a supported reader mixed with PointGet fails closed, and DML with unexcludable ancillary reads marks only reader transport unknown. A ROLLBACK case proves the explicit unsupported status and absence of units. Missing root stats, incomplete cop summaries, ambiguous scan details, missing reader transport, invalid expression structure, invalid mutation normalization, negative inputs, overflow, NaN, and infinity all fail closed with bounded reasons. SELECT produces no partial billable total. DML preserves complete independent units but does not claim a complete statement total. @@ -597,3 +601,5 @@ Revision note (2026-07-23): real TiKV proved that non-Scan execution summaries w Revision note (2026-07-24): PointGet and BatchPointGet now consume only the existing statement-scoped TiKV read-RPC counter. The plan adds one synthetic point-lookup publisher with a closed producer set and retains fail-closed handling wherever the counter may include locking, DML, cop-reader, MPP, or unknown work. No CPU or byte estimate, runtime field, protocol change, or new weight-bearing unit was introduced. Revision note (2026-07-24): UnionScan now reuses the existing `cpu_work` semantic unit with `input_source=runtime_child_act_rows` and value equal to the direct child's actual rows. This intentionally simple first formula does not multiply by UnionScan conditions and does not estimate its transaction mem-buffer input. + +Revision note (2026-07-24): DML PointGet and BatchPointGet no longer fail solely because they execute under DML. Their existing plan-local snapshot runtime stats provide structural `CmdGet`/`CmdBatchGet` counts, while the DML statement producer set remains open and its RUv2 read counter is not assigned to PointGet. This lets the lookup report its own requests without absorbing pessimistic-lock writes or unrelated ancillary reads. diff --git a/pkg/executor/explain_test.go b/pkg/executor/explain_test.go index 074c387d9759a..037d4197e0a0b 100644 --- a/pkg/executor/explain_test.go +++ b/pkg/executor/explain_test.go @@ -917,16 +917,18 @@ func TestExplainAnalyzeFormatRUWriteDML(t *testing.T) { require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "key_bytes"), rows) require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "value_bytes"), rows) requireExplainRUOperatorUnitAbsent(t, rows, "tidb/kv_mutation", "cpu_work") - requireExplainRUUnitAbsent(t, rows, "write_request_count") - require.Contains(t, fmt.Sprint(rows[0][16]), "partial_missing_write_rpc_count") + require.Zero(t, explainRUCountUnitValue(t, rows, "tikv/kv_write", "write_request_count"), rows) tk.MustQuery("select * from explain_ru_write_v4").Check(testkit.Rows("1 one")) tk.MustExec("begin pessimistic") rows, err = queryExplainRURowsOrErr(t, tk, "explain analyze format='ru' update explain_ru_write_v4 set b = 'two' where a = 1") require.NoError(t, err) + requireExplainRUOperatorClass(t, rows, "tikv/kv_point_lookup") + require.GreaterOrEqual(t, explainRUCountUnitValue(t, rows, "tikv/kv_point_lookup", "read_request_count"), 0.0) + requireExplainRUUnitSource(t, rows, "tikv/kv_point_lookup", "read_request_count", "snapshot_runtime_stats") require.Positive(t, explainRUCountUnitValue(t, rows, "tidb/kv_mutation", "encoded_mutation_count"), rows) requireExplainRUOperatorUnitAbsent(t, rows, "tidb/kv_mutation", "cpu_work") - requireExplainRUUnitAbsent(t, rows, "write_request_count") + require.Zero(t, explainRUCountUnitValue(t, rows, "tikv/kv_write", "write_request_count"), rows) tk.MustExec("commit") tk.MustQuery("select * from explain_ru_write_v4").Check(testkit.Rows("1 two")) } @@ -940,6 +942,17 @@ func requireExplainRUUnitAbsent(t *testing.T, rows [][]any, unit string) { } } +func requireExplainRUUnitSource(t *testing.T, rows [][]any, operatorClass, unit, source string) { + t.Helper() + for _, row := range rows { + if len(row) == 17 && row[0] == "plan" && row[3] == operatorClass && row[11] == unit { + require.Equal(t, source, row[15], rows) + return + } + } + require.Failf(t, "missing RU unit", "operator_class=%s unit=%s rows=%v", operatorClass, unit, rows) +} + func requireExplainRUOperatorUnitAbsent(t *testing.T, rows [][]any, operatorClass, unit string) { t.Helper() for _, row := range rows { diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index b62e14732b951..05a570b9a0d45 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -209,6 +209,7 @@ go_library( "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//kv", "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//util", "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_atomic//:atomic", @@ -339,6 +340,7 @@ go_test( "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_tipb//go-tipb", "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//util", "@com_github_tikv_pd_client//resource_group/controller", "@org_uber_go_goleak//:goleak", diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index ec1bf6bed225b..33ad763a3e021 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/tidb/pkg/util/stmtsummary" "github.com/pingcap/tipb/go-tipb" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikvrpc" tikvutil "github.com/tikv/client-go/v2/util" rmclient "github.com/tikv/pd/client/resource_group/controller" ) @@ -323,6 +324,42 @@ func readBillingDemoResolveWeights(site, opClass, version string) (legacyReadBil return w, true } +type readBillingDemoPointLookupRPCStatsForTest struct { + counts map[tikvrpc.CmdType]int64 +} + +func (*readBillingDemoPointLookupRPCStatsForTest) String() string { + return "" +} + +func (s *readBillingDemoPointLookupRPCStatsForTest) Merge(other execdetails.RuntimeStats) { + otherStats, ok := other.(*readBillingDemoPointLookupRPCStatsForTest) + if !ok { + return + } + for cmd, count := range otherStats.counts { + s.counts[cmd] += count + } +} + +func (s *readBillingDemoPointLookupRPCStatsForTest) Clone() execdetails.RuntimeStats { + cloned := &readBillingDemoPointLookupRPCStatsForTest{ + counts: make(map[tikvrpc.CmdType]int64, len(s.counts)), + } + for cmd, count := range s.counts { + cloned.counts[cmd] = count + } + return cloned +} + +func (*readBillingDemoPointLookupRPCStatsForTest) Tp() int { + return execdetails.TpRuntimeStatsWithSnapshot +} + +func (s *readBillingDemoPointLookupRPCStatsForTest) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 { + return s.counts[cmd] +} + func TestReadBillingDemoV4FormulaContract(t *testing.T) { weights := readBillingDemoWeights{ Version: "test-v4-calibrated", @@ -465,7 +502,7 @@ func TestReadBillingDemoV4FormulaContract(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - op, present := readBillingDemoPointLookupTransport(tc.flat, metrics, false) + op, present := readBillingDemoPointLookupTransport(tc.flat, nil, metrics, false) require.True(t, present) require.Equal(t, "point_lookup@statement", op.id) require.Equal(t, readBillingDemoStatusOperatorOK, op.status) @@ -478,31 +515,57 @@ func TestReadBillingDemoV4FormulaContract(t *testing.T) { }) } - zeroOp, present := readBillingDemoPointLookupTransport(testCases[0].flat, execdetails.NewRUV2Metrics(), false) + zeroOp, present := readBillingDemoPointLookupTransport(testCases[0].flat, nil, execdetails.NewRUV2Metrics(), false) require.True(t, present) require.Equal(t, readBillingDemoStatusOperatorOK, zeroOp.status) require.Zero(t, readBillingDemoUnitValue(zeroOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) - missingOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, false) + missingOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, nil, false) require.Equal(t, readBillingDemoReasonMissingReaderTransport, missingOp.reason) bypassedMetrics := execdetails.NewRUV2Metrics() bypassedMetrics.SetBypass(true) - bypassedOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, bypassedMetrics, false) + bypassedOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, bypassedMetrics, false) require.Equal(t, readBillingDemoReasonMissingReaderTransport, bypassedOp.reason) - dmlOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, metrics, true) - require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, dmlOp.reason) + + dmlRuntimeStats := execdetails.NewRuntimeStatsColl(nil) + dmlRuntimeStats.RegisterStats(point.ID(), &readBillingDemoPointLookupRPCStatsForTest{ + counts: map[tikvrpc.CmdType]int64{tikvrpc.CmdGet: 3}, + }) + dmlRuntimeStats.RegisterStats(batch.ID(), &readBillingDemoPointLookupRPCStatsForTest{ + counts: map[tikvrpc.CmdType]int64{tikvrpc.CmdBatchGet: 3}, + }) + for i, expectedRequests := range []int64{3, 3, 6} { + dmlOp, present := readBillingDemoPointLookupTransport(testCases[i].flat, dmlRuntimeStats, nil, true) + require.True(t, present) + require.Equal(t, readBillingDemoStatusOperatorOK, dmlOp.status) + require.Equal(t, float64(expectedRequests), readBillingDemoUnitValue(dmlOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceSnapshotRuntimeStats, dmlOp.units[0].source) + } + missingDMLStatsOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, metrics, true) + require.Equal(t, readBillingDemoReasonMissingReaderTransport, missingDMLStatsOp.reason) + mismatchedDMLMetrics := execdetails.NewRUV2Metrics() + mismatchedDMLMetrics.AddResourceManagerReadCnt(4) + mismatchedDMLOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, dmlRuntimeStats, mismatchedDMLMetrics, true) + require.Equal(t, readBillingDemoStatusOperatorOK, mismatchedDMLOp.status) + require.Equal(t, 3.0, readBillingDemoUnitValue(mismatchedDMLOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceSnapshotRuntimeStats, mismatchedDMLOp.units[0].source) point.Lock = true - lockOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, metrics, false) + lockOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, nil, metrics, false) require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, lockOp.reason) + dmlLockOp, _ := readBillingDemoPointLookupTransport(testCases[0].flat, dmlRuntimeStats, nil, true) + require.Equal(t, readBillingDemoStatusOperatorOK, dmlLockOp.status) point.Lock = false reader := physicalop.PhysicalTableReader{StoreType: kv.TiKV}.Init(ctx, 0) mixedReaderFlat := &FlatPhysicalPlan{ Main: testCases[0].flat.Main, ScalarSubQueries: []FlatPlanTree{{{Origin: reader, IsRoot: true, StoreType: kv.TiDB}}}, } - mixedReaderOp, _ := readBillingDemoPointLookupTransport(mixedReaderFlat, metrics, false) + mixedReaderOp, _ := readBillingDemoPointLookupTransport(mixedReaderFlat, nil, metrics, false) require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, mixedReaderOp.reason) + mixedReaderDMLOp, _ := readBillingDemoPointLookupTransport(mixedReaderFlat, dmlRuntimeStats, nil, true) + require.Equal(t, readBillingDemoStatusOperatorOK, mixedReaderDMLOp.status) + require.Equal(t, 3.0, readBillingDemoUnitValue(mixedReaderDMLOp.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) physicalOp, supported, reason := readBillingDemoClassifyOperator(testCases[0].flat.Main[0]) physicalOp.id = point.ExplainID().String() diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index bfd2f66975027..f2e8b04e28b24 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/plancodec" "github.com/pingcap/tidb/pkg/util/stmtsummary" + "github.com/tikv/client-go/v2/tikvrpc" tikvutil "github.com/tikv/client-go/v2/util" rmclient "github.com/tikv/pd/client/resource_group/controller" ) @@ -175,6 +176,7 @@ const ( readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" readBillingDemoInputSourceCommitDetail = "commit_detail" readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" + readBillingDemoInputSourceSnapshotRuntimeStats = "snapshot_runtime_stats" readBillingDemoInputSourcePhysicalPlan = "physical_plan" readBillingDemoInputSourceHashJoinRuntime = "hash_join_runtime_stats" readBillingDemoInputSideAll = "all" @@ -445,7 +447,7 @@ func buildReadBillingDemoResult(sctx base.PlanContext, plan base.Plan, stmt ast. } result.operators = append(result.operators, op) } - if op, present := readBillingDemoPointLookupTransport(flat, ruv2Metrics, false); present { + if op, present := readBillingDemoPointLookupTransport(flat, runtimeStats, ruv2Metrics, false); present { if op.status != readBillingDemoStatusOperatorOK { return readBillingDemoFailedOperator(readBillingDemoStatusUnknownInput, op) } @@ -535,7 +537,7 @@ func appendReadBillingDemoDMLPlan(result *readBillingDemoResult, sctx base.PlanC if op, present := readBillingDemoReaderTransport(flat, runtimeStats, ruv2Metrics, true); present { result.operators = append(result.operators, op) } - if op, present := readBillingDemoPointLookupTransport(flat, ruv2Metrics, true); present { + if op, present := readBillingDemoPointLookupTransport(flat, runtimeStats, ruv2Metrics, true); present { result.operators = append(result.operators, op) } } @@ -747,7 +749,16 @@ func readBillingDemoReaderTransport(flat *FlatPhysicalPlan, runtimeStats *execde return op, true } -func readBillingDemoPointLookupTransport(flat *FlatPhysicalPlan, ruv2Metrics *execdetails.RUV2Metrics, dml bool) (readBillingDemoOperatorResult, bool) { +type readBillingDemoPointLookupRPCStats interface { + GetCmdRPCCount(tikvrpc.CmdType) int64 +} + +func readBillingDemoPointLookupTransport( + flat *FlatPhysicalPlan, + runtimeStats *execdetails.RuntimeStatsColl, + ruv2Metrics *execdetails.RUV2Metrics, + dml bool, +) (readBillingDemoOperatorResult, bool) { op := readBillingDemoOperatorResult{ id: "point_lookup@statement", site: readBillingDemoSiteTiKV, @@ -756,6 +767,7 @@ func readBillingDemoPointLookupTransport(flat *FlatPhysicalPlan, ruv2Metrics *ex emitStatusRow: true, } kinds := make(map[string]struct{}) + pointLookupPlans := make(map[int]tikvrpc.CmdType) openProducerSet := dml for _, tree := range readBillingDemoAllTrees(flat) { for _, node := range tree { @@ -765,9 +777,11 @@ func readBillingDemoPointLookupTransport(flat *FlatPhysicalPlan, ruv2Metrics *ex switch plan := node.Origin.(type) { case *physicalop.PointGetPlan: kinds["point_get"] = struct{}{} + pointLookupPlans[plan.ID()] = tikvrpc.CmdGet openProducerSet = openProducerSet || plan.Lock case *physicalop.BatchPointGetPlan: kinds["batch_point_get"] = struct{}{} + pointLookupPlans[plan.ID()] = tikvrpc.CmdBatchGet openProducerSet = openProducerSet || plan.Lock case *physicalop.PhysicalTableReader: openProducerSet = true @@ -786,6 +800,21 @@ func readBillingDemoPointLookupTransport(flat *FlatPhysicalPlan, ruv2Metrics *ex op.operatorKind = kind } } + if dml { + requests, ok := readBillingDemoPointLookupRPCCount(runtimeStats, pointLookupPlans) + if !ok { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{{ + unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceSnapshotRuntimeStats, + side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable, + }} + return op, true + } if openProducerSet { op.status = readBillingDemoStatusUnknownInput op.reason = readBillingDemoReasonAmbiguousReaderTransport @@ -811,6 +840,36 @@ func readBillingDemoPointLookupTransport(flat *FlatPhysicalPlan, ruv2Metrics *ex return op, true } +func readBillingDemoPointLookupRPCCount(runtimeStats *execdetails.RuntimeStatsColl, plans map[int]tikvrpc.CmdType) (int64, bool) { + if runtimeStats == nil { + return 0, false + } + var total int64 + for planID, cmd := range plans { + if !runtimeStats.ExistsRootStats(planID) { + return 0, false + } + _, groups := runtimeStats.GetRootStats(planID).MergeStats() + found := false + for _, group := range groups { + rpcStats, ok := group.(readBillingDemoPointLookupRPCStats) + if !ok { + continue + } + found = true + count := rpcStats.GetCmdRPCCount(cmd) + if count < 0 || count > math.MaxInt64-total { + return 0, false + } + total += count + } + if !found { + return 0, false + } + } + return total, true +} + func readBillingDemoWeightsValid(weights readBillingDemoWeights) bool { if weights.Version == "" || weights.Version == readBillingDemoWeightVersion || !weights.Calibrated || weights.MutationBytesPerCPUUnit <= 0 || math.IsNaN(weights.MutationBytesPerCPUUnit) || math.IsInf(weights.MutationBytesPerCPUUnit, 0) { From 89d0d658ab924b47da8d3d0f8ff59c3ec20464c4 Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Fri, 24 Jul 2026 13:08:15 +0800 Subject: [PATCH 25/25] planner: count DML cop read requests from runtime stats --- .../references/distsql-case-map.md | 2 +- .../references/planner-case-map.md | 2 +- pkg/distsql/BUILD.bazel | 1 + pkg/distsql/select_result.go | 9 +++ pkg/distsql/select_result_test.go | 3 + pkg/planner/core/common_plans_test.go | 43 ++++++++--- pkg/planner/core/explain_ru.go | 74 ++++++++++++++++++- 7 files changed, 118 insertions(+), 16 deletions(-) diff --git a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md index 7b5f37c91c3a3..21769fb42abe1 100644 --- a/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/distsql-case-map.md @@ -14,7 +14,7 @@ - `pkg/distsql/distsql_test.go` - Tests normal select. - `pkg/distsql/main_test.go` - Configures default goleak settings and registers testdata. - `pkg/distsql/request_builder_test.go` - Tests table handles to KV ranges. -- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership/provenance counts, multi-response aggregation, independent expected-task coverage, and close-time stats-only handling that cannot replay stale summaries. +- `pkg/distsql/select_result_test.go` - Tests coprocessor runtime-stat updates, including exact plan-ID row summaries, last-plan scan-detail ownership/provenance counts, multi-response aggregation, plan-local Cop/CopStream RPC access, independent expected-task coverage, and close-time stats-only handling that cannot replay stale summaries. ## pkg/distsql/context diff --git a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md index 96af105cbc566..80ea7934e8851 100644 --- a/.agents/skills/tidb-test-guidelines/references/planner-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/planner-case-map.md @@ -82,7 +82,7 @@ ### Tests - `pkg/planner/core/binary_plan_test.go` - planner/core: Tests binary plan generation and size limits. - `pkg/planner/core/cbo_test.go` - planner/core: Benchmarks optimizer plan selection. -- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, private v4 semantic weights and overflow rejection, independent read/write request units and weights, expression-slot counts for Agg/Join/Window families, Sort CPU work plus TopN input saturation/zero-count/checked bounds, UnionScan direct-child CPU work, reader transport attribution, plan-local DML PointGet RPC ownership and isolation from statement counters, real distsql ScanDetail ownership/provenance and fail-closed coverage, HashJoin state inputs, mutation normalization into dimension-qualified `cpu_work` with raw diagnostics, and statement-local DML/COMMIT request ownership. +- `pkg/planner/core/common_plans_test.go` - planner/core: Tests common plans plus preview RU gating, private v4 semantic weights and overflow rejection, independent read/write request units and weights, expression-slot counts for Agg/Join/Window families, Sort CPU work plus TopN input saturation/zero-count/checked bounds, UnionScan direct-child CPU work, plan-local DML cop-reader and PointGet RPC ownership with isolation from statement counters, real distsql ScanDetail ownership/provenance and fail-closed coverage, HashJoin state inputs, mutation normalization into dimension-qualified `cpu_work` with raw diagnostics, and statement-local DML/COMMIT request ownership. - `pkg/planner/core/enforce_mpp_test.go` - planner/core: Tests row-size impact on TiFlash MPP cost. - `pkg/planner/core/exhaust_physical_plans_test.go` - planner/core: Tests index join lookup filter analysis and range building. - `pkg/planner/core/expression_test.go` - planner/core: Tests AST expression eval (between/case/cast). diff --git a/pkg/distsql/BUILD.bazel b/pkg/distsql/BUILD.bazel index cfd5e77f3a4b7..2252000f73550 100644 --- a/pkg/distsql/BUILD.bazel +++ b/pkg/distsql/BUILD.bazel @@ -47,6 +47,7 @@ go_library( "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//metrics", "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//tikvrpc/interceptor", "@com_github_tikv_client_go_v2//util", "@org_golang_google_grpc//metadata", diff --git a/pkg/distsql/select_result.go b/pkg/distsql/select_result.go index 2f55030c3d8eb..15dc58dff13b0 100644 --- a/pkg/distsql/select_result.go +++ b/pkg/distsql/select_result.go @@ -47,6 +47,7 @@ import ( "github.com/pingcap/tipb/go-tipb" tikvmetrics "github.com/tikv/client-go/v2/metrics" "github.com/tikv/client-go/v2/tikv" + "github.com/tikv/client-go/v2/tikvrpc" clientutil "github.com/tikv/client-go/v2/util" "go.uber.org/zap" ) @@ -1051,6 +1052,14 @@ func (s *selectResultRuntimeStats) mergeCopRuntimeStats(copStats *copr.CopRuntim } } +// GetCmdRPCCount returns the number of RPCs issued by this SelectResult for cmd. +func (s *selectResultRuntimeStats) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 { + if s == nil || s.reqStat == nil { + return 0 + } + return int64(s.reqStat.GetCmdRPCCount(cmd)) +} + func (s *selectResultRuntimeStats) Clone() execdetails.RuntimeStats { newRs := selectResultRuntimeStats{ copRespTime: execdetails.Percentile[execdetails.Duration]{}, diff --git a/pkg/distsql/select_result_test.go b/pkg/distsql/select_result_test.go index 5726983ddf3e6..a1184501304ae 100644 --- a/pkg/distsql/select_result_test.go +++ b/pkg/distsql/select_result_test.go @@ -212,6 +212,7 @@ func TestUpdateCopRuntimeStats(t *testing.T) { parentStats = ctx.GetSessionVars().StmtCtx.RuntimeStatsColl.GetCopStats(parentPlanID) reqStats := tikv.NewRegionRequestRuntimeStats() reqStats.RecordRPCRuntimeStats(tikvrpc.CmdCop, time.Millisecond) + reqStats.RecordRPCRuntimeStats(tikvrpc.CmdCopStream, time.Millisecond) closeErr := fmt.Errorf("close failed") resp := &closeAppendingRuntimeStatsResponse{ statsOnClose: &copr.CopRuntimeStats{ReqStats: reqStats}, @@ -222,6 +223,8 @@ func TestUpdateCopRuntimeStats(t *testing.T) { require.True(t, resp.closed) require.Equal(t, 1, resp.collectCalls) require.Equal(t, uint32(1), sr.stats.reqStat.GetCmdRPCCount(tikvrpc.CmdCop)) + require.Equal(t, int64(1), sr.stats.GetCmdRPCCount(tikvrpc.CmdCop)) + require.Equal(t, int64(1), sr.stats.GetCmdRPCCount(tikvrpc.CmdCopStream)) require.Equal(t, int64(4), scanStats.GetActRows()) require.Equal(t, int32(1), scanStats.GetTasks()) require.Equal(t, int64(2), parentStats.GetActRows()) diff --git a/pkg/planner/core/common_plans_test.go b/pkg/planner/core/common_plans_test.go index 33ad763a3e021..f6af045a54da5 100644 --- a/pkg/planner/core/common_plans_test.go +++ b/pkg/planner/core/common_plans_test.go @@ -324,16 +324,17 @@ func readBillingDemoResolveWeights(site, opClass, version string) (legacyReadBil return w, true } -type readBillingDemoPointLookupRPCStatsForTest struct { +type readBillingDemoRPCStatsForTest struct { counts map[tikvrpc.CmdType]int64 + tp int } -func (*readBillingDemoPointLookupRPCStatsForTest) String() string { +func (*readBillingDemoRPCStatsForTest) String() string { return "" } -func (s *readBillingDemoPointLookupRPCStatsForTest) Merge(other execdetails.RuntimeStats) { - otherStats, ok := other.(*readBillingDemoPointLookupRPCStatsForTest) +func (s *readBillingDemoRPCStatsForTest) Merge(other execdetails.RuntimeStats) { + otherStats, ok := other.(*readBillingDemoRPCStatsForTest) if !ok { return } @@ -342,9 +343,10 @@ func (s *readBillingDemoPointLookupRPCStatsForTest) Merge(other execdetails.Runt } } -func (s *readBillingDemoPointLookupRPCStatsForTest) Clone() execdetails.RuntimeStats { - cloned := &readBillingDemoPointLookupRPCStatsForTest{ +func (s *readBillingDemoRPCStatsForTest) Clone() execdetails.RuntimeStats { + cloned := &readBillingDemoRPCStatsForTest{ counts: make(map[tikvrpc.CmdType]int64, len(s.counts)), + tp: s.tp, } for cmd, count := range s.counts { cloned.counts[cmd] = count @@ -352,11 +354,14 @@ func (s *readBillingDemoPointLookupRPCStatsForTest) Clone() execdetails.RuntimeS return cloned } -func (*readBillingDemoPointLookupRPCStatsForTest) Tp() int { +func (s *readBillingDemoRPCStatsForTest) Tp() int { + if s.tp != 0 { + return s.tp + } return execdetails.TpRuntimeStatsWithSnapshot } -func (s *readBillingDemoPointLookupRPCStatsForTest) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 { +func (s *readBillingDemoRPCStatsForTest) GetCmdRPCCount(cmd tikvrpc.CmdType) int64 { return s.counts[cmd] } @@ -461,8 +466,22 @@ func TestReadBillingDemoV4FormulaContract(t *testing.T) { runtimeStats.RecordExpectedCopTasks([]int{scan.ID()}) op, _ = readBillingDemoReaderTransport(flat, runtimeStats, execdetails.NewRUV2Metrics(), false) require.Equal(t, readBillingDemoReasonMissingReaderTransport, op.reason) - op, _ = readBillingDemoReaderTransport(flat, runtimeStats, metrics, true) - require.Equal(t, readBillingDemoReasonAmbiguousReaderTransport, op.reason) + + runtimeStats.RegisterStats(reader.ID(), &readBillingDemoRPCStatsForTest{ + counts: map[tikvrpc.CmdType]int64{ + tikvrpc.CmdCop: 2, + tikvrpc.CmdCopStream: 1, + }, + tp: execdetails.TpSelectResultRuntimeStats, + }) + dmlMetrics := execdetails.NewRUV2Metrics() + dmlMetrics.AddResourceManagerReadCnt(99) + dmlMetrics.AddTiKVCoprocessorResponseBytes(128) + op, _ = readBillingDemoReaderTransport(flat, runtimeStats, dmlMetrics, true) + require.Equal(t, readBillingDemoStatusOperatorOK, op.status) + require.Equal(t, 128.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitNetBytes, readBillingDemoInputSideAll)) + require.Equal(t, 3.0, readBillingDemoUnitValue(op.units, readBillingDemoUnitReadRequestCount, readBillingDemoInputSideAll)) + require.Equal(t, readBillingDemoInputSourceDistSQLRuntimeStats, op.units[1].source) }) t.Run("point lookup transport is rpc only and emitted once", func(t *testing.T) { @@ -528,10 +547,10 @@ func TestReadBillingDemoV4FormulaContract(t *testing.T) { require.Equal(t, readBillingDemoReasonMissingReaderTransport, bypassedOp.reason) dmlRuntimeStats := execdetails.NewRuntimeStatsColl(nil) - dmlRuntimeStats.RegisterStats(point.ID(), &readBillingDemoPointLookupRPCStatsForTest{ + dmlRuntimeStats.RegisterStats(point.ID(), &readBillingDemoRPCStatsForTest{ counts: map[tikvrpc.CmdType]int64{tikvrpc.CmdGet: 3}, }) - dmlRuntimeStats.RegisterStats(batch.ID(), &readBillingDemoPointLookupRPCStatsForTest{ + dmlRuntimeStats.RegisterStats(batch.ID(), &readBillingDemoRPCStatsForTest{ counts: map[tikvrpc.CmdType]int64{tikvrpc.CmdBatchGet: 3}, }) for i, expectedRequests := range []int64{3, 3, 6} { diff --git a/pkg/planner/core/explain_ru.go b/pkg/planner/core/explain_ru.go index f2e8b04e28b24..c2065558e8677 100644 --- a/pkg/planner/core/explain_ru.go +++ b/pkg/planner/core/explain_ru.go @@ -176,6 +176,7 @@ const ( readBillingDemoInputSourceStmtMemDBMutation = "stmt_memdb_mutation_calls" readBillingDemoInputSourceCommitDetail = "commit_detail" readBillingDemoInputSourceRUV2Metrics = "ruv2_metrics" + readBillingDemoInputSourceDistSQLRuntimeStats = "distsql_runtime_stats" readBillingDemoInputSourceSnapshotRuntimeStats = "snapshot_runtime_stats" readBillingDemoInputSourcePhysicalPlan = "physical_plan" readBillingDemoInputSourceHashJoinRuntime = "hash_join_runtime_stats" @@ -723,6 +724,35 @@ func readBillingDemoReaderTransport(flat *FlatPhysicalPlan, runtimeStats *execde op.operatorKind = kind } } + if dml { + if ruv2Metrics == nil || ruv2Metrics.Bypass() { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + requests, ok := readBillingDemoCopRPCCount(flat, runtimeStats) + if !ok { + if hasTasks || !allReaderRowsZero { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + requests = 0 + } + netBytes := ruv2Metrics.TiKVCoprocessorResponseBytes() + if netBytes < 0 || requests < 0 || (netBytes > 0 && requests == 0) { + op.status = readBillingDemoStatusUnknownInput + op.reason = readBillingDemoReasonMissingReaderTransport + return op, true + } + op.status = readBillingDemoStatusOperatorOK + op.reason = readBillingDemoReasonNone + op.units = []readBillingDemoUnit{ + {unit: readBillingDemoUnitNetBytes, source: readBillingDemoInputSourceRUV2Metrics, side: readBillingDemoInputSideAll, value: float64(netBytes), widthSource: explainRUWidthSourceNotApplicable}, + {unit: readBillingDemoUnitReadRequestCount, source: readBillingDemoInputSourceDistSQLRuntimeStats, side: readBillingDemoInputSideAll, value: float64(requests), widthSource: explainRUWidthSourceNotApplicable}, + } + return op, true + } if openProducerSet { op.status = readBillingDemoStatusUnknownInput op.reason = readBillingDemoReasonAmbiguousReaderTransport @@ -749,10 +779,50 @@ func readBillingDemoReaderTransport(flat *FlatPhysicalPlan, runtimeStats *execde return op, true } -type readBillingDemoPointLookupRPCStats interface { +type readBillingDemoRPCStats interface { GetCmdRPCCount(tikvrpc.CmdType) int64 } +func readBillingDemoCopRPCCount(flat *FlatPhysicalPlan, runtimeStats *execdetails.RuntimeStatsColl) (int64, bool) { + if flat == nil || runtimeStats == nil { + return 0, false + } + planIDs := make(map[int]struct{}) + for _, tree := range readBillingDemoAllTrees(flat) { + for _, node := range tree { + if node != nil && node.Origin != nil { + planIDs[node.Origin.ID()] = struct{}{} + } + } + } + var total int64 + found := false + for planID := range planIDs { + if !runtimeStats.ExistsRootStats(planID) { + continue + } + _, groups := runtimeStats.GetRootStats(planID).MergeStats() + for _, group := range groups { + if group.Tp() != execdetails.TpSelectResultRuntimeStats { + continue + } + rpcStats, ok := group.(readBillingDemoRPCStats) + if !ok { + return 0, false + } + found = true + for _, cmd := range []tikvrpc.CmdType{tikvrpc.CmdCop, tikvrpc.CmdCopStream} { + count := rpcStats.GetCmdRPCCount(cmd) + if count < 0 || count > math.MaxInt64-total { + return 0, false + } + total += count + } + } + } + return total, found +} + func readBillingDemoPointLookupTransport( flat *FlatPhysicalPlan, runtimeStats *execdetails.RuntimeStatsColl, @@ -852,7 +922,7 @@ func readBillingDemoPointLookupRPCCount(runtimeStats *execdetails.RuntimeStatsCo _, groups := runtimeStats.GetRootStats(planID).MergeStats() found := false for _, group := range groups { - rpcStats, ok := group.(readBillingDemoPointLookupRPCStats) + rpcStats, ok := group.(readBillingDemoRPCStats) if !ok { continue }