diff --git a/docs/use-cases/observability/clickstack/ingesting-data/schemas.md b/docs/use-cases/observability/clickstack/ingesting-data/schemas.md index 572ef81a6d8..df64eca6633 100644 --- a/docs/use-cases/observability/clickstack/ingesting-data/schemas.md +++ b/docs/use-cases/observability/clickstack/ingesting-data/schemas.md @@ -20,43 +20,48 @@ The following tables are created for each data type in the `default` database. Y ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs ( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), + `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(3)), + `TraceId` String CODEC(ZSTD(3)), + `SpanId` String CODEC(ZSTD(3)), `TraceFlags` UInt8, - `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityText` LowCardinality(String) CODEC(ZSTD(3)), `SeverityNumber` UInt8, - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `Body` String CODEC(ZSTD(1)), - `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `EventName` String CODEC(ZSTD(1)), - `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), - `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `Body` String CODEC(ZSTD(3)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(3)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `EventName` String CODEC(ZSTD(3)), + `__hdx_materialized_k8s.cluster.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(3)), + `__hdx_materialized_k8s.container.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(3)), + `__hdx_materialized_k8s.deployment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(3)), + `__hdx_materialized_k8s.namespace.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(3)), + `__hdx_materialized_k8s.node.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(3)), + `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)), + `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)), + `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), + `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), + `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))), + `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) -TTL toDateTime(Timestamp) + ${TABLES_TTL} +TTL toDateTime(Timestamp) + ${LOGS_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1; ``` @@ -68,43 +73,45 @@ SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_ ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces ( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), - `ParentSpanId` String CODEC(ZSTD(1)), - `TraceState` String CODEC(ZSTD(1)), - `SpanName` LowCardinality(String) CODEC(ZSTD(1)), - `SpanKind` LowCardinality(String) CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `Duration` UInt64 CODEC(ZSTD(1)), - `StatusCode` LowCardinality(String) CODEC(ZSTD(1)), - `StatusMessage` String CODEC(ZSTD(1)), - `Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)), - `Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)), - `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Links.TraceId` Array(String) CODEC(ZSTD(1)), - `Links.SpanId` Array(String) CODEC(ZSTD(1)), - `Links.TraceState` Array(String) CODEC(ZSTD(1)), - `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)), - `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(3)), + `TraceId` String CODEC(ZSTD(3)), + `SpanId` String CODEC(ZSTD(3)), + `ParentSpanId` String CODEC(ZSTD(3)), + `TraceState` String CODEC(ZSTD(3)), + `SpanName` LowCardinality(String) CODEC(ZSTD(3)), + `SpanKind` LowCardinality(String) CODEC(ZSTD(3)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` String CODEC(ZSTD(3)), + `SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `Duration` UInt64 CODEC(ZSTD(3)), + `StatusCode` LowCardinality(String) CODEC(ZSTD(3)), + `StatusMessage` String CODEC(ZSTD(3)), + `Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(3)), + `Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(3)), + `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(3)), + `Links.TraceId` Array(String) CODEC(ZSTD(3)), + `Links.SpanId` Array(String) CODEC(ZSTD(3)), + `Links.TraceState` Array(String) CODEC(ZSTD(3)), + `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(3)), + `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), + `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(3)), + `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), + `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))), + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, - INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 + INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) -TTL toDate(Timestamp) + ${TABLES_TTL} +TTL toDate(Timestamp) + ${TRACES_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ``` @@ -118,38 +125,39 @@ SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge ( - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ResourceSchemaUrl` String CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), - `ScopeSchemaUrl` String CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), - `MetricDescription` String CODEC(ZSTD(1)), - `MetricUnit` String CODEC(ZSTD(1)), - `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `Value` Float64 CODEC(ZSTD(1)), - `Flags` UInt32 CODEC(ZSTD(1)), - `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), - `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), - `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), - `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ResourceSchemaUrl` String CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` String CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(3)), + `ScopeSchemaUrl` String CODEC(ZSTD(3)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricDescription` String CODEC(ZSTD(3)), + `MetricUnit` String CODEC(ZSTD(3)), + `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `Value` Float64 CODEC(ZSTD(3)), + `Flags` UInt32 CODEC(ZSTD(3)), + `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(3)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(3)), + `Exemplars.Value` Array(Float64) CODEC(ZSTD(3)), + `Exemplars.SpanId` Array(String) CODEC(ZSTD(3)), + `Exemplars.TraceId` Array(String) CODEC(ZSTD(3)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -161,40 +169,41 @@ SETTINGS ttl_only_drop_parts = 1; ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum ( - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ResourceSchemaUrl` String CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), - `ScopeSchemaUrl` String CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), - `MetricDescription` String CODEC(ZSTD(1)), - `MetricUnit` String CODEC(ZSTD(1)), - `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `Value` Float64 CODEC(ZSTD(1)), - `Flags` UInt32 CODEC(ZSTD(1)), - `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), - `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), - `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), - `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), - `AggregationTemporality` Int32 CODEC(ZSTD(1)), - `IsMonotonic` Bool CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ResourceSchemaUrl` String CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` String CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(3)), + `ScopeSchemaUrl` String CODEC(ZSTD(3)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricDescription` String CODEC(ZSTD(3)), + `MetricUnit` String CODEC(ZSTD(3)), + `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `Value` Float64 CODEC(ZSTD(3)), + `Flags` UInt32 CODEC(ZSTD(3)), + `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(3)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(3)), + `Exemplars.Value` Array(Float64) CODEC(ZSTD(3)), + `Exemplars.SpanId` Array(String) CODEC(ZSTD(3)), + `Exemplars.TraceId` Array(String) CODEC(ZSTD(3)), + `AggregationTemporality` Int32 CODEC(ZSTD(3)), + `IsMonotonic` Bool CODEC(ZSTD(3)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -206,44 +215,45 @@ SETTINGS ttl_only_drop_parts = 1; ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram ( - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ResourceSchemaUrl` String CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), - `ScopeSchemaUrl` String CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), - `MetricDescription` String CODEC(ZSTD(1)), - `MetricUnit` String CODEC(ZSTD(1)), - `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `Count` UInt64 CODEC(Delta(8), ZSTD(1)), - `Sum` Float64 CODEC(ZSTD(1)), - `BucketCounts` Array(UInt64) CODEC(ZSTD(1)), - `ExplicitBounds` Array(Float64) CODEC(ZSTD(1)), - `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), - `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), - `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), - `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), - `Flags` UInt32 CODEC(ZSTD(1)), - `Min` Float64 CODEC(ZSTD(1)), - `Max` Float64 CODEC(ZSTD(1)), - `AggregationTemporality` Int32 CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ResourceSchemaUrl` String CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` String CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(3)), + `ScopeSchemaUrl` String CODEC(ZSTD(3)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricDescription` String CODEC(ZSTD(3)), + `MetricUnit` String CODEC(ZSTD(3)), + `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `Count` UInt64 CODEC(Delta, ZSTD(3)), + `Sum` Float64 CODEC(ZSTD(3)), + `BucketCounts` Array(UInt64) CODEC(ZSTD(3)), + `ExplicitBounds` Array(Float64) CODEC(ZSTD(3)), + `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(3)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(3)), + `Exemplars.Value` Array(Float64) CODEC(ZSTD(3)), + `Exemplars.SpanId` Array(String) CODEC(ZSTD(3)), + `Exemplars.TraceId` Array(String) CODEC(ZSTD(3)), + `Flags` UInt32 CODEC(ZSTD(3)), + `Min` Float64 CODEC(ZSTD(3)), + `Max` Float64 CODEC(ZSTD(3)), + `AggregationTemporality` Int32 CODEC(ZSTD(3)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -259,48 +269,49 @@ HyperDX doesn't support fetching/displaying exponential histogram metrics yet. Y ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram ( - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ResourceSchemaUrl` String CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), - `ScopeSchemaUrl` String CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), - `MetricDescription` String CODEC(ZSTD(1)), - `MetricUnit` String CODEC(ZSTD(1)), - `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `Count` UInt64 CODEC(Delta(8), ZSTD(1)), - `Sum` Float64 CODEC(ZSTD(1)), - `Scale` Int32 CODEC(ZSTD(1)), - `ZeroCount` UInt64 CODEC(ZSTD(1)), - `PositiveOffset` Int32 CODEC(ZSTD(1)), - `PositiveBucketCounts` Array(UInt64) CODEC(ZSTD(1)), - `NegativeOffset` Int32 CODEC(ZSTD(1)), - `NegativeBucketCounts` Array(UInt64) CODEC(ZSTD(1)), - `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), - `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), - `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), - `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), - `Flags` UInt32 CODEC(ZSTD(1)), - `Min` Float64 CODEC(ZSTD(1)), - `Max` Float64 CODEC(ZSTD(1)), - `AggregationTemporality` Int32 CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ResourceSchemaUrl` String CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` String CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(3)), + `ScopeSchemaUrl` String CODEC(ZSTD(3)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricDescription` String CODEC(ZSTD(3)), + `MetricUnit` String CODEC(ZSTD(3)), + `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `Count` UInt64 CODEC(Delta, ZSTD(3)), + `Sum` Float64 CODEC(ZSTD(3)), + `Scale` Int32 CODEC(ZSTD(3)), + `ZeroCount` UInt64 CODEC(ZSTD(3)), + `PositiveOffset` Int32 CODEC(ZSTD(3)), + `PositiveBucketCounts` Array(UInt64) CODEC(ZSTD(3)), + `NegativeOffset` Int32 CODEC(ZSTD(3)), + `NegativeBucketCounts` Array(UInt64) CODEC(ZSTD(3)), + `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(3)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(3)), + `Exemplars.Value` Array(Float64) CODEC(ZSTD(3)), + `Exemplars.SpanId` Array(String) CODEC(ZSTD(3)), + `Exemplars.TraceId` Array(String) CODEC(ZSTD(3)), + `Flags` UInt32 CODEC(ZSTD(3)), + `Min` Float64 CODEC(ZSTD(3)), + `Max` Float64 CODEC(ZSTD(3)), + `AggregationTemporality` Int32 CODEC(ZSTD(3)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -312,36 +323,37 @@ SETTINGS ttl_only_drop_parts = 1; ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary ( - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ResourceSchemaUrl` String CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` String CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), - `ScopeSchemaUrl` String CODEC(ZSTD(1)), - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), - `MetricDescription` String CODEC(ZSTD(1)), - `MetricUnit` String CODEC(ZSTD(1)), - `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `Count` UInt64 CODEC(Delta(8), ZSTD(1)), - `Sum` Float64 CODEC(ZSTD(1)), - `ValueAtQuantiles.Quantile` Array(Float64) CODEC(ZSTD(1)), - `ValueAtQuantiles.Value` Array(Float64) CODEC(ZSTD(1)), - `Flags` UInt32 CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ResourceSchemaUrl` String CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` String CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(3)), + `ScopeSchemaUrl` String CODEC(ZSTD(3)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricName` LowCardinality(String) CODEC(ZSTD(3)), + `MetricDescription` String CODEC(ZSTD(3)), + `MetricUnit` String CODEC(ZSTD(3)), + `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(3)), + `Count` UInt64 CODEC(Delta, ZSTD(3)), + `Sum` Float64 CODEC(ZSTD(3)), + `ValueAtQuantiles.Quantile` Array(Float64) CODEC(ZSTD(3)), + `ValueAtQuantiles.Value` Array(Float64) CODEC(ZSTD(3)), + `Flags` UInt32 CODEC(ZSTD(3)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -353,23 +365,27 @@ SETTINGS ttl_only_drop_parts = 1; ```sql CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions ( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(3)), `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), + `TraceId` String CODEC(ZSTD(3)), + `SpanId` String CODEC(ZSTD(3)), `TraceFlags` UInt8, - `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityText` LowCardinality(String) CODEC(ZSTD(3)), `SeverityNumber` UInt8, - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `Body` String CODEC(ZSTD(1)), - `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `Body` String CODEC(ZSTD(3)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(3)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), + `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, @@ -382,7 +398,7 @@ ENGINE = MergeTree PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, TimestampTime) ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + ${TABLES_TTL} +TTL TimestampTime + ${SESSIONS_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ```