diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java index 47865e1..c9ef254 100644 --- a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java @@ -47,10 +47,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.locks.Lock; /** @@ -146,16 +148,17 @@ * * *

The key-discovery SPI methods ({@link #findAllKeysByEntityIds}, {@link - * #findAllKeysByEntityIdsAsync}; full derived discovery is a follow-up) and the batch {@link - * #findLatestByEntityIds}/{@link #findLatestByEntityIdsAsync} pair (new in ThingsBoard v4.3.1.2; - * derived batch read is a follow-up) return an empty list rather than throwing, because they are - * reachable in normal operation — {@code findAllKeysByEntityIdsAsync}/{@code - * findLatestByEntityIdsAsync} back the dashboard {@code POST /api/entitiesQuery/find/keys} lookup - * (DefaultEntityQueryService#fetchTimeseriesKeys), and the sync {@code findAllKeysByEntityIds} - * backs entity-delete housekeeping — where a thrown {@link UnsupportedOperationException} would - * surface as an HTTP 500 / a failed cleanup task. This matches the official {@code - * CassandraBaseTimeseriesLatestDao}, which returns empty for all four, and {@link - * #findAllKeysByDeviceProfileId}. + * #findAllKeysByEntityIdsAsync}) derive the DISTINCT telemetry keys for an entity set from BOTH the + * historical {@code telemetry} table AND the {@code telemetry_latest} overlay (the two DISTINCT-key + * reads are merged), so key discovery returns the same key universe as {@link #findAllLatest}: a + * latest-only key written via {@code saveLatest} with no paired {@code save} lives only in the + * overlay and must still be discoverable. {@link #findAllKeysByDeviceProfileId} returns the + * tenant-wide DISTINCT keys (again unioned across both tables) for a null device profile (the "all + * profiles" path) while returning an empty list for a specific profile (neither table carries + * device-profile membership). The batch {@link #findLatestByEntityIds}/{@link + * #findLatestByEntityIdsAsync} pair (new in ThingsBoard v4.3.1.2) returns an empty list rather than + * throwing, matching the official {@code CassandraBaseTimeseriesLatestDao}; the derived batch read + * is a follow-up. */ @Slf4j @Repository @@ -341,32 +344,53 @@ public ListenableFuture removeLatest( @Override public List findAllKeysByDeviceProfileId( TenantId tenantId, DeviceProfileId deviceProfileId) { - // Config-time UI key enumeration (GET /api/deviceProfile/devices/keys/timeseries, - // TENANT_ADMIN). Return empty rather than throwing so the endpoint degrades gracefully instead - // of returning 500 — matching the sibling IoTDBTableAttributesDao and the non-relational - // ThingsBoard backends (e.g. CassandraBaseTimeseriesLatestDao.findAllKeysByDeviceProfileId - // returns an empty list). Tenant-wide DISTINCT-key discovery (the null-deviceProfileId branch) - // is a follow-up. + Objects.requireNonNull(tenantId, "tenantId"); + // Mirroring the reference SqlTimeseriesLatestDao: a null deviceProfileId is the "all profiles" + // path and must return the tenant-wide distinct keys, which the telemetry table CAN derive. + if (deviceProfileId == null) { + try { + return doFindAllKeysByTenant(tenantId); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException("Failed to read latest telemetry keys by tenant", e); + } + } + // Non-null profile lookup stays deferred for a structural reason, not a missing implementation: + // the IoTDB Table Mode telemetry table is tagged only by (tenant_id, entity_type, entity_id, + // key) -- it carries NO device_profile_id column (see schema-iotdb-table.sql), so the + // membership + // "which devices belong to profile P" simply does not exist in this store. ThingsBoard's + // relational backend answers this by JOINing the time-series keys against the relational device + // table (device.device_profile_id), which lives in ThingsBoard's entity database, not in IoTDB. + // Faking a tenant-wide or empty-but-pretending answer would silently widen or narrow attribute + // discovery, so the honest behaviour is to return no keys for a specific profile and let the + // caller's relational path own profile membership. (The null "all profiles" branch above is + // fully derivable from the telemetry table and is implemented.) return Collections.emptyList(); } @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { - // Reachable in normal operation (entity-delete housekeeping, TelemetryDeletionTaskProcessor), - // so degrade gracefully rather than throw: the official CassandraBaseTimeseriesLatestDao also - // returns an empty list. Full derived DISTINCT-key discovery over the telemetry table - // is a follow-up. - return Collections.emptyList(); + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityIds, "entityIds"); + // Synchronous SPI method: it runs on the calling thread (not the read executor), so the checked + // session/query failure is surfaced to the caller as an unchecked exception. + try { + return doFindAllKeysByEntityIds(tenantId, entityIds); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException("Failed to read latest telemetry keys by entity ids", e); + } } @Override public ListenableFuture> findAllKeysByEntityIdsAsync( TenantId tenantId, List entityIds) { - // Backs POST /api/entitiesQuery/find/keys (DefaultEntityQueryService#fetchTimeseriesKeys), the - // dashboard "available telemetry keys" lookup — a synchronous throw here would surface as an - // HTTP 500. Mirror CassandraBaseTimeseriesLatestDao and return an empty future; full derived - // DISTINCT-key discovery is a follow-up. - return Futures.immediateFuture(Collections.emptyList()); + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityIds, "entityIds"); + return submitReadTask(() -> doFindAllKeysByEntityIds(tenantId, entityIds)); } @Override @@ -669,6 +693,74 @@ private Optional doFindHistoryBefore( return readLatestRowB1Lenient(buildRewriteHistorySql(tenantId, entityId, key, startTs), key); } + // ---- telemetry key discovery (DISTINCT keys from telemetry + the telemetry_latest overlay) ---- + + private List doFindAllKeysByEntityIds(TenantId tenantId, List entityIds) + throws Exception { + if (entityIds.isEmpty()) { + return List.of(); + } + // saveLatest writes ONLY the telemetry_latest overlay, so a latest-only key (no paired save) + // never reaches the telemetry table. Mirror doFindAllLatest's two-read style: collect DISTINCT + // keys from BOTH tables into a LinkedHashSet (no IoTDB cross-table UNION) so key discovery + // returns the same key universe as findAllLatest. The same identity predicate works on both + // tables (shared tag columns) and the overlay holds at most one row per identity, so the union + // cannot over-report stale keys. + Set keys = new LinkedHashSet<>(); + collectKeys(keys, buildFindAllKeysByEntityIdsSql(TABLE_NAME, tenantId, entityIds)); + collectKeys(keys, buildFindAllKeysByEntityIdsSql(TABLE_LATEST, tenantId, entityIds)); + return new ArrayList<>(keys); + } + + private List doFindAllKeysByTenant(TenantId tenantId) throws Exception { + // Tenant-wide discovery unions both tables for the same reason as the entity-set path above, so + // findAllKeysByDeviceProfileId(null) surfaces overlay-only latest keys too. + Set keys = new LinkedHashSet<>(); + collectKeys(keys, buildFindAllKeysByTenantSql(TABLE_NAME, tenantId)); + collectKeys(keys, buildFindAllKeysByTenantSql(TABLE_LATEST, tenantId)); + return new ArrayList<>(keys); + } + + private void collectKeys(Set into, String sql) throws Exception { + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + into.add(row.getString("key")); + } + } + } + + private String buildFindAllKeysByTenantSql(String table, TenantId tenantId) { + return "SELECT DISTINCT key FROM " + + table + + " WHERE tenant_id=" + + sqlString(tenantId.getId().toString()); + } + + private String buildFindAllKeysByEntityIdsSql( + String table, TenantId tenantId, List entityIds) { + StringBuilder sql = + new StringBuilder("SELECT DISTINCT key FROM ") + .append(table) + .append(" WHERE tenant_id=") + .append(sqlString(tenantId.getId().toString())) + .append(" AND ("); + for (int i = 0; i < entityIds.size(); i++) { + EntityId entityId = Objects.requireNonNull(entityIds.get(i), "entityId"); + if (i > 0) { + sql.append(" OR "); + } + sql.append("(entity_type=") + .append(sqlString(entityId.getEntityType().name())) + .append(" AND entity_id=") + .append(sqlString(entityId.getId().toString())) + .append(")"); + } + sql.append(")"); + return sql.toString(); + } + // ---- SQL builders ---- private String buildFindLatestSql(TenantId tenantId, EntityId entityId, String key) { diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java index 9dff89c..216ffcd 100644 --- a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java @@ -36,11 +36,18 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.util.TimeUtils; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -56,11 +63,13 @@ *

Strategy F consumes ThingsBoard common-data types from the compile classpath and binds the * real historical {@link TimeseriesDao} SPI. * - *

This initial implementation delivers the batch WRITE path ({@link #save}), the RAW - * (non-aggregated) historical READ path ({@link #findAllAsync}) and the DELETE path ({@link - * #remove}), all driven through a bounded read thread-pool. The time-bucketed aggregation read path - * is not implemented; a positive-interval aggregation query still throws {@link - * UnsupportedOperationException}. + *

This implementation delivers the batch WRITE path ({@link #save}), the RAW (non-aggregated) + * historical READ path, the time-bucketed aggregation READ path ({@link #findAllAsync}) and the + * DELETE path ({@link #remove}), all driven through a bounded read thread-pool. Aggregation + * supports BOTH the fixed-width millisecond {@code date_bin} path (buckets anchored at the query + * start timestamp) AND the timezone-aware per-bucket calendar path ({@code WEEK}/{@code + * WEEK_ISO}/{@code MONTH}/{@code QUARTER}), which walks calendar boundaries in Java and runs one + * bounded aggregate query per bucket. */ @Slf4j @Repository @@ -71,6 +80,38 @@ public class IoTDBTableTimeseriesDao extends IoTDBTableBaseDao private static final long SECONDS_PER_DAY = 86400L; private static final String TABLE_NAME = IoTDBTableTimeseriesWriter.TABLE_NAME; + // Result-set column aliases for the time-bucketed aggregation read path (design doc §3.3). + private static final String BUCKET_TS_COLUMN = "bucket_ts"; + private static final String AGG_NUM_COLUMN = "agg_num"; + private static final String AGG_STR_COLUMN = "agg_str"; + private static final String MAX_TS_COLUMN = "max_ts"; + // Typed COUNT columns: one non-null counter per FIELD type, matching ThingsBoard's per-type + // SUM(CASE WHEN IS NOT NULL THEN 1 ELSE 0 END) and dominant-column selection. + private static final String COUNT_BOOL_COLUMN = "count_bool"; + private static final String COUNT_STR_COLUMN = "count_str"; + private static final String COUNT_JSON_COLUMN = "count_json"; + private static final String COUNT_LONG_COLUMN = "count_long"; + private static final String COUNT_DOUBLE_COLUMN = "count_double"; + // Per-type SUM aliases: ThingsBoard 4.3.1.2 keeps a SUM result LONG-typed when only long values + // participate in the bucket and promotes to DOUBLE only when a double participates, so the SUM + // path projects both partial sums and lets the row mapper pick the TB-faithful result type. + private static final String SUM_LONG_COLUMN = "sum_long"; + private static final String SUM_DOUBLE_COLUMN = "sum_double"; + // Direct long MIN/MAX channels: MIN/MAX over the raw long_v column SELECT a stored long value + // with + // no accumulation, so they are exact for every long (even > 2^53). The long-only MIN/MAX mapping + // reads these instead of the COALESCE->DOUBLE agg_num, which would round-trip a large long + // through + // a double and lose precision. They are also reused as the SUM exactness-bound inputs. + private static final String MIN_LONG_COLUMN = "min_long"; + private static final String MAX_LONG_COLUMN = "max_long"; + // Mixed-type numeric promotion: exactly one of double_v / long_v is non-null per row. + private static final String NUMERIC_VALUE = "COALESCE(double_v, CAST(long_v AS DOUBLE))"; + // Doubles represent every integer in [-2^53, 2^53] exactly; beyond that the gap grows. IoTDB + // computes SUM(INT64) with a DOUBLE accumulator, so a long-only SUM is only provably bit-exact + // while every partial sum stays within this range (see aggregatedSumEntry). + private static final long DOUBLE_EXACT_INTEGER_LIMIT = 9007199254740992L; // 2^53 + private final IoTDBTableTimeseriesWriter timeseriesWriter; private final long defaultTtlSeconds; @@ -226,11 +267,7 @@ private ReadTsKvQueryResult readQuery(TenantId tenantId, EntityId entityId, Read if (aggregationOf(query) == Aggregation.NONE || query.getInterval() < 1L) { return readRawQuery(tenantId, entityId, query); } - // The positive-interval, time-bucketed aggregation read path is not implemented; only the RAW - // (Aggregation.NONE or interval < 1) branch is implemented now. - throw new UnsupportedOperationException( - "Time-bucketed aggregation is not supported by this incremental IoTDB Table Mode backend" - + " yet; raw read, write and delete are available."); + return readAggregatedQuery(tenantId, entityId, query); } private ReadTsKvQueryResult readRawQuery( @@ -264,6 +301,189 @@ private ReadTsKvQueryResult readRawQuery( return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); } + /** + * Routes calendar {@link IntervalType}s to {@link #readCalendarAggregatedQuery} and {@code + * MILLISECONDS}/{@code null} to {@link #readMillisecondsAggregatedQuery}. + */ + private ReadTsKvQueryResult readAggregatedQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + if (isCalendarInterval(query)) { + return readCalendarAggregatedQuery(tenantId, entityId, query); + } + return readMillisecondsAggregatedQuery(tenantId, entityId, query); + } + + /** + * Time-bucketed aggregation read path matching ThingsBoard 4.3.1.2's {@code + * AbstractChunkedAggregationTimeseriesDao.findAllAsync} contract (verified against tag v4.3.1.2). + * + *

ThingsBoard walks {@code [startTs, endPeriod)} (where {@code endPeriod = max(startTs + 1, + * endTs)}) in fixed-width {@code interval}-millisecond buckets anchored at {@code + * startTs} (NOT epoch 1970), runs one aggregate per bucket, skips empty buckets, and stamps + * each emitted entry at the bucket midpoint {@code bucketStart + (bucketEnd - + * bucketStart) / 2} with {@code bucketEnd = min(bucketStart + interval, endPeriod)} (integer + * division; the last, end-clamped bucket therefore has an earlier-than-full-width midpoint). The + * query order and limit are ignored for aggregation: all non-empty buckets are returned in + * ascending time order. The result's {@code lastEntryTs} is the maximum underlying data timestamp + * across every bucket, falling back to {@code startTs} when no data matched. + * + *

The {@code startTs}-anchored buckets come from IoTDB 2.0.8 Table Mode's three-argument + * {@code date_bin(ms, time, )} primitive (origin = {@code startTs}); see + * {@link #buildAggregationSql}. + */ + private ReadTsKvQueryResult readMillisecondsAggregatedQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + String key = requireTelemetryKey(query.getKey()); + Aggregation aggregation = aggregationOf(query); + long interval = query.getInterval(); + if (interval <= 0L) { + throw new IllegalArgumentException( + "IoTDB Table Mode aggregation requires a positive interval; got " + interval); + } + long startTs = query.getStartTs(); + // ThingsBoard's endPeriod guards a zero-width range so a single bucket is still walked. + long endPeriod = Math.max(startTs + 1, query.getEndTs()); + + String sql = buildAggregationSql(tenantId, entityId, query, aggregation, interval, endPeriod); + List entries = new ArrayList<>(); + long lastEntryTs = startTs; + boolean hasEntry = false; + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + long bucketStart = row.getTimestamp(BUCKET_TS_COLUMN).getTime(); + // Clamp to endPeriod exactly like the aggregate query's `time < endPeriod` filter. The SUM + // re-sum fallback re-queries this same [bucketStart, bucketEnd) window, so it covers + // exactly + // the rows date_bin aggregated; using the un-clamped bucketStart+interval would let a last, + // end-clamped bucket re-sum rows beyond the query window. + long bucketEnd = Math.min(bucketStart + interval, endPeriod); + KvEntry value = + aggregatedEntry( + aggregation, + row, + new SumReSumContext(tenantId, entityId, key, bucketStart, bucketEnd)); + if (value == null) { + // Defensive: a bucket with only NULL value columns produces no entry. + continue; + } + long bucketTs = bucketStart + (bucketEnd - bucketStart) / 2; + entries.add(new BasicTsKvEntry(bucketTs, value)); + // ThingsBoard reports MAX(ts) of the underlying data, not the bucket midpoint. + long maxDataTs = row.getTimestamp(MAX_TS_COLUMN).getTime(); + if (!hasEntry || maxDataTs > lastEntryTs) { + lastEntryTs = maxDataTs; + hasEntry = true; + } + } + } + return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); + } + + /** + * Calendar-interval (non-{@code MILLISECONDS}) aggregation read path matching ThingsBoard + * 4.3.1.2's {@code AbstractChunkedAggregationTimeseriesDao.findAllAsync} contract for {@code + * WEEK}/{@code WEEK_ISO}/{@code MONTH}/{@code QUARTER} buckets (verified against tag v4.3.1.2). + * + *

IoTDB 2.0.8's native {@code date_bin} calendar primitive cannot reproduce ThingsBoard's + * boundaries: it anchors each calendar bucket on the origin's day-of-month (so {@code + * date_bin(1mo, time, startTs)} from a mid-month {@code startTs} steps day-15 → day-15, not to + * the 1st of each month) and it exposes no timezone argument (it computes in the + * server's UTC zone only). ThingsBoard instead advances {@code startTs} to the start of the next + * calendar unit in {@code tzId} via {@link TimeUtils#calculateIntervalEnd}, so the first bucket + * is the partial {@code [startTs, nextCalendarBoundary)} and later buckets are full calendar + * units. This path therefore reproduces ThingsBoard exactly the way ThingsBoard itself does: it + * walks the calendar boundaries in Java and issues one bounded aggregate query per bucket + * (ThingsBoard issues one future per bucket), reusing the very same projection, row mapper and + * typed-COUNT logic as the {@code MILLISECONDS} path. + * + *

Walking {@code [startTs, endPeriod)} where {@code endPeriod = max(startTs + 1, endTs)}: each + * iteration takes {@code bucketStart = startPeriod}, {@code bucketEnd = min(calculateIntervalEnd( + * bucketStart, intervalType, tzId), endPeriod)}, stamps the emitted entry at the integer midpoint + * {@code bucketStart + (bucketEnd - bucketStart) / 2}, skips empty buckets, and advances {@code + * startPeriod = bucketEnd}. Query order and limit are ignored (aggregation always returns every + * non-empty bucket ascending). {@code lastEntryTs} is the maximum underlying data timestamp + * across all buckets, falling back to {@code startTs} when nothing matched. + */ + private ReadTsKvQueryResult readCalendarAggregatedQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + String key = requireTelemetryKey(query.getKey()); + Aggregation aggregation = aggregationOf(query); + IntervalType intervalType = query.getAggParameters().getIntervalType(); + ZoneId tzId = calendarZone(query); + long startTs = query.getStartTs(); + // ThingsBoard clamps endPeriod = max(startTs + 1, endTs) so a zero-width range still walks one + // bucket; the final calendar bucket is clamped to endPeriod just like the milliseconds path. + long endPeriod = Math.max(startTs + 1, query.getEndTs()); + + List entries = new ArrayList<>(); + long lastEntryTs = startTs; + boolean hasEntry = false; + try (ITableSession session = tableSessionPool.getSession()) { + long startPeriod = startTs; + while (startPeriod < endPeriod) { + long bucketStart = startPeriod; + long bucketEnd = + Math.min(TimeUtils.calculateIntervalEnd(bucketStart, intervalType, tzId), endPeriod); + // Defensive: calculateIntervalEnd always advances, but guard against a degenerate boundary + // (e.g. a clamp that did not move) so the loop cannot spin forever. + if (bucketEnd <= bucketStart) { + bucketEnd = endPeriod; + } + long bucketTs = bucketStart + (bucketEnd - bucketStart) / 2; + String sql = + buildBucketAggregationSql(tenantId, entityId, key, aggregation, bucketStart, bucketEnd); + try (SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + // Each calendar bucket is its own bounded aggregate query with no GROUP BY, so an EMPTY + // window still returns one row (COUNT=0, AVG/MIN/MAX/SUM=NULL, MAX(time)=NULL). The + // milliseconds GROUP BY path drops empty buckets implicitly; here we must skip them + // explicitly. MAX(time) is NULL iff the window matched zero rows (time is never null), so + // it is the robust empty-bucket test across every aggregation type, including COUNT + // (which + // would otherwise emit a spurious 0 and violate ThingsBoard's "skip empty buckets"). + if (row.next() && !row.isNull(MAX_TS_COLUMN)) { + KvEntry value = + aggregatedEntry( + aggregation, + row, + new SumReSumContext(tenantId, entityId, key, bucketStart, bucketEnd)); + if (value != null) { + entries.add(new BasicTsKvEntry(bucketTs, value)); + // The enclosing guard already proved MAX_TS_COLUMN is non-null (an empty bucket was + // skipped), so read the max underlying data timestamp directly like the ms path. + long maxDataTs = row.getTimestamp(MAX_TS_COLUMN).getTime(); + if (!hasEntry || maxDataTs > lastEntryTs) { + lastEntryTs = maxDataTs; + hasEntry = true; + } + } + } + } + startPeriod = bucketEnd; + } + } + return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); + } + + private static boolean isCalendarInterval(ReadTsKvQuery query) { + var params = query.getAggParameters(); + if (params == null) { + return false; + } + IntervalType intervalType = params.getIntervalType(); + // A null IntervalType defaults to MILLISECONDS semantics (fixed-width date_bin bucketing). + return intervalType != null && intervalType != IntervalType.MILLISECONDS; + } + + private static ZoneId calendarZone(ReadTsKvQuery query) { + ZoneId tzId = query.getAggParameters().getTzId(); + // ThingsBoard always supplies a zone for calendar aggregation (AggregationParams.calendar + // resolves it); default to the system zone defensively so calculateIntervalEnd never NPEs. + return tzId != null ? tzId : ZoneId.systemDefault(); + } + private String buildReadSql( TenantId tenantId, EntityId entityId, ReadTsKvQuery query, String order) { String key = requireTelemetryKey(query.getKey()); @@ -311,6 +531,380 @@ private static Aggregation aggregationOf(ReadTsKvQuery query) { return aggregation == null ? Aggregation.NONE : aggregation; } + /** + * Builds the {@code startTs}-anchored, time-bucketed aggregation SQL matching ThingsBoard + * 4.3.1.2. Buckets are anchored at {@code startTs} via the three-argument {@code + * date_bin(ms, time, )} primitive (origin = {@code startTs}) so bucket {@code + * k} spans {@code [startTs + k*interval, startTs + (k+1)*interval)} rather than the epoch-1970 + * alignment of the two-argument form. An explicit milliseconds literal ({@code ms}) + * avoids the {@code 1m}/{@code 1M} minute-vs-month parsing ambiguity flagged for IoTDB 2.0.x; + * {@code ReadTsKvQuery.getInterval()} is already expressed in milliseconds. + * + *

Every projection also selects {@code MAX(time)} so the reader can report the maximum + * underlying data timestamp as {@code lastEntryTs}. Results are ordered by the bucket key + * ascending; the query's order and limit are intentionally ignored for aggregation (only the raw + * {@code Aggregation.NONE} path honours them). Mixed-type numeric aggregates promote {@code + * long_v} to DOUBLE via {@code COALESCE(double_v, CAST(long_v AS DOUBLE))}; {@code COUNT} counts + * non-null typed values per FIELD column (see {@link #countProjection()}). {@code MIN}/{@code + * MAX} project both a numeric and a string aggregate so the row mapper can pick the populated one + * (IoTDB performs lexicographic MIN/MAX over STRING natively). + */ + private String buildAggregationSql( + TenantId tenantId, + EntityId entityId, + ReadTsKvQuery query, + Aggregation aggregation, + long interval, + long endExclusive) { + String key = requireTelemetryKey(query.getKey()); + String dateBin = "date_bin(" + interval + "ms, time, " + query.getStartTs() + ")"; + StringBuilder sql = new StringBuilder("SELECT ").append(dateBin).append(" AS "); + sql.append(BUCKET_TS_COLUMN).append(", ").append(aggregationProjection(aggregation)); + sql.append(", MAX(time) AS ").append(MAX_TS_COLUMN); + sql.append(" FROM ").append(TABLE_NAME); + sql.append(" WHERE tenant_id=").append(sqlString(tenantId.getId().toString())); + sql.append(" AND entity_type=").append(sqlString(entityId.getEntityType().name())); + sql.append(" AND entity_id=").append(sqlString(entityId.getId().toString())); + sql.append(" AND key=").append(sqlString(key)); + sql.append(" AND time >= ").append(query.getStartTs()); + // Upper bound is ThingsBoard's clamped endPeriod (max(startTs+1, endTs)), not the raw endTs, so + // a zero-width [startTs, startTs] query still walks the single [startTs, startTs+1) bucket and + // includes a point at startTs (matching AbstractChunkedAggregationTimeseriesDao). + sql.append(" AND time < ").append(endExclusive); + sql.append(" GROUP BY 1 ORDER BY 1 ASC"); + return sql.toString(); + } + + /** + * Builds the single-bucket aggregate SQL for one calendar bucket {@code [bucketStart, + * bucketEnd)}. Unlike {@link #buildAggregationSql}, there is no {@code date_bin}/{@code GROUP + * BY}: ThingsBoard computes calendar bucket boundaries in Java (timezone-aware, + * calendar-start-aligned) and runs one bounded aggregate per bucket, so the half-open {@code time + * >= bucketStart AND time < bucketEnd} window is the bucket. The same aggregate + * projection, {@code MAX(time) AS max_ts} and typed-COUNT logic as the {@code MILLISECONDS} path + * are reused; the caller derives the bucket midpoint timestamp and skips empty buckets in Java. + */ + private String buildBucketAggregationSql( + TenantId tenantId, + EntityId entityId, + String key, + Aggregation aggregation, + long bucketStart, + long bucketEnd) { + StringBuilder sql = new StringBuilder("SELECT ").append(aggregationProjection(aggregation)); + sql.append(", MAX(time) AS ").append(MAX_TS_COLUMN); + sql.append(" FROM ").append(TABLE_NAME); + sql.append(" WHERE tenant_id=").append(sqlString(tenantId.getId().toString())); + sql.append(" AND entity_type=").append(sqlString(entityId.getEntityType().name())); + sql.append(" AND entity_id=").append(sqlString(entityId.getId().toString())); + sql.append(" AND key=").append(sqlString(key)); + sql.append(" AND time >= ").append(bucketStart); + sql.append(" AND time < ").append(bucketEnd); + return sql.toString(); + } + + private static String aggregationProjection(Aggregation aggregation) { + return switch (aggregation) { + case AVG -> "AVG(" + NUMERIC_VALUE + ") AS " + AGG_NUM_COLUMN; + // SUM keeps the ThingsBoard 4.3.1.2 result type: long-only buckets stay LONG, mixed buckets + // promote to DOUBLE. Project the partial long/double sums plus the long/double non-null + // counts so the row mapper can pick the type without re-reading the raw rows. + case SUM -> + // IoTDB 2.0.8 computes SUM over an INT64 column with a DOUBLE accumulator and returns a + // DOUBLE. Project the long partial as SUM(CAST(long_v AS DOUBLE)) -- a plain DOUBLE -- + // and + // NEVER cast it back to INT64 in SQL: CAST(SUM(long_v) AS INT64) THROWS a "Double value + // out of range of long value" error at the IoTDB level when the long-only sum exceeds + // Long.MAX, which would fail the whole aggregate query before the Java + // bound-check/fallback + // could run. The DOUBLE accumulator only keeps the sum bit-exact while every partial sum + // stays within +/-2^53, so the row mapper reads MIN(long_v)/MAX(long_v) to bound the sum, + // returns the DOUBLE cast back to long (lossless within the bound) for the provably-exact + // long-only case, and falls back to an exact Java re-sum when the bound exceeds 2^53 (see + // aggregatedSumEntry). The double partial stays DOUBLE. + "SUM(CAST(long_v AS DOUBLE)) AS " + + SUM_LONG_COLUMN + + ", SUM(double_v) AS " + + SUM_DOUBLE_COLUMN + + ", MIN(long_v) AS " + + MIN_LONG_COLUMN + + ", MAX(long_v) AS " + + MAX_LONG_COLUMN + + ", " + + numericCountProjection(); + case COUNT -> countProjection(); + // MIN/MAX keep the mixed/double numeric value via MIN/MAX(NUMERIC_VALUE) (the COALESCE + // promotes long_v to DOUBLE, correct for mixed and double-only buckets) and the string + // fallback via MIN/MAX(str_v). A long-only bucket instead reads the direct MIN(long_v)/ + // MAX(long_v) channel, which SELECTs a stored long with no accumulation and is therefore + // exact for every long (even > 2^53) -- routing it through agg_num's DOUBLE would + // round-trip + // a large long and lose precision. The long/double non-null counts pick the populated + // channel. + case MIN -> + "MIN(" + + NUMERIC_VALUE + + ") AS " + + AGG_NUM_COLUMN + + ", MIN(long_v) AS " + + MIN_LONG_COLUMN + + ", MIN(str_v) AS " + + AGG_STR_COLUMN + + ", " + + numericCountProjection(); + case MAX -> + "MAX(" + + NUMERIC_VALUE + + ") AS " + + AGG_NUM_COLUMN + + ", MAX(long_v) AS " + + MAX_LONG_COLUMN + + ", MAX(str_v) AS " + + AGG_STR_COLUMN + + ", " + + numericCountProjection(); + case NONE -> + throw new IllegalArgumentException("Aggregation.NONE has no aggregation projection"); + }; + } + + /** + * Projects the long/double non-null counters that the SUM and MIN/MAX row mappers use to decide + * the ThingsBoard-faithful result type: a bucket with only long values ({@code count_long > 0 && + * count_double == 0}) yields a {@code LongDataEntry}; any participating double yields a {@code + * DoubleDataEntry}. The schema stores {@code long_v} XOR {@code double_v} per row, so these + * counters cleanly partition the numeric rows. + */ + private static String numericCountProjection() { + return typedCount("long_v", COUNT_LONG_COLUMN) + + ", " + + typedCount("double_v", COUNT_DOUBLE_COLUMN); + } + + /** + * ThingsBoard's COUNT counts non-null typed values per FIELD column ({@code SUM(CASE + * WHEN IS NOT NULL THEN 1 ELSE 0 END)}) rather than {@code COUNT(*)}, then reports the + * first non-zero counter in dominant-column priority order (boolean, string, json, then + * long+double). Each per-type SUM is cast to {@code INT64} because IoTDB returns the {@code CASE} + * sum as DOUBLE. + */ + private static String countProjection() { + return typedCount("bool_v", COUNT_BOOL_COLUMN) + + ", " + + typedCount("str_v", COUNT_STR_COLUMN) + + ", " + + typedCount("json_v", COUNT_JSON_COLUMN) + + ", " + + typedCount("long_v", COUNT_LONG_COLUMN) + + ", " + + typedCount("double_v", COUNT_DOUBLE_COLUMN); + } + + private static String typedCount(String column, String alias) { + return "CAST(SUM(CASE WHEN " + column + " IS NOT NULL THEN 1 ELSE 0 END) AS INT64) AS " + alias; + } + + private KvEntry aggregatedEntry( + Aggregation aggregation, SessionDataSet.DataIterator row, SumReSumContext sumContext) + throws Exception { + String key = sumContext.key(); + return switch (aggregation) { + case AVG -> { + // AVG is always DOUBLE in ThingsBoard, regardless of the participating value types. + if (row.isNull(AGG_NUM_COLUMN)) { + yield null; + } + yield new DoubleDataEntry(key, row.getDouble(AGG_NUM_COLUMN)); + } + case SUM -> aggregatedSumEntry(row, sumContext); + case COUNT -> new LongDataEntry(key, typedCount(row)); + case MIN, MAX -> { + long countLong = countColumn(row, COUNT_LONG_COLUMN); + long countDouble = countColumn(row, COUNT_DOUBLE_COLUMN); + if (countLong > 0L && countDouble == 0L) { + // Long-only bucket: read the direct MIN(long_v)/MAX(long_v) channel, which is exact for + // every long (MIN/MAX SELECT a stored value with no accumulation). Routing it through + // agg_num's COALESCE->DOUBLE would round a long > 2^53 down to the nearest double; this + // keeps the LONG result bit-exact, matching ThingsBoard 4.3.1.2. + String longColumn = aggregation == Aggregation.MIN ? MIN_LONG_COLUMN : MAX_LONG_COLUMN; + if (!row.isNull(longColumn)) { + yield new LongDataEntry(key, row.getLong(longColumn)); + } + } + if (!row.isNull(AGG_NUM_COLUMN)) { + // Any participating double promotes the result to DOUBLE; the mixed/double-only value is + // byte-for-byte unchanged from the prior DoubleDataEntry behaviour. + yield new DoubleDataEntry(key, row.getDouble(AGG_NUM_COLUMN)); + } + if (!row.isNull(AGG_STR_COLUMN)) { + yield new StringDataEntry(key, row.getString(AGG_STR_COLUMN)); + } + yield null; + } + case NONE -> throw new IllegalArgumentException("Aggregation.NONE is not an aggregate"); + }; + } + + /** + * Maps a SUM bucket to its ThingsBoard-faithful entry. A mixed/double bucket promotes to DOUBLE + * (unchanged). A long-only bucket keeps the LONG type but must be EXACT: IoTDB computes {@code + * SUM(long_v)} with a DOUBLE accumulator (projected here as {@code SUM(CAST(long_v AS DOUBLE))} + * to avoid the INT64-cast overflow error), so the double sum is only bit-exact while every + * partial sum stays within {@code +/-2^53}. Because {@code |sum| <= count_long * maxAbs} and + * every partial sum is bounded the same way (where {@code maxAbs = max(|min_long|,|max_long|)}), + * the double sum is provably exact iff {@code count_long * maxAbs <= 2^53}. When that bound may + * be exceeded we cannot trust the double sum, so we re-query the bucket's raw {@code long_v} + * values and accumulate them in Java as {@code long} (natural overflow to 2^63, matching + * ThingsBoard's long arithmetic). The fast SQL path handles every normal bucket; only genuinely + * huge buckets re-query. + */ + private KvEntry aggregatedSumEntry(SessionDataSet.DataIterator row, SumReSumContext sumContext) + throws Exception { + String key = sumContext.key(); + long countLong = countColumn(row, COUNT_LONG_COLUMN); + long countDouble = countColumn(row, COUNT_DOUBLE_COLUMN); + if (countLong == 0L && countDouble == 0L) { + // No numeric rows in the bucket: emit nothing (matches the prior NULL-agg behaviour). + return null; + } + // The long partial is projected as SUM(CAST(long_v AS DOUBLE)) -- a DOUBLE -- so it never + // throws + // the INT64 out-of-range error a SQL CAST would (see aggregationProjection); read it as a + // double. + double sumLong = nullableDouble(row, SUM_LONG_COLUMN); + if (countDouble > 0L) { + // Mixed (or double-only) bucket: ThingsBoard promotes to DOUBLE, summing the long and double + // partials together (the long partial is 0 for a double-only bucket); both partials are + // already doubles. + return new DoubleDataEntry(key, nullableDouble(row, SUM_DOUBLE_COLUMN) + sumLong); + } + // Long-only bucket: ThingsBoard keeps the SUM LONG-typed and sums longs EXACTLY. + long minLong = nullableLong(row, MIN_LONG_COLUMN); + long maxLong = nullableLong(row, MAX_LONG_COLUMN); + if (sumIsProvablyExact(countLong, minLong, maxLong)) { + // The double accumulator never lost a bit and the bound guarantees the sum is an exact + // integer + // in [-2^53, 2^53], so casting the DOUBLE partial back to long is lossless. + return new LongDataEntry(key, (long) sumLong); + } + // The bound may exceed 2^53, so the DOUBLE accumulator may have rounded: recompute exactly. + return new LongDataEntry(key, exactLongSum(sumContext)); + } + + /** + * Conservative, overflow-free exactness check for a long-only {@code SUM(long_v)} computed by + * IoTDB's DOUBLE accumulator. With {@code maxAbs = max(|min_long|, |max_long|)}, the final sum + * and every partial sum satisfy {@code |partial| <= count_long * maxAbs}; if that product is + * {@code <= 2^53} the accumulator stayed in double's exact-integer range and never lost a bit. + * The product is tested via DIVISION ({@code count_long <= 2^53 / maxAbs}) so the check itself + * cannot overflow. If {@code min_long == Long.MIN_VALUE} its absolute value is not representable + * as a positive long, so we conservatively treat the bound as exceeded and fall back to the exact + * Java re-sum. + */ + private static boolean sumIsProvablyExact(long countLong, long minLong, long maxLong) { + if (minLong == Long.MIN_VALUE) { + // |Long.MIN_VALUE| overflows a positive long; cannot prove exactness, force the Java re-sum. + return false; + } + long maxAbs = Math.max(Math.abs(minLong), Math.abs(maxLong)); + if (maxAbs == 0L) { + // Every value is 0, so the sum is exactly 0. + return true; + } + // No multiplication: count_long * maxAbs <= 2^53 <=> count_long <= 2^53 / maxAbs. + return countLong <= DOUBLE_EXACT_INTEGER_LIMIT / maxAbs; + } + + /** + * Re-queries a long-only bucket's raw {@code long_v} values and accumulates them in Java as + * {@code long}, with natural overflow to 2^63 exactly the way ThingsBoard sums longs. Used only + * when the bucket's values are large enough that IoTDB's DOUBLE SUM accumulator may have lost + * precision (see {@link #sumIsProvablyExact}); the bounded window {@code [bucketStart, + * bucketEnd)} reuses the same tenant/entity/key identity predicate as the aggregate query, on its + * own pooled {@link ITableSession} so it never opens a second result set on the session that is + * iterating the outer aggregate. + */ + private long exactLongSum(SumReSumContext sumContext) throws Exception { + String sql = + "SELECT long_v FROM " + + TABLE_NAME + + " WHERE tenant_id=" + + sqlString(sumContext.tenantId().getId().toString()) + + " AND entity_type=" + + sqlString(sumContext.entityId().getEntityType().name()) + + " AND entity_id=" + + sqlString(sumContext.entityId().getId().toString()) + + " AND key=" + + sqlString(sumContext.key()) + + " AND time >= " + + sumContext.bucketStart() + + " AND time < " + + sumContext.bucketEnd() + + " AND long_v IS NOT NULL"; + long total = 0L; + // Use a SEPARATE pooled session rather than the one iterating the outer aggregate result set: + // IoTDB Table Mode does not guarantee two concurrently open result sets on a single session, so + // re-using it could throw or silently close the outer result set and corrupt the remaining + // bucket rows. The re-sum is a rare fallback, so the extra pool checkout is negligible. + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + if (!row.isNull("long_v")) { + total += row.getLong("long_v"); + } + } + } + return total; + } + + private static long nullableLong(SessionDataSet.DataIterator row, String column) + throws Exception { + return row.isNull(column) ? 0L : row.getLong(column); + } + + private static double nullableDouble(SessionDataSet.DataIterator row, String column) + throws Exception { + return row.isNull(column) ? 0.0D : row.getDouble(column); + } + + /** + * Selects ThingsBoard's dominant typed COUNT for a bucket: the first non-zero per-type counter in + * priority order boolean, string, json, then the long+double numeric pair (a numeric value lands + * in exactly one of {@code long_v}/{@code double_v}, so summing both yields the numeric row + * count). For our normal single-typed-column rows this equals the row count; the priority only + * matters for multi-typed (stale) buckets. + */ + private static long typedCount(SessionDataSet.DataIterator row) throws Exception { + long countBool = countColumn(row, COUNT_BOOL_COLUMN); + if (countBool > 0L) { + return countBool; + } + long countStr = countColumn(row, COUNT_STR_COLUMN); + if (countStr > 0L) { + return countStr; + } + long countJson = countColumn(row, COUNT_JSON_COLUMN); + if (countJson > 0L) { + return countJson; + } + return countColumn(row, COUNT_LONG_COLUMN) + countColumn(row, COUNT_DOUBLE_COLUMN); + } + + private static long countColumn(SessionDataSet.DataIterator row, String column) throws Exception { + return row.isNull(column) ? 0L : row.getLong(column); + } + + /** + * Carries the identity and bucket bounds a long-only SUM bucket needs to re-query its raw {@code + * long_v} values for an exact Java re-sum when the IoTDB DOUBLE accumulator may have lost + * precision (see {@link #aggregatedSumEntry}); {@link #exactLongSum} runs that re-query on its + * own pooled session. The same instance also supplies the telemetry {@code key} every aggregation + * mapping stamps onto its emitted {@link KvEntry}. + */ + private record SumReSumContext( + TenantId tenantId, EntityId entityId, String key, long bucketStart, long bucketEnd) {} + private static String sqlOrder(String order) { String normalized = Objects.requireNonNull(order, "order").trim().toUpperCase(Locale.ROOT); if (!"ASC".equals(normalized) && !"DESC".equals(normalized)) { diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java new file mode 100644 index 0000000..c6a02cb --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (tag commit c37fb509). +package org.thingsboard.server.dao.util; + +import org.thingsboard.server.common.data.kv.IntervalType; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.IsoFields; +import java.time.temporal.WeekFields; + +/** + * Strategy F provided-surface stub mirroring ThingsBoard {@code + * org.thingsboard.server.dao.util.TimeUtils}. Provided on the compile classpath only and excluded + * from the published jar, exactly like the other {@code src/provided} ThingsBoard types. + * + *

{@link #calculateIntervalEnd(long, IntervalType, ZoneId)} is the calendar-bucket boundary + * primitive that {@code AbstractChunkedAggregationTimeseriesDao.findAllAsync} uses for every + * non-{@code MILLISECONDS} {@link IntervalType}: it advances {@code startTs} to the start of the + * next calendar unit (next week, next 1st-of-month, next quarter start) in {@code tzId}. The IoTDB + * Table Mode calendar aggregation path ({@code IoTDBTableTimeseriesDao}) calls this so its bucket + * boundaries are identical to ThingsBoard's, because IoTDB 2.0.8's native {@code date_bin} calendar + * primitive anchors on the origin's day-of-month and exposes no timezone argument and therefore + * cannot reproduce ThingsBoard's timezone-aware, calendar-start-aligned boundaries. + */ +public final class TimeUtils { + + private TimeUtils() {} + + public static long calculateIntervalEnd(long startTs, IntervalType intervalType, ZoneId tzId) { + var startTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTs), tzId); + switch (intervalType) { + case WEEK: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .with(WeekFields.SUNDAY_START.dayOfWeek(), 1) + .plusDays(7) + .toInstant() + .toEpochMilli(); + case WEEK_ISO: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .with(WeekFields.ISO.dayOfWeek(), 1) + .plusDays(7) + .toInstant() + .toEpochMilli(); + case MONTH: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .withDayOfMonth(1) + .plusMonths(1) + .toInstant() + .toEpochMilli(); + case QUARTER: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .with(IsoFields.DAY_OF_QUARTER, 1) + .plusMonths(3) + .toInstant() + .toEpochMilli(); + default: + throw new RuntimeException("Not supported!"); + } + } + + public static ZonedDateTime toZonedDateTime(long ts, ZoneId zoneId) { + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(ts), zoneId); + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java index 7d56ee0..a4c329a 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java @@ -729,6 +729,163 @@ void concurrentSaveLatestSameIdentity_convergesToOneRowMaxTsWins() throws Except } } + @Test + void findAllKeysByEntityIds_returnsDistinctKeysForEntities() throws Exception { + TestScope scope = + scope( + "latest_keys", + "55555555-5555-5555-5555-555555555504", + "66666666-6666-6666-6666-666666666604"); + bootstrapSchema(scope.database()); + // Key discovery now also reads the telemetry_latest overlay, so the overlay table must exist. + bootstrapLatestSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(6); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao tsDao = new IoTDBTableTimeseriesDao(pool, writer, config); + IoTDBTableLatestDao latestDao = new IoTDBTableLatestDao(pool, config); + try { + saveAll( + tsDao, + scope, + List.of( + entry(3000L, "temperature", DataType.DOUBLE, 1.0D), + entry(3001L, "temperature", DataType.DOUBLE, 2.0D), + entry(3000L, "humidity", DataType.LONG, 50L), + entry(3000L, "status", DataType.STRING, "ok"))); + + List keys = + latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(scope.entityId())); + List sorted = new ArrayList<>(keys); + sorted.sort(Comparator.naturalOrder()); + assertEquals(List.of("humidity", "status", "temperature"), sorted); + + List async = + latestDao + .findAllKeysByEntityIdsAsync(scope.tenantId(), List.of(scope.entityId())) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + async.sort(Comparator.naturalOrder()); + assertEquals(List.of("humidity", "status", "temperature"), async); + } finally { + latestDao.destroy(); + tsDao.destroy(); + writer.destroy(); + } + } + } + + @Test + void keyDiscovery_scopesDistinctKeysByEntitySetTenantAndDefersDeviceProfile() throws Exception { + TestScope scope = + scope( + "latest_kscope", + "55555555-5555-5555-5555-555555555510", + "66666666-6666-6666-6666-666666666610"); + // A second entity under the SAME tenant with a partly-overlapping key set, and an entity under + // a + // DIFFERENT tenant whose keys must never leak into the first tenant's discovery. + EntityId secondEntity = + new TestEntityId(UUID.fromString("66666666-6666-6666-6666-666666666611"), EntityType.ASSET); + TenantId otherTenant = new TenantId(UUID.fromString("55555555-5555-5555-5555-555555555599")); + EntityId otherTenantEntity = + new TestEntityId( + UUID.fromString("66666666-6666-6666-6666-666666666699"), EntityType.DEVICE); + bootstrapSchema(scope.database()); + // Key discovery now also reads the telemetry_latest overlay, so the overlay table must exist. + bootstrapLatestSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao tsDao = new IoTDBTableTimeseriesDao(pool, writer, config); + IoTDBTableLatestDao latestDao = new IoTDBTableLatestDao(pool, config); + try { + // Entity 1 (DEVICE): temperature, humidity. Entity 2 (ASSET, same tenant): humidity, power. + // Other-tenant entity: leaked (must be excluded by tenant scoping). + saveOne(tsDao, scope.tenantId(), scope.entityId(), 7000L, "temperature", 1.0D); + saveOne(tsDao, scope.tenantId(), scope.entityId(), 7000L, "humidity", 40L); + saveOne(tsDao, scope.tenantId(), secondEntity, 7000L, "humidity", 41L); + saveOne(tsDao, scope.tenantId(), secondEntity, 7000L, "power", 9.9D); + saveOne(tsDao, otherTenant, otherTenantEntity, 7000L, "leaked", 7L); + + // Entity-set union across both same-tenant entities -> deduplicated humidity. + assertEquals( + List.of("humidity", "power", "temperature"), + sorted( + latestDao.findAllKeysByEntityIds( + scope.tenantId(), List.of(scope.entityId(), secondEntity)))); + + // Single-entity scope returns only that entity's keys. + assertEquals( + List.of("humidity", "temperature"), + sorted(latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(scope.entityId())))); + assertEquals( + List.of("humidity", "power"), + sorted(latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(secondEntity)))); + + // Null deviceProfileId -> tenant-wide distinct keys, never crossing the tenant boundary. + List tenantWide = + sorted(latestDao.findAllKeysByDeviceProfileId(scope.tenantId(), null)); + assertEquals(List.of("humidity", "power", "temperature"), tenantWide); + assertTrue(!tenantWide.contains("leaked"), "tenant scoping must exclude other-tenant keys"); + + // Non-null deviceProfileId stays deferred (telemetry table has no device_profile_id tag); + // the structural deferral returns no keys rather than faking profile membership. + assertEquals( + List.of(), + latestDao.findAllKeysByDeviceProfileId( + scope.tenantId(), + new org.thingsboard.server.common.data.id.DeviceProfileId( + UUID.fromString("44444444-4444-4444-4444-444444444444")))); + } finally { + latestDao.destroy(); + tsDao.destroy(); + writer.destroy(); + } + } + } + + @Test + void findAllKeys_includesOverlayOnlyLatestKey() throws Exception { + // Regression: a latest-only key written via saveLatest with NO paired tsDao.save() lives ONLY + // in + // the telemetry_latest overlay. Key discovery unions telemetry + the overlay, so that + // overlay-only key must surface in BOTH findAllKeysByEntityIds AND the tenant-wide + // findAllKeysByDeviceProfileId(null) path (otherwise key discovery would return a narrower key + // universe than findAllLatest). + TestScope scope = + scope( + "lt_keys_ov", + "55555555-5555-5555-5555-555555555518", + "66666666-6666-6666-6666-666666666618"); + bootstrapSchema(scope.database()); + bootstrapLatestSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(2); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao tsDao = new IoTDBTableTimeseriesDao(pool, writer, config); + IoTDBTableLatestDao latestDao = new IoTDBTableLatestDao(pool, config); + try { + // "historical" lands in telemetry; "overlayOnly" is written via saveLatest with no save(). + saveAll(tsDao, scope, List.of(entry(1000L, "historical", DataType.DOUBLE, 1.0D))); + saveLatest(latestDao, scope, entry(2000L, "overlayOnly", DataType.LONG, 9L)); + + // Entity-set key discovery unions both tables -> the overlay-only key is present. + assertEquals( + List.of("historical", "overlayOnly"), + sorted(latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(scope.entityId())))); + + // Tenant-wide (null deviceProfileId) discovery also unions both tables. + assertEquals( + List.of("historical", "overlayOnly"), + sorted(latestDao.findAllKeysByDeviceProfileId(scope.tenantId(), null))); + } finally { + latestDao.destroy(); + tsDao.destroy(); + writer.destroy(); + } + } + } + private void assertLatest( IoTDBTableLatestDao latestDao, TestScope scope, @@ -859,6 +1016,30 @@ private void saveAll(IoTDBTableTimeseriesDao dao, TestScope scope, List sorted(List keys) { + List copy = new ArrayList<>(keys); + copy.sort(Comparator.naturalOrder()); + return copy; + } + private void saveLatest(IoTDBTableLatestDao dao, TestScope scope, TestTsKvEntry entry) throws Exception { // saveLatest writes the overlay only (no paired tsDao.save()) and returns a null version. diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java index 8bc7b33..39c5959 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java @@ -33,6 +33,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; @@ -72,6 +73,8 @@ class IoTDBTableLatestDaoTest { new TenantId(UUID.fromString("11111111-1111-1111-1111-111111111111")); private static final EntityId ENTITY_ID = new TestEntityId(UUID.fromString("22222222-2222-2222-2222-222222222222"), EntityType.DEVICE); + private static final EntityId SECOND_ENTITY_ID = + new TestEntityId(UUID.fromString("33333333-3333-3333-3333-333333333333"), EntityType.ASSET); private static final String DERIVED_SQL_PREFIX = "SELECT time, bool_v, long_v, double_v, str_v, json_v FROM telemetry " @@ -734,27 +737,110 @@ void removeLatest_b1OverlayRow_doesNotWedge() throws Exception { assertTrue(result.isRemoved()); } - // ---- key discovery / batch latest: graceful empty (reachable paths must not 500) ---- + // ---- key discovery: DISTINCT keys unioned across telemetry + the telemetry_latest overlay ---- @Test - void findAllKeysByEntityIds_returnsEmptyWithoutThrowing() { + void findAllKeysByEntityIds_buildsDistinctKeySqlAndCollectsKeys() throws Exception { TestContext context = newContext(); - assertTrue(context.dao().findAllKeysByEntityIds(TENANT_ID, List.of(ENTITY_ID)).isEmpty()); - verifyNoSession(context); + // Key discovery unions telemetry + the telemetry_latest overlay (two reads); the overlay read + // returns no rows here, so the discovered key set stays {temperature, humidity}. + stubKeyReads(context.session(), dataSet(keyRow("temperature"), keyRow("humidity"))); + + List keys = + context.dao().findAllKeysByEntityIds(TENANT_ID, List.of(ENTITY_ID, SECOND_ENTITY_ID)); + + assertEquals(List.of("temperature", "humidity"), keys); + + List queries = captureQueries(context.session(), 2); + String entityPredicate = + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND ((entity_type='DEVICE' AND entity_id='22222222-2222-2222-2222-222222222222') " + + "OR (entity_type='ASSET' AND entity_id='33333333-3333-3333-3333-333333333333'))"; + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry " + entityPredicate), + "telemetry key SQL: " + queries); + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry_latest " + entityPredicate), + "overlay key SQL: " + queries); } @Test - void findAllKeysByEntityIdsAsync_returnsImmediateEmpty() throws Exception { + void findAllKeysByEntityIds_emptyListReturnsEmptyAndSkipsQuery() throws Exception { TestContext context = newContext(); - assertTrue( + + assertEquals(List.of(), context.dao().findAllKeysByEntityIds(TENANT_ID, List.of())); + + verify(context.pool(), never()).getSession(); + } + + @Test + void findAllKeysByEntityIdsAsync_runsOnReadExecutor() throws Exception { + TestContext context = newContext(); + // Unions telemetry + the overlay; the overlay read returns no rows, so keys stay {speed}. + stubKeyReads(context.session(), dataSet(keyRow("speed"))); + + List keys = context .dao() .findAllKeysByEntityIdsAsync(TENANT_ID, List.of(ENTITY_ID)) - .get(3, TimeUnit.SECONDS) - .isEmpty()); - verifyNoSession(context); + .get(3, TimeUnit.SECONDS); + + assertEquals(List.of("speed"), keys); + List queries = captureQueries(context.session(), 2); + String entityPredicate = + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND ((entity_type='DEVICE' AND entity_id='22222222-2222-2222-2222-222222222222'))"; + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry " + entityPredicate), + "telemetry key SQL: " + queries); + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry_latest " + entityPredicate), + "overlay key SQL: " + queries); + } + + @Test + void findAllKeysByDeviceProfileId_returnsEmptyDeferred() throws Exception { + TestContext context = newContext(); + + List keys = + context + .dao() + .findAllKeysByDeviceProfileId( + TENANT_ID, + new DeviceProfileId(UUID.fromString("44444444-4444-4444-4444-444444444444"))); + + assertEquals(List.of(), keys); + verify(context.pool(), never()).getSession(); } + @Test + void findAllKeysByDeviceProfileId_nullProfileReturnsTenantWideDistinctKeys() throws Exception { + TestContext context = newContext(); + // A null deviceProfileId is the "all profiles" path: return tenant-wide distinct keys, now + // unioned across telemetry + the telemetry_latest overlay (mirroring the reference + // SqlTimeseriesLatestDao.getKeysByTenantId for the historical side). The overlay read returns + // no + // rows here, so the discovered key set stays {temperature, humidity}. + stubKeyReads(context.session(), dataSet(keyRow("temperature"), keyRow("humidity"))); + + List keys = context.dao().findAllKeysByDeviceProfileId(TENANT_ID, null); + + assertEquals(List.of("temperature", "humidity"), keys); + List queries = captureQueries(context.session(), 2); + assertTrue( + queries.contains( + "SELECT DISTINCT key FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111'"), + "telemetry tenant SQL: " + queries); + assertTrue( + queries.contains( + "SELECT DISTINCT key FROM telemetry_latest " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111'"), + "overlay tenant SQL: " + queries); + } + + // ---- batch latest: graceful empty (reachable paths must not 500) ---- + @Test void findLatestByEntityIds_returnsEmptyWithoutThrowing() { TestContext context = newContext(); @@ -877,6 +963,24 @@ private void stubReads( } } + /** + * Stubs the two reads key discovery now issues (telemetry + the telemetry_latest overlay): the + * telemetry query returns {@code telemetryKeys}, the overlay query returns an empty dataset, so a + * test that exercises only the historical keys keeps its original key-set assertion. + */ + private void stubKeyReads(ITableSession session, SessionDataSet telemetryKeys) { + try { + when(session.executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + String sql = invocation.getArgument(0); + return sql.contains("telemetry_latest") ? dataSet() : telemetryKeys; + }); + } catch (IoTDBConnectionException | StatementExecutionException e) { + throw new AssertionError(e); + } + } + private List captureQueries(ITableSession session, int expected) throws Exception { ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); verify(session, timeout(3000).times(expected)).executeQueryStatement(sql.capture()); @@ -941,6 +1045,13 @@ private MockRow row(long ts, String column, Object value) { return new MockRow(columns); } + /** A DISTINCT-key discovery row exposing only the {@code key} column. */ + private MockRow keyRow(String key) { + Map columns = new HashMap<>(); + columns.put("key", key); + return new MockRow(columns); + } + /** A single typed-value row with explicit typed columns (for B1 multi-column cases). */ private MockRow rowOf(long ts, Map typed) { Map columns = new HashMap<>(typed); diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java new file mode 100644 index 0000000..3e65e50 --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java @@ -0,0 +1,1032 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; + +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AggregationParams; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Real-Docker integration test that proves the IoTDB 2.0.8 Table Mode native three-argument {@code + * date_bin(ms, time, )} + {@code GROUP BY} time-bucketed aggregation path + * matches ThingsBoard 4.3.1.2's contract: buckets anchored at {@code startTs} (not epoch 1970), + * entries stamped at the bucket midpoint, every non-empty bucket returned ascending regardless of + * query order/limit, and typed COUNT semantics -- all against hand-computed expected values. Reuses + * the testcontainer harness from {@link IoTDBTableTimeseriesDaoIT}: {@code + * apache/iotdb:2.0.8-standalone}, {@code dn_rpc_address=0.0.0.0}, exposed port 6667, short-prefix + * unique database, schema bootstrap from {@code schema-iotdb-table.sql}. + */ +@Tag("integration") +@Testcontainers(disabledWithoutDocker = true) +class IoTDBTableTimeseriesAggregationIT { + private static final int FUTURE_TIMEOUT_SECONDS = 30; + private static final Duration IOTDB_STARTUP_TIMEOUT = Duration.ofMinutes(3); + private static final Duration IOTDB_READY_TIMEOUT = Duration.ofSeconds(60); + private static final Duration IOTDB_READY_POLL_INTERVAL = Duration.ofMillis(500); + private static final long INTERVAL = 1000L; + + @Container + static final GenericContainer IOTDB = + new GenericContainer<>(DockerImageName.parse("apache/iotdb:2.0.8-standalone")) + .withExposedPorts(6667) + .withEnv("dn_rpc_address", "0.0.0.0") + .waitingFor(Wait.forListeningPort().withStartupTimeout(IOTDB_STARTUP_TIMEOUT)); + + @Test + void numericAggregationsBucketCorrectlyAgainstHandComputedValues() throws Exception { + TestScope scope = + scope( + "agg_numeric", + "55555555-5555-5555-5555-555555555501", + "66666666-6666-6666-6666-666666666601"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // Query [1000,3000), interval 1000, startTs-anchored buckets (origin=1000): + // Bucket [1000,2000): 10, 20, 5.5 -> midpoint 1500; sum 35.5, count 3, avg 11.8333, + // min 5.5, max 20.0 + // Bucket [2000,3000): 30, 40 -> midpoint 2500; sum 70.0, count 2, avg 35.0, + // min 30.0, max 40.0 + // Bucket [3000,4000): empty -> no entry emitted + // lastEntryTs = MAX(underlying time) = 2700 (ThingsBoard reports the max data ts). + saveAll( + dao, + scope, + List.of( + entry(1000L, "n", DataType.LONG, 10L), + entry(1200L, "n", DataType.LONG, 20L), + entry(1500L, "n", DataType.DOUBLE, 5.5D), + entry(2100L, "n", DataType.LONG, 30L), + entry(2700L, "n", DataType.LONG, 40L))); + + // Result TYPE: bucket [1000,2000) is MIXED (has the 5.5 double) so SUM/MIN/MAX stay DOUBLE; + // bucket [2000,3000) is LONG-ONLY (30, 40) so SUM/MIN/MAX come back LONG-typed (TB 4.3.1.2 + // keeps a long-only SUM/MIN/MAX LONG). AVG is always DOUBLE; COUNT is always LONG. + ReadTsKvQueryResult avg = aggregate(dao, scope, "n", Aggregation.AVG); + assertDoubleBuckets( + avg, new long[] {1500L, 2500L}, new double[] {35.5D / 3D, 35.0D}, 2700L); + + ReadTsKvQueryResult sum = aggregate(dao, scope, "n", Aggregation.SUM); + assertNumericBuckets( + sum, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.LONG}, + new double[] {35.5D, 70.0D}, + 2700L); + + ReadTsKvQueryResult count = aggregate(dao, scope, "n", Aggregation.COUNT); + assertLongBuckets(count, new long[] {1500L, 2500L}, new long[] {3L, 2L}); + + ReadTsKvQueryResult min = aggregate(dao, scope, "n", Aggregation.MIN); + assertNumericBuckets( + min, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.LONG}, + new double[] {5.5D, 30.0D}, + 2700L); + + ReadTsKvQueryResult max = aggregate(dao, scope, "n", Aggregation.MAX); + assertNumericBuckets( + max, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.LONG}, + new double[] {20.0D, 40.0D}, + 2700L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void longOnlyAndMixedBucketsKeepThingsBoardResultTypeAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_resulttype", + "55555555-5555-5555-5555-555555555509", + "66666666-6666-6666-6666-666666666609"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end proof of the result TYPE contract against REAL IoTDB: + // Bucket [1000,2000): LONG-ONLY data 4,6,10 -> midpoint 1500; + // sum 20, min 4, max 10 -- all LONG-typed (TB 4.3.1.2 keeps a long-only result LONG). + // Bucket [2000,3000): MIXED data 3 (long) + 2.5 (double) -> midpoint 2500; + // sum 5.5, min 2.5, max 3.0 -- all DOUBLE-typed (a participating double promotes). + // AVG is always DOUBLE; COUNT is always LONG. + saveAll( + dao, + scope, + List.of( + entry(1000L, "m", DataType.LONG, 4L), + entry(1300L, "m", DataType.LONG, 6L), + entry(1700L, "m", DataType.LONG, 10L), + entry(2100L, "m", DataType.LONG, 3L), + entry(2400L, "m", DataType.DOUBLE, 2.5D))); + + ReadTsKvQueryResult sum = aggregate(dao, scope, "m", Aggregation.SUM); + assertNumericBuckets( + sum, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {20.0D, 5.5D}, + 2400L); + + ReadTsKvQueryResult min = aggregate(dao, scope, "m", Aggregation.MIN); + assertNumericBuckets( + min, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {4.0D, 2.5D}, + 2400L); + + ReadTsKvQueryResult max = aggregate(dao, scope, "m", Aggregation.MAX); + assertNumericBuckets( + max, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {10.0D, 3.0D}, + 2400L); + + // AVG stays DOUBLE even for the long-only bucket; COUNT stays LONG everywhere. + ReadTsKvQueryResult avg = aggregate(dao, scope, "m", Aggregation.AVG); + assertDoubleBuckets( + avg, new long[] {1500L, 2500L}, new double[] {20.0D / 3D, 2.75D}, 2400L); + + ReadTsKvQueryResult count = aggregate(dao, scope, "m", Aggregation.COUNT); + assertLongBuckets(count, new long[] {1500L, 2500L}, new long[] {3L, 2L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void sumLongOnlyExceedingLongMaxDoesNotCrashAndReSumsExactlyAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_overflow", + "55555555-5555-5555-5555-555555555511", + "66666666-6666-6666-6666-666666666611"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end proof the unsafe INT64-cast path is GONE. Two longs near Long.MAX land in ONE + // long-only bucket [1000,2000); their true sum 18446744073709550000 exceeds Long.MAX + // (9223372036854775807). Against REAL IoTDB the old projection CAST(SUM(long_v) AS INT64) + // THROWS "Double value ... out of range of long value" and FAILS the whole aggregate query + // before the Java bound-check/fallback can run. The new SUM(CAST(long_v AS DOUBLE)) partial + // never throws, the bound (count_long=2, maxAbs ~ 9.2e18 -> 2 > 2^53/maxAbs) forces the + // exact + // Java re-sum, and the DAO returns the bucket with the EXACT long value -- the same natural + // 2^64 overflow ThingsBoard's long arithmetic produces: -1616. + long nearMax = 9223372036854775000L; // sum of two exceeds Long.MAX + long overflowSum = nearMax + nearMax; // -1616 under Java natural long overflow (== TB) + saveAll( + dao, + scope, + List.of( + entry(1000L, "o", DataType.LONG, nearMax), + entry(1500L, "o", DataType.LONG, nearMax))); + + ReadTsKvQuery query = + new BaseReadTsKvQuery("o", 1000L, 2000L, INTERVAL, 100, Aggregation.SUM, "ASC"); + ReadTsKvQueryResult sum = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // The query did NOT crash (it returned), and the long-only bucket comes back at midpoint + // 1500 + // with the EXACT Java re-sum (-1616), proving the fallback handled the > Long.MAX sum. + assertEquals(1, sum.getData().size(), "exactly one long-only bucket returned (no crash)"); + assertExactNumericBuckets( + sum, + new long[] {1500L}, + new DataType[] {DataType.LONG}, + new long[] {overflowSum}, + new double[] {0D}, + 1500L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void longAggregationsStayBitExactAbove2Pow53AgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_precision", + "55555555-5555-5555-5555-555555555510", + "66666666-6666-6666-6666-666666666610"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end PRECISION proof against REAL IoTDB for the two > 2^53 cases. Query + // [1000,5000), interval 1000, three non-empty buckets: + // Bucket [1000,2000): LONG-ONLY 9007199254740993 (= 2^53 + 1) and 1000 -> midpoint 1500. + // MIN = 1000, MAX = 9007199254740993 (EXACT: routed through MIN(long_v)/MAX(long_v), + // NOT the COALESCE->DOUBLE agg_num which would round MAX to ...992). SUM = + // 9007199254741993 (EXACT: the bound exceeds 2^53 so the DAO re-sums the raw long_v + // in + // Java; IoTDB's DOUBLE SUM accumulator would have returned ...992). + // Bucket [2000,3000): LONG-ONLY small 4, 6 -> midpoint 2500. Fast path (bound <= 2^53): + // SUM = 10, MIN = 4, MAX = 6, all EXACT LONG. + // Bucket [3000,4000): MIXED 3 (long) + 2.5 (double) -> midpoint 3500. DOUBLE everywhere: + // SUM = 5.5, MIN = 2.5, MAX = 3.0 (unchanged mixed behaviour). + long big = 9007199254740993L; // 2^53 + 1, NOT representable as a double + long bigSum = 9007199254741993L; // big + 1000, exact long arithmetic + saveAll( + dao, + scope, + List.of( + entry(1000L, "p", DataType.LONG, big), + entry(1500L, "p", DataType.LONG, 1000L), + entry(2100L, "p", DataType.LONG, 4L), + entry(2400L, "p", DataType.LONG, 6L), + entry(3100L, "p", DataType.LONG, 3L), + entry(3400L, "p", DataType.DOUBLE, 2.5D))); + + // MIN: long-only buckets EXACT (incl. the > 2^53 MAX channel); mixed bucket DOUBLE. + ReadTsKvQueryResult min = precisionAggregate(dao, scope, "p", Aggregation.MIN); + assertExactNumericBuckets( + min, + new long[] {1500L, 2500L, 3500L}, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new long[] {1000L, 4L, 0L}, + new double[] {0D, 0D, 2.5D}, + 3400L); + + // MAX: the long-only > 2^53 value comes back EXACT (9007199254740993, not ...992). + ReadTsKvQueryResult max = precisionAggregate(dao, scope, "p", Aggregation.MAX); + assertExactNumericBuckets( + max, + new long[] {1500L, 2500L, 3500L}, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new long[] {big, 6L, 0L}, + new double[] {0D, 0D, 3.0D}, + 3400L); + + // SUM: the > 2^53 long-only bucket re-sums EXACTLY (9007199254741993, not ...992); the + // small + // long-only bucket takes the fast path; the mixed bucket promotes to DOUBLE. + ReadTsKvQueryResult sum = precisionAggregate(dao, scope, "p", Aggregation.SUM); + assertExactNumericBuckets( + sum, + new long[] {1500L, 2500L, 3500L}, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new long[] {bigSum, 10L, 0L}, + new double[] {0D, 0D, 5.5D}, + 3400L); + + // AVG stays DOUBLE; COUNT stays LONG (unchanged). + ReadTsKvQueryResult count = precisionAggregate(dao, scope, "p", Aggregation.COUNT); + assertLongBuckets(count, new long[] {1500L, 2500L, 3500L}, new long[] {2L, 2L, 2L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void stringMinMaxBucketLexicographically() throws Exception { + TestScope scope = + scope( + "agg_string", + "55555555-5555-5555-5555-555555555502", + "66666666-6666-6666-6666-666666666602"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // startTs-anchored buckets (origin=1000), midpoints 1500 / 2500: + // Bucket [1000,2000): 'banana','apple' -> midpoint 1500; min 'apple', max 'banana' + // Bucket [2000,3000): 'cherry' -> midpoint 2500; min/max 'cherry' + saveAll( + dao, + scope, + List.of( + entry(1000L, "s", DataType.STRING, "banana"), + entry(1200L, "s", DataType.STRING, "apple"), + entry(2100L, "s", DataType.STRING, "cherry"))); + + ReadTsKvQueryResult min = aggregate(dao, scope, "s", Aggregation.MIN); + assertStringBuckets(min, new long[] {1500L, 2500L}, new String[] {"apple", "cherry"}); + + ReadTsKvQueryResult max = aggregate(dao, scope, "s", Aggregation.MAX); + assertStringBuckets(max, new long[] {1500L, 2500L}, new String[] {"banana", "cherry"}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void aggregationIgnoresLimitOrderAndReturnsAllBucketsAscending() throws Exception { + TestScope scope = + scope( + "agg_limit", + "55555555-5555-5555-5555-555555555503", + "66666666-6666-6666-6666-666666666603"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(6); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // Query [1000,5000), interval 1000, startTs-anchored buckets (origin=1000): + // [1000,2000) data 1000 -> midpoint 1500 + // [2000,3000) data 2000 -> midpoint 2500 + // [3000,4000) empty -> skipped + // [4000,5000) data 4000 -> midpoint 4500 + // ThingsBoard ignores query order/limit for aggregation: every non-empty bucket is + // returned in ascending time order regardless of LIMIT 2 / ASC-vs-DESC. + saveAll( + dao, + scope, + List.of( + entry(1000L, "n", DataType.LONG, 1L), + entry(2000L, "n", DataType.LONG, 2L), + entry(4000L, "n", DataType.LONG, 4L))); + + long[] expectedTs = {1500L, 2500L, 4500L}; + // Long-only data -> the SUM result is LONG-typed in every bucket. + DataType[] expectedTypes = {DataType.LONG, DataType.LONG, DataType.LONG}; + double[] expectedValues = {1.0D, 2.0D, 4.0D}; + long expectedLastTs = 4000L; // MAX(underlying time) + + // ASC + LIMIT 2: limit and order are ignored; all three buckets come back ascending. + ReadTsKvQuery asc = + new BaseReadTsKvQuery("n", 1000L, 5000L, INTERVAL, 2, Aggregation.SUM, "ASC"); + ReadTsKvQueryResult ascResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(asc)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertNumericBuckets(ascResult, expectedTs, expectedTypes, expectedValues, expectedLastTs); + + // DESC + LIMIT 2: identical result -- order and limit have no effect on aggregation. + ReadTsKvQuery desc = + new BaseReadTsKvQuery("n", 1000L, 5000L, INTERVAL, 2, Aggregation.SUM, "DESC"); + ReadTsKvQueryResult descResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(desc)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertNumericBuckets(descResult, expectedTs, expectedTypes, expectedValues, expectedLastTs); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void nonAlignedStartTsBucketsAnchorAtStartTsNotEpochZero() throws Exception { + TestScope scope = + scope( + "agg_nonalign", + "55555555-5555-5555-5555-555555555505", + "66666666-6666-6666-6666-666666666605"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // startTs=1001 is NOT a multiple of interval=1000. With epoch-0 (2-arg date_bin) the point + // at 2000 would land in bucket [2000,3000); with startTs-anchored buckets (3-arg origin) + // it lands in [1001,2001) -> midpoint 1001 + (2001-1001)/2 = 1501. A second point at 2500 + // lands in [2001,3001) -> midpoint 2501. This proves epoch-0 misalignment is gone. + saveAll( + dao, + scope, + List.of(entry(2000L, "n", DataType.LONG, 7L), entry(2500L, "n", DataType.LONG, 9L))); + + ReadTsKvQuery query = + new BaseReadTsKvQuery("n", 1001L, 3001L, INTERVAL, 100, Aggregation.SUM, "ASC"); + ReadTsKvQueryResult result = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // Bucket [1001,2001) midpoint 1501 holds 7; bucket [2001,3001) midpoint 2501 holds 9. + // Long-only data -> LONG-typed SUM in both buckets. + assertNumericBuckets( + result, + new long[] {1501L, 2501L}, + new DataType[] {DataType.LONG, DataType.LONG}, + new double[] {7.0D, 9.0D}, + 2500L); + + // COUNT on the same non-aligned range yields the same midpoints and counts of 1 each. + ReadTsKvQuery countQuery = + new BaseReadTsKvQuery("n", 1001L, 3001L, INTERVAL, 100, Aggregation.COUNT, "ASC"); + ReadTsKvQueryResult countResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(countQuery)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertLongBuckets(countResult, new long[] {1501L, 2501L}, new long[] {1L, 1L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void rawPathStillWorksWhenAggregationIsNone() throws Exception { + TestScope scope = + scope( + "agg_none", + "55555555-5555-5555-5555-555555555504", + "66666666-6666-6666-6666-666666666604"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(3); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(1000L, "n", DataType.LONG, 7L), + entry(1500L, "n", DataType.LONG, 8L), + entry(2100L, "n", DataType.LONG, 9L))); + + ReadTsKvQuery raw = new BaseReadTsKvQuery("n", 1000L, 3000L, 10, "ASC"); + ReadTsKvQueryResult rawResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(raw)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // Aggregation.NONE returns every raw row (not bucketed) at its own timestamp. + assertEquals(3, rawResult.getData().size()); + assertEquals(1000L, rawResult.getData().get(0).getTs()); + assertEquals(1500L, rawResult.getData().get(1).getTs()); + assertEquals(2100L, rawResult.getData().get(2).getTs()); + assertEquals(9L, rawResult.getData().get(2).getValue()); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void calendarMonthBucketsLandOnTbFaithfulCalendarBoundariesWithMidpoints() throws Exception { + TestScope scope = + scope( + "agg_month", + "55555555-5555-5555-5555-555555555506", + "66666666-6666-6666-6666-666666666606"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // UTC calendar MONTH buckets for [2023-01-01, 2023-04-01): three variable-width months + // Jan [1672531200000,1675209600000) 31d -> midpoint 1673870400000 (2023-01-16T12:00Z) + // Feb [1675209600000,1677628800000) 28d -> midpoint 1676419200000 (2023-02-15T00:00Z) + // Mar [1677628800000,1680307200000) 31d -> midpoint 1678968000000 (2023-03-16T12:00Z) + // The differing midpoints (16th-noon vs 15th-midnight) prove TRUE calendar widths, not a + // fixed 30-day step; the Java-side bucketing reproduces TimeUtils.calculateIntervalEnd. + saveAll( + dao, + scope, + List.of( + entry(1673308800000L, "n", DataType.LONG, 10L), // 2023-01-10 + entry(1674172800000L, "n", DataType.LONG, 20L), // 2023-01-20 + entry(1676419200000L, "n", DataType.LONG, 100L), // 2023-02-15 + entry(1677974400000L, "n", DataType.DOUBLE, 5.0D), // 2023-03-05 + entry(1679702400000L, "n", DataType.DOUBLE, 7.0D))); // 2023-03-25 + + long startTs = 1672531200000L; // 2023-01-01T00:00Z + long endTs = 1680307200000L; // 2023-04-01T00:00Z + long[] midpoints = {1673870400000L, 1676419200000L, 1678968000000L}; + + // Result TYPE per calendar bucket: Jan (10,20) and Feb (100) are LONG-only -> LONG SUM/MAX; + // Mar (5.0, 7.0) is double-only -> DOUBLE SUM/MAX. AVG is always DOUBLE; COUNT LONG. + ReadTsKvQueryResult sum = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.SUM); + // Jan SUM=30, Feb SUM=100, Mar SUM=12; lastEntryTs = MAX(data ts) = 2023-03-25. + assertNumericBuckets( + sum, + midpoints, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new double[] {30.0D, 100.0D, 12.0D}, + 1679702400000L); + + ReadTsKvQueryResult count = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.COUNT); + assertLongBuckets(count, midpoints, new long[] {2L, 1L, 2L}); + + ReadTsKvQueryResult avg = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.AVG); + assertDoubleBuckets(avg, midpoints, new double[] {15.0D, 100.0D, 6.0D}, 1679702400000L); + + ReadTsKvQueryResult max = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.MAX); + assertNumericBuckets( + max, + midpoints, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new double[] {20.0D, 100.0D, 7.0D}, + 1679702400000L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void calendarMonthFirstBucketIsPartialFromMidMonthStart() throws Exception { + TestScope scope = + scope( + "agg_partial", + "55555555-5555-5555-5555-555555555507", + "66666666-6666-6666-6666-666666666607"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // startTs = 2023-01-15 (NOT a month boundary). ThingsBoard advances to the next calendar + // boundary (Feb 1), so the FIRST bucket is the partial [Jan15, Feb1) with midpoint + // 1674475200000 (2023-01-23T12:00Z), then the full calendar month [Feb1, Mar1). + saveAll( + dao, + scope, + List.of( + entry(1673740800000L, "n", DataType.LONG, 3L), // 2023-01-15 (bucket start) + entry(1676419200000L, "n", DataType.LONG, 9L))); // 2023-02-15 + + // Long-only data -> LONG-typed SUM in both the partial and full calendar buckets. + ReadTsKvQueryResult sum = + calendarAggregate(dao, scope, "n", 1673740800000L, 1677628800000L, Aggregation.SUM); + assertNumericBuckets( + sum, + new long[] {1674475200000L, 1676419200000L}, + new DataType[] {DataType.LONG, DataType.LONG}, + new double[] {3.0D, 9.0D}, + 1676419200000L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void calendarCountSkipsEmptyMiddleMonthAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_emptymid", + "55555555-5555-5555-5555-555555555508", + "66666666-6666-6666-6666-666666666608"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end empty-bucket proof against REAL IoTDB: write rows in Jan and Mar 2023 but NONE + // in Feb, then run a UTC calendar MONTH COUNT spanning all three months. For the empty Feb + // bucket the bounded per-bucket aggregate REALLY returns one row with COUNT columns = 0 and + // MAX(time) = NULL (the shape the unit mocks replicate via emptyAggRow). The DAO's + // `!isNull(max_ts)` guard must skip it, so EXACTLY the two non-empty months come back with + // their correct counts and calendar midpoints; the empty Feb month is ABSENT (not a + // spurious + // count-0 entry). + // Jan [1672531200000,1675209600000) 31d -> midpoint 1673870400000, count 2 + // Feb [1675209600000,1677628800000) 28d -> EMPTY -> skipped (no entry) + // Mar [1677628800000,1680307200000) 31d -> midpoint 1678968000000, count 2 + saveAll( + dao, + scope, + List.of( + entry(1673308800000L, "n", DataType.LONG, 10L), // 2023-01-10 + entry(1674172800000L, "n", DataType.LONG, 20L), // 2023-01-20 + entry(1677974400000L, "n", DataType.LONG, 5L), // 2023-03-05 + entry(1679702400000L, "n", DataType.LONG, 7L))); // 2023-03-25 + + long startTs = 1672531200000L; // 2023-01-01T00:00Z + long endTs = 1680307200000L; // 2023-04-01T00:00Z + + ReadTsKvQueryResult count = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.COUNT); + // EXACTLY the two non-empty months at their calendar midpoints; Feb is absent. + assertLongBuckets(count, new long[] {1673870400000L, 1678968000000L}, new long[] {2L, 2L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + private ReadTsKvQueryResult calendarAggregate( + IoTDBTableTimeseriesDao dao, + TestScope scope, + String key, + long startTs, + long endTs, + Aggregation aggregation) + throws Exception { + ReadTsKvQuery query = + new BaseReadTsKvQuery( + key, + startTs, + endTs, + AggregationParams.calendar(aggregation, IntervalType.MONTH, "UTC"), + 100, + "ASC"); + return dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + } + + private ReadTsKvQueryResult aggregate( + IoTDBTableTimeseriesDao dao, TestScope scope, String key, Aggregation aggregation) + throws Exception { + ReadTsKvQuery query = + new BaseReadTsKvQuery(key, 1000L, 3000L, INTERVAL, 100, aggregation, "ASC"); + return dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + } + + private ReadTsKvQueryResult precisionAggregate( + IoTDBTableTimeseriesDao dao, TestScope scope, String key, Aggregation aggregation) + throws Exception { + ReadTsKvQuery query = + new BaseReadTsKvQuery(key, 1000L, 5000L, INTERVAL, 100, aggregation, "ASC"); + return dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + } + + private void assertDoubleBuckets( + ReadTsKvQueryResult result, + long[] expectedTs, + double[] expectedValues, + long expectedLastEntryTs) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(DataType.DOUBLE, entry.getDataType(), "data type at index " + i); + assertTrue(entry.getDoubleValue().isPresent(), "double value present at index " + i); + assertEquals( + expectedValues[i], entry.getDoubleValue().get(), 1e-9, "double value at index " + i); + } + // ThingsBoard reports lastEntryTs as MAX(underlying data ts), not a bucket midpoint. + assertEquals(expectedLastEntryTs, result.getLastEntryTs(), "lastEntryTs"); + } + + private void assertLongBuckets( + ReadTsKvQueryResult result, long[] expectedTs, long[] expectedValues) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(DataType.LONG, entry.getDataType(), "data type at index " + i); + assertEquals( + Optional.of(expectedValues[i]), entry.getLongValue(), "long value at index " + i); + } + } + + private void assertStringBuckets( + ReadTsKvQueryResult result, long[] expectedTs, String[] expectedValues) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(DataType.STRING, entry.getDataType(), "data type at index " + i); + assertEquals( + Optional.of(expectedValues[i]), entry.getStrValue(), "string value at index " + i); + } + } + + /** + * Asserts numeric buckets whose per-bucket result TYPE varies: a long-only SUM/MIN/MAX bucket + * comes back {@link DataType#LONG}, a bucket with any participating double comes back {@link + * DataType#DOUBLE}. {@code expectedTypes[i]} must be LONG or DOUBLE; the value is compared via + * the matching typed getter (exact for LONG, 1e-9 tolerance for DOUBLE). + */ + private void assertNumericBuckets( + ReadTsKvQueryResult result, + long[] expectedTs, + DataType[] expectedTypes, + double[] expectedValues, + long expectedLastEntryTs) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(expectedTypes[i], entry.getDataType(), "data type at index " + i); + if (expectedTypes[i] == DataType.LONG) { + assertEquals( + Optional.of((long) expectedValues[i]), + entry.getLongValue(), + "long value at index " + i); + } else { + assertTrue(entry.getDoubleValue().isPresent(), "double value present at index " + i); + assertEquals( + expectedValues[i], entry.getDoubleValue().get(), 1e-9, "double value at index " + i); + } + } + assertEquals(expectedLastEntryTs, result.getLastEntryTs(), "lastEntryTs"); + } + + /** + * Like {@link #assertNumericBuckets} but keeps the expected LONG values in a {@code long[]} so a + * value > 2^53 cannot be silently rounded by the test itself (the {@code double[]}-based helper + * would lose the low bit of {@code 9007199254740993L}). For a LONG bucket the value is compared + * EXACTLY via {@code getLongValue()}; for a DOUBLE bucket {@code expectedDoubleValues[i]} is + * compared with a 1e-9 tolerance. This is the assertion that proves the precision contract: under + * a COALESCE->DOUBLE round-trip (MIN/MAX) or a DOUBLE SUM accumulator the long results would come + * back as {@code ...992} and fail here. + */ + private void assertExactNumericBuckets( + ReadTsKvQueryResult result, + long[] expectedTs, + DataType[] expectedTypes, + long[] expectedLongValues, + double[] expectedDoubleValues, + long expectedLastEntryTs) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(expectedTypes[i], entry.getDataType(), "data type at index " + i); + if (expectedTypes[i] == DataType.LONG) { + assertTrue(entry.getLongValue().isPresent(), "long value present at index " + i); + assertEquals( + expectedLongValues[i], + entry.getLongValue().get().longValue(), + "exact long value at index " + i); + } else { + assertTrue(entry.getDoubleValue().isPresent(), "double value present at index " + i); + assertEquals( + expectedDoubleValues[i], + entry.getDoubleValue().get(), + 1e-9, + "double value at index " + i); + } + } + assertEquals(expectedLastEntryTs, result.getLastEntryTs(), "lastEntryTs"); + } + + private ITableSessionPool newPool(String database) { + TableSessionPoolBuilder builder = + new TableSessionPoolBuilder() + .nodeUrls(List.of("127.0.0.1:" + IOTDB.getMappedPort(6667))) + .user("root") + .password("root") + .maxSize(4); + if (database != null) { + builder.database(database); + } + return builder.build(); + } + + private void bootstrapSchema(String database) throws Exception { + awaitIoTDBReady(database); + + String schema; + try (InputStream stream = + IoTDBTableTimeseriesAggregationIT.class + .getClassLoader() + .getResourceAsStream("schema-iotdb-table.sql")) { + schema = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + schema = + schema + .replace( + "CREATE DATABASE IF NOT EXISTS thingsboard;", + "CREATE DATABASE IF NOT EXISTS " + database + ";") + .replace("USE thingsboard;", "USE " + database + ";"); + schema = schema.replaceAll("(?s)/\\*.*?\\*/", "").replaceAll("(?m)--.*$", ""); + try (ITableSessionPool bootstrapPool = newPool(null)) { + try (ITableSession session = bootstrapPool.getSession()) { + for (String statement : schema.split(";")) { + String trimmed = statement.trim(); + if (!trimmed.isEmpty()) { + session.executeNonQueryStatement(trimmed); + } + } + } + } + } + + private void awaitIoTDBReady(String database) throws Exception { + long deadlineNanos = System.nanoTime() + IOTDB_READY_TIMEOUT.toNanos(); + Exception lastFailure = null; + while (System.nanoTime() < deadlineNanos) { + try (ITableSessionPool bootstrapPool = newPool(null); + ITableSession session = bootstrapPool.getSession()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS " + database); + return; + } catch (Exception e) { + lastFailure = e; + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + if (remainingMillis <= 0) { + break; + } + Thread.sleep(Math.min(IOTDB_READY_POLL_INTERVAL.toMillis(), remainingMillis)); + } + } + throw new IllegalStateException( + "IoTDB did not accept table-session statements within " + IOTDB_READY_TIMEOUT, lastFailure); + } + + private IoTDBTableConfig config(int batchSize) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.getTs().getSave().setBatchSize(batchSize); + config.getTs().getSave().setMaxLingerMs(20L); + config.getTs().getSave().setRetryInitialBackoffMs(1L); + config.getTs().getSave().setRetryMaxBackoffMs(1L); + config.getTs().getRead().setThreads(1); + return config; + } + + private void saveAll(IoTDBTableTimeseriesDao dao, TestScope scope, List entries) + throws Exception { + List> futures = new ArrayList<>(); + for (TestTsKvEntry entry : entries) { + futures.add(dao.save(scope.tenantId(), scope.entityId(), entry, 0)); + } + for (ListenableFuture future : futures) { + assertEquals(1, future.get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + } + + private TestScope scope(String databasePrefix, String tenantId, String entityId) { + return new TestScope( + uniqueDatabase(databasePrefix), + new TenantId(UUID.fromString(tenantId)), + new TestEntityId(UUID.fromString(entityId), EntityType.DEVICE)); + } + + private String uniqueDatabase(String prefix) { + String shortPrefix = prefix.length() > 12 ? prefix.substring(0, 12) : prefix; + String shortUuid = UUID.randomUUID().toString().replace("-", "").substring(0, 16); + return "tb_it_" + shortPrefix + "_" + shortUuid; + } + + private TestTsKvEntry entry(long ts, String key, DataType dataType, Object value) { + return new TestTsKvEntry(ts, key, dataType, value); + } + + private record TestScope(String database, TenantId tenantId, EntityId entityId) {} + + private record TestEntityId(UUID id, EntityType entityType) implements EntityId { + @Override + public UUID getId() { + return id; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + } + + private record TestTsKvEntry(long ts, String key, DataType dataType, Object value) + implements TsKvEntry { + @Override + public long getTs() { + return ts; + } + + @Override + public String getKey() { + return key; + } + + @Override + public DataType getDataType() { + return dataType; + } + + @Override + public Optional getBooleanValue() { + return dataType == DataType.BOOLEAN ? Optional.of((Boolean) value) : Optional.empty(); + } + + @Override + public Optional getLongValue() { + return dataType == DataType.LONG ? Optional.of((Long) value) : Optional.empty(); + } + + @Override + public Optional getDoubleValue() { + return dataType == DataType.DOUBLE ? Optional.of((Double) value) : Optional.empty(); + } + + @Override + public Optional getStrValue() { + return dataType == DataType.STRING ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public Optional getJsonValue() { + return dataType == DataType.JSON ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public String getValueAsString() { + return String.valueOf(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public Long getVersion() { + return null; + } + + @Override + public int getDataPoints() { + return 1; + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java index 1f5dcb2..46e0aad 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; @@ -60,8 +61,8 @@ /** * Integration tests for the IoTDB Table Mode timeseries DAO against a real IoTDB 2.0.8 container: * the WRITE path (verified by reading the telemetry table back through raw table-session SQL) plus - * the RAW (non-aggregated) READ path and the DELETE path exercised through the DAO. The - * time-bucketed aggregation read path is not implemented and is not exercised here. + * the RAW (non-aggregated) READ path, a millisecond time-bucketed aggregation smoke read and the + * DELETE path exercised through the DAO. */ @Tag("integration") @Testcontainers(disabledWithoutDocker = true) @@ -497,6 +498,48 @@ void keyEscaping_roundTrip() throws Exception { } } + @Test + void aggregationCountReturnsBucketedResult() throws Exception { + TestScope scope = + scope( + "aggregation_count", + "33333333-3333-3333-3333-333333333310", + "44444444-4444-4444-4444-444444444410"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(2); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(9000L, "agg", DataType.LONG, 9L), entry(9200L, "agg", DataType.LONG, 11L))); + + ReadTsKvQuery query = + new BaseReadTsKvQuery("agg", 9000L, 10000L, 1000L, 10, Aggregation.COUNT, "ASC"); + ReadTsKvQueryResult result = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // startTs-anchored single bucket [9000,10000) -> ThingsBoard midpoint + // 9000 + (10000-9000)/2 = 9500. Typed COUNT of two long_v rows = 2. + assertEquals(1, result.getData().size()); + TsKvEntry bucket = result.getData().get(0); + assertEquals(9500L, bucket.getTs()); + assertEquals(DataType.LONG, bucket.getDataType()); + assertEquals(Optional.of(2L), bucket.getLongValue()); + // lastEntryTs = MAX(underlying time) = 9200. + assertEquals(9200L, result.getLastEntryTs()); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + private ITableSessionPool newPool(String database) { TableSessionPoolBuilder builder = new TableSessionPoolBuilder() diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java index 4a92165..9672f50 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java @@ -36,10 +36,16 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AggregationParams; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -81,8 +87,8 @@ /** * Unit tests for the IoTDB Table Mode timeseries DAO: the WRITE path (multi-row Tablet mapping, * batch flushing, connection retry, back-pressure rejection and graceful-shutdown drain) plus the - * RAW (non-aggregated) READ path, the DELETE path and the bounded read thread-pool. The - * time-bucketed aggregation read path is not implemented and is not exercised here. + * RAW (non-aggregated) READ path, the millisecond time-bucketed aggregation READ path, the DELETE + * path and the bounded read thread-pool. */ class IoTDBTableTimeseriesDaoTest { private static final TenantId TENANT_ID = @@ -1178,6 +1184,1063 @@ void saveReturnsFailedFutureAfterDestroy() throws Exception { verify(context.session(), never()).insert(any(Tablet.class)); } + @Test + void findAllAsync_calendarSumKeepsLongTypeForLongOnlyBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // On the calendar per-bucket path a long-only calendar bucket keeps the LONG SUM type, a mixed + // calendar bucket promotes to DOUBLE. WEEK_ISO from startTs=0 yields two buckets. + SessionDataSet week0 = aggDataSet(MockAggBucket.sum(0L, 100000000L, 30.0D, null, 2L, 0L)); + SessionDataSet week1 = aggDataSet(MockAggBucket.sum(0L, 600000000L, 4.0D, 2.5D, 1L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(week0, week1); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 950400000L, IntervalType.WEEK_ISO, "UTC", Aggregation.SUM); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertMappedEntry(data.get(0), 172800000L, "k", DataType.LONG, 30L); + assertInstanceOf(DoubleDataEntry.class, innerKv(data.get(1))); + assertMappedEntry(data.get(1), 648000000L, "k", DataType.DOUBLE, 6.5D); + } + + @Test + void findAllAsync_calendarMonthBuildsBoundedPerBucketSqlWithoutDateBin() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0 (1970-01-01T00:00Z), UTC MONTH buckets: [0,Feb1), [Feb1,Mar1), [Mar1,Apr1). + // endTs=Apr1 => endPeriod=Apr1. Three calendar buckets => THREE bounded aggregate queries, + // each with NO date_bin / NO GROUP BY, bounded by the calendar boundary, MAX(time) projected. + SessionDataSet bucket0 = aggDataSet(); + SessionDataSet bucket1 = aggDataSet(); + SessionDataSet bucket2 = aggDataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(bucket0, bucket1, bucket2); + + ReadTsKvQuery query = + calendarQuery("temperature", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(3)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + assertEquals( + "SELECT AVG(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num, " + + "MAX(time) AS max_ts FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND entity_type='DEVICE' " + + "AND entity_id='22222222-2222-2222-2222-222222222222' " + + "AND key='temperature' AND time >= 0 AND time < 2678400000", + statements.get(0)); + assertTrue( + statements.get(1).contains("AND time >= 2678400000 AND time < 5097600000"), + statements.get(1)); + assertTrue( + statements.get(2).contains("AND time >= 5097600000 AND time < 7776000000"), + statements.get(2)); + for (String statement : statements) { + // Calendar buckets are computed in Java; the per-bucket SQL must never use date_bin/GROUP BY. + assertFalse(statement.contains("date_bin"), statement); + assertFalse(statement.contains("GROUP BY"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + } + + @Test + void findAllAsync_calendarMonthMapsBucketsToCalendarMidpointEntries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // UTC MONTH from startTs=0: bucket midpoints Jan-mid=1339200000, Feb-mid=3888000000. + // The Mar bucket is empty (single-row dataset with NULL agg_num) and must be skipped. + // ThingsBoard stamps each entry at the integer calendar-bucket midpoint and reports + // lastEntryTs = MAX(underlying ts) across all buckets. + SessionDataSet janBucket = aggDataSet(numericBucket(0L, 2000000000L, 11.5D)); + SessionDataSet febBucket = aggDataSet(numericBucket(0L, 4000000000L, 22.0D)); + SessionDataSet marBucket = aggDataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(janBucket, febBucket, marBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.AVG); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.DOUBLE, 11.5D); + assertMappedEntry(data.get(1), 3888000000L, "k", DataType.DOUBLE, 22.0D); + assertEquals(4000000000L, result.getLastEntryTs()); + } + + @Test + void findAllAsync_calendarMonthFirstBucketIsPartialFromMidMonthStart() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=Jan15 1970 (1209600000) is NOT a month boundary. ThingsBoard's calculateIntervalEnd + // advances to the START of the next month (Feb1), so the FIRST bucket is the partial + // [Jan15, Feb1) with midpoint 1944000000 (NOT a 30-day fixed-width step). The second bucket is + // the full calendar month [Feb1, Mar1) midpoint 3888000000. + // SUM over double-valued data: each bucket carries a double partial sum (DoubleDataEntry). + SessionDataSet partialBucket = + aggDataSet(MockAggBucket.sum(0L, 1500000000L, null, 1.0D, 0L, 1L)); + SessionDataSet fullBucket = aggDataSet(MockAggBucket.sum(0L, 4000000000L, null, 2.0D, 0L, 1L)); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(partialBucket, fullBucket); + + ReadTsKvQuery query = + calendarQuery("k", 1209600000L, 5097600000L, IntervalType.MONTH, "UTC", Aggregation.SUM); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)).executeQueryStatement(sql.capture()); + assertTrue( + sql.getAllValues().get(0).contains("AND time >= 1209600000 AND time < 2678400000"), + sql.getAllValues().get(0)); + assertTrue( + sql.getAllValues().get(1).contains("AND time >= 2678400000 AND time < 5097600000"), + sql.getAllValues().get(1)); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1944000000L, "k", DataType.DOUBLE, 1.0D); + assertMappedEntry(data.get(1), 3888000000L, "k", DataType.DOUBLE, 2.0D); + } + + @Test + void findAllAsync_calendarWeekUsesSundayAlignedBoundaries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // 1970-01-01 is a Thursday. WEEK (Sunday-start) from startTs=0: the first (partial) bucket runs + // to the next Sunday 1970-01-04 (259200000), then full 7-day weeks. Midpoints 129600000 / + // 561600000. + SessionDataSet week0 = aggDataSet(countBucket(0L, 3L)); + SessionDataSet week1 = aggDataSet(countBucket(0L, 2L)); + SessionDataSet week2 = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(week0, week1, week2); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 1468800000L, IntervalType.WEEK, "UTC", Aggregation.COUNT); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(3)).executeQueryStatement(sql.capture()); + assertTrue( + sql.getAllValues().get(0).contains("AND time >= 0 AND time < 259200000"), + sql.getAllValues().get(0)); + assertTrue( + sql.getAllValues().get(1).contains("AND time >= 259200000 AND time < 864000000"), + sql.getAllValues().get(1)); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 129600000L, "k", DataType.LONG, 3L); + assertMappedEntry(data.get(1), 561600000L, "k", DataType.LONG, 2L); + } + + @Test + void findAllAsync_calendarCountAppliesDominantTypePriority() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // The calendar path reuses the typed-COUNT dominant-column priority (bool > str > json > + // long+double). Bucket 0 string wins (countStr=2 despite numeric); bucket 1 numeric only. + SessionDataSet strDominant = + aggDataSet(MockAggBucket.typedCount(0L, 2000000000L, 0L, 2L, 0L, 9L, 9L)); + SessionDataSet numericOnly = + aggDataSet(MockAggBucket.typedCount(0L, 4000000000L, 0L, 0L, 0L, 4L, 1L)); + SessionDataSet emptyBucket = aggDataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(strDominant, numericOnly, emptyBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.COUNT); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.LONG, 2L); + assertMappedEntry(data.get(1), 3888000000L, "k", DataType.LONG, 5L); + } + + @Test + void findAllAsync_calendarEmptyResultFallsBackToStartTsAndIgnoresLimitOrder() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // No data in any calendar bucket: lastEntryTs falls back to startTs and order/limit are ignored + // (a DESC + LIMIT 0 calendar query still walks and queries every bucket). + SessionDataSet empty0 = aggDataSet(); + SessionDataSet empty1 = aggDataSet(); + SessionDataSet empty2 = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(empty0, empty1, empty2); + + ReadTsKvQuery query = + new BaseReadTsKvQuery( + "k", + 0L, + 7776000000L, + AggregationParams.calendar(Aggregation.AVG, IntervalType.MONTH, "UTC"), + 0, + "DESC"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + assertEquals(List.of(), result.getData()); + assertEquals(0L, result.getLastEntryTs()); + // One bounded query per calendar bucket (3): the zero limit did not short-circuit. + verify(context.session(), times(3)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_millisecondsIntervalStillRoutesToDateBinPath() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet groupedDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(groupedDataSet); + + // An explicit MILLISECONDS interval type must keep the single grouped date_bin SQL (one query, + // GROUP BY, ORDER BY 1 ASC) and must NOT be routed to the per-bucket calendar path. + ReadTsKvQuery query = calendarLikeMilliseconds("k", 0L, 100L, 25L, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(1)).executeQueryStatement(sql.capture()); + String statement = sql.getValue(); + assertTrue(statement.contains("date_bin(25ms, time, 0) AS bucket_ts"), statement); + assertTrue(statement.endsWith("GROUP BY 1 ORDER BY 1 ASC"), statement); + } + + @Test + void findAllAsync_calendarCountSkipsRealShapedEmptyMiddleBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // UTC MONTH over [0, Apr1=7776000000): three calendar buckets Jan/Feb/Mar. The MIDDLE month + // (Feb) is the REAL empty shape IoTDB returns for an empty bounded window: ONE row with every + // typed COUNT = 0 and MAX(time) NULL (emptyAggRow). The other two months have data. The + // calendar reader's `!isNull(max_ts)` guard skips the empty middle bucket, so the spurious + // COUNT=0 LongDataEntry is NOT emitted. Without that guard COUNT would leak a third entry + // (count 0) at the Feb midpoint -> 3 entries instead of 2. + SessionDataSet janBucket = aggDataSet(countBucket(0L, 3L)); + SessionDataSet febEmpty = aggDataSet(emptyAggRow(2678400000L)); + SessionDataSet marBucket = aggDataSet(countBucket(5097600000L, 5L)); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(janBucket, febEmpty, marBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.COUNT); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + // EXACTLY two entries: the empty Feb bucket is SKIPPED, never emitted as count 0. + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.LONG, 3L); // Jan midpoint + assertMappedEntry(data.get(1), 6436800000L, "k", DataType.LONG, 5L); // Mar midpoint + // All three calendar buckets were queried (the empty one was queried but dropped in Java). + verify(context.session(), times(3)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_calendarAvgSkipsRealShapedEmptyMiddleBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Regression guard that the universal empty-bucket skip did not break the AVG path: the REAL + // empty middle bucket here has NULL agg_num AND NULL max_ts (emptyAggRow). The reader skips it + // on the `!isNull(max_ts)` guard exactly as it does for COUNT, so AVG returns the two non-empty + // months only. + SessionDataSet janBucket = aggDataSet(numericBucket(0L, 2000000000L, 11.5D)); + SessionDataSet febEmpty = aggDataSet(emptyAggRow(2678400000L)); + SessionDataSet marBucket = aggDataSet(numericBucket(5097600000L, 6000000000L, 22.0D)); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(janBucket, febEmpty, marBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.AVG); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.DOUBLE, 11.5D); // Jan midpoint + assertMappedEntry(data.get(1), 6436800000L, "k", DataType.DOUBLE, 22.0D); // Mar midpoint + // lastEntryTs = MAX(underlying ts) across non-empty buckets; the empty Feb row never updates + // it. + assertEquals(6000000000L, result.getLastEntryTs()); + verify(context.session(), times(3)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_millisecondsFactoryRoutesToDateBinPath() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet groupedDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(groupedDataSet); + + // Real-TB MILLISECONDS routing: a query built the way ThingsBoard actually builds a fixed-width + // aggregation (AggregationParams.milliseconds carries IntervalType.MILLISECONDS + a positive + // interval) routes to the date_bin MILLISECONDS path. A null-IntervalType non-NONE aggregation + // is NOT a real-TB scenario (real TB always pairs a non-NONE aggregation with a concrete + // IntervalType), and getInterval() returns 0L for a null type matching real TB v4.3.1.2, so the + // MS path's interval<=0 guard would (correctly) reject it; this test therefore exercises the + // REAL milliseconds() factory instead. + AggregationParams msParams = AggregationParams.milliseconds(Aggregation.AVG, 25); + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 100L, msParams, 10); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(1)).executeQueryStatement(sql.capture()); + String statement = sql.getValue(); + // MILLISECONDS path with the 25ms interval, anchored at startTs=0. + assertTrue(statement.contains("date_bin(25ms, time, 0) AS bucket_ts"), statement); + assertTrue(statement.endsWith("GROUP BY 1 ORDER BY 1 ASC"), statement); + } + + @Test + void findAllAsync_calendarWeekIsoRoutesToBoundedPerBucketSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // 1970-01-01 is a Thursday. WEEK_ISO (Monday-start) from startTs=0: first ISO boundary is + // Monday + // 1970-01-05 (345600000), then the full ISO week to 1970-01-12 (950400000). Two buckets => + // TWO bounded aggregate queries, each with NO date_bin / NO GROUP BY, bounded by the ISO + // boundaries from TimeUtils.calculateIntervalEnd. Midpoints 172800000 / 648000000. + SessionDataSet week0 = aggDataSet(countBucket(0L, 4L)); + SessionDataSet week1 = aggDataSet(countBucket(0L, 2L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(week0, week1); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 950400000L, IntervalType.WEEK_ISO, "UTC", Aggregation.COUNT); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + assertTrue(statements.get(0).contains("AND time >= 0 AND time < 345600000"), statements.get(0)); + assertTrue( + statements.get(1).contains("AND time >= 345600000 AND time < 950400000"), + statements.get(1)); + for (String statement : statements) { + assertFalse(statement.contains("date_bin"), statement); + assertFalse(statement.contains("GROUP BY"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 172800000L, "k", DataType.LONG, 4L); + assertMappedEntry(data.get(1), 648000000L, "k", DataType.LONG, 2L); + } + + @Test + void findAllAsync_calendarQuarterRoutesToBoundedPerBucketSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // QUARTER from startTs=0 (1970-01-01 = Q1 start): next quarter boundary is 1970-04-01 + // (7776000000), then 1970-07-01 (15638400000). Two buckets => TWO bounded aggregate queries, + // each with NO date_bin / NO GROUP BY, bounded by the quarter boundaries from + // TimeUtils.calculateIntervalEnd. Midpoints 3888000000 / 11707200000. + // SUM over double-valued data: each quarter carries a double partial sum (DoubleDataEntry). + SessionDataSet q0 = aggDataSet(MockAggBucket.sum(0L, 1000000000L, null, 10.0D, 0L, 1L)); + SessionDataSet q1 = aggDataSet(MockAggBucket.sum(0L, 9000000000L, null, 20.0D, 0L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(q0, q1); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 15638400000L, IntervalType.QUARTER, "UTC", Aggregation.SUM); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + assertTrue( + statements.get(0).contains("AND time >= 0 AND time < 7776000000"), statements.get(0)); + assertTrue( + statements.get(1).contains("AND time >= 7776000000 AND time < 15638400000"), + statements.get(1)); + for (String statement : statements) { + assertFalse(statement.contains("date_bin"), statement); + assertFalse(statement.contains("GROUP BY"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 3888000000L, "k", DataType.DOUBLE, 10.0D); + assertMappedEntry(data.get(1), 11707200000L, "k", DataType.DOUBLE, 20.0D); + } + + @Test + void findAllAsync_avgBuildsStartTsAnchoredBucketedSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + // DESC order + LIMIT 10 are intentionally ignored for aggregation (ThingsBoard contract): + // buckets are anchored at startTs=0, MAX(time) is projected for lastEntryTs, and the SQL is + // ordered ascending with no LIMIT. + ReadTsKvQuery query = new BaseReadTsKvQuery("temperature", 0L, 100L, 25L, 10, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertEquals( + "SELECT date_bin(25ms, time, 0) AS bucket_ts, " + + "AVG(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num, " + + "MAX(time) AS max_ts FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND entity_type='DEVICE' " + + "AND entity_id='22222222-2222-2222-2222-222222222222' " + + "AND key='temperature' AND time >= 0 AND time < 100 " + + "GROUP BY 1 ORDER BY 1 ASC", + sql.getValue()); + } + + @Test + void findAllAsync_aggregationZeroWidthRangeStillWalksOneBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + // ThingsBoard clamps endPeriod = max(startTs + 1, endTs); a zero-width [50, 50] aggregation + // query + // must still scan [50, 51) so a point at startTs is included rather than dropped by time < 50. + ReadTsKvQuery query = new BaseReadTsKvQuery("temperature", 50L, 50L, 25L, 10, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertTrue(sql.getValue().contains("AND time >= 50 AND time < 51 "), sql.getValue()); + } + + @Test + void findAllAsync_sumCountMinMaxBuildSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet sumCountMinMaxDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(sumCountMinMaxDataSet); + + ReadTsKvQuery sum = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.SUM, "ASC"); + ReadTsKvQuery count = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.COUNT, "ASC"); + ReadTsKvQuery min = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery max = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MAX, "ASC"); + + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(sum, count, min, max)) + .get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(4)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + // SUM projects per-type partial sums (the long partial as SUM(CAST(long_v AS DOUBLE)) -- a + // DOUBLE, NEVER CAST(SUM AS INT64) which would THROW an out-of-range error when the long-only + // sum + // exceeds Long.MAX) + long/double non-null counts so the row mapper can keep a long-only SUM + // LONG-typed and promote a mixed bucket to DOUBLE. It also projects MIN(long_v)/MAX(long_v) so + // the mapper can bound the long sum and trust the double partial only while count_long + // * maxAbs <= 2^53, falling back to an exact Java re-sum otherwise. + assertTrue( + statements.get(0).contains("SUM(CAST(long_v AS DOUBLE)) AS sum_long") + && statements.get(0).contains("SUM(double_v) AS sum_double") + && statements.get(0).contains("MIN(long_v) AS min_long") + && statements.get(0).contains("MAX(long_v) AS max_long") + && statements + .get(0) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(0) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(0)); + assertFalse( + statements.get(0).contains("SUM(COALESCE(double_v, CAST(long_v AS DOUBLE)))"), + statements.get(0)); + // COUNT projects ThingsBoard's per-type non-null counters (not COUNT(*)). + assertTrue( + statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN bool_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_bool") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN str_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_str") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN json_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_json") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(1)); + assertFalse(statements.get(1).contains("COUNT(*)"), statements.get(1)); + // MIN/MAX project the numeric + string aggregates AND a direct MIN(long_v)/MAX(long_v) channel + // AND the long/double non-null counts. The long channel is EXACT for a long-only bucket (it + // SELECTs a stored long, no double round-trip); the counts let the row mapper keep a + // long-only MIN/MAX LONG-typed and promote a mixed bucket to DOUBLE. + assertTrue( + statements.get(2).contains("MIN(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num") + && statements.get(2).contains("MIN(long_v) AS min_long") + && statements.get(2).contains("MIN(str_v) AS agg_str") + && statements + .get(2) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(2) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(2)); + assertTrue( + statements.get(3).contains("MAX(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num") + && statements.get(3).contains("MAX(long_v) AS max_long") + && statements.get(3).contains("MAX(str_v) AS agg_str") + && statements + .get(3) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(3) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(3)); + for (String statement : statements) { + // startTs=0 anchored buckets, MAX(time) for lastEntryTs, ascending, no LIMIT (order/limit + // ignored for aggregation). + assertTrue(statement.contains("date_bin(30ms, time, 0) AS bucket_ts"), statement); + assertTrue(statement.contains("MAX(time) AS max_ts"), statement); + assertTrue(statement.endsWith("GROUP BY 1 ORDER BY 1 ASC"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + } + + @Test + void findAllAsync_avgMapsNumericBucketsToMidpointDoubleEntries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0, interval=30, endTs=90 -> bucket starts 0/30/60, all full width. + // TB midpoints: [0,30)->15, [30,60)->45, [60,90)->75. lastEntryTs = MAX(time) = 80. + SessionDataSet dataSet = + aggDataSet( + numericBucket(0L, 25L, 11.5D), + numericBucket(30L, 55L, 30.0D), + numericBucket(60L, 80L, 7.25D)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 90L, 30L, 10, Aggregation.AVG, "ASC"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + assertEquals(3, data.size()); + assertMappedEntry(data.get(0), 15L, "k", DataType.DOUBLE, 11.5D); + assertMappedEntry(data.get(1), 45L, "k", DataType.DOUBLE, 30.0D); + assertMappedEntry(data.get(2), 75L, "k", DataType.DOUBLE, 7.25D); + assertEquals(80L, result.getLastEntryTs()); + } + + @Test + void findAllAsync_lastBucketMidpointIsClampedToEndTs() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0, interval=30, endTs=50 -> endPeriod = max(1, 50) = 50. + // Bucket [0,30): full width -> midpoint 0 + (30-0)/2 = 15. + // Bucket [30,50): END-CLAMPED -> bucketEnd = min(30+30, 50) = 50, + // midpoint 30 + (50-30)/2 = 40 (NOT the full-width 45). + SessionDataSet dataSet = + aggDataSet(numericBucket(0L, 20L, 1.0D), numericBucket(30L, 45L, 2.0D)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 50L, 30L, 10, Aggregation.AVG, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 15L, "k", DataType.DOUBLE, 1.0D); + assertMappedEntry(data.get(1), 40L, "k", DataType.DOUBLE, 2.0D); + } + + @Test + void findAllAsync_countMapsTypedCountToMidpointLongEntries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0, interval=30, endTs=60 -> bucket starts 0/30 -> TB midpoints 15/45. + // Each single-typed (long_v) bucket's typed count equals its row count. + SessionDataSet dataSet = aggDataSet(countBucket(0L, 3L), countBucket(30L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.COUNT, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 3L); + assertMappedEntry(data.get(1), 45L, "k", DataType.LONG, 1L); + } + + @Test + void findAllAsync_countAppliesThingsBoardDominantTypePriority() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // ThingsBoard reports the FIRST non-zero typed counter in priority order: + // boolean -> string -> json -> (long + double). + // Bucket 0: only boolean populated (countBool=4) despite numeric counters set -> boolean wins. + // Bucket 1: no boolean, string populated (countStr=2) despite json/long/double -> string wins. + // Bucket 2: no bool/str, json populated (countJson=5) despite long/double -> json wins. + // Bucket 3: only numeric populated (long=2, double=3) -> long+double=5. + SessionDataSet dataSet = + aggDataSet( + MockAggBucket.typedCount(0L, 0L, 4L, 9L, 9L, 9L, 9L), + MockAggBucket.typedCount(30L, 30L, 0L, 2L, 9L, 9L, 9L), + MockAggBucket.typedCount(60L, 60L, 0L, 0L, 5L, 9L, 9L), + MockAggBucket.typedCount(90L, 90L, 0L, 0L, 0L, 2L, 3L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 120L, 30L, 10, Aggregation.COUNT, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(4, data.size()); + // startTs=0, interval=30, endTs=120 -> midpoints 15/45/75/105. + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 4L); + assertMappedEntry(data.get(1), 45L, "k", DataType.LONG, 2L); + assertMappedEntry(data.get(2), 75L, "k", DataType.LONG, 5L); + assertMappedEntry(data.get(3), 105L, "k", DataType.LONG, 5L); + } + + @Test + void findAllAsync_minMaxPickNumericOrStringPerBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet numericDataSet = aggDataSet(numericBucket(0L, 5.5D), numericBucket(30L, 30.0D)); + SessionDataSet stringDataSet = + aggDataSet(stringBucket(0L, "apple"), stringBucket(30L, "cherry")); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(numericDataSet, stringDataSet); + + ReadTsKvQuery numeric = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery string = new BaseReadTsKvQuery("sk", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(numeric, string)) + .get(3, TimeUnit.SECONDS); + + // startTs=0, interval=30, endTs=60 -> bucket starts 0/30 -> TB midpoints 15/45. + List numericData = results.get(0).getData(); + assertEquals(2, numericData.size()); + assertMappedEntry(numericData.get(0), 15L, "k", DataType.DOUBLE, 5.5D); + assertMappedEntry(numericData.get(1), 45L, "k", DataType.DOUBLE, 30.0D); + + List stringData = results.get(1).getData(); + assertEquals(2, stringData.size()); + assertMappedEntry(stringData.get(0), 15L, "sk", DataType.STRING, "apple"); + assertMappedEntry(stringData.get(1), 45L, "sk", DataType.STRING, "cherry"); + } + + @Test + void findAllAsync_sumKeepsLongTypeForLongOnlyBucketAndDoubleForMixed() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // ThingsBoard 4.3.1.2 returns a LONG-typed SUM when only long values participate in a + // bucket and a DOUBLE only when a double participates. + // Bucket [0,30): long-only -> sum_long=5, no doubles -> LongDataEntry(5) + // Bucket [30,60): mixed -> sum_long=4, sum_double=1.5, 1 double row -> + // DoubleDataEntry(5.5) + SessionDataSet dataSet = + aggDataSet( + MockAggBucket.sum(0L, 20L, 5.0D, null, 2L, 0L), + MockAggBucket.sum(30L, 55L, 4.0D, 1.5D, 1L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + // Long-only SUM keeps the LONG type. + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertEquals(DataType.LONG, data.get(0).getDataType()); + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 5L); + // Mixed bucket promotes to DOUBLE, summing the long and double partials (4 + 1.5 = 5.5). + assertInstanceOf(DoubleDataEntry.class, innerKv(data.get(1))); + assertEquals(DataType.DOUBLE, data.get(1).getDataType()); + assertMappedEntry(data.get(1), 45L, "k", DataType.DOUBLE, 5.5D); + } + + @Test + void findAllAsync_minMaxKeepLongTypeForLongOnlyBucketAndDoubleForMixed() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Long-only MIN/MAX keep the LONG type; a bucket with any participating double stays + // DOUBLE with the SAME numeric value as before. + // Bucket [0,30): long-only (countLong=3, countDouble=0) -> LongDataEntry((long) agg_num) + // Bucket [30,60): mixed (countLong=2, countDouble=1) -> DoubleDataEntry(agg_num) + SessionDataSet minDataSet = + aggDataSet( + MockAggBucket.typedNumeric(0L, 20L, 5.0D, 3L, 0L), + MockAggBucket.typedNumeric(30L, 55L, 7.5D, 2L, 1L)); + SessionDataSet maxDataSet = + aggDataSet( + MockAggBucket.typedNumeric(0L, 20L, 9.0D, 3L, 0L), + MockAggBucket.typedNumeric(30L, 55L, 12.5D, 2L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(minDataSet, maxDataSet); + + ReadTsKvQuery min = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery max = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MAX, "ASC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(min, max)) + .get(3, TimeUnit.SECONDS); + + List minData = results.get(0).getData(); + assertEquals(2, minData.size()); + assertInstanceOf(LongDataEntry.class, innerKv(minData.get(0))); + assertMappedEntry(minData.get(0), 15L, "k", DataType.LONG, 5L); + assertInstanceOf(DoubleDataEntry.class, innerKv(minData.get(1))); + assertMappedEntry(minData.get(1), 45L, "k", DataType.DOUBLE, 7.5D); + + List maxData = results.get(1).getData(); + assertEquals(2, maxData.size()); + assertInstanceOf(LongDataEntry.class, innerKv(maxData.get(0))); + assertMappedEntry(maxData.get(0), 15L, "k", DataType.LONG, 9L); + assertInstanceOf(DoubleDataEntry.class, innerKv(maxData.get(1))); + assertMappedEntry(maxData.get(1), 45L, "k", DataType.DOUBLE, 12.5D); + } + + @Test + void findAllAsync_minMaxLongOnlyReadExactLongChannelAbove2Pow53() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // A long-only MIN/MAX must come back EXACT even above 2^53. The stored long is + // 9007199254740993 (= 2^53 + 1), which is NOT representable as a double: COALESCE(double_v, + // CAST(long_v AS DOUBLE)) would round it to 9007199254740992.0 (agg_num below). The DAO must + // read the direct MIN(long_v)/MAX(long_v) channel instead, yielding the exact long. The + // (long) getDouble(agg_num) round-trip would return ...992 and FAIL this test. + long exact = 9007199254740993L; // 2^53 + 1 + double roundedAggNum = 9007199254740992.0D; // what the COALESCE->DOUBLE path would expose + SessionDataSet minDataSet = + aggDataSet(MockAggBucket.minMaxLong(0L, 20L, roundedAggNum, exact, 1L)); + SessionDataSet maxDataSet = + aggDataSet(MockAggBucket.minMaxLong(0L, 20L, roundedAggNum, exact, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(minDataSet, maxDataSet); + + ReadTsKvQuery min = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery max = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.MAX, "ASC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(min, max)) + .get(3, TimeUnit.SECONDS); + + TsKvEntry minEntry = results.get(0).getData().get(0); + assertInstanceOf(LongDataEntry.class, innerKv(minEntry)); + assertEquals(DataType.LONG, minEntry.getDataType()); + assertEquals( + Optional.of(exact), minEntry.getLongValue(), "MIN must be the exact long, not ...992"); + TsKvEntry maxEntry = results.get(1).getData().get(0); + assertInstanceOf(LongDataEntry.class, innerKv(maxEntry)); + assertEquals(DataType.LONG, maxEntry.getDataType()); + assertEquals( + Optional.of(exact), maxEntry.getLongValue(), "MAX must be the exact long, not ...992"); + } + + @Test + void findAllAsync_sumLongOnlyFastPathWhenBoundWithin2Pow53() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Fast path: a long-only bucket whose conservative bound count_long * maxAbs stays + // within 2^53 is provably exact, so the DAO trusts the DOUBLE-projected sum_long (cast back to + // long, lossless within the bound) and does NOT re-query. count_long=3, maxAbs=40 -> 120 <= + // 2^53, + // fast path. Exactly ONE aggregate query. + SessionDataSet dataSet = aggDataSet(MockAggBucket.sumWithBound(0L, 20L, 90.0D, 3L, 20L, 40L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 90L); + // Fast path: no raw long_v re-query (exactly one aggregate statement was issued). + verify(context.session(), times(1)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_sumLongOnlyReSumsExactlyWhenBoundExceeds2Pow53() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Fallback: a long-only bucket whose bound count_long * maxAbs MAY exceed 2^53 cannot + // trust the DOUBLE-accumulated SUM, so the DAO re-queries the raw long_v values and sums them + // in + // Java as long. Here the two stored values are 9007199254740993 (2^53 + 1) and 1000; their + // EXACT + // long sum is 9007199254741993. IoTDB's double accumulator would have returned 9007199254741992 + // (the rounded sum_long we deliberately mock), so trusting it would FAIL. maxAbs = + // 9007199254740993 + // and count_long = 2, so count_long > 2^53 / maxAbs -> the bound check forces the fallback. + long bigValue = 9007199254740993L; // 2^53 + 1 + long exactSum = 9007199254741993L; // bigValue + 1000, exact long arithmetic + long roundedDoubleSum = + 9007199254741992L; // what IoTDB's DOUBLE accumulator would have produced + SessionDataSet aggDataSet = + aggDataSet(MockAggBucket.sumWithBound(0L, 20L, roundedDoubleSum, 2L, 1000L, bigValue)); + SessionDataSet rawLong = rawLongDataSet(bigValue, 1000L); + when(context.session().executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + String sql = invocation.getArgument(0); + // The aggregate uses GROUP BY; the exact re-sum selects the raw long_v column. + return sql.contains("GROUP BY") ? aggDataSet : rawLong; + }); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertEquals(DataType.LONG, data.get(0).getDataType()); + // EXACT Java re-sum, not the rounded double sum the accumulator would have produced. + assertEquals( + Optional.of(exactSum), + data.get(0).getLongValue(), + "long-only SUM > 2^53 must be the exact Java re-sum, not the rounded double sum"); + // The fallback issued the raw long_v re-query in the same bucket window. + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), times(2)).executeQueryStatement(sql.capture()); + String reQuery = + sql.getAllValues().stream().filter(s -> !s.contains("GROUP BY")).findFirst().orElseThrow(); + assertTrue(reQuery.contains("SELECT long_v FROM telemetry"), reQuery); + assertTrue(reQuery.contains("long_v IS NOT NULL"), reQuery); + // The re-query window is the FULL date_bin bucket [startTs, startTs + interval) = [0, 30). + assertTrue(reQuery.contains("time >= 0") && reQuery.contains("time < 30"), reQuery); + } + + @Test + void findAllAsync_sumFallbackClampsFinalBucketReQueryToEndPeriodNotFullWidth() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Regression for a final, end-clamped MS bucket that triggers the SUM re-sum fallback: the + // re-query MUST cover exactly the rows the date_bin aggregate counted ([bucketStart, + // endPeriod)), + // not the full physical bucket [bucketStart, bucketStart + interval). Query [0, 2500) interval + // 1000 -> the last bucket date_bin=2000 is clamped to endPeriod=2500; a full-width [2000, 3000) + // re-query would wrongly include rows beyond endTs and over-count the sum. + long bigValue = 9007199254740993L; // 2^53 + 1 -> forces the fallback + long exactSum = 9007199254741993L; // bigValue + 1000 + long roundedDoubleSum = 9007199254741992L; + SessionDataSet aggDataSet = + aggDataSet(MockAggBucket.sumWithBound(2000L, 2400L, roundedDoubleSum, 2L, 1000L, bigValue)); + SessionDataSet rawLong = rawLongDataSet(bigValue, 1000L); + when(context.session().executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + String sql = invocation.getArgument(0); + return sql.contains("GROUP BY") ? aggDataSet : rawLong; + }); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 2500L, 1000L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertEquals(Optional.of(exactSum), data.get(0).getLongValue()); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), times(2)).executeQueryStatement(sql.capture()); + String reQuery = + sql.getAllValues().stream().filter(s -> !s.contains("GROUP BY")).findFirst().orElseThrow(); + // CLAMPED to endPeriod (2500), matching the aggregate's `time < endPeriod` filter -- NOT the + // full-width bucketStart + interval (3000). + assertTrue(reQuery.contains("time >= 2000") && reQuery.contains("time < 2500"), reQuery); + assertFalse(reQuery.contains("time < 3000"), reQuery); + } + + @Test + void findAllAsync_avgStaysDoubleForLongOnlyBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // AVG is ALWAYS DOUBLE in ThingsBoard, even for a long-only bucket. + SessionDataSet dataSet = aggDataSet(MockAggBucket.typedNumeric(0L, 20L, 7.0D, 3L, 0L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.AVG, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertInstanceOf(DoubleDataEntry.class, innerKv(data.get(0))); + assertMappedEntry(data.get(0), 15L, "k", DataType.DOUBLE, 7.0D); + } + + @Test + void findAllAsync_aggregationSkipsEmptyBucketsAndIgnoresLimit() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + ReadTsKvQuery noBuckets = new BaseReadTsKvQuery("k", 100L, 400L, 100L, 10, Aggregation.AVG); + ReadTsKvQueryResult emptyResult = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(noBuckets)) + .get(3, TimeUnit.SECONDS) + .get(0); + assertEquals(List.of(), emptyResult.getData()); + // No data matched -> lastEntryTs falls back to startTs (ThingsBoard contract). + assertEquals(100L, emptyResult.getLastEntryTs()); + + // ThingsBoard ignores the query limit for aggregation: a zero limit must NOT short-circuit; + // the aggregate is still issued and returns every non-empty bucket (here: none). + ReadTsKvQuery zeroLimit = new BaseReadTsKvQuery("k", 100L, 400L, 100L, 0, Aggregation.AVG); + ReadTsKvQueryResult zeroLimitResult = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(zeroLimit)) + .get(3, TimeUnit.SECONDS) + .get(0); + assertEquals(List.of(), zeroLimitResult.getData()); + assertEquals(100L, zeroLimitResult.getLastEntryTs()); + // Both queries issue SQL: limit is not consulted for aggregation. + verify(context.session(), times(2)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_aggregationWithSubOneIntervalRoutesToRawLikeThingsBoard() throws Exception { + // ThingsBoard 4.3.1.2 (AbstractChunkedAggregationTimeseriesDao.findAllAsync) routes to the RAW + // findAllWithLimit path when aggregation == NONE OR interval < 1. An AVG query with interval 0 + // must + // therefore return RAW telemetry (a plain typed-column SELECT with ORDER BY time + LIMIT), NOT + // be + // rejected and NOT build a date_bin aggregation. + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet rawDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(rawDataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 100L, 0L, 10, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + String captured = sql.getValue(); + assertTrue( + captured.contains("SELECT time, bool_v, long_v, double_v, str_v, json_v FROM telemetry"), + captured); + assertTrue(captured.contains("LIMIT 10"), captured); + assertTrue(!captured.contains("date_bin"), captured); + } + + @Test + void findAllAsync_aggregationEscapesKeyAndIgnoresQueryOrder() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet escapeDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(escapeDataSet); + + ReadTsKvQuery escaped = new BaseReadTsKvQuery("a'b", 1L, 10L, 5L, 10, Aggregation.SUM, "asc"); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(escaped)).get(3, TimeUnit.SECONDS); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertTrue(sql.getValue().contains("key='a''b'")); + + // ThingsBoard ignores the query order for aggregation: an unrecognised order string is NOT + // rejected (the aggregate is always emitted ascending), unlike the raw Aggregation.NONE path. + ReadTsKvQuery ignoredOrder = + new BaseReadTsKvQuery("k", 1L, 10L, 5L, 10, Aggregation.SUM, "sideways"); + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(ignoredOrder)) + .get(3, TimeUnit.SECONDS); + ArgumentCaptor ignoredOrderSql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)) + .executeQueryStatement(ignoredOrderSql.capture()); + String emitted = ignoredOrderSql.getAllValues().get(1); + assertTrue(emitted.endsWith("GROUP BY 1 ORDER BY 1 ASC"), emitted); + assertFalse(emitted.contains("sideways"), emitted); + } + private TestContext newContext(IoTDBTableConfig config, boolean startWorker) throws IoTDBConnectionException { ITableSessionPool pool = mock(ITableSessionPool.class); @@ -1321,6 +2384,369 @@ private int tbDataPoints(String value) { return Math.max(1, (value.length() + 511) / 512); } + /** Unwraps the inner {@link KvEntry} so a test can assert the concrete data-entry type. */ + private KvEntry innerKv(TsKvEntry entry) { + return assertInstanceOf(BasicTsKvEntry.class, entry).getKv(); + } + + private SessionDataSet aggDataSet(MockAggBucket... buckets) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = mock(SessionDataSet.class); + SessionDataSet.DataIterator iterator = mock(SessionDataSet.DataIterator.class); + AtomicInteger index = new AtomicInteger(-1); + when(dataSet.iterator()).thenReturn(iterator); + when(iterator.next()).thenAnswer(invocation -> index.incrementAndGet() < buckets.length); + when(iterator.isNull(anyString())) + .thenAnswer(invocation -> buckets[index.get()].isNull(invocation.getArgument(0))); + // date_bin emits the startTs-anchored bucket START; the DAO derives the TB midpoint in Java. + when(iterator.getTimestamp("bucket_ts")) + .thenAnswer(invocation -> new Timestamp(buckets[index.get()].bucketStart())); + // MAX(time) of the underlying data drives lastEntryTs; default to the bucket start. + when(iterator.getTimestamp("max_ts")) + .thenAnswer(invocation -> new Timestamp(buckets[index.get()].maxTs())); + when(iterator.getDouble("agg_num")).thenAnswer(invocation -> buckets[index.get()].numeric()); + // sum_long is projected as SUM(CAST(long_v AS DOUBLE)) -- a DOUBLE -- so the DAO reads it via + // getDouble. + when(iterator.getDouble("sum_long")).thenAnswer(invocation -> buckets[index.get()].sumLong()); + when(iterator.getDouble("sum_double")) + .thenAnswer(invocation -> buckets[index.get()].sumDouble()); + when(iterator.getString("agg_str")).thenAnswer(invocation -> buckets[index.get()].string()); + when(iterator.getLong(anyString())) + .thenAnswer(invocation -> buckets[index.get()].longColumn(invocation.getArgument(0))); + return dataSet; + } + + /** + * Models the raw {@code SELECT long_v ... AND long_v IS NOT NULL} re-query the SUM mapper issues + * for a long-only bucket whose double-accumulated sum may have lost precision. Each supplied + * value is one non-null {@code long_v} row; the DAO accumulates them in Java as {@code long}, so + * the assertion proves the EXACT long sum (not the rounded double SUM). + */ + private SessionDataSet rawLongDataSet(long... values) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = mock(SessionDataSet.class); + SessionDataSet.DataIterator iterator = mock(SessionDataSet.DataIterator.class); + AtomicInteger index = new AtomicInteger(-1); + when(dataSet.iterator()).thenReturn(iterator); + when(iterator.next()).thenAnswer(invocation -> index.incrementAndGet() < values.length); + when(iterator.isNull("long_v")).thenReturn(false); + when(iterator.getLong("long_v")).thenAnswer(invocation -> values[index.get()]); + return dataSet; + } + + private MockAggBucket numericBucket(long bucketStart, double numeric) { + return MockAggBucket.numeric(bucketStart, bucketStart, numeric); + } + + private MockAggBucket numericBucket(long bucketStart, long maxTs, double numeric) { + return MockAggBucket.numeric(bucketStart, maxTs, numeric); + } + + private MockAggBucket stringBucket(long bucketStart, String value) { + return MockAggBucket.string(bucketStart, bucketStart, value); + } + + /** A COUNT bucket whose value lands entirely in the long_v column (normal single-typed row). */ + private MockAggBucket countBucket(long bucketStart, long count) { + return MockAggBucket.typedCount(bucketStart, bucketStart, 0L, 0L, 0L, count, 0L); + } + + /** + * The REAL row IoTDB returns for an EMPTY bounded calendar bucket: a single row whose typed COUNT + * columns are all 0 and whose MAX(time)/aggregates are NULL (so {@code isNull("max_ts")} is + * true). This is what the bounded per-bucket calendar path actually receives for an empty window, + * unlike {@code aggDataSet()} (zero rows) which only the GROUP-BY milliseconds path produces. + */ + private MockAggBucket emptyAggRow(long bucketStart) { + return MockAggBucket.emptyAggRow(bucketStart); + } + + private ReadTsKvQuery calendarQuery( + String key, + long startTs, + long endTs, + IntervalType intervalType, + String tzId, + Aggregation aggregation) { + return new BaseReadTsKvQuery( + key, + startTs, + endTs, + AggregationParams.calendar(aggregation, intervalType, tzId), + 100, + "ASC"); + } + + private ReadTsKvQuery calendarLikeMilliseconds( + String key, long startTs, long endTs, long interval, Aggregation aggregation) { + // Explicit MILLISECONDS IntervalType (with a tz set) must still route to the date_bin path. + return new BaseReadTsKvQuery( + key, + startTs, + endTs, + AggregationParams.of( + aggregation, IntervalType.MILLISECONDS, java.time.ZoneId.of("UTC"), interval), + 100, + "DESC"); + } + + /** + * Mocks one aggregation result row. {@code countBool/countStr/countJson/countLong/countDouble} + * model ThingsBoard's per-type {@code SUM(CASE WHEN IS NOT NULL THEN 1 ELSE 0 END)} + * counters; the DAO selects the dominant one. {@code maxTs} models {@code MAX(time)} of the + * underlying data for {@code lastEntryTs}. {@code emptyAgg} models the row IoTDB returns for an + * empty bounded calendar bucket (one row whose {@code MAX(time)} is NULL and whose typed COUNT + * columns are all 0); see {@link #emptyAggRow(long)}. + */ + private record MockAggBucket( + long bucketStart, + long maxTs, + Double numeric, + String string, + Long countBool, + Long countStr, + Long countJson, + Long countLong, + Long countDouble, + Double sumLong, + Double sumDouble, + Long minLong, + Long maxLong, + boolean emptyAgg) { + + private static MockAggBucket numeric(long bucketStart, long maxTs, double numeric) { + // AVG/MIN/MAX numeric bucket with no recorded long/double participation; the SUM and + // MIN/MAX type-selection columns default to null (so isNull(...) is true), exercising the AVG + // and string-fallback paths that do not depend on the typed counts. + return new MockAggBucket( + bucketStart, + maxTs, + numeric, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false); + } + + /** + * A MIN/MAX/AVG numeric bucket that also records the long/double participation counts, so the + * result-type selector (long-only -> LONG, any double -> DOUBLE) can be exercised. {@code + * countLong}/{@code countDouble} drive both the COUNT path and the MIN/MAX result type; {@code + * numeric} is the MIN/MAX/AVG value read from {@code agg_num}. For a long-only bucket the + * direct MIN(long_v)/MAX(long_v) channel is set to {@code (long) numeric} so the exact + * long-channel MIN/MAX mapping reads the matching value; for a bucket with a participating + * double the long channel is left null because the DOUBLE mapping ignores it. + */ + private static MockAggBucket typedNumeric( + long bucketStart, long maxTs, double numeric, long countLong, long countDouble) { + Long longChannel = countDouble == 0L ? (long) numeric : null; + return new MockAggBucket( + bucketStart, + maxTs, + numeric, + null, + null, + null, + null, + countLong, + countDouble, + null, + null, + longChannel, + longChannel, + false); + } + + /** + * A long-only MIN/MAX bucket that proves the EXACT long channel: {@code agg_num} holds the + * value IoTDB's {@code COALESCE(double_v, CAST(long_v AS DOUBLE))} would produce (a double, + * which silently rounds a long > 2^53), while {@code min_long}/{@code max_long} hold the EXACT + * stored long. The DAO must read the long channel, so the entry's long value equals {@code + * exactLong} (not the rounded {@code roundedAggNum}). A bucket is long-only (countDouble == 0). + */ + private static MockAggBucket minMaxLong( + long bucketStart, long maxTs, double roundedAggNum, long exactLong, long countLong) { + return new MockAggBucket( + bucketStart, + maxTs, + roundedAggNum, + null, + null, + null, + null, + countLong, + 0L, + null, + null, + exactLong, + exactLong, + false); + } + + /** + * A SUM bucket: the DAO reads {@code sum_long}/{@code sum_double} plus the long/double counts + * (NOT {@code agg_num}). A long-only bucket records {@code countDouble == 0} and {@code + * sumDouble == null}; a bucket with any double records {@code countDouble > 0} and a non-null + * {@code sumDouble}. The {@code min_long}/{@code max_long} bound channels default to null (read + * as 0), so a long-only bucket built this way always takes the exact-fast-path (maxAbs == 0 + * proves the double sum lost no bits); use {@link #sumWithBound} to drive the > 2^53 re-sum + * fallback. + */ + private static MockAggBucket sum( + long bucketStart, + long maxTs, + Double sumLong, + Double sumDouble, + long countLong, + long countDouble) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + null, + null, + null, + null, + countLong, + countDouble, + sumLong, + sumDouble, + null, + null, + false); + } + + /** + * A long-only SUM bucket that also records the {@code min_long}/{@code max_long} bound + * channels. The DAO computes {@code maxAbs = max(|min_long|, |max_long|)} and trusts the + * DOUBLE-accumulated {@code sum_long} only while {@code count_long * maxAbs <= 2^53}; otherwise + * it re-queries the raw {@code long_v} values and sums them in Java. Used to drive both the + * fast path (small bound) and the re-sum fallback (large bound). + */ + private static MockAggBucket sumWithBound( + long bucketStart, long maxTs, double sumLong, long countLong, long minLong, long maxLong) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + null, + null, + null, + null, + countLong, + 0L, + sumLong, + null, + minLong, + maxLong, + false); + } + + private static MockAggBucket string(long bucketStart, long maxTs, String value) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + value, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false); + } + + private static MockAggBucket typedCount( + long bucketStart, + long maxTs, + long countBool, + long countStr, + long countJson, + long countLong, + long countDouble) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + null, + countBool, + countStr, + countJson, + countLong, + countDouble, + null, + null, + null, + null, + false); + } + + /** + * Models the REAL row that IoTDB returns for an EMPTY bounded calendar bucket. Unlike {@code + * aggDataSet()} with zero rows (which the GROUP-BY milliseconds path produces but the bounded + * per-bucket calendar path never does), a single bounded aggregate over a window matching zero + * rows still returns ONE row: every typed COUNT is 0 and every other aggregate (AVG/SUM/MIN/MAX + * and MAX(time)) is NULL. The calendar reader's empty-bucket guard keys off {@code + * isNull("max_ts")} (time is never null, so MAX(time) is NULL iff the window was empty), so + * this row MUST report {@code isNull("max_ts") == true} while its typed-COUNT columns read 0. + * Without the {@code !isNull(max_ts)} guard, COUNT's {@code LongDataEntry(0)} would leak as a + * spurious empty bucket. + */ + private static MockAggBucket emptyAggRow(long bucketStart) { + return new MockAggBucket( + bucketStart, bucketStart, null, null, 0L, 0L, 0L, 0L, 0L, null, null, null, null, true); + } + + private long longColumn(String column) { + return switch (column) { + case "count_bool" -> orZero(countBool); + case "count_str" -> orZero(countStr); + case "count_json" -> orZero(countJson); + case "count_long" -> orZero(countLong); + case "count_double" -> orZero(countDouble); + case "min_long" -> orZero(minLong); + case "max_long" -> orZero(maxLong); + default -> 0L; + }; + } + + private static long orZero(Long value) { + return value == null ? 0L : value; + } + + private boolean isNull(String column) { + return switch (column) { + case "agg_num" -> numeric == null; + case "agg_str" -> string == null; + case "count_bool" -> countBool == null; + case "count_str" -> countStr == null; + case "count_json" -> countJson == null; + case "count_long" -> countLong == null; + case "count_double" -> countDouble == null; + case "sum_long" -> sumLong == null; + case "sum_double" -> sumDouble == null; + case "min_long" -> minLong == null; + case "max_long" -> maxLong == null; + // MAX(time) is NULL iff the bounded window matched zero rows (time is never null). A real + // empty calendar bucket (emptyAggRow) returns one row with MAX(time) NULL; every other + // bucket has matching data, so its max_ts is non-null. + case "max_ts" -> emptyAgg; + default -> true; + }; + } + } + private record MockTelemetryRow(long ts, String valueColumn, Object value) { private boolean isNull(String column) { return !valueColumn.equals(column);