diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java index 216ffcd..37c244c 100644 --- a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java @@ -652,10 +652,24 @@ private static String aggregationProjection(Aggregation aggregation) { + AGG_STR_COLUMN + ", " + numericCountProjection(); + // The numeric MAX is projected as -MIN(-x) rather than MAX(x). IoTDB's GROUPED max + // accumulator seeds FLOAT/DOUBLE state with Float/Double.MIN_VALUE -- the smallest + // POSITIVE value, not the most negative one -- and only marks a group initialized when + // `value >= state`, so a bucket whose numeric maximum is zero or negative comes back + // NULL. This DAO reads a NULL aggregate as "empty bucket" and skips it, so a MAX + // downsampling query over e.g. a sub-zero sensor would silently lose whole buckets + // instead of failing. Every release up to and including 2.0.10 is affected; fixed on + // master by apache/iotdb#18300, which is not in a released version yet. The grouped MIN + // accumulator seeds with MAX_VALUE and is not affected, and IEEE-754 negation is exact, + // so -MIN(-x) is an exact substitute for MAX(x) over finite values, on affected and + // fixed servers alike. This projection is shared with the calendar path, whose + // non-grouped accumulators track an explicit initialized flag rather than a sentinel, so + // the substitution is exact there too and an empty bucket still yields NULL. MAX(long_v) + // and MAX(str_v) are already correct (true Long.MIN_VALUE seed / flag-based) and stay. case MAX -> - "MAX(" + "-1 * MIN(-1 * (" + NUMERIC_VALUE - + ") AS " + + ")) AS " + AGG_NUM_COLUMN + ", MAX(long_v) AS " + MAX_LONG_COLUMN diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java index 3e65e50..302f4fa 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java @@ -694,6 +694,104 @@ void calendarCountSkipsEmptyMiddleMonthAgainstRealIoTDB() throws Exception { } } + @Test + void maxKeepsBucketsWhoseValuesAreAllNonPositiveAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_nonpositive", + "55555555-5555-5555-5555-555555555511", + "66666666-6666-6666-6666-666666666611"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // Regression for apache/iotdb#18300 against REAL IoTDB. The server's GROUPED max + // accumulator seeds FLOAT/DOUBLE state with Double.MIN_VALUE -- the smallest POSITIVE + // value -- so on every release up to and including 2.0.10 a bucket whose maximum is zero + // or negative makes MAX() return NULL. This DAO reads a NULL aggregate as an + // empty bucket and skips it, so such buckets used to VANISH from the series: a MAX + // downsampling query over a sub-zero sensor silently returned fewer points, with no + // error anywhere. The projection computes -MIN(-x) instead, which the unaffected grouped + // MIN accumulator evaluates exactly. Every bucket here has a non-positive maximum, so on + // the old projection this test fails with "bucket count expected 2 but was 0". + // Bucket [1000,2000): doubles -5.0, -3.0 -> midpoint 1500, MAX = -3.0, MIN = -5.0 + // Bucket [2000,3000): doubles 0.0, 0.0 -> midpoint 2500, MAX = 0.0, MIN = 0.0 + saveAll( + dao, + scope, + List.of( + entry(1000L, "np", DataType.DOUBLE, -5.0D), + entry(1500L, "np", DataType.DOUBLE, -3.0D), + entry(2100L, "np", DataType.DOUBLE, 0.0D), + entry(2400L, "np", DataType.DOUBLE, 0.0D))); + + ReadTsKvQueryResult max = aggregate(dao, scope, "np", Aggregation.MAX); + assertNumericBuckets( + max, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.DOUBLE}, + new double[] {-3.0D, 0.0D}, + 2400L); + + // MIN is not affected by the upstream bug; it must be byte-for-byte unchanged. + ReadTsKvQueryResult min = aggregate(dao, scope, "np", Aggregation.MIN); + assertNumericBuckets( + min, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.DOUBLE}, + new double[] {-5.0D, 0.0D}, + 2400L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void maxOverNonPositiveLongOnlyAndMixedBucketsKeepsResultTypeAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_nonpositive_types", + "55555555-5555-5555-5555-555555555512", + "66666666-6666-6666-6666-666666666612"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // Same regression across the two typed channels, since they take different code paths: + // Bucket [1000,2000): LONG-ONLY -5, -3 -> midpoint 1500, MAX = -3 kept LONG-typed via + // the direct MAX(long_v) channel (LongBigArray seeds with the true Long.MIN_VALUE, + // so that channel was always correct). + // Bucket [2000,3000): MIXED long -5 + double -3.0 -> midpoint 2500, MAX = -3.0 promoted + // to DOUBLE, which reads the numeric channel and therefore exercises the fix. + saveAll( + dao, + scope, + List.of( + entry(1000L, "nt", DataType.LONG, -5L), + entry(1500L, "nt", DataType.LONG, -3L), + entry(2100L, "nt", DataType.LONG, -5L), + entry(2400L, "nt", DataType.DOUBLE, -3.0D))); + + ReadTsKvQueryResult max = aggregate(dao, scope, "nt", Aggregation.MAX); + assertNumericBuckets( + max, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {-3.0D, -3.0D}, + 2400L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + private ReadTsKvQueryResult calendarAggregate( IoTDBTableTimeseriesDao dao, TestScope scope, diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java index 9672f50..66fa857 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java @@ -1737,8 +1737,16 @@ void findAllAsync_sumCountMinMaxBuildSql() throws Exception { "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + "AS count_double"), statements.get(2)); + // The numeric MAX is projected as -MIN(-x): IoTDB's grouped max accumulator seeds + // FLOAT/DOUBLE state with Double.MIN_VALUE (the smallest POSITIVE value), so MAX over a + // bucket whose maximum is zero or negative returns NULL and the bucket would be dropped + // (apache/iotdb#18300). The grouped MIN accumulator is unaffected and IEEE-754 negation is + // exact, so this is an exact substitute over finite values. The long and string channels are + // already correct and keep using MAX. assertTrue( - statements.get(3).contains("MAX(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num") + statements + .get(3) + .contains("-1 * MIN(-1 * (COALESCE(double_v, CAST(long_v AS DOUBLE)))) AS agg_num") && statements.get(3).contains("MAX(long_v) AS max_long") && statements.get(3).contains("MAX(str_v) AS agg_str") && statements