From 97062d865ef999c17f2dfac85045920059e8dee0 Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Fri, 12 Jun 2026 15:15:56 -0500 Subject: [PATCH 1/6] Fix pseudo-irregular time series direct reads --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 184 +++++++++++++++++- .../cda/api/TimeSeriesDirectReadParityIT.java | 49 ++++- 2 files changed, 226 insertions(+), 7 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index d7ba53637..33130ae6d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -47,6 +47,8 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.time.Duration; @@ -59,6 +61,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -67,6 +70,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -177,9 +181,18 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private final Histogram getRequestedTimeSeriesResultsReturnedHistogram; @NotNull private final Histogram getRequestedTimeSeriesRequestWindowMillisHistogram; + @NotNull + private final MetricRegistry metrics; + private final boolean forceOldLrtsFormatting; public TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics) { + this(dsl, metrics, false); + } + + private TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics, boolean forceOldLrtsFormatting) { super(dsl); + this.metrics = metrics; + this.forceOldLrtsFormatting = forceOldLrtsFormatting; String className = this.getClass().getName(); CacheStats stats = isVersionedCache.stats(); @@ -669,6 +682,50 @@ private TimeSeries buildTimeSeriesFromMetadata(Record tsMetadata, @Nullable Inte private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, @NotNull TimeSeriesRequestParameters requestParameters) { + if (isPseudoIrregularOldStyleLocalRegularId(requestParameters)) { + return getRequestedTimeSeriesDirectWithOldLrtsFormatting(page, pageSize, requestParameters); + } + return getRequestedTimeSeriesDirectForSession(page, pageSize, requestParameters); + } + + private TimeSeries getRequestedTimeSeriesDirectWithOldLrtsFormatting(String page, int pageSize, + @NotNull TimeSeriesRequestParameters + requestParameters) { + return connectionResult(dsl, conn -> { + DSLContext oldLrtsDsl = DSL.using(conn, SQLDialect.ORACLE18C); + setOldLrtsFormatting(oldLrtsDsl); + TimeSeriesDaoImpl oldLrtsDao = new TimeSeriesDaoImpl(oldLrtsDsl, metrics, true); + return oldLrtsDao.getRequestedTimeSeriesDirectForSession(page, pageSize, requestParameters); + }); + } + + private boolean isPseudoIrregularOldStyleLocalRegularId(@NotNull TimeSeriesRequestParameters requestParameters) { + if (!isOldStyleLocalRegularId(requestParameters.getNames())) { + return false; + } + return connectionResult(dsl, conn -> { + DSLContext oldLrtsDsl = DSL.using(conn, SQLDialect.ORACLE18C); + setOldLrtsFormatting(oldLrtsDsl); + TimeSeriesDaoImpl oldLrtsDao = new TimeSeriesDaoImpl(oldLrtsDsl, metrics, true); + DirectReadMetadata metadata = oldLrtsDao.fetchRequestedTimeSeriesMetadataRecord(requestParameters); + return metadata != null && metadata.intervalUtcOffset == UTC_OFFSET_IRREGULAR; + }); + } + + private static void setOldLrtsFormatting(DSLContext oldLrtsDsl) { + CWMS_UTIL_PACKAGE.call_SET_SESSION_INFO(oldLrtsDsl.configuration(), + SESSION_USE_LRTS_ID_FORMAT, formatBool(false), REQUIRE_OLD_LRTS_ID_FORMAT); + } + + private static boolean isOldStyleLocalRegularId(String tsId) { + String[] parts = splitTimeSeriesId(tsId); + String intervalPart = getTimeSeriesIdPart(parts, 3); + return intervalPart != null && intervalPart.startsWith("~"); + } + + private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageSize, + @NotNull TimeSeriesRequestParameters + requestParameters) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); String requestedUnits = requestParameters.getUnits(); @@ -679,6 +736,10 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, String cursor = null; Timestamp tsCursor = null; + if (forceOldLrtsFormatting) { + setOldLrtsFormatting(dsl); + } + if (page != null && !page.isEmpty()) { final String[] parts = CwmsDTOPaginated.decodeCursor(page); @@ -727,10 +788,13 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, // Pagination happens after regular-interval gap rows are merged // fetch the full raw window first + if (forceOldLrtsFormatting) { + setOldLrtsFormatting(dsl); + } List rawRows = fetchRequestedTimeSeriesRows(tsCode, metadataOfficeId, nativeUnits, metadataUnits, requestParameters, includeEntryDate); long effectiveIntervalOffset = intervalOffset; - if (isRegularSeries(intervalMinutes, intervalPart)) { + if (isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { effectiveIntervalOffset = resolveIntervalOffset(intervalOffset, timeZoneId, intervalPart, isLrts, rawRows); } @@ -747,7 +811,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, beginTime, endTime, metadataUnits, - resolveIntervalDuration(intervalMinutes, intervalPart), + resolveIntervalDuration(intervalMinutes, intervalOffset, intervalPart, isLrts), verticalDatumInfo, effectiveIntervalOffset, timeZoneId, @@ -843,6 +907,11 @@ private List fetchRequestedTimeSeriesRows(long tsCode, String String requestedUnits, TimeSeriesRequestParameters requestParameters, boolean includeEntryDate) { + if (forceOldLrtsFormatting) { + return fetchRequestedTimeSeriesRowsWithJdbc(tsCode, officeId, nativeUnits, requestedUnits, + requestParameters, includeEntryDate); + } + ZonedDateTime beginTime = requestParameters.getBeginTime(); ZonedDateTime endTime = requestParameters.getEndTime(); ZonedDateTime versionDate = requestParameters.getVersionDate(); @@ -890,6 +959,103 @@ private List fetchRequestedTimeSeriesRows(long tsCode, String }); } + private List fetchRequestedTimeSeriesRowsWithJdbc( + long tsCode, String officeId, String nativeUnits, String requestedUnits, + TimeSeriesRequestParameters requestParameters, boolean includeEntryDate) { + return connectionResult(dsl, conn -> { + setOldLrtsFormatting(DSL.using(conn, SQLDialect.ORACLE18C)); + ZonedDateTime versionDate = requestParameters.getVersionDate(); + String sql = versionDate != null + ? buildVersionedRowsSql(includeEntryDate) + : buildMaxVersionRowsSql(includeEntryDate); + try (PreparedStatement statement = conn.prepareStatement(sql)) { + bindDirectRowQuery(statement, tsCode, officeId, nativeUnits, requestedUnits, + requestParameters, versionDate); + try (ResultSet resultSet = statement.executeQuery()) { + List rows = new ArrayList<>(); + Calendar utcCalendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + while (resultSet.next()) { + Timestamp dateTime = resultSet.getTimestamp(DATE_TIME, utcCalendar); + Double value = resultSet.getDouble(VALUE); + if (resultSet.wasNull()) { + value = null; + } + int qualityCode = resultSet.getInt("quality_norm"); + Timestamp dataEntryDate = includeEntryDate + ? resultSet.getTimestamp(DATA_ENTRY_DATE, utcCalendar) + : null; + if (includeEntryDate) { + rows.add(new TimeSeries.Record(dateTime, value, qualityCode, dataEntryDate)); + } else { + rows.add(new TimeSeries.Record(dateTime, value, qualityCode)); + } + } + return rows; + } + } catch (SQLException ex) { + throw new DataAccessException("Unable to fetch direct time series rows", ex); + } + }); + } + + private static String buildVersionedRowsSql(boolean includeEntryDate) { + return "select date_time," + + " cwms_20.cwms_util.convert_units(value, unit_id, ?) value," + + " cwms_20.cwms_ts.normalize_quality(nvl(cast(quality_code as number), 5)) quality_norm," + + (includeEntryDate ? " data_entry_date" : " cast(null as timestamp) data_entry_date") + + " from cwms_20.av_tsv_dqu" + + " where aliased_item is null" + + " and ts_code = ?" + + " and office_id = ?" + + " and lower(unit_id) = lower(?)" + + " and date_time >= ?" + + " and date_time <= ?" + + " and start_date <= ?" + + " and end_date > ?" + + " and version_date = ?" + + " order by date_time"; + } + + private static String buildMaxVersionRowsSql(boolean includeEntryDate) { + return "select date_time, value, quality_norm, data_entry_date from (" + + " select date_time," + + " cwms_20.cwms_util.convert_units(value, unit_id, ?) value," + + " cwms_20.cwms_ts.normalize_quality(nvl(cast(quality_code as number), 5)) quality_norm," + + (includeEntryDate ? " data_entry_date" : " cast(null as timestamp) data_entry_date") + + ", row_number() over (partition by date_time order by version_date desc, data_entry_date desc)" + + " version_rank" + + " from cwms_20.av_tsv_dqu" + + " where aliased_item is null" + + " and ts_code = ?" + + " and office_id = ?" + + " and lower(unit_id) = lower(?)" + + " and date_time >= ?" + + " and date_time <= ?" + + " and start_date <= ?" + + " and end_date > ?" + + ") where version_rank = 1" + + " order by date_time"; + } + + private static void bindDirectRowQuery(PreparedStatement statement, long tsCode, String officeId, + String nativeUnits, String requestedUnits, + TimeSeriesRequestParameters requestParameters, + ZonedDateTime versionDate) throws SQLException { + Timestamp beginTimestamp = Timestamp.from(requestParameters.getBeginTime().toInstant()); + Timestamp endTimestamp = Timestamp.from(requestParameters.getEndTime().toInstant()); + statement.setString(1, requestedUnits); + statement.setLong(2, tsCode); + statement.setString(3, officeId); + statement.setString(4, nativeUnits); + statement.setTimestamp(5, beginTimestamp); + statement.setTimestamp(6, endTimestamp); + statement.setTimestamp(7, endTimestamp); + statement.setTimestamp(8, beginTimestamp); + if (versionDate != null) { + statement.setTimestamp(9, Timestamp.from(versionDate.toInstant())); + } + } + private ResultQuery> buildVersionedRowsQuery( AV_TSV_DQU view, Field value, @@ -951,7 +1117,7 @@ private List fetchExpectedRegularTimes(long intervalMinutes, long int TimeSeriesRequestParameters requestParameters, List rawRows) { boolean shouldTrim = requestParameters.isShouldTrim(); - if (!isRegularSeries(intervalMinutes, intervalPart)) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { return Collections.emptyList(); } // Trimmed requests collapse to the observed data window @@ -1029,11 +1195,17 @@ private long resolveIntervalOffset(long intervalOffset, String timeZoneId, return (rawRows.get(0).getDateTime().getTime() - topOfInterval.getTime()) / TimeUnit.MINUTES.toMillis(1); } - private boolean isRegularSeries(long intervalMinutes, String intervalPart) { - return intervalMinutes != 0L || isLocalRegularInterval(intervalPart); + private boolean isRegularSeries(long intervalMinutes, long intervalOffset, String intervalPart, boolean isLrts) { + return intervalOffset != UTC_OFFSET_IRREGULAR + && (intervalMinutes != 0L || (isLrts && isLocalRegularInterval(intervalPart))); } - private Duration resolveIntervalDuration(long intervalMinutes, String intervalPart) { + private Duration resolveIntervalDuration(long intervalMinutes, long intervalOffset, + String intervalPart, boolean isLrts) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { + return Duration.ZERO; + } + if (intervalMinutes != 0L) { return Duration.ofMinutes(intervalMinutes); } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java index 231b9cec5..98c5c8d8a 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.ApiServlet; import cwms.cda.api.enums.VersionType; import cwms.cda.data.dto.TimeSeries; import cwms.cda.formatters.Formats; @@ -159,6 +160,32 @@ void irregularReadMatchesRetrieveTs() throws Exception { ); } + @Test + void pseudoIrregularReadWithLrtsHeaderMatchesRetrieveTs() throws Exception { + String seriesId = "ITPARPIRR.Flow.Inst.~15Minutes.0.BENCH"; + Instant beginTime = Instant.parse("2024-01-05T12:00:00Z"); + Instant endTime = Instant.parse("2024-01-05T13:00:00Z"); + List rows = List.of( + row("2024-01-05T12:00:00Z", 10.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T12:17:00Z", 20.0, 0, "2024-01-06T00:01:00Z", null), + row("2024-01-05T12:45:00Z", 30.0, 0, "2024-01-06T00:02:00Z", null) + ); + seedTimeSeries("ITPARPIRR", seriesId, rows, false, null); + + List expectedRows = fetchOracleRows(seriesId, "cfs", beginTime, endTime, + false, null); + TimeSeries actualResponse = fetchCdaRowsWithPageSize(seriesId, "cfs", beginTime, endTime, + 1000, false, null, true, true); + + assertEquals(expectedRows.size(), actualResponse.getTotal(), "total"); + assertEquals(expectedRows.size(), actualResponse.getValues().size(), "values size"); + assertEquals(Duration.ZERO, actualResponse.getInterval(), "interval"); + assertEquals((long) Integer.MIN_VALUE, actualResponse.getIntervalOffset(), "interval offset"); + for (int index = 0; index < expectedRows.size(); index++) { + assertRecordsEqual(expectedRows.get(index), actualResponse.getValues().get(index), index); + } + } + @Test void dstWindowRegularReadMatchesRetrieveTs() throws Exception { Instant dstStart = Instant.parse("2024-03-09T00:00:00Z"); @@ -420,8 +447,17 @@ private static void assertRecordsEqual(TimeSeries.Record expected, TimeSeries.Re private static void seedTimeSeries(String locationId, String seriesId, List rows, boolean versioned) throws SQLException { + seedTimeSeries(locationId, seriesId, rows, versioned, 0); + } + + private static void seedTimeSeries(String locationId, String seriesId, List rows, + boolean versioned, Integer intervalOffset) throws SQLException { createLocation(locationId, true, OFFICE); - createTimeseries(OFFICE, seriesId, 0); + if (intervalOffset != null) { + createTimeseries(OFFICE, seriesId, intervalOffset); + } else { + createTimeseries(OFFICE, seriesId); + } CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); database.connection(connection -> { @@ -650,6 +686,14 @@ private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units Instant endTime, int pageSize, boolean includeEntryDate, Instant versionDate, boolean trim) throws Exception { + return fetchCdaRowsWithPageSize(seriesId, units, beginTime, endTime, pageSize, includeEntryDate, + versionDate, trim, null); + } + + private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units, Instant beginTime, + Instant endTime, int pageSize, boolean includeEntryDate, + Instant versionDate, boolean trim, Boolean lrtsFormatting) + throws Exception { RequestSpecification request = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) @@ -661,6 +705,9 @@ private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units .queryParam(Controllers.PAGE_SIZE, pageSize) .queryParam(Controllers.TRIM, trim) .queryParam(Controllers.INCLUDE_ENTRY_DATE, includeEntryDate); + if (lrtsFormatting != null) { + request = request.header(ApiServlet.IS_NEW_LRTS, lrtsFormatting); + } if (versionDate != null) { request = request.queryParam(Controllers.VERSION_DATE, versionDate.toString()); } From 2260736eef76a05956a9b0312c0d06840ab832b0 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Sat, 20 Jun 2026 09:51:00 -0500 Subject: [PATCH 2/6] Fix pseudo-irregular direct time-series reads Signed-off-by: Charles Graham, SWT --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 262 +++++++----------- .../cda/api/TimeSeriesDirectReadParityIT.java | 27 +- 2 files changed, 109 insertions(+), 180 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 33130ae6d..13b0faf55 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -47,8 +47,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.time.Duration; @@ -61,7 +59,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; -import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -70,7 +67,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TimeZone; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -183,16 +179,10 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private final Histogram getRequestedTimeSeriesRequestWindowMillisHistogram; @NotNull private final MetricRegistry metrics; - private final boolean forceOldLrtsFormatting; public TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics) { - this(dsl, metrics, false); - } - - private TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics, boolean forceOldLrtsFormatting) { super(dsl); this.metrics = metrics; - this.forceOldLrtsFormatting = forceOldLrtsFormatting; String className = this.getClass().getName(); CacheStats stats = isVersionedCache.stats(); @@ -691,12 +681,12 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, private TimeSeries getRequestedTimeSeriesDirectWithOldLrtsFormatting(String page, int pageSize, @NotNull TimeSeriesRequestParameters requestParameters) { - return connectionResult(dsl, conn -> { + DirectReadMetadata metadata = connectionResult(dsl, conn -> { DSLContext oldLrtsDsl = DSL.using(conn, SQLDialect.ORACLE18C); setOldLrtsFormatting(oldLrtsDsl); - TimeSeriesDaoImpl oldLrtsDao = new TimeSeriesDaoImpl(oldLrtsDsl, metrics, true); - return oldLrtsDao.getRequestedTimeSeriesDirectForSession(page, pageSize, requestParameters); + return fetchRequestedTimeSeriesMetadataRecord(oldLrtsDsl, requestParameters); }); + return getRequestedTimeSeriesDirectForSession(page, pageSize, requestParameters, metadata); } private boolean isPseudoIrregularOldStyleLocalRegularId(@NotNull TimeSeriesRequestParameters requestParameters) { @@ -706,8 +696,7 @@ private boolean isPseudoIrregularOldStyleLocalRegularId(@NotNull TimeSeriesReque return connectionResult(dsl, conn -> { DSLContext oldLrtsDsl = DSL.using(conn, SQLDialect.ORACLE18C); setOldLrtsFormatting(oldLrtsDsl); - TimeSeriesDaoImpl oldLrtsDao = new TimeSeriesDaoImpl(oldLrtsDsl, metrics, true); - DirectReadMetadata metadata = oldLrtsDao.fetchRequestedTimeSeriesMetadataRecord(requestParameters); + DirectReadMetadata metadata = fetchRequestedTimeSeriesMetadataRecord(oldLrtsDsl, requestParameters); return metadata != null && metadata.intervalUtcOffset == UTC_OFFSET_IRREGULAR; }); } @@ -717,6 +706,10 @@ private static void setOldLrtsFormatting(DSLContext oldLrtsDsl) { SESSION_USE_LRTS_ID_FORMAT, formatBool(false), REQUIRE_OLD_LRTS_ID_FORMAT); } + private static void clearLrtsFormatting(DSLContext lrtsDsl) { + CWMS_UTIL_PACKAGE.call_RESET_SESSION_INFO(lrtsDsl.configuration(), SESSION_USE_LRTS_ID_FORMAT); + } + private static boolean isOldStyleLocalRegularId(String tsId) { String[] parts = splitTimeSeriesId(tsId); String intervalPart = getTimeSeriesIdPart(parts, 3); @@ -726,6 +719,12 @@ private static boolean isOldStyleLocalRegularId(String tsId) { private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageSize, @NotNull TimeSeriesRequestParameters requestParameters) { + return getRequestedTimeSeriesDirectForSession(page, pageSize, requestParameters, null); + } + + private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageSize, + @NotNull TimeSeriesRequestParameters requestParameters, + DirectReadMetadata suppliedMetadata) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); String requestedUnits = requestParameters.getUnits(); @@ -736,10 +735,6 @@ private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageS String cursor = null; Timestamp tsCursor = null; - if (forceOldLrtsFormatting) { - setOldLrtsFormatting(dsl); - } - if (page != null && !page.isEmpty()) { final String[] parts = CwmsDTOPaginated.decodeCursor(page); @@ -759,17 +754,17 @@ private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageS } } - DirectReadMetadata metadata = fetchRequestedTimeSeriesMetadataRecord(requestParameters); + DirectReadMetadata metadata = suppliedMetadata != null + ? suppliedMetadata + : fetchRequestedTimeSeriesMetadataRecord(requestParameters); if (metadata == null) { throw new DataAccessException("Unable to resolve time series metadata for " + names); } - long tsCode = metadata.tsCode; String tsId = metadata.tsId; String[] tsIdParts = splitTimeSeriesId(tsId); String metadataOfficeId = metadata.officeId; String metadataUnits = metadata.units; - String nativeUnits = metadata.nativeUnits; String locPart = getTimeSeriesIdPart(tsIdParts, 0); String parmPart = getTimeSeriesIdPart(tsIdParts, 1); String intervalPart = getTimeSeriesIdPart(tsIdParts, 3); @@ -788,11 +783,8 @@ private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageS // Pagination happens after regular-interval gap rows are merged // fetch the full raw window first - if (forceOldLrtsFormatting) { - setOldLrtsFormatting(dsl); - } - List rawRows = fetchRequestedTimeSeriesRows(tsCode, metadataOfficeId, nativeUnits, - metadataUnits, requestParameters, includeEntryDate); + List rawRows = fetchRequestedTimeSeriesRows(tsCode, metadataOfficeId, + metadataUnits, requestParameters, includeEntryDate, intervalOffset == UTC_OFFSET_IRREGULAR); long effectiveIntervalOffset = intervalOffset; if (isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { effectiveIntervalOffset = resolveIntervalOffset(intervalOffset, timeZoneId, intervalPart, isLrts, rawRows); @@ -829,6 +821,11 @@ private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageS private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( TimeSeriesRequestParameters requestParameters) { + return fetchRequestedTimeSeriesMetadataRecord(dsl, requestParameters); + } + + private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord(DSLContext metadataDsl, + TimeSeriesRequestParameters requestParameters) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); String units = requestParameters.getUnits(); @@ -867,13 +864,12 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( var tsIdView = AV_CWMS_TS_ID.AV_CWMS_TS_ID; SelectJoinStep metadataQuery = - dsl.with(valid) + metadataDsl.with(valid) .select( valid.field("tscode", BigDecimal.class).as("tscode"), valid.field("tsid", String.class).as("tsid"), valid.field("office_id", String.class).as("office_id"), valid.field("units", String.class).as("units"), - tsIdView.UNIT_ID.as("native_units"), valid.field("interval", BigDecimal.class).as("interval"), tsIdView.INTERVAL_UTC_OFFSET.as("interval_utc_offset"), tsIdView.TIME_ZONE_ID.as("time_zone_id"), @@ -890,7 +886,6 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( record.getValue("tsid", String.class), record.getValue("office_id", String.class), record.getValue("units", String.class), - record.getValue("native_units", String.class), record.getValue("interval", BigDecimal.class) == null ? 0L : record.getValue("interval", BigDecimal.class).longValue(), @@ -903,198 +898,131 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( record.getValue("version_flag", String.class))); } - private List fetchRequestedTimeSeriesRows(long tsCode, String officeId, String nativeUnits, - String requestedUnits, + private List fetchRequestedTimeSeriesRows(long tsCode, String officeId, String requestedUnits, TimeSeriesRequestParameters requestParameters, - boolean includeEntryDate) { - if (forceOldLrtsFormatting) { - return fetchRequestedTimeSeriesRowsWithJdbc(tsCode, officeId, nativeUnits, requestedUnits, - requestParameters, includeEntryDate); + boolean includeEntryDate, boolean irregular) { + if (irregular) { + return connectionResult(dsl, conn -> { + DSLContext rowDsl = DSL.using(conn, SQLDialect.ORACLE18C); + clearLrtsFormatting(rowDsl); + return fetchRequestedTimeSeriesRows(rowDsl, tsCode, officeId, requestedUnits, requestParameters, + includeEntryDate); + }); } + return fetchRequestedTimeSeriesRows(dsl, tsCode, officeId, requestedUnits, requestParameters, includeEntryDate); + } + private List fetchRequestedTimeSeriesRows(DSLContext rowDsl, long tsCode, String officeId, + String requestedUnits, + TimeSeriesRequestParameters requestParameters, + boolean includeEntryDate) { ZonedDateTime beginTime = requestParameters.getBeginTime(); ZonedDateTime endTime = requestParameters.getEndTime(); ZonedDateTime versionDate = requestParameters.getVersionDate(); Timestamp beginTimestamp = Timestamp.from(beginTime.toInstant()); Timestamp endTimestamp = Timestamp.from(endTime.toInstant()); + String beginTimestampText = beginTimestamp.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + String endTimestampText = endTimestamp.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; - Field qualityForNormalization = DSL.nvl( - view.QUALITY_CODE.cast(BigDecimal.class), - DSL.val(BigDecimal.valueOf(5)) - ); - Field normalizedQuality = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - qualityForNormalization).as("quality_norm"); - Field convertedValue = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( - view.VALUE, view.UNIT_ID, DSL.val(requestedUnits, String.class)).as(VALUE); + Field dateTimeField = field(name("CWMS_20", "AV_TSV_DQU", DATE_TIME), Timestamp.class); + Field versionDateField = field(name("CWMS_20", "AV_TSV_DQU", VERSION_DATE), Timestamp.class); + Field qualityCode = view.QUALITY_CODE.cast(BigDecimal.class).as(QUALITY_CODE); + Field value = view.VALUE.as(VALUE); Condition baseCondition = view.ALIASED_ITEM.isNull() .and(view.TS_CODE.eq(tsCode)) .and(view.OFFICE_ID.eq(officeId)) - .and(view.UNIT_ID.equalIgnoreCase(nativeUnits)) - .and(view.DATE_TIME.ge(beginTimestamp)) - .and(view.DATE_TIME.le(endTimestamp)) - .and(view.START_DATE.le(endTimestamp)) - .and(view.END_DATE.gt(beginTimestamp)); + .and(view.UNIT_ID.equalIgnoreCase(requestedUnits)) + .and(DSL.condition("{0} >= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(beginTimestampText))) + .and(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(endTimestampText))) + .and(view.START_DATE.isNull() + .or(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + view.START_DATE, DSL.val(endTimestampText)))) + .and(view.END_DATE.isNull() + .or(DSL.condition("{0} > to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + view.END_DATE, DSL.val(beginTimestampText)))); ResultQuery> query; if (versionDate != null) { - query = buildVersionedRowsQuery(view, convertedValue, normalizedQuality, baseCondition, versionDate, + query = buildVersionedRowsQuery(rowDsl, view, dateTimeField, versionDateField, value, qualityCode, + baseCondition, versionDate, includeEntryDate); } else { - query = buildMaxVersionRowsQuery(view, convertedValue, normalizedQuality, baseCondition, includeEntryDate); + query = buildMaxVersionRowsQuery(rowDsl, view, dateTimeField, versionDateField, value, qualityCode, + baseCondition, includeEntryDate); } logger.atFine().log("%s", lazy(() -> query.getSQL(ParamType.INLINED))); return query.fetch(record -> { Timestamp dateTime = record.getValue(0, Timestamp.class); - Double value = record.getValue(1, Double.class); - int qualityCode = record.getValue(2, BigDecimal.class).intValue(); + Double dataValue = record.getValue(1, Double.class); + int quality = normalizeQualityCode(record.getValue(2, BigDecimal.class)); Timestamp dataEntryDate = record.getValue(3, Timestamp.class); if (dataEntryDate != null) { - return new TimeSeries.Record(dateTime, value, qualityCode, dataEntryDate); + return new TimeSeries.Record(dateTime, dataValue, quality, dataEntryDate); } - return new TimeSeries.Record(dateTime, value, qualityCode); + return new TimeSeries.Record(dateTime, dataValue, quality); }); } - private List fetchRequestedTimeSeriesRowsWithJdbc( - long tsCode, String officeId, String nativeUnits, String requestedUnits, - TimeSeriesRequestParameters requestParameters, boolean includeEntryDate) { - return connectionResult(dsl, conn -> { - setOldLrtsFormatting(DSL.using(conn, SQLDialect.ORACLE18C)); - ZonedDateTime versionDate = requestParameters.getVersionDate(); - String sql = versionDate != null - ? buildVersionedRowsSql(includeEntryDate) - : buildMaxVersionRowsSql(includeEntryDate); - try (PreparedStatement statement = conn.prepareStatement(sql)) { - bindDirectRowQuery(statement, tsCode, officeId, nativeUnits, requestedUnits, - requestParameters, versionDate); - try (ResultSet resultSet = statement.executeQuery()) { - List rows = new ArrayList<>(); - Calendar utcCalendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - while (resultSet.next()) { - Timestamp dateTime = resultSet.getTimestamp(DATE_TIME, utcCalendar); - Double value = resultSet.getDouble(VALUE); - if (resultSet.wasNull()) { - value = null; - } - int qualityCode = resultSet.getInt("quality_norm"); - Timestamp dataEntryDate = includeEntryDate - ? resultSet.getTimestamp(DATA_ENTRY_DATE, utcCalendar) - : null; - if (includeEntryDate) { - rows.add(new TimeSeries.Record(dateTime, value, qualityCode, dataEntryDate)); - } else { - rows.add(new TimeSeries.Record(dateTime, value, qualityCode)); - } - } - return rows; - } - } catch (SQLException ex) { - throw new DataAccessException("Unable to fetch direct time series rows", ex); - } - }); - } - - private static String buildVersionedRowsSql(boolean includeEntryDate) { - return "select date_time," - + " cwms_20.cwms_util.convert_units(value, unit_id, ?) value," - + " cwms_20.cwms_ts.normalize_quality(nvl(cast(quality_code as number), 5)) quality_norm," - + (includeEntryDate ? " data_entry_date" : " cast(null as timestamp) data_entry_date") - + " from cwms_20.av_tsv_dqu" - + " where aliased_item is null" - + " and ts_code = ?" - + " and office_id = ?" - + " and lower(unit_id) = lower(?)" - + " and date_time >= ?" - + " and date_time <= ?" - + " and start_date <= ?" - + " and end_date > ?" - + " and version_date = ?" - + " order by date_time"; - } - - private static String buildMaxVersionRowsSql(boolean includeEntryDate) { - return "select date_time, value, quality_norm, data_entry_date from (" - + " select date_time," - + " cwms_20.cwms_util.convert_units(value, unit_id, ?) value," - + " cwms_20.cwms_ts.normalize_quality(nvl(cast(quality_code as number), 5)) quality_norm," - + (includeEntryDate ? " data_entry_date" : " cast(null as timestamp) data_entry_date") - + ", row_number() over (partition by date_time order by version_date desc, data_entry_date desc)" - + " version_rank" - + " from cwms_20.av_tsv_dqu" - + " where aliased_item is null" - + " and ts_code = ?" - + " and office_id = ?" - + " and lower(unit_id) = lower(?)" - + " and date_time >= ?" - + " and date_time <= ?" - + " and start_date <= ?" - + " and end_date > ?" - + ") where version_rank = 1" - + " order by date_time"; - } - - private static void bindDirectRowQuery(PreparedStatement statement, long tsCode, String officeId, - String nativeUnits, String requestedUnits, - TimeSeriesRequestParameters requestParameters, - ZonedDateTime versionDate) throws SQLException { - Timestamp beginTimestamp = Timestamp.from(requestParameters.getBeginTime().toInstant()); - Timestamp endTimestamp = Timestamp.from(requestParameters.getEndTime().toInstant()); - statement.setString(1, requestedUnits); - statement.setLong(2, tsCode); - statement.setString(3, officeId); - statement.setString(4, nativeUnits); - statement.setTimestamp(5, beginTimestamp); - statement.setTimestamp(6, endTimestamp); - statement.setTimestamp(7, endTimestamp); - statement.setTimestamp(8, beginTimestamp); - if (versionDate != null) { - statement.setTimestamp(9, Timestamp.from(versionDate.toInstant())); + private static int normalizeQualityCode(BigDecimal qualityCode) { + long quality = qualityCode == null ? 5L : qualityCode.longValue(); + if (quality > Integer.MAX_VALUE) { + quality -= 4_294_967_296L; } + return (int) quality; } private ResultQuery> buildVersionedRowsQuery( + DSLContext rowDsl, AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, ZonedDateTime versionDate, boolean includeEntryDate) { - Field versionTimestamp = CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.val(versionDate.toInstant().toEpochMilli())); + String versionTimestampText = Timestamp.from(versionDate.toInstant()).toLocalDateTime() + .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); Field dataEntryDateField = includeEntryDate ? view.DATA_ENTRY_DATE : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE); - return dsl.select( - view.DATE_TIME, + return rowDsl.select( + dateTime, value, - normalizedQuality, + qualityCode, dataEntryDateField) .from(view) - .where(baseCondition.and(view.VERSION_DATE.eq(versionTimestamp))) - .orderBy(view.DATE_TIME.asc()); + .where(baseCondition.and(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + versionDateField, DSL.val(versionTimestampText)))) + .orderBy(dateTime.asc()); } private ResultQuery> buildMaxVersionRowsQuery( + DSLContext rowDsl, AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, boolean includeEntryDate) { - var rankedRows = dsl.select( - view.DATE_TIME.as(DATE_TIME), + var rankedRows = rowDsl.select( + dateTime.as(DATE_TIME), value, - normalizedQuality, + qualityCode, includeEntryDate ? view.DATA_ENTRY_DATE.as(DATA_ENTRY_DATE) : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE), DSL.rowNumber() - .over(partitionBy(view.DATE_TIME) - .orderBy(view.VERSION_DATE.desc(), view.DATA_ENTRY_DATE.desc())) + .over(partitionBy(dateTime) + .orderBy(versionDateField.desc(), view.DATA_ENTRY_DATE.desc())) .as("version_rank")) .from(view) .where(baseCondition) @@ -1102,11 +1030,11 @@ private ResultQuery> buildMaxV Field dateTimeCol = rankedRows.field(DATE_TIME, Timestamp.class); Field valueCol = rankedRows.field(VALUE, Double.class); - Field qualityCol = rankedRows.field("quality_norm", BigDecimal.class); + Field qualityCol = rankedRows.field(QUALITY_CODE, BigDecimal.class); Field dataEntryDateCol = rankedRows.field(DATA_ENTRY_DATE, Timestamp.class); Field versionRankCol = rankedRows.field("version_rank", Integer.class); - return dsl.select(dateTimeCol, valueCol, qualityCol, dataEntryDateCol) + return rowDsl.select(dateTimeCol, valueCol, qualityCol, dataEntryDateCol) .from(rankedRows) .where(versionRankCol.eq(1)) .orderBy(dateTimeCol.asc()); @@ -2583,20 +2511,18 @@ private static final class DirectReadMetadata { private final String tsId; private final String officeId; private final String units; - private final String nativeUnits; private final long intervalMinutes; private final long intervalUtcOffset; private final String timeZoneId; private final String versionFlag; - private DirectReadMetadata(long tsCode, String tsId, String officeId, String units, String nativeUnits, + private DirectReadMetadata(long tsCode, String tsId, String officeId, String units, long intervalMinutes, long intervalUtcOffset, String timeZoneId, String versionFlag) { this.tsCode = tsCode; this.tsId = tsId; this.officeId = officeId; this.units = units; - this.nativeUnits = nativeUnits; this.intervalMinutes = intervalMinutes; this.intervalUtcOffset = intervalUtcOffset; this.timeZoneId = timeZoneId; diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java index 98c5c8d8a..a0643121e 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java @@ -26,8 +26,6 @@ import java.sql.Types; import java.time.Duration; import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; @@ -211,13 +209,13 @@ void localRegularGapReadMatchesRetrieveTs() throws Exception { "ITPARLCL", "ITPARLCL.Flow.Inst.~1Day.0.BENCH", "cfs", - Instant.parse("2024-01-01T00:00:00Z"), - Instant.parse("2024-01-05T00:00:00Z"), + Instant.parse("2024-01-01T06:00:00Z"), + Instant.parse("2024-01-05T06:00:00Z"), List.of( - row("2024-01-01T00:00:00Z", 1.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-02T00:00:00Z", 2.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-04T00:00:00Z", 4.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-05T00:00:00Z", 5.0, 0, "2024-01-06T00:00:00Z", null) + row("2024-01-01T06:00:00Z", 1.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-02T06:00:00Z", 2.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-04T06:00:00Z", 4.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T06:00:00Z", 5.0, 0, "2024-01-06T00:00:00Z", null) ), false, false, @@ -297,8 +295,9 @@ void trimmedResponseWindowMatchesReturnedValues() throws Exception { assertNotNull(response.getBegin(), "begin"); assertNotNull(response.getEnd(), "end"); - assertEquals(Instant.parse("2024-01-01T00:00:00Z"), response.getBegin().toInstant(), "begin"); - assertEquals(Instant.parse("2024-01-01T00:09:00Z"), response.getEnd().toInstant(), "end"); + assertEquals(response.getValues().get(0).getDateTime().toInstant(), response.getBegin().toInstant(), "begin"); + assertEquals(response.getValues().get(response.getValues().size() - 1).getDateTime().toInstant(), + response.getEnd().toInstant(), "end"); } private static void assertDirectReadMatchesOracle(String locationId, String seriesId, String units, @@ -471,7 +470,7 @@ private static void seedTimeSeries(String locationId, String seriesId, List years = rows.stream() - .map(seedRow -> OffsetDateTime.ofInstant(seedRow.dateTime, ZoneOffset.UTC).getYear()) + .map(seedRow -> storageYear(seedRow.dateTime)) .distinct() .collect(Collectors.toList()); @@ -521,7 +520,7 @@ private static void insertScenarioRows(Connection connection, long tsCode, List< Map> rowsByYear = new LinkedHashMap<>(); for (SeedRow row: sortedRows) { - int year = OffsetDateTime.ofInstant(row.dateTime, ZoneOffset.UTC).getYear(); + int year = storageYear(row.dateTime); rowsByYear.computeIfAbsent(year, ignored -> new ArrayList<>()).add(row); } @@ -564,6 +563,10 @@ private static void bindScenarioInsert(PreparedStatement statement, long tsCode, statement.setInt(6, row.qualityCode); } + private static int storageYear(Instant instant) { + return Timestamp.from(instant).toLocalDateTime().getYear(); + } + private static void updateScenarioExtents(Connection connection, long tsCode, List rows) throws SQLException { Set distinctVersionDates = rows.stream() From 5ef0b0ce304d0e22ad1fc00c3636d040fff4c053 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Sat, 20 Jun 2026 11:34:35 -0500 Subject: [PATCH 3/6] Restore direct-read unit validation Signed-off-by: Charles Graham, SWT --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 13b0faf55..7c028870b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -119,6 +119,8 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; private static final String CWMS_20 = "CWMS_20"; private static final String UNIT_ID = "UNIT_ID"; + private static final DateTimeFormatter ORACLE_DATE_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; @@ -765,6 +767,7 @@ private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageS String[] tsIdParts = splitTimeSeriesId(tsId); String metadataOfficeId = metadata.officeId; String metadataUnits = metadata.units; + String nativeUnits = metadata.nativeUnits; String locPart = getTimeSeriesIdPart(tsIdParts, 0); String parmPart = getTimeSeriesIdPart(tsIdParts, 1); String intervalPart = getTimeSeriesIdPart(tsIdParts, 3); @@ -777,6 +780,7 @@ private TimeSeries getRequestedTimeSeriesDirectForSession(String page, int pageS if (shouldFetchVerticalDatum(parmPart)) { verticalDatumInfo = fetchVerticalDatumInfoSeparately(locPart, requestedUnits, office); } + validateRequestedUnits(nativeUnits, requestedUnits); VersionType finalDateVersionType = getDirectReadVersionType( metadata.versionFlag, versionDate != null); @@ -870,6 +874,7 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord(DSLContext met valid.field("tsid", String.class).as("tsid"), valid.field("office_id", String.class).as("office_id"), valid.field("units", String.class).as("units"), + tsIdView.UNIT_ID.as("native_units"), valid.field("interval", BigDecimal.class).as("interval"), tsIdView.INTERVAL_UTC_OFFSET.as("interval_utc_offset"), tsIdView.TIME_ZONE_ID.as("time_zone_id"), @@ -886,6 +891,7 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord(DSLContext met record.getValue("tsid", String.class), record.getValue("office_id", String.class), record.getValue("units", String.class), + record.getValue("native_units", String.class), record.getValue("interval", BigDecimal.class) == null ? 0L : record.getValue("interval", BigDecimal.class).longValue(), @@ -921,8 +927,8 @@ private List fetchRequestedTimeSeriesRows(DSLContext rowDsl, ZonedDateTime versionDate = requestParameters.getVersionDate(); Timestamp beginTimestamp = Timestamp.from(beginTime.toInstant()); Timestamp endTimestamp = Timestamp.from(endTime.toInstant()); - String beginTimestampText = beginTimestamp.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); - String endTimestampText = endTimestamp.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + String beginTimestampText = beginTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); + String endTimestampText = endTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; Field dateTimeField = field(name("CWMS_20", "AV_TSV_DQU", DATE_TIME), Timestamp.class); @@ -988,7 +994,7 @@ private ResultQuery> buildVers ZonedDateTime versionDate, boolean includeEntryDate) { String versionTimestampText = Timestamp.from(versionDate.toInstant()).toLocalDateTime() - .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + .format(ORACLE_DATE_FORMATTER); Field dataEntryDateField = includeEntryDate ? view.DATA_ENTRY_DATE : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE); @@ -1040,6 +1046,12 @@ private ResultQuery> buildMaxV .orderBy(dateTimeCol.asc()); } + private void validateRequestedUnits(String nativeUnits, String requestedUnits) { + if (nativeUnits != null && requestedUnits != null) { + CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(dsl.configuration(), 0.0D, nativeUnits, requestedUnits); + } + } + private List fetchExpectedRegularTimes(long intervalMinutes, long intervalOffset, String timeZoneId, String intervalPart, boolean isLrts, TimeSeriesRequestParameters requestParameters, @@ -2511,18 +2523,20 @@ private static final class DirectReadMetadata { private final String tsId; private final String officeId; private final String units; + private final String nativeUnits; private final long intervalMinutes; private final long intervalUtcOffset; private final String timeZoneId; private final String versionFlag; - private DirectReadMetadata(long tsCode, String tsId, String officeId, String units, + private DirectReadMetadata(long tsCode, String tsId, String officeId, String units, String nativeUnits, long intervalMinutes, long intervalUtcOffset, String timeZoneId, String versionFlag) { this.tsCode = tsCode; this.tsId = tsId; this.officeId = officeId; this.units = units; + this.nativeUnits = nativeUnits; this.intervalMinutes = intervalMinutes; this.intervalUtcOffset = intervalUtcOffset; this.timeZoneId = timeZoneId; From 631dc015300b9daa6d8be9fa96e4ef3e1053d742 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Thu, 25 Jun 2026 22:15:39 -0500 Subject: [PATCH 4/6] Align direct TSV date predicates with retrieve_ts Signed-off-by: Charles Graham, SWT --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 349fbf72e..20c2fd56c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -129,6 +129,8 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; private static final String CWMS_20 = "CWMS_20"; private static final String UNIT_ID = "UNIT_ID"; + private static final DateTimeFormatter ORACLE_DATE_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; @@ -240,8 +242,12 @@ private ResultQuery> buildTsvD Timestamp beginTimestamp = Timestamp.from(beginTime.toInstant()); Timestamp endTimestamp = Timestamp.from(endTime.toInstant()); + String beginTimestampText = beginTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); + String endTimestampText = endTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; + Field dateTimeField = field(name("CWMS_20", "AV_TSV_DQU", DATE_TIME), Timestamp.class); + Field versionDateField = field(name("CWMS_20", "AV_TSV_DQU", VERSION_DATE), Timestamp.class); Field qualityForNormalization = DSL.nvl( view.QUALITY_CODE.cast(BigDecimal.class), @@ -258,15 +264,23 @@ private ResultQuery> buildTsvD .and(view.TS_CODE.eq(tsCode)) .and(view.OFFICE_ID.eq(officeId)) .and(view.UNIT_ID.equalIgnoreCase(units)) - .and(view.DATE_TIME.ge(beginTimestamp)) - .and(view.DATE_TIME.le(endTimestamp)) - .and(view.START_DATE.le(endTimestamp)) - .and(view.END_DATE.gt(beginTimestamp)); + .and(DSL.condition("{0} >= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(beginTimestampText))) + .and(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(endTimestampText))) + .and(view.START_DATE.isNull() + .or(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + view.START_DATE, DSL.val(endTimestampText)))) + .and(view.END_DATE.isNull() + .or(DSL.condition("{0} > to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + view.END_DATE, DSL.val(beginTimestampText)))); ResultQuery> query; if (versionDate != null) { query = buildVersionedRowsQuery( view, + dateTimeField, + versionDateField, value, normalizedQuality, baseCondition, @@ -276,6 +290,8 @@ private ResultQuery> buildTsvD } else { query = buildMaxVersionRowsQuery( view, + dateTimeField, + versionDateField, value, normalizedQuality, baseCondition, @@ -1011,43 +1027,48 @@ private List fetchRequestedTimeSeriesRows(long tsCode, String private ResultQuery> buildVersionedRowsQuery( AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, Field normalizedQuality, Condition baseCondition, ZonedDateTime versionDate, boolean includeEntryDate) { - Field versionTimestamp = CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( - DSL.val(versionDate.toInstant().toEpochMilli())); + String versionTimestampText = Timestamp.from(versionDate.toInstant()).toLocalDateTime() + .format(ORACLE_DATE_FORMATTER); Field dataEntryDateField = includeEntryDate ? view.DATA_ENTRY_DATE : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE); return dsl.select( - view.DATE_TIME, + dateTime, value, normalizedQuality, dataEntryDateField) .from(view) - .where(baseCondition.and(view.VERSION_DATE.eq(versionTimestamp))) - .orderBy(view.DATE_TIME.asc()); + .where(baseCondition.and(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + versionDateField, DSL.val(versionTimestampText)))) + .orderBy(dateTime.asc()); } private ResultQuery> buildMaxVersionRowsQuery( AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, Field normalizedQuality, Condition baseCondition, boolean includeEntryDate) { var rankedRows = dsl.select( - view.DATE_TIME.as(DATE_TIME), + dateTime.as(DATE_TIME), value, normalizedQuality, includeEntryDate ? view.DATA_ENTRY_DATE.as(DATA_ENTRY_DATE) : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE), DSL.rowNumber() - .over(partitionBy(view.DATE_TIME) - .orderBy(view.VERSION_DATE.desc(), view.DATA_ENTRY_DATE.desc())) + .over(partitionBy(dateTime) + .orderBy(versionDateField.desc(), view.DATA_ENTRY_DATE.desc())) .as("version_rank")) .from(view) .where(baseCondition) From cea2f38e24a71e27bc5056695782b84e4a5300df Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Thu, 25 Jun 2026 22:16:02 -0500 Subject: [PATCH 5/6] Simplify direct read regularity handling Signed-off-by: Charles Graham, SWT --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 77 ++++++++++++------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 20c2fd56c..661a91f9a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -189,9 +189,12 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private final Histogram getRequestedTimeSeriesResultsReturnedHistogram; @NotNull private final Histogram getRequestedTimeSeriesRequestWindowMillisHistogram; + @NotNull + private final MetricRegistry metrics; public TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics) { super(dsl); + this.metrics = metrics; String className = this.getClass().getName(); CacheStats stats = isVersionedCache.stats(); @@ -248,16 +251,7 @@ private ResultQuery> buildTsvD AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; Field dateTimeField = field(name("CWMS_20", "AV_TSV_DQU", DATE_TIME), Timestamp.class); Field versionDateField = field(name("CWMS_20", "AV_TSV_DQU", VERSION_DATE), Timestamp.class); - - Field qualityForNormalization = DSL.nvl( - view.QUALITY_CODE.cast(BigDecimal.class), - DSL.val(BigDecimal.valueOf(5)) - ); - - Field normalizedQuality = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - qualityForNormalization - ).as("quality_norm"); - + Field qualityCode = view.QUALITY_CODE.cast(BigDecimal.class).as(QUALITY_CODE); Field value = view.VALUE.as(VALUE); Condition baseCondition = view.ALIASED_ITEM.isNull() @@ -282,7 +276,7 @@ private ResultQuery> buildTsvD dateTimeField, versionDateField, value, - normalizedQuality, + qualityCode, baseCondition, versionDate, includeEntryDate @@ -293,7 +287,7 @@ private ResultQuery> buildTsvD dateTimeField, versionDateField, value, - normalizedQuality, + qualityCode, baseCondition, includeEntryDate ); @@ -839,7 +833,6 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (metadata == null) { throw new DataAccessException("Unable to resolve time series metadata for " + names); } - long tsCode = metadata.tsCode; String tsId = metadata.tsId; String[] tsIdParts = splitTimeSeriesId(tsId); @@ -858,6 +851,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (shouldFetchVerticalDatum(parmPart)) { verticalDatumInfo = fetchVerticalDatumInfoSeparately(locPart, requestedUnits, office); } + validateRequestedUnits(nativeUnits, requestedUnits); VersionType finalDateVersionType = getDirectReadVersionType( metadata.versionFlag, versionDate != null); @@ -867,7 +861,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, List rawRows = fetchRequestedTimeSeriesRows(tsCode, metadataOfficeId, metadataUnits, requestParameters, includeEntryDate); long effectiveIntervalOffset = intervalOffset; - if (isRegularSeries(intervalMinutes, intervalPart)) { + if (isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { effectiveIntervalOffset = resolveIntervalOffset(intervalOffset, timeZoneId, intervalPart, isLrts, rawRows); } @@ -884,7 +878,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, beginTime, endTime, metadataUnits, - resolveIntervalDuration(intervalMinutes, intervalPart), + resolveIntervalDuration(intervalMinutes, intervalOffset, intervalPart, isLrts), verticalDatumInfo, effectiveIntervalOffset, timeZoneId, @@ -902,6 +896,11 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( TimeSeriesRequestParameters requestParameters) { + return fetchRequestedTimeSeriesMetadataRecord(dsl, requestParameters); + } + + private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord(DSLContext metadataDsl, + TimeSeriesRequestParameters requestParameters) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); String units = requestParameters.getUnits(); @@ -942,7 +941,7 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( var tsIdView = AV_CWMS_TS_ID.AV_CWMS_TS_ID; SelectJoinStep metadataQuery = - dsl.with(valid) + metadataDsl.with(valid) .select( valid.field("tscode", BigDecimal.class).as("tscode"), valid.field("tsid", String.class).as("tsid"), @@ -1015,22 +1014,30 @@ private List fetchRequestedTimeSeriesRows(long tsCode, String return query.fetch(record -> { Timestamp dateTime = record.getValue(0, Timestamp.class); - Double value = record.getValue(1, Double.class); - int qualityCode = record.getValue(2, BigDecimal.class).intValue(); + Double dataValue = record.getValue(1, Double.class); + int quality = normalizeQualityCode(record.getValue(2, BigDecimal.class)); Timestamp dataEntryDate = record.getValue(3, Timestamp.class); if (dataEntryDate != null) { - return new TimeSeries.Record(dateTime, value, qualityCode, dataEntryDate); + return new TimeSeries.Record(dateTime, dataValue, quality, dataEntryDate); } - return new TimeSeries.Record(dateTime, value, qualityCode); + return new TimeSeries.Record(dateTime, dataValue, quality); }); } + private static int normalizeQualityCode(BigDecimal qualityCode) { + long quality = qualityCode == null ? 5L : qualityCode.longValue(); + if (quality > Integer.MAX_VALUE) { + quality -= 4_294_967_296L; + } + return (int) quality; + } + private ResultQuery> buildVersionedRowsQuery( AV_TSV_DQU view, Field dateTime, Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, ZonedDateTime versionDate, boolean includeEntryDate) { @@ -1043,7 +1050,7 @@ private ResultQuery> buildVers return dsl.select( dateTime, value, - normalizedQuality, + qualityCode, dataEntryDateField) .from(view) .where(baseCondition.and(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", @@ -1056,13 +1063,13 @@ private ResultQuery> buildMaxV Field dateTime, Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, boolean includeEntryDate) { var rankedRows = dsl.select( dateTime.as(DATE_TIME), value, - normalizedQuality, + qualityCode, includeEntryDate ? view.DATA_ENTRY_DATE.as(DATA_ENTRY_DATE) : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE), @@ -1076,7 +1083,7 @@ private ResultQuery> buildMaxV Field dateTimeCol = rankedRows.field(DATE_TIME, Timestamp.class); Field valueCol = rankedRows.field(VALUE, Double.class); - Field qualityCol = rankedRows.field("quality_norm", BigDecimal.class); + Field qualityCol = rankedRows.field(QUALITY_CODE, BigDecimal.class); Field dataEntryDateCol = rankedRows.field(DATA_ENTRY_DATE, Timestamp.class); Field versionRankCol = rankedRows.field("version_rank", Integer.class); @@ -1086,12 +1093,18 @@ private ResultQuery> buildMaxV .orderBy(dateTimeCol.asc()); } + private void validateRequestedUnits(String nativeUnits, String requestedUnits) { + if (nativeUnits != null && requestedUnits != null) { + CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(dsl.configuration(), 0.0D, nativeUnits, requestedUnits); + } + } + private List fetchExpectedRegularTimes(long intervalMinutes, long intervalOffset, String timeZoneId, String intervalPart, boolean isLrts, TimeSeriesRequestParameters requestParameters, List rawRows) { boolean shouldTrim = requestParameters.isShouldTrim(); - if (!isRegularSeries(intervalMinutes, intervalPart)) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { return Collections.emptyList(); } // Trimmed requests collapse to the observed data window @@ -1169,11 +1182,17 @@ private long resolveIntervalOffset(long intervalOffset, String timeZoneId, return (rawRows.get(0).getDateTime().getTime() - topOfInterval.getTime()) / TimeUnit.MINUTES.toMillis(1); } - private boolean isRegularSeries(long intervalMinutes, String intervalPart) { - return intervalMinutes != 0L || isLocalRegularInterval(intervalPart); + private boolean isRegularSeries(long intervalMinutes, long intervalOffset, String intervalPart, boolean isLrts) { + return intervalOffset != UTC_OFFSET_IRREGULAR + && (intervalMinutes != 0L || (isLrts && isLocalRegularInterval(intervalPart))); } - private Duration resolveIntervalDuration(long intervalMinutes, String intervalPart) { + private Duration resolveIntervalDuration(long intervalMinutes, long intervalOffset, + String intervalPart, boolean isLrts) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { + return Duration.ZERO; + } + if (intervalMinutes != 0L) { return Duration.ofMinutes(intervalMinutes); } From 530030f2140b8da6ba0dce86dce1381ad1c90d95 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Thu, 25 Jun 2026 23:54:31 -0500 Subject: [PATCH 6/6] Fix direct read unit and version filters Signed-off-by: Charles Graham, SWT --- .../main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 661a91f9a..d258ded52 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -851,7 +851,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (shouldFetchVerticalDatum(parmPart)) { verticalDatumInfo = fetchVerticalDatumInfoSeparately(locPart, requestedUnits, office); } - validateRequestedUnits(nativeUnits, requestedUnits); + validateRequestedUnits(nativeUnits, metadataUnits); VersionType finalDateVersionType = getDirectReadVersionType( metadata.versionFlag, versionDate != null); @@ -1041,8 +1041,13 @@ private ResultQuery> buildVers Condition baseCondition, ZonedDateTime versionDate, boolean includeEntryDate) { + Field versionTimestamp = CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.val(versionDate.toInstant().toEpochMilli())); String versionTimestampText = Timestamp.from(versionDate.toInstant()).toLocalDateTime() .format(ORACLE_DATE_FORMATTER); + Condition versionDateCondition = versionDateField.eq(versionTimestamp) + .or(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + versionDateField, DSL.val(versionTimestampText))); Field dataEntryDateField = includeEntryDate ? view.DATA_ENTRY_DATE : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE); @@ -1053,8 +1058,7 @@ private ResultQuery> buildVers qualityCode, dataEntryDateField) .from(view) - .where(baseCondition.and(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", - versionDateField, DSL.val(versionTimestampText)))) + .where(baseCondition.and(versionDateCondition)) .orderBy(dateTime.asc()); }