diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 349fbf72e..d258ded52 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -129,6 +129,8 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; private static final String CWMS_20 = "CWMS_20"; private static final String UNIT_ID = "UNIT_ID"; + private static final DateTimeFormatter ORACLE_DATE_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; @@ -187,9 +189,12 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private final Histogram getRequestedTimeSeriesResultsReturnedHistogram; @NotNull private final Histogram getRequestedTimeSeriesRequestWindowMillisHistogram; + @NotNull + private final MetricRegistry metrics; public TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics) { super(dsl); + this.metrics = metrics; String className = this.getClass().getName(); CacheStats stats = isVersionedCache.stats(); @@ -240,35 +245,38 @@ private ResultQuery> buildTsvD Timestamp beginTimestamp = Timestamp.from(beginTime.toInstant()); Timestamp endTimestamp = Timestamp.from(endTime.toInstant()); + String beginTimestampText = beginTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); + String endTimestampText = endTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; - - Field qualityForNormalization = DSL.nvl( - view.QUALITY_CODE.cast(BigDecimal.class), - DSL.val(BigDecimal.valueOf(5)) - ); - - Field normalizedQuality = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - qualityForNormalization - ).as("quality_norm"); - + Field dateTimeField = field(name("CWMS_20", "AV_TSV_DQU", DATE_TIME), Timestamp.class); + Field versionDateField = field(name("CWMS_20", "AV_TSV_DQU", VERSION_DATE), Timestamp.class); + Field qualityCode = view.QUALITY_CODE.cast(BigDecimal.class).as(QUALITY_CODE); Field value = view.VALUE.as(VALUE); Condition baseCondition = view.ALIASED_ITEM.isNull() .and(view.TS_CODE.eq(tsCode)) .and(view.OFFICE_ID.eq(officeId)) .and(view.UNIT_ID.equalIgnoreCase(units)) - .and(view.DATE_TIME.ge(beginTimestamp)) - .and(view.DATE_TIME.le(endTimestamp)) - .and(view.START_DATE.le(endTimestamp)) - .and(view.END_DATE.gt(beginTimestamp)); + .and(DSL.condition("{0} >= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(beginTimestampText))) + .and(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(endTimestampText))) + .and(view.START_DATE.isNull() + .or(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + view.START_DATE, DSL.val(endTimestampText)))) + .and(view.END_DATE.isNull() + .or(DSL.condition("{0} > to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + view.END_DATE, DSL.val(beginTimestampText)))); ResultQuery> query; if (versionDate != null) { query = buildVersionedRowsQuery( view, + dateTimeField, + versionDateField, value, - normalizedQuality, + qualityCode, baseCondition, versionDate, includeEntryDate @@ -276,8 +284,10 @@ private ResultQuery> buildTsvD } else { query = buildMaxVersionRowsQuery( view, + dateTimeField, + versionDateField, value, - normalizedQuality, + qualityCode, baseCondition, includeEntryDate ); @@ -823,7 +833,6 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (metadata == null) { throw new DataAccessException("Unable to resolve time series metadata for " + names); } - long tsCode = metadata.tsCode; String tsId = metadata.tsId; String[] tsIdParts = splitTimeSeriesId(tsId); @@ -842,6 +851,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (shouldFetchVerticalDatum(parmPart)) { verticalDatumInfo = fetchVerticalDatumInfoSeparately(locPart, requestedUnits, office); } + validateRequestedUnits(nativeUnits, metadataUnits); VersionType finalDateVersionType = getDirectReadVersionType( metadata.versionFlag, versionDate != null); @@ -851,7 +861,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, List rawRows = fetchRequestedTimeSeriesRows(tsCode, metadataOfficeId, metadataUnits, requestParameters, includeEntryDate); long effectiveIntervalOffset = intervalOffset; - if (isRegularSeries(intervalMinutes, intervalPart)) { + if (isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { effectiveIntervalOffset = resolveIntervalOffset(intervalOffset, timeZoneId, intervalPart, isLrts, rawRows); } @@ -868,7 +878,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, beginTime, endTime, metadataUnits, - resolveIntervalDuration(intervalMinutes, intervalPart), + resolveIntervalDuration(intervalMinutes, intervalOffset, intervalPart, isLrts), verticalDatumInfo, effectiveIntervalOffset, timeZoneId, @@ -886,6 +896,11 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( TimeSeriesRequestParameters requestParameters) { + return fetchRequestedTimeSeriesMetadataRecord(dsl, requestParameters); + } + + private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord(DSLContext metadataDsl, + TimeSeriesRequestParameters requestParameters) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); String units = requestParameters.getUnits(); @@ -926,7 +941,7 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( var tsIdView = AV_CWMS_TS_ID.AV_CWMS_TS_ID; SelectJoinStep metadataQuery = - dsl.with(valid) + metadataDsl.with(valid) .select( valid.field("tscode", BigDecimal.class).as("tscode"), valid.field("tsid", String.class).as("tsid"), @@ -999,55 +1014,72 @@ private List fetchRequestedTimeSeriesRows(long tsCode, String return query.fetch(record -> { Timestamp dateTime = record.getValue(0, Timestamp.class); - Double value = record.getValue(1, Double.class); - int qualityCode = record.getValue(2, BigDecimal.class).intValue(); + Double dataValue = record.getValue(1, Double.class); + int quality = normalizeQualityCode(record.getValue(2, BigDecimal.class)); Timestamp dataEntryDate = record.getValue(3, Timestamp.class); if (dataEntryDate != null) { - return new TimeSeries.Record(dateTime, value, qualityCode, dataEntryDate); + return new TimeSeries.Record(dateTime, dataValue, quality, dataEntryDate); } - return new TimeSeries.Record(dateTime, value, qualityCode); + return new TimeSeries.Record(dateTime, dataValue, quality); }); } + private static int normalizeQualityCode(BigDecimal qualityCode) { + long quality = qualityCode == null ? 5L : qualityCode.longValue(); + if (quality > Integer.MAX_VALUE) { + quality -= 4_294_967_296L; + } + return (int) quality; + } + private ResultQuery> buildVersionedRowsQuery( AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, ZonedDateTime versionDate, boolean includeEntryDate) { Field versionTimestamp = CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( DSL.val(versionDate.toInstant().toEpochMilli())); + String versionTimestampText = Timestamp.from(versionDate.toInstant()).toLocalDateTime() + .format(ORACLE_DATE_FORMATTER); + Condition versionDateCondition = versionDateField.eq(versionTimestamp) + .or(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + versionDateField, DSL.val(versionTimestampText))); Field dataEntryDateField = includeEntryDate ? view.DATA_ENTRY_DATE : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE); return dsl.select( - view.DATE_TIME, + dateTime, value, - normalizedQuality, + qualityCode, dataEntryDateField) .from(view) - .where(baseCondition.and(view.VERSION_DATE.eq(versionTimestamp))) - .orderBy(view.DATE_TIME.asc()); + .where(baseCondition.and(versionDateCondition)) + .orderBy(dateTime.asc()); } private ResultQuery> buildMaxVersionRowsQuery( AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, boolean includeEntryDate) { var rankedRows = dsl.select( - view.DATE_TIME.as(DATE_TIME), + dateTime.as(DATE_TIME), value, - normalizedQuality, + qualityCode, includeEntryDate ? view.DATA_ENTRY_DATE.as(DATA_ENTRY_DATE) : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE), DSL.rowNumber() - .over(partitionBy(view.DATE_TIME) - .orderBy(view.VERSION_DATE.desc(), view.DATA_ENTRY_DATE.desc())) + .over(partitionBy(dateTime) + .orderBy(versionDateField.desc(), view.DATA_ENTRY_DATE.desc())) .as("version_rank")) .from(view) .where(baseCondition) @@ -1055,7 +1087,7 @@ private ResultQuery> buildMaxV Field dateTimeCol = rankedRows.field(DATE_TIME, Timestamp.class); Field valueCol = rankedRows.field(VALUE, Double.class); - Field qualityCol = rankedRows.field("quality_norm", BigDecimal.class); + Field qualityCol = rankedRows.field(QUALITY_CODE, BigDecimal.class); Field dataEntryDateCol = rankedRows.field(DATA_ENTRY_DATE, Timestamp.class); Field versionRankCol = rankedRows.field("version_rank", Integer.class); @@ -1065,12 +1097,18 @@ private ResultQuery> buildMaxV .orderBy(dateTimeCol.asc()); } + private void validateRequestedUnits(String nativeUnits, String requestedUnits) { + if (nativeUnits != null && requestedUnits != null) { + CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(dsl.configuration(), 0.0D, nativeUnits, requestedUnits); + } + } + private List fetchExpectedRegularTimes(long intervalMinutes, long intervalOffset, String timeZoneId, String intervalPart, boolean isLrts, TimeSeriesRequestParameters requestParameters, List rawRows) { boolean shouldTrim = requestParameters.isShouldTrim(); - if (!isRegularSeries(intervalMinutes, intervalPart)) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { return Collections.emptyList(); } // Trimmed requests collapse to the observed data window @@ -1148,11 +1186,17 @@ private long resolveIntervalOffset(long intervalOffset, String timeZoneId, return (rawRows.get(0).getDateTime().getTime() - topOfInterval.getTime()) / TimeUnit.MINUTES.toMillis(1); } - private boolean isRegularSeries(long intervalMinutes, String intervalPart) { - return intervalMinutes != 0L || isLocalRegularInterval(intervalPart); + private boolean isRegularSeries(long intervalMinutes, long intervalOffset, String intervalPart, boolean isLrts) { + return intervalOffset != UTC_OFFSET_IRREGULAR + && (intervalMinutes != 0L || (isLrts && isLocalRegularInterval(intervalPart))); } - private Duration resolveIntervalDuration(long intervalMinutes, String intervalPart) { + private Duration resolveIntervalDuration(long intervalMinutes, long intervalOffset, + String intervalPart, boolean isLrts) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { + return Duration.ZERO; + } + if (intervalMinutes != 0L) { return Duration.ofMinutes(intervalMinutes); } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java index 231b9cec5..a0643121e 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.ApiServlet; import cwms.cda.api.enums.VersionType; import cwms.cda.data.dto.TimeSeries; import cwms.cda.formatters.Formats; @@ -25,8 +26,6 @@ import java.sql.Types; import java.time.Duration; import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; @@ -159,6 +158,32 @@ void irregularReadMatchesRetrieveTs() throws Exception { ); } + @Test + void pseudoIrregularReadWithLrtsHeaderMatchesRetrieveTs() throws Exception { + String seriesId = "ITPARPIRR.Flow.Inst.~15Minutes.0.BENCH"; + Instant beginTime = Instant.parse("2024-01-05T12:00:00Z"); + Instant endTime = Instant.parse("2024-01-05T13:00:00Z"); + List rows = List.of( + row("2024-01-05T12:00:00Z", 10.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T12:17:00Z", 20.0, 0, "2024-01-06T00:01:00Z", null), + row("2024-01-05T12:45:00Z", 30.0, 0, "2024-01-06T00:02:00Z", null) + ); + seedTimeSeries("ITPARPIRR", seriesId, rows, false, null); + + List expectedRows = fetchOracleRows(seriesId, "cfs", beginTime, endTime, + false, null); + TimeSeries actualResponse = fetchCdaRowsWithPageSize(seriesId, "cfs", beginTime, endTime, + 1000, false, null, true, true); + + assertEquals(expectedRows.size(), actualResponse.getTotal(), "total"); + assertEquals(expectedRows.size(), actualResponse.getValues().size(), "values size"); + assertEquals(Duration.ZERO, actualResponse.getInterval(), "interval"); + assertEquals((long) Integer.MIN_VALUE, actualResponse.getIntervalOffset(), "interval offset"); + for (int index = 0; index < expectedRows.size(); index++) { + assertRecordsEqual(expectedRows.get(index), actualResponse.getValues().get(index), index); + } + } + @Test void dstWindowRegularReadMatchesRetrieveTs() throws Exception { Instant dstStart = Instant.parse("2024-03-09T00:00:00Z"); @@ -184,13 +209,13 @@ void localRegularGapReadMatchesRetrieveTs() throws Exception { "ITPARLCL", "ITPARLCL.Flow.Inst.~1Day.0.BENCH", "cfs", - Instant.parse("2024-01-01T00:00:00Z"), - Instant.parse("2024-01-05T00:00:00Z"), + Instant.parse("2024-01-01T06:00:00Z"), + Instant.parse("2024-01-05T06:00:00Z"), List.of( - row("2024-01-01T00:00:00Z", 1.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-02T00:00:00Z", 2.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-04T00:00:00Z", 4.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-05T00:00:00Z", 5.0, 0, "2024-01-06T00:00:00Z", null) + row("2024-01-01T06:00:00Z", 1.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-02T06:00:00Z", 2.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-04T06:00:00Z", 4.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T06:00:00Z", 5.0, 0, "2024-01-06T00:00:00Z", null) ), false, false, @@ -270,8 +295,9 @@ void trimmedResponseWindowMatchesReturnedValues() throws Exception { assertNotNull(response.getBegin(), "begin"); assertNotNull(response.getEnd(), "end"); - assertEquals(Instant.parse("2024-01-01T00:00:00Z"), response.getBegin().toInstant(), "begin"); - assertEquals(Instant.parse("2024-01-01T00:09:00Z"), response.getEnd().toInstant(), "end"); + assertEquals(response.getValues().get(0).getDateTime().toInstant(), response.getBegin().toInstant(), "begin"); + assertEquals(response.getValues().get(response.getValues().size() - 1).getDateTime().toInstant(), + response.getEnd().toInstant(), "end"); } private static void assertDirectReadMatchesOracle(String locationId, String seriesId, String units, @@ -420,8 +446,17 @@ private static void assertRecordsEqual(TimeSeries.Record expected, TimeSeries.Re private static void seedTimeSeries(String locationId, String seriesId, List rows, boolean versioned) throws SQLException { + seedTimeSeries(locationId, seriesId, rows, versioned, 0); + } + + private static void seedTimeSeries(String locationId, String seriesId, List rows, + boolean versioned, Integer intervalOffset) throws SQLException { createLocation(locationId, true, OFFICE); - createTimeseries(OFFICE, seriesId, 0); + if (intervalOffset != null) { + createTimeseries(OFFICE, seriesId, intervalOffset); + } else { + createTimeseries(OFFICE, seriesId); + } CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); database.connection(connection -> { @@ -435,7 +470,7 @@ private static void seedTimeSeries(String locationId, String seriesId, List years = rows.stream() - .map(seedRow -> OffsetDateTime.ofInstant(seedRow.dateTime, ZoneOffset.UTC).getYear()) + .map(seedRow -> storageYear(seedRow.dateTime)) .distinct() .collect(Collectors.toList()); @@ -485,7 +520,7 @@ private static void insertScenarioRows(Connection connection, long tsCode, List< Map> rowsByYear = new LinkedHashMap<>(); for (SeedRow row: sortedRows) { - int year = OffsetDateTime.ofInstant(row.dateTime, ZoneOffset.UTC).getYear(); + int year = storageYear(row.dateTime); rowsByYear.computeIfAbsent(year, ignored -> new ArrayList<>()).add(row); } @@ -528,6 +563,10 @@ private static void bindScenarioInsert(PreparedStatement statement, long tsCode, statement.setInt(6, row.qualityCode); } + private static int storageYear(Instant instant) { + return Timestamp.from(instant).toLocalDateTime().getYear(); + } + private static void updateScenarioExtents(Connection connection, long tsCode, List rows) throws SQLException { Set distinctVersionDates = rows.stream() @@ -650,6 +689,14 @@ private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units Instant endTime, int pageSize, boolean includeEntryDate, Instant versionDate, boolean trim) throws Exception { + return fetchCdaRowsWithPageSize(seriesId, units, beginTime, endTime, pageSize, includeEntryDate, + versionDate, trim, null); + } + + private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units, Instant beginTime, + Instant endTime, int pageSize, boolean includeEntryDate, + Instant versionDate, boolean trim, Boolean lrtsFormatting) + throws Exception { RequestSpecification request = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) @@ -661,6 +708,9 @@ private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units .queryParam(Controllers.PAGE_SIZE, pageSize) .queryParam(Controllers.TRIM, trim) .queryParam(Controllers.INCLUDE_ENTRY_DATE, includeEntryDate); + if (lrtsFormatting != null) { + request = request.header(ApiServlet.IS_NEW_LRTS, lrtsFormatting); + } if (versionDate != null) { request = request.queryParam(Controllers.VERSION_DATE, versionDate.toString()); }