diff --git a/cwms-data-api/build.gradle b/cwms-data-api/build.gradle index ac38b5518e..b25d16bc74 100644 --- a/cwms-data-api/build.gradle +++ b/cwms-data-api/build.gradle @@ -291,6 +291,35 @@ task integrationTests(type: Test) { jvmArgs += "-Dcatalina.base=$buildDir/tomcat" } +task timeseriesReadBenchmark(type: JavaExec) { + group "verification" + description = "Run the local time-series read benchmark harness" + dependsOn generateConfig + dependsOn war + dependsOn testClasses + + workingDir = projectDir + classpath = sourceSets.test.runtimeClasspath + classpath += configurations.baseLibs + classpath += configurations.tomcatLibs + + mainClass = "helpers.TimeSeriesReadBenchmark" + + systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") && !k.startsWith("RADAR_JDBC") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") && !k.startsWith("CDA_JDBC") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("testcontainer") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("benchmark.") } + + jvmArgs += "-DwarFile=$buildDir/libs/${project.name}-${project.version}.war" + jvmArgs += "-DwarContext=/cwms-data" + jvmArgs += "-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager" + jvmArgs += "-Djava.util.logging.config.file=$projectDir/logging.properties" + jvmArgs += "-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" + jvmArgs += "-Dcwms.dataapi.access.provider=MultipleAccessManager" + jvmArgs += "-Dcwms.dataapi.access.providers=KeyAccessManager,CwmsAccessManager" + jvmArgs += "-Dcatalina.base=$buildDir/tomcat" +} + task prepareDockerBuild(type: Copy, dependsOn: war) { doFirst { project.mkdir("$buildDir/docker") diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 6dabcad233..1204be255d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -52,7 +52,9 @@ import java.time.Duration; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -63,6 +65,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -104,6 +107,7 @@ import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU; import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN; import usace.cwms.db.jooq.codegen.udt.records.DATE_TABLE_TYPE; +import usace.cwms.db.jooq.codegen.udt.records.DATE_RANGE_T; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_ARRAY; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_TYPE; @@ -135,6 +139,9 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries public static final String PROP_BASE = "cwms.cda.data.dao.ts"; public static final String VERSIONED_NAME = "isVersioned"; + private static final long UTC_OFFSET_IRREGULAR = -2147483648L; + private static final long UTC_OFFSET_UNDEFINED = 2147483647L; + private static final String UTC = "UTC"; /** To be able to use a named inner table (otherwise JOOQ creates a random alias which messes * with the planner) we need to use fixed names to be able to reference the required columns. @@ -243,8 +250,18 @@ public FilteredTimeSeries getTimeseries(String page, int pageSize, TimeSeriesReq return fts; } - protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull TimeSeriesRequestParameters requestParameters, - @Nullable FilteredTimeSeriesParameters fp) { + protected TimeSeries getRequestedTimeSeries(String page, int pageSize, + @NotNull TimeSeriesRequestParameters requestParameters, + @Nullable FilteredTimeSeriesParameters fp) { + if (fp != null) { + return getRequestedTimeSeriesLegacy(page, pageSize, requestParameters, fp); + } + return getRequestedTimeSeriesDirect(page, pageSize, requestParameters); + } + + protected TimeSeries getRequestedTimeSeriesLegacy(String page, int pageSize, + @NotNull TimeSeriesRequestParameters requestParameters, + @Nullable FilteredTimeSeriesParameters fp) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); @@ -540,6 +557,597 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull return retVal; } + private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, + @NotNull TimeSeriesRequestParameters requestParameters) { + String names = requestParameters.getNames(); + String office = requestParameters.getOffice(); + String requestedUnits = requestParameters.getUnits(); + ZonedDateTime beginTime = requestParameters.getBeginTime(); + ZonedDateTime endTime = requestParameters.getEndTime(); + ZonedDateTime versionDate = requestParameters.getVersionDate(); + boolean includeEntryDate = requestParameters.isIncludeEntryDate(); + String cursor = null; + Timestamp tsCursor = null; + + validateEntryDateSupport(includeEntryDate); + + if (page != null && !page.isEmpty()) { + final String[] parts = CwmsDTOPaginated.decodeCursor(page); + + logger.atFine().log("Decoded cursor"); + logger.atFinest().log("%s", lazy(() -> { + StringBuilder sb = new StringBuilder(); + for (String p : parts) { + sb.append(p).append("\n"); + } + return sb.toString(); + })); + + if (parts.length > 1) { + cursor = parts[0]; + tsCursor = Timestamp.from(Instant.ofEpochMilli(Long.parseLong(parts[0]))); + pageSize = Integer.parseInt(parts[parts.length - 1]); + } + } + + RequestedTimeSeriesMetadata metadata = fetchRequestedTimeSeriesMetadata(requestParameters); + if (metadata == null) { + throw new DataAccessException("Unable to resolve time series metadata for " + names); + } + + String parmPart = metadata.getParmPart(); + String locPart = metadata.getLocPart(); + VerticalDatumInfo verticalDatumInfo = null; + if (shouldFetchVerticalDatum(parmPart)) { + verticalDatumInfo = fetchVerticalDatumInfoSeparately(locPart, requestedUnits, office); + } + + VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); + if (pageSize == 0) { + return null; + } + + List rawRows = fetchRequestedTimeSeriesRows(metadata, requestParameters); + List expectedTimes = fetchExpectedRegularTimes(metadata, requestParameters, rawRows); + int total = countMergedRows(rawRows, expectedTimes); + + TimeSeries timeseries = new TimeSeries( + cursor, + pageSize, + total, + metadata.getTsId(), + metadata.getOfficeId(), + beginTime, + endTime, + metadata.getUnits(), + Duration.ofMinutes(metadata.getIntervalMinutes()), + verticalDatumInfo, + metadata.getIntervalOffset(), + metadata.getTimeZoneId(), + versionDate, + finalDateVersionType + ); + + populateTimeSeriesValues(timeseries, rawRows, expectedTimes, tsCursor, includeEntryDate); + return timeseries; + } + + private RequestedTimeSeriesMetadata fetchRequestedTimeSeriesMetadata( + TimeSeriesRequestParameters requestParameters) { + String names = requestParameters.getNames(); + String office = requestParameters.getOffice(); + String units = requestParameters.getUnits(); + + final Field officeId = CWMS_UTIL_PACKAGE.call_GET_DB_OFFICE_ID( + office != null ? DSL.val(office) : CWMS_UTIL_PACKAGE.call_USER_OFFICE_ID()); + final Field tsId = CWMS_TS_PACKAGE.call_GET_TS_ID__2(DSL.val(names), officeId); + final Field tsCode = CWMS_TS_PACKAGE.call_GET_TS_CODE__2(DSL.val(names), officeId); + + Table> validTs = + select(tsCode.as("tscode"), + tsId.as("tsid"), + officeId.as("office_id")) + .asTable("validts"); + + Field loc = CWMS_UTIL_PACKAGE.call_SPLIT_TEXT( + validTs.field("tsid", String.class), + DSL.val(BigInteger.valueOf(1L)), DSL.val("."), + DSL.val(BigInteger.valueOf(6L))); + Field param = DSL.upper(CWMS_UTIL_PACKAGE.call_SPLIT_TEXT( + validTs.field("tsid", String.class), + DSL.val(BigInteger.valueOf(2L)), DSL.val("."), + DSL.val(BigInteger.valueOf(6L)))); + Field intervalPart = CWMS_UTIL_PACKAGE.call_SPLIT_TEXT( + validTs.field("tsid", String.class), + DSL.val(BigInteger.valueOf(4L)), DSL.val("."), + DSL.val(BigInteger.valueOf(6L))); + + Field unit = units.compareToIgnoreCase("SI") == 0 + || units.compareToIgnoreCase("EN") == 0 + ? CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(tsCode), + DSL.val(units, String.class)) + : DSL.val(units, String.class); + + Field interval = CWMS_TS_PACKAGE.call_GET_TS_INTERVAL__2(validTs.field("tsid", String.class)); + + CommonTableExpression valid = + name("valid").fields("tscode", "tsid", "office_id", "loc_part", "units", + "interval", "parm_part", "interval_part") + .as( + select( + validTs.field("tscode", BigDecimal.class).as("tscode"), + validTs.field("tsid", String.class).as("tsid"), + validTs.field("office_id", String.class).as("office_id"), + loc.as("loc_part"), + unit.as("units"), + interval.as("interval"), + param.as("parm_part"), + intervalPart.as("interval_part")) + .from(validTs)); + + SelectJoinStep metadataQuery = + dsl.with(valid) + .select( + valid.field("tscode", BigDecimal.class).as("tscode"), + valid.field("tsid", String.class).as("tsid"), + valid.field("office_id", String.class).as("office_id"), + valid.field("units", String.class).as("units"), + AV_CWMS_TS_ID2.UNIT_ID.as("source_unit"), + valid.field("interval", BigDecimal.class).as("interval"), + valid.field("loc_part", String.class).as("loc_part"), + valid.field("parm_part", String.class).as("parm_part"), + valid.field("interval_part", String.class).as("interval_part"), + AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET, + AV_CWMS_TS_ID2.TIME_ZONE_ID) + .from(valid) + .leftOuterJoin(AV_CWMS_TS_ID2) + .on(AV_CWMS_TS_ID2.DB_OFFICE_ID.eq(valid.field("office_id", String.class)) + .and(AV_CWMS_TS_ID2.TS_CODE.eq(valid.field("tscode", BigDecimal.class))) + .and(AV_CWMS_TS_ID2.ALIASED_ITEM.isNull())); + + logger.atFine().log("%s", lazy(() -> metadataQuery.getSQL(ParamType.INLINED))); + + return metadataQuery.fetchOne(tsMetadata -> { + BigDecimal intervalValue = tsMetadata.getValue("interval", BigDecimal.class); + Number offsetValue = tsMetadata.getValue(AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET); + BigDecimal tsCodeValue = tsMetadata.getValue("tscode", BigDecimal.class); + long tsCodeLong = tsCodeValue.longValue(); + String requestedUnit = tsMetadata.getValue("units", String.class); + String sourceUnit = tsMetadata.getValue("source_unit", String.class); + validateRequestedUnits(sourceUnit, requestedUnit); + boolean isLrts = parseBool(CWMS_TS_PACKAGE.call_IS_LRTS__2(dsl.configuration(), tsCodeLong)); + return new RequestedTimeSeriesMetadata( + tsCodeLong, + tsMetadata.getValue("tsid", String.class), + tsMetadata.getValue("office_id", String.class), + requestedUnit, + intervalValue == null ? 0L : intervalValue.longValue(), + offsetValue == null ? UTC_OFFSET_IRREGULAR : offsetValue.longValue(), + tsMetadata.getValue(AV_CWMS_TS_ID2.TIME_ZONE_ID) == null + ? UTC + : tsMetadata.getValue(AV_CWMS_TS_ID2.TIME_ZONE_ID), + tsMetadata.getValue("loc_part", String.class), + tsMetadata.getValue("parm_part", String.class), + tsMetadata.getValue("interval_part", String.class), + isLrts + ); + }); + } + + private List fetchRequestedTimeSeriesRows(RequestedTimeSeriesMetadata metadata, + TimeSeriesRequestParameters requestParameters) { + ZonedDateTime beginTime = requestParameters.getBeginTime(); + ZonedDateTime endTime = requestParameters.getEndTime(); + ZonedDateTime versionDate = requestParameters.getVersionDate(); + Timestamp beginTimestamp = Timestamp.from(beginTime.toInstant()); + Timestamp endTimestamp = Timestamp.from(endTime.toInstant()); + + AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; + Field qualityForNormalization = DSL.nvl( + view.QUALITY_CODE.cast(BigDecimal.class), + DSL.val(BigDecimal.valueOf(5)) + ); + Field normalizedQuality = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( + qualityForNormalization).as("quality_norm"); + + Condition baseCondition = view.ALIASED_ITEM.isNull() + .and(view.TS_CODE.eq(metadata.getTsCode())) + .and(view.OFFICE_ID.eq(metadata.getOfficeId())) + .and(view.UNIT_ID.equalIgnoreCase(metadata.getUnits())) + .and(view.DATE_TIME.ge(beginTimestamp)) + .and(view.DATE_TIME.le(endTimestamp)) + .and(view.START_DATE.le(endTimestamp)) + .and(view.END_DATE.gt(beginTimestamp)); + + SelectConditionStep> query; + if (versionDate != null) { + Field versionTimestamp = CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( + DSL.val(versionDate.toInstant().toEpochMilli())); + query = dsl.select( + view.DATE_TIME, + view.VALUE, + normalizedQuality, + view.DATA_ENTRY_DATE) + .from(view) + .where(baseCondition.and(view.VERSION_DATE.eq(versionTimestamp))); + } else { + Table rankedRows = dsl.select( + view.DATE_TIME.as(DATE_TIME), + view.VALUE.as(VALUE), + normalizedQuality, + view.DATA_ENTRY_DATE.as(DATA_ENTRY_DATE), + DSL.rowNumber() + .over(partitionBy(view.DATE_TIME) + .orderBy(view.VERSION_DATE.desc(), view.DATA_ENTRY_DATE.desc())) + .as("version_rank")) + .from(view) + .where(baseCondition) + .asTable("ranked_rows"); + + Field dateTimeCol = rankedRows.field(DATE_TIME, Timestamp.class); + Field valueCol = rankedRows.field(VALUE, Double.class); + Field qualityCol = rankedRows.field("quality_norm", BigDecimal.class); + Field dataEntryDateCol = rankedRows.field(DATA_ENTRY_DATE, Timestamp.class); + Field versionRankCol = rankedRows.field("version_rank", Integer.class); + + query = dsl.select(dateTimeCol, valueCol, qualityCol, dataEntryDateCol) + .from(rankedRows) + .where(versionRankCol.eq(1)); + } + + query.orderBy(field(DATE_TIME, Timestamp.class).asc()); + logger.atFine().log("%s", lazy(() -> query.getSQL(ParamType.INLINED))); + + return query.fetch(record -> new RetrievedTimeSeriesValue( + record.getValue(0, Timestamp.class), + record.getValue(1, Double.class), + record.getValue(2, BigDecimal.class).intValue(), + record.getValue(3, Timestamp.class) + )); + } + + private List fetchExpectedRegularTimes(RequestedTimeSeriesMetadata metadata, + TimeSeriesRequestParameters requestParameters, + List rawRows) { + if (!isRegularSeries(metadata)) { + return Collections.emptyList(); + } + if (rawRows.isEmpty() && requestParameters.isShouldTrim()) { + return Collections.emptyList(); + } + + Timestamp rangeStart = requestParameters.isShouldTrim() + ? rawRows.get(0).getDateTime() + : Timestamp.from(requestParameters.getBeginTime().toInstant()); + Timestamp rangeEnd = requestParameters.isShouldTrim() + ? rawRows.get(rawRows.size() - 1).getDateTime() + : Timestamp.from(requestParameters.getEndTime().toInstant()); + + long offsetMinutes = resolveIntervalOffset(metadata, rawRows); + if (canGenerateExpectedTimesInJava(metadata)) { + return buildExpectedRegularTimesUtc(rangeStart, rangeEnd, metadata.getIntervalMinutes(), offsetMinutes); + } + + String intervalTimeZone = metadata.isLrts() ? metadata.getTimeZoneId() : UTC; + DATE_RANGE_T dateRange = new DATE_RANGE_T(rangeStart, rangeEnd, UTC, "T", "T", null); + DATE_TABLE_TYPE expectedTimeTable = CWMS_TS_PACKAGE.call_GET_REG_TS_TIMES_UTC_F( + dsl.configuration(), + dateRange, + metadata.getIntervalPart(), + String.valueOf(offsetMinutes), + intervalTimeZone + ); + + List retVal = new ArrayList<>(); + if (expectedTimeTable != null) { + expectedTimeTable.forEach(timestamp -> { + if (timestamp != null) { + retVal.add(normalizeOracleUtcTimestamp(timestamp)); + } + }); + } + return retVal; + } + + private long resolveIntervalOffset(RequestedTimeSeriesMetadata metadata, + List rawRows) { + long intervalOffset = metadata.getIntervalOffset(); + if (intervalOffset != UTC_OFFSET_UNDEFINED) { + return intervalOffset; + } + if (rawRows.isEmpty()) { + return 0L; + } + + if (canGenerateExpectedTimesInJava(metadata)) { + long intervalMillis = TimeUnit.MINUTES.toMillis(metadata.getIntervalMinutes()); + return TimeUnit.MILLISECONDS.toMinutes(Math.floorMod(rawRows.get(0).getDateTime().getTime(), intervalMillis)); + } + + String intervalTimeZone = metadata.isLrts() ? metadata.getTimeZoneId() : UTC; + Timestamp topOfInterval = normalizeOracleUtcTimestamp(CWMS_TS_PACKAGE.call_TOP_OF_INTERVAL_UTC( + dsl.configuration(), + rawRows.get(0).getDateTime(), + metadata.getIntervalPart(), + intervalTimeZone, + "F" + )); + return (rawRows.get(0).getDateTime().getTime() - topOfInterval.getTime()) / TimeUnit.MINUTES.toMillis(1); + } + + private boolean isRegularSeries(RequestedTimeSeriesMetadata metadata) { + return metadata.getIntervalMinutes() != 0L || metadata.getIntervalOffset() != UTC_OFFSET_IRREGULAR; + } + + private int countMergedRows(List rawRows, List expectedTimes) { + if (expectedTimes.isEmpty()) { + return rawRows.size(); + } + + int total = 0; + int rawIndex = 0; + int expectedIndex = 0; + while (rawIndex < rawRows.size() || expectedIndex < expectedTimes.size()) { + Timestamp rawTime = rawIndex < rawRows.size() ? rawRows.get(rawIndex).getDateTime() : null; + Timestamp expectedTime = expectedIndex < expectedTimes.size() ? expectedTimes.get(expectedIndex) : null; + + if (rawTime == null) { + expectedIndex++; + } else if (expectedTime == null) { + rawIndex++; + } else { + int compare = compareTimestampOrder(expectedTime, rawTime); + if (compare < 0) { + expectedIndex++; + } else if (compare > 0) { + rawIndex++; + } else { + expectedIndex++; + rawIndex++; + } + } + total++; + } + return total; + } + + private void populateTimeSeriesValues(TimeSeries timeseries, + List rawRows, + List expectedTimes, + Timestamp tsCursor, + boolean includeEntryDate) { + int rawIndex = 0; + int expectedIndex = 0; + int collected = 0; + int maxRecords = timeseries.getPageSize() > 0 ? timeseries.getPageSize() + 1 : Integer.MAX_VALUE; + + while ((rawIndex < rawRows.size() || expectedIndex < expectedTimes.size()) && collected < maxRecords) { + RetrievedTimeSeriesValue rawRow = rawIndex < rawRows.size() ? rawRows.get(rawIndex) : null; + Timestamp expectedTime = expectedIndex < expectedTimes.size() ? expectedTimes.get(expectedIndex) : null; + + Timestamp candidateTime; + RetrievedTimeSeriesValue candidateRow = null; + boolean syntheticRow = false; + + if (rawRow == null) { + candidateTime = expectedTime; + syntheticRow = true; + expectedIndex++; + } else if (expectedTime == null) { + candidateTime = rawRow.getDateTime(); + candidateRow = rawRow; + rawIndex++; + } else { + int compare = compareTimestampOrder(expectedTime, rawRow.getDateTime()); + if (compare < 0) { + candidateTime = expectedTime; + syntheticRow = true; + expectedIndex++; + } else if (compare > 0) { + candidateTime = rawRow.getDateTime(); + candidateRow = rawRow; + rawIndex++; + } else { + candidateTime = rawRow.getDateTime(); + candidateRow = rawRow; + rawIndex++; + expectedIndex++; + } + } + + if (tsCursor != null && compareTimestampOrder(candidateTime, tsCursor) < 0) { + continue; + } + + if (syntheticRow) { + if (includeEntryDate) { + timeseries.addValue(candidateTime, null, 0, null); + } else { + timeseries.addValue(candidateTime, null, 0); + } + } else if (includeEntryDate) { + timeseries.addValue(candidateRow.getDateTime(), candidateRow.getValue(), + candidateRow.getQualityCode(), candidateRow.getDataEntryDate()); + } else { + timeseries.addValue(candidateRow.getDateTime(), candidateRow.getValue(), + candidateRow.getQualityCode()); + } + collected++; + } + } + + private int compareTimestampOrder(Timestamp left, Timestamp right) { + return Long.compare(left.getTime(), right.getTime()); + } + + private Timestamp normalizeOracleUtcTimestamp(Timestamp timestamp) { + LocalDateTime utcWallTime = timestamp.toLocalDateTime(); + return Timestamp.from(utcWallTime.toInstant(ZoneOffset.UTC)); + } + + private boolean canGenerateExpectedTimesInJava(RequestedTimeSeriesMetadata metadata) { + if (metadata.isLrts() || metadata.getIntervalMinutes() <= 0L) { + return false; + } + + String intervalPart = metadata.getIntervalPart(); + if (intervalPart == null) { + return false; + } + + String normalizedInterval = intervalPart.toLowerCase(Locale.ENGLISH); + return normalizedInterval.endsWith("minute") + || normalizedInterval.endsWith("minutes") + || normalizedInterval.endsWith("hour") + || normalizedInterval.endsWith("hours") + || normalizedInterval.endsWith("day") + || normalizedInterval.endsWith("days") + || normalizedInterval.endsWith("week") + || normalizedInterval.endsWith("weeks"); + } + + private List buildExpectedRegularTimesUtc(Timestamp rangeStart, + Timestamp rangeEnd, + long intervalMinutes, + long offsetMinutes) { + long intervalMillis = TimeUnit.MINUTES.toMillis(intervalMinutes); + long offsetMillis = TimeUnit.MINUTES.toMillis(Math.floorMod(offsetMinutes, intervalMinutes)); + long startMillis = rangeStart.getTime(); + long endMillis = rangeEnd.getTime(); + long firstMillis = alignToInterval(startMillis, intervalMillis, offsetMillis); + + List expectedTimes = new ArrayList<>(); + for (long millis = firstMillis; millis <= endMillis; millis += intervalMillis) { + expectedTimes.add(new Timestamp(millis)); + } + return expectedTimes; + } + + private long alignToInterval(long timestampMillis, long intervalMillis, long offsetMillis) { + long remainder = Math.floorMod(timestampMillis - offsetMillis, intervalMillis); + if (remainder == 0L) { + return timestampMillis; + } + return timestampMillis + (intervalMillis - remainder); + } + + private void validateRequestedUnits(String sourceUnit, String requestedUnit) { + if (sourceUnit == null || requestedUnit == null || sourceUnit.equalsIgnoreCase(requestedUnit)) { + return; + } + dsl.select(CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( + DSL.val(0.0d), + DSL.val(sourceUnit), + DSL.val(requestedUnit))) + .fetchOne(0, Double.class); + } + + private static final class RequestedTimeSeriesMetadata { + private final long tsCode; + private final String tsId; + private final String officeId; + private final String units; + private final long intervalMinutes; + private final long intervalOffset; + private final String timeZoneId; + private final String locPart; + private final String parmPart; + private final String intervalPart; + private final boolean isLrts; + + private RequestedTimeSeriesMetadata(long tsCode, String tsId, String officeId, String units, + long intervalMinutes, long intervalOffset, String timeZoneId, + String locPart, String parmPart, String intervalPart, + boolean isLrts) { + this.tsCode = tsCode; + this.tsId = tsId; + this.officeId = officeId; + this.units = units; + this.intervalMinutes = intervalMinutes; + this.intervalOffset = intervalOffset; + this.timeZoneId = timeZoneId; + this.locPart = locPart; + this.parmPart = parmPart; + this.intervalPart = intervalPart; + this.isLrts = isLrts; + } + + private long getTsCode() { + return tsCode; + } + + private String getTsId() { + return tsId; + } + + private String getOfficeId() { + return officeId; + } + + private String getUnits() { + return units; + } + + private long getIntervalMinutes() { + return intervalMinutes; + } + + private long getIntervalOffset() { + return intervalOffset; + } + + private String getTimeZoneId() { + return timeZoneId; + } + + private String getLocPart() { + return locPart; + } + + private String getParmPart() { + return parmPart; + } + + private String getIntervalPart() { + return intervalPart; + } + + private boolean isLrts() { + return isLrts; + } + } + + private static final class RetrievedTimeSeriesValue { + private final Timestamp dateTime; + private final Double value; + private final int qualityCode; + private final Timestamp dataEntryDate; + + private RetrievedTimeSeriesValue(Timestamp dateTime, Double value, int qualityCode, Timestamp dataEntryDate) { + this.dateTime = dateTime; + this.value = value; + this.qualityCode = qualityCode; + this.dataEntryDate = dataEntryDate; + } + + private Timestamp getDateTime() { + return dateTime; + } + + private Double getValue() { + return value; + } + + private int getQualityCode() { + return qualityCode; + } + + private Timestamp getDataEntryDate() { + return dataEntryDate; + } + } + private boolean shouldFetchVerticalDatum(String parmPart) { // Check if parameter requires vertical datum (e.g., "ELEV") if (parmPart == null) { diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java new file mode 100644 index 0000000000..b3e9d68834 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java @@ -0,0 +1,600 @@ +package cwms.cda.api; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.formatters.Formats; +import fixtures.CwmsDataApiSetupCallback; +import io.restassured.filter.log.LogDetail; +import io.restassured.response.ExtractableResponse; +import io.restassured.response.Response; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.IntStream; +import javax.servlet.http.HttpServletResponse; +import mil.army.usace.hec.test.database.CwmsDatabaseContainer; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; +import io.restassured.specification.RequestSpecification; + +@Tag("integration") +final class TimeSeriesDirectReadParityIT extends DataApiTestIT { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String OFFICE = "SPK"; + private static final double DOUBLE_TOLERANCE = 1e-9; + + @ParameterizedTest(name = "{0}") + @MethodSource("scenarios") + void directReadMatchesOracleRetrieveTs(Scenario scenario) throws Exception { + seedScenario(scenario); + + List expectedRows = fetchOracleRows(scenario); + TimeSeriesResponse actualResponse = fetchCdaRows(scenario); + String mismatchSummary = buildMismatchSummary(expectedRows, actualResponse); + + assertEquals(expectedRows.size(), actualResponse.total, "Reported total " + mismatchSummary); + assertEquals(scenario.expectedDateVersionType, actualResponse.dateVersionType, "Date version type"); + assertEquals(scenario.expectedInterval, actualResponse.interval, "Interval"); + assertEquals(scenario.expectedIntervalOffset, actualResponse.intervalOffset, "Interval offset"); + + if (scenario.versionDate != null) { + assertNotNull(actualResponse.versionDate, "Version date"); + assertEquals(scenario.versionDate, actualResponse.versionDate, "Version date"); + } else { + assertNull(actualResponse.versionDate, "Version date"); + } + + assertEquals(expectedRows.size(), actualResponse.rows.size(), "Row count " + mismatchSummary); + for (int i = 0; i < expectedRows.size(); i++) { + assertRowsEqual(expectedRows.get(i), actualResponse.rows.get(i), i); + } + } + + private static String buildMismatchSummary(List expectedRows, TimeSeriesResponse actualResponse) { + return "expectedRows=" + summarizeRows(expectedRows) + + " actualRows=" + summarizeRows(actualResponse.rows) + + " actualTotal=" + actualResponse.total; + } + + private static String summarizeRows(List rows) { + return rows.stream() + .limit(12) + .map(row -> "{t=" + row.dateTimeMillis + + ",v=" + row.value + + ",q=" + row.qualityCode + + ",e=" + row.dataEntryDateMillis + + "}") + .collect(Collectors.joining(", ", "[", rows.size() > 12 ? ", ...]" : "]")); + } + + private static Stream scenarios() { + Instant olderVersion = Instant.parse("2024-06-20T08:00:00Z"); + Instant newerVersion = Instant.parse("2024-06-21T08:00:00Z"); + + List denseRows = List.of( + row("2024-01-01T00:00:00Z", 1.0, 0, "2024-01-02T00:00:00Z", null), + row("2024-01-01T00:01:00Z", 2.0, 0, "2024-01-02T00:01:00Z", null), + row("2024-01-01T00:02:00Z", 3.0, 0, "2024-01-02T00:02:00Z", null), + row("2024-01-01T00:03:00Z", 4.0, 0, "2024-01-02T00:03:00Z", null), + row("2024-01-01T00:04:00Z", 5.0, 0, "2024-01-02T00:04:00Z", null), + row("2024-01-01T00:05:00Z", 6.0, 0, "2024-01-02T00:05:00Z", null) + ); + + List gapRows = List.of( + row("2024-01-01T00:00:00Z", 1.0, 0, "2024-01-03T00:00:00Z", null), + row("2024-01-01T00:01:00Z", 2.0, 0, "2024-01-03T00:01:00Z", null), + row("2024-01-01T00:02:00Z", 3.0, 0, "2024-01-03T00:02:00Z", null), + row("2024-01-01T00:05:00Z", 6.0, 0, "2024-01-03T00:05:00Z", null), + row("2024-01-01T00:06:00Z", 7.0, 0, "2024-01-03T00:06:00Z", null), + row("2024-01-01T00:07:00Z", 8.0, 0, "2024-01-03T00:07:00Z", null), + row("2024-01-01T00:08:00Z", 9.0, 0, "2024-01-03T00:08:00Z", null), + row("2024-01-01T00:09:00Z", 10.0, 0, "2024-01-03T00:09:00Z", null) + ); + + List versionedRows = List.of( + row("2024-05-01T15:00:00Z", 4.0, 0, "2024-06-20T09:00:00Z", olderVersion), + row("2024-05-01T16:00:00Z", 4.0, 0, "2024-06-20T09:01:00Z", olderVersion), + row("2024-05-01T17:00:00Z", 4.0, 0, "2024-06-20T09:02:00Z", olderVersion), + row("2024-05-01T18:00:00Z", 3.0, 0, "2024-06-20T09:03:00Z", olderVersion), + row("2024-05-01T15:00:00Z", 1.0, 0, "2024-06-21T09:00:00Z", newerVersion), + row("2024-05-01T16:00:00Z", 1.0, 0, "2024-06-21T09:01:00Z", newerVersion), + row("2024-05-01T17:00:00Z", 1.0, 0, "2024-06-21T09:02:00Z", newerVersion) + ); + + List irregularRows = List.of( + row("2024-01-05T12:00:00Z", 10.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T12:07:20Z", 20.0, 0, "2024-01-06T00:01:00Z", null), + row("2024-01-05T12:19:45Z", 30.0, 0, "2024-01-06T00:02:00Z", null), + row("2024-01-05T12:33:10Z", 40.0, 0, "2024-01-06T00:03:00Z", null) + ); + + Instant dstStart = Instant.parse("2024-03-09T00:00:00Z"); + List dstRows = regularRows(dstStart, 5000, 1.0, Duration.ofDays(1)); + + return Stream.of( + new Scenario("dense-regular", + "ITPARREG", + "ITPARREG.Stage.Inst.1Minute.0.BENCH", + "ft", + Instant.parse("2024-01-01T00:00:00Z"), + Instant.parse("2024-01-01T00:05:00Z"), + denseRows, + false, + false, + "UNVERSIONED", + "PT1M", + 0L, + null), + new Scenario("dense-regular-entry-date", + "ITPARREG", + "ITPARREG.Stage.Inst.1Minute.0.BENCH", + "ft", + Instant.parse("2024-01-01T00:00:00Z"), + Instant.parse("2024-01-01T00:05:00Z"), + denseRows, + false, + true, + "UNVERSIONED", + "PT1M", + 0L, + null), + new Scenario("gap-regular", + "ITPARGAP", + "ITPARGAP.Stage.Inst.1Minute.0.BENCH", + "ft", + Instant.parse("2024-01-01T00:00:00Z"), + Instant.parse("2024-01-01T00:09:00Z"), + gapRows, + false, + false, + "UNVERSIONED", + "PT1M", + 0L, + null), + new Scenario("versioned-max", + "ITPARVER", + "ITPARVER.Flow.Inst.1Hour.0.BENCH", + "cfs", + Instant.parse("2024-05-01T15:00:00Z"), + Instant.parse("2024-05-01T18:00:00Z"), + versionedRows, + true, + false, + "MAX_AGGREGATE", + "PT1H", + 0L, + null), + new Scenario("versioned-single", + "ITPARVER", + "ITPARVER.Flow.Inst.1Hour.0.BENCH", + "cfs", + Instant.parse("2024-05-01T15:00:00Z"), + Instant.parse("2024-05-01T18:00:00Z"), + versionedRows, + true, + false, + "SINGLE_VERSION", + "PT1H", + 0L, + newerVersion), + new Scenario("irregular", + "ITPARIRR", + "ITPARIRR.Flow.Inst.0.0.BENCH", + "cfs", + Instant.parse("2024-01-05T12:00:00Z"), + Instant.parse("2024-01-05T12:33:10Z"), + irregularRows, + false, + false, + "UNVERSIONED", + "PT0S", + Integer.MIN_VALUE, + null), + new Scenario("dense-regular-dst-window", + "ITPARDST", + "ITPARDST.Stage.Inst.1Minute.0.BENCH", + "ft", + dstStart, + dstStart.plus(Duration.ofMinutes(4999)), + dstRows, + false, + false, + "UNVERSIONED", + "PT1M", + 0L, + null) + ); + } + + private static SeedRow row(String dateTime, Double value, int qualityCode, String dataEntryDate, Instant versionDate) { + return new SeedRow( + Instant.parse(dateTime), + value, + qualityCode, + Instant.parse(dataEntryDate), + versionDate + ); + } + + private static List regularRows(Instant start, int count, double firstValue, Duration entryDateOffset) { + return IntStream.range(0, count) + .mapToObj(index -> new SeedRow( + start.plusSeconds(index * 60L), + firstValue + index, + 0, + start.plus(entryDateOffset).plusSeconds(index * 60L), + null + )) + .collect(Collectors.toList()); + } + + private static void assertRowsEqual(RetrievedRow expected, RetrievedRow actual, int index) { + assertEquals(expected.dateTimeMillis, actual.dateTimeMillis, "Row " + index + " timestamp"); + assertEquals(expected.qualityCode, actual.qualityCode, "Row " + index + " quality"); + + if (expected.value == null) { + assertNull(actual.value, "Row " + index + " value"); + } else { + assertNotNull(actual.value, "Row " + index + " value"); + assertEquals(expected.value, actual.value, DOUBLE_TOLERANCE, "Row " + index + " value"); + } + + if (expected.dataEntryDateMillis == null) { + assertNull(actual.dataEntryDateMillis, "Row " + index + " entry date"); + } else { + assertEquals(expected.dataEntryDateMillis, actual.dataEntryDateMillis, "Row " + index + " entry date"); + } + } + + private static void seedScenario(Scenario scenario) throws SQLException { + createLocation(scenario.locationId, true, OFFICE); + createTimeseries(OFFICE, scenario.seriesId, 0); + + CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); + database.connection(connection -> { + try { + CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED(DSL.using(connection).configuration(), + scenario.seriesId, + scenario.versioned ? "T" : "F", + OFFICE); + + long tsCode = findTsCode(connection, scenario.seriesId); + List years = scenario.rows.stream() + .map(row -> OffsetDateTime.ofInstant(row.dateTime, ZoneOffset.UTC).getYear()) + .distinct() + .collect(Collectors.toList()); + + clearScenarioRows(connection, tsCode, years); + insertScenarioRows(connection, tsCode, scenario.rows); + updateScenarioExtents(connection, tsCode, scenario.rows); + } catch (SQLException e) { + throw new RuntimeException("Unable to seed scenario " + scenario.name, e); + } + }, "cwms_20"); + } + + private static long findTsCode(Connection connection, String seriesId) throws SQLException { + String sql = "select ts_code from at_cwms_ts_id where db_office_id = ? and cwms_ts_id = ?"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, OFFICE); + statement.setString(2, seriesId); + try (ResultSet resultSet = statement.executeQuery()) { + if (!resultSet.next()) { + throw new IllegalStateException("Unable to find ts_code for " + seriesId); + } + return resultSet.getLong(1); + } + } + } + + private static void clearScenarioRows(Connection connection, long tsCode, List years) throws SQLException { + for (Integer year : years) { + try (PreparedStatement statement = connection.prepareStatement( + "delete from at_tsv_" + year + " where ts_code = ?")) { + statement.setLong(1, tsCode); + statement.executeUpdate(); + } + } + + try (PreparedStatement statement = connection.prepareStatement( + "delete from at_ts_extents where ts_code = ?")) { + statement.setLong(1, tsCode); + statement.executeUpdate(); + } + } + + private static void insertScenarioRows(Connection connection, long tsCode, List rows) throws SQLException { + List sortedRows = new ArrayList<>(rows); + sortedRows.sort(Comparator.comparing(seedRow -> seedRow.dateTime)); + + for (SeedRow row : sortedRows) { + int year = OffsetDateTime.ofInstant(row.dateTime, ZoneOffset.UTC).getYear(); + String sql = "insert into at_tsv_" + year + + " (ts_code, date_time, version_date, data_entry_date, value, quality_code, dest_flag)" + + " values (" + + tsCode + ", " + + toOracleDateExpression(row.dateTime) + ", " + + (row.versionDate != null ? toOracleDateExpression(row.versionDate) : "date '1111-11-11'") + ", " + + (row.dataEntryDate != null ? toOracleTimestampExpression(row.dataEntryDate) : "null") + ", " + + (row.value != null ? Double.toString(row.value) : "null") + ", " + + row.qualityCode + + ", 0)"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.executeUpdate(); + } + } + } + + private static void updateScenarioExtents(Connection connection, long tsCode, List rows) throws SQLException { + Set distinctVersionDates = rows.stream() + .map(seedRow -> seedRow.versionDate) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + if (distinctVersionDates.isEmpty()) { + updateTsExtents(connection, tsCode, "date '1111-11-11'"); + return; + } + + for (Instant versionDate : distinctVersionDates) { + updateTsExtents(connection, tsCode, toOracleDateExpression(versionDate)); + } + } + + private static void updateTsExtents(Connection connection, long tsCode, String versionDateExpression) throws SQLException { + String sql = "begin cwms_ts.update_ts_extents(" + tsCode + ", " + versionDateExpression + "); end;"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.execute(); + } + } + + private static List fetchOracleRows(Scenario scenario) throws SQLException { + CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); + return database.connection(connection -> { + try { + String functionName = scenario.includeEntryDate + ? "cwms_20.cwms_ts.retrieve_ts_entry_out_tab" + : "cwms_20.cwms_ts.retrieve_ts_out_tab"; + String rowProjection = scenario.includeEntryDate + ? ", case when data_entry_date is null then null else round((cast(data_entry_date as date) - date '1970-01-01') * 86400000) end as data_entry_date_ms" + : ""; + String versionDateExpression = scenario.versionDate != null + ? toOracleDateExpression(scenario.versionDate) + : "null"; + String maxVersionFlag = scenario.versionDate != null ? "'F'" : "'T'"; + String sql = "select round((date_time - date '1970-01-01') * 86400000) as date_time_ms," + + " value," + + " quality_code" + + rowProjection + + " from table(" + functionName + "(" + + toSqlStringLiteral(scenario.seriesId) + ", " + + toSqlStringLiteral(scenario.units) + ", " + + toOracleDateExpression(scenario.beginTime) + ", " + + toOracleDateExpression(scenario.endTime) + ", " + + "'UTC', 'T', 'T', 'T', 'F', 'F', " + + versionDateExpression + ", " + + maxVersionFlag + ", " + + toSqlStringLiteral(OFFICE) + + "))" + + " order by date_time"; + + try (PreparedStatement statement = connection.prepareStatement(sql)) { + try (ResultSet resultSet = statement.executeQuery()) { + List rows = new ArrayList<>(); + while (resultSet.next()) { + Double value = resultSet.getDouble("value"); + if (resultSet.wasNull()) { + value = null; + } + + Long dataEntryDateMillis = null; + if (scenario.includeEntryDate) { + long entryMillis = resultSet.getLong("data_entry_date_ms"); + if (!resultSet.wasNull()) { + dataEntryDateMillis = entryMillis; + } + } + + rows.add(new RetrievedRow( + resultSet.getLong("date_time_ms"), + value, + resultSet.getInt("quality_code"), + dataEntryDateMillis + )); + } + return rows; + } + } + } catch (SQLException e) { + throw new RuntimeException("Unable to fetch Oracle rows for " + scenario.name, e); + } + }, "cwms_20"); + } + + private static TimeSeriesResponse fetchCdaRows(Scenario scenario) throws Exception { + int pageSize = Math.max(1000, scenario.rows.size() * 2); + RequestSpecification request = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(Controllers.OFFICE, OFFICE) + .queryParam(Controllers.NAME, scenario.seriesId) + .queryParam(Controllers.UNIT, scenario.units) + .queryParam(Controllers.BEGIN, scenario.beginTime.toString()) + .queryParam(Controllers.END, scenario.endTime.toString()) + .queryParam("page-size", pageSize) + .queryParam(Controllers.INCLUDE_ENTRY_DATE, scenario.includeEntryDate); + if (scenario.versionDate != null) { + request = request.queryParam(Controllers.VERSION_DATE, scenario.versionDate.toString()); + } + + ExtractableResponse response = request.when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .extract(); + + JsonNode payload = OBJECT_MAPPER.readTree(response.asString()); + List rows = new ArrayList<>(); + for (JsonNode entry : payload.get("values")) { + Double value = entry.get(1).isNull() ? null : entry.get(1).asDouble(); + Long dataEntryDateMillis = null; + if (scenario.includeEntryDate && entry.size() > 3 && !entry.get(3).isNull()) { + dataEntryDateMillis = entry.get(3).asLong(); + } + rows.add(new RetrievedRow( + entry.get(0).asLong(), + value, + entry.get(2).asInt(), + dataEntryDateMillis + )); + } + + Instant versionDate = null; + JsonNode versionDateNode = payload.get("version-date"); + if (versionDateNode != null && !versionDateNode.isNull()) { + versionDate = OffsetDateTime.parse(versionDateNode.asText()).toInstant(); + } + + return new TimeSeriesResponse( + rows, + payload.get("total").asInt(), + payload.get("date-version-type").asText(), + payload.get("interval").asText(), + payload.get("interval-offset").asLong(), + versionDate + ); + } + + private static String toSqlStringLiteral(String value) { + return "'" + value.replace("'", "''") + "'"; + } + + private static String toOracleDateExpression(Instant instant) { + LocalDateTime utc = LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + return "to_date('" + utc.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + + "', 'yyyy-mm-dd hh24:mi:ss')"; + } + + private static String toOracleTimestampExpression(Instant instant) { + LocalDateTime utc = LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + return "to_timestamp('" + utc.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + + "', 'yyyy-mm-dd hh24:mi:ss')"; + } + + private static final class Scenario { + private final String name; + private final String locationId; + private final String seriesId; + private final String units; + private final Instant beginTime; + private final Instant endTime; + private final List rows; + private final boolean versioned; + private final boolean includeEntryDate; + private final String expectedDateVersionType; + private final String expectedInterval; + private final long expectedIntervalOffset; + private final Instant versionDate; + + private Scenario(String name, String locationId, String seriesId, String units, Instant beginTime, + Instant endTime, List rows, boolean versioned, boolean includeEntryDate, + String expectedDateVersionType, String expectedInterval, long expectedIntervalOffset, + Instant versionDate) { + this.name = name; + this.locationId = locationId; + this.seriesId = seriesId; + this.units = units; + this.beginTime = beginTime; + this.endTime = endTime; + this.rows = rows; + this.versioned = versioned; + this.includeEntryDate = includeEntryDate; + this.expectedDateVersionType = expectedDateVersionType; + this.expectedInterval = expectedInterval; + this.expectedIntervalOffset = expectedIntervalOffset; + this.versionDate = versionDate; + } + + @Override + public String toString() { + return name; + } + } + + private static final class SeedRow { + private final Instant dateTime; + private final Double value; + private final int qualityCode; + private final Instant dataEntryDate; + private final Instant versionDate; + + private SeedRow(Instant dateTime, Double value, int qualityCode, Instant dataEntryDate, + Instant versionDate) { + this.dateTime = dateTime; + this.value = value; + this.qualityCode = qualityCode; + this.dataEntryDate = dataEntryDate; + this.versionDate = versionDate; + } + } + + private static final class RetrievedRow { + private final long dateTimeMillis; + private final Double value; + private final int qualityCode; + private final Long dataEntryDateMillis; + + private RetrievedRow(long dateTimeMillis, Double value, int qualityCode, Long dataEntryDateMillis) { + this.dateTimeMillis = dateTimeMillis; + this.value = value; + this.qualityCode = qualityCode; + this.dataEntryDateMillis = dataEntryDateMillis; + } + } + + private static final class TimeSeriesResponse { + private final List rows; + private final int total; + private final String dateVersionType; + private final String interval; + private final long intervalOffset; + private final Instant versionDate; + + private TimeSeriesResponse(List rows, int total, String dateVersionType, + String interval, long intervalOffset, Instant versionDate) { + this.rows = rows; + this.total = total; + this.dateVersionType = dateVersionType; + this.interval = interval; + this.intervalOffset = intervalOffset; + this.versionDate = versionDate; + } + } +} diff --git a/cwms-data-api/src/test/java/fixtures/CwmsDataApiSetupCallback.java b/cwms-data-api/src/test/java/fixtures/CwmsDataApiSetupCallback.java index 7781b7ca0c..03994a2321 100644 --- a/cwms-data-api/src/test/java/fixtures/CwmsDataApiSetupCallback.java +++ b/cwms-data-api/src/test/java/fixtures/CwmsDataApiSetupCallback.java @@ -266,6 +266,39 @@ public static CwmsDatabaseContainer getDatabaseLink() { return cwmsDb; } + public static void shutdown() throws Exception { + Exception failure = null; + if (cdaInstance != null) { + try { + cdaInstance.stop(); + } catch (Exception e) { + failure = e; + } finally { + cdaInstance = null; + } + } + + if (cwmsDb != null) { + try { + cwmsDb.stop(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } finally { + cwmsDb = null; + } + } + + webUser = null; + + if (failure != null) { + throw failure; + } + } + private String loadResourceAsString(String fileName) { try { return IOUtils.toString( diff --git a/cwms-data-api/src/test/java/fixtures/KeyCloakExtension.java b/cwms-data-api/src/test/java/fixtures/KeyCloakExtension.java index 4949b27186..70a0bd2232 100644 --- a/cwms-data-api/src/test/java/fixtures/KeyCloakExtension.java +++ b/cwms-data-api/src/test/java/fixtures/KeyCloakExtension.java @@ -121,6 +121,16 @@ public static String getCodeUrl() { public static String getTokenUrl() { return tokenUrl; } + + public static void shutdown() { + if (kcc.isRunning()) { + kcc.stop(); + } + authUrl = null; + issuer = null; + codeUrl = null; + tokenUrl = null; + } /** * Retrieve the Access token for the user. diff --git a/cwms-data-api/src/test/java/fixtures/MinIOExtension.java b/cwms-data-api/src/test/java/fixtures/MinIOExtension.java index 15dce3f721..8eeacb4455 100644 --- a/cwms-data-api/src/test/java/fixtures/MinIOExtension.java +++ b/cwms-data-api/src/test/java/fixtures/MinIOExtension.java @@ -52,5 +52,11 @@ private static void createTestBucket() { } } + public static void shutdown() { + if (MINIO_CONTAINER.isRunning()) { + MINIO_CONTAINER.stop(); + } + } + } diff --git a/cwms-data-api/src/test/java/helpers/TimeSeriesReadBenchmark.java b/cwms-data-api/src/test/java/helpers/TimeSeriesReadBenchmark.java new file mode 100644 index 0000000000..1ae74aae4f --- /dev/null +++ b/cwms-data-api/src/test/java/helpers/TimeSeriesReadBenchmark.java @@ -0,0 +1,705 @@ +package helpers; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; +import fixtures.CwmsDataApiSetupCallback; +import fixtures.KeyCloakExtension; +import fixtures.MinIOExtension; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import mil.army.usace.hec.test.database.CwmsDatabaseContainer; +import org.jooq.impl.DSL; +import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; + +public final class TimeSeriesReadBenchmark { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final JsonFactory JSON_FACTORY = new JsonFactory(); + private static final DateTimeFormatter REQUEST_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'").withZone(ZoneOffset.UTC); + private static final DateTimeFormatter ORACLE_DATE_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneOffset.UTC); + private static final String ACCEPT_JSON_V2 = "application/json;version=2"; + private static final String NON_VERSIONED_DATE_SQL = "date '1111-11-11'"; + + private TimeSeriesReadBenchmark() { + } + + public static void main(String[] args) throws Exception { + BenchmarkConfig config = BenchmarkConfig.fromSystemProperties(); + System.out.println("Starting benchmark fixtures..."); + + try { + new KeyCloakExtension().beforeAll(null); + new MinIOExtension().beforeAll(null); + new CwmsDataApiSetupCallback().beforeAll(null); + + System.out.println("Running benchmark..."); + BenchmarkReport report = runBenchmark(config); + + Files.createDirectories(config.resultsDir); + Path resultFile = config.resultsDir.resolve("timeseries-read-benchmark-" + + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").withZone(ZoneOffset.UTC).format(Instant.now()) + + ".json"); + + OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValue(resultFile.toFile(), report); + OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValue(System.out, report); + System.out.println(); + System.out.println("Benchmark report written to " + resultFile); + + for (BenchmarkRun run : report.runs) { + if (run.httpCode != 200) { + throw new IllegalStateException( + "Benchmark completed with HTTP failures. Results saved to " + resultFile); + } + } + } finally { + System.out.println("Shutting down benchmark fixtures..."); + shutdownFixtures(); + } + } + + private static BenchmarkReport runBenchmark(BenchmarkConfig config) throws Exception { + Files.createDirectories(config.resultsDir); + Files.createDirectories(config.responsesDir); + + SeedInfo seed = ensureBenchmarkSeed(config); + if (seed.pointCount != config.pointCount) { + throw new IllegalStateException("Expected " + config.pointCount + " seeded points but found " + + seed.pointCount); + } + + waitForCdaReady(config); + if (config.warmup) { + Path warmupFile = config.responsesDir.resolve("warmup.json"); + executeRequest(config, warmupFile); + if (!config.keepResponses) { + Files.deleteIfExists(warmupFile); + } + } + + List runs = new ArrayList<>(); + for (int runIndex = 1; runIndex <= config.runs; runIndex++) { + runs.add(executeRun(config, runIndex)); + } + + return new BenchmarkReport( + "timeseries-read", + Instant.now().toString(), + resolveGitValue("git", "branch", "--show-current"), + resolveGitValue("git", "rev-parse", "HEAD"), + config.office, + config.locationId, + config.seriesId, + config.units, + config.startTime.toString(), + config.endTime.toString(), + config.pointCount, + config.pageSize, + config.requestUrl().toString(), + seed, + BenchmarkSummary.fromRuns(runs), + runs + ); + } + + private static SeedInfo ensureBenchmarkSeed(BenchmarkConfig config) throws SQLException { + long existingCount = getSeededPointCount(config); + if (config.skipSeed) { + return new SeedInfo(false, existingCount); + } + if (!config.forceReseed && existingCount == config.pointCount) { + return new SeedInfo(false, existingCount); + } + + CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); + database.connection(connection -> { + try { + ensureLocationExists(connection, config); + ensureTimeSeriesExists(connection, config); + CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED( + DSL.using(connection).configuration(), config.seriesId, "F", config.office); + + long tsCode = findTsCode(connection, config.office, config.seriesId); + List segments = buildYearSegments(config.startTime, config.pointCount); + clearSeededRows(connection, tsCode, segments); + insertSeededRows(connection, tsCode, segments); + updateTsExtents(connection, tsCode); + if (!connection.getAutoCommit()) { + connection.commit(); + } + } catch (SQLException e) { + throw new RuntimeException("Unable to seed benchmark series " + config.seriesId, e); + } + }, "cwms_20"); + + return new SeedInfo(true, getSeededPointCount(config)); + } + + private static void ensureLocationExists(Connection connection, BenchmarkConfig config) throws SQLException { + String sql = "declare " + + "location_exists exception; " + + "pragma exception_init(location_exists, -20026); " + + "begin " + + "cwms_loc.create_location(?, ?, null, null, null, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); " + + "exception when location_exists then null; " + + "end;"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, config.locationId); + statement.setString(2, "SITE"); + statement.setDouble(3, 38.0d); + statement.setDouble(4, -90.0d); + statement.setString(5, "NAD83"); + statement.setString(6, config.locationId); + statement.setString(7, config.locationId + " Benchmark Location"); + statement.setString(8, "Performance benchmark location"); + statement.setString(9, "UTC"); + statement.setString(10, null); + statement.setString(11, null); + statement.setString(12, "T"); + statement.setString(13, config.office); + statement.execute(); + } + } + + private static void ensureTimeSeriesExists(Connection connection, BenchmarkConfig config) throws SQLException { + String sql = "declare " + + "ts_exists exception; " + + "pragma exception_init(ts_exists, -20003); " + + "begin " + + "cwms_ts.create_ts(?, ?, 0); " + + "exception when ts_exists then null; " + + "end;"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setString(1, config.office); + statement.setString(2, config.seriesId); + statement.execute(); + } + } + + private static long findTsCode(Connection connection, String office, String seriesId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "select ts_code from at_cwms_ts_id where db_office_id = ? and cwms_ts_id = ?")) { + statement.setString(1, office); + statement.setString(2, seriesId); + try (ResultSet resultSet = statement.executeQuery()) { + if (!resultSet.next()) { + throw new IllegalStateException("Unable to find ts_code for " + seriesId); + } + return resultSet.getLong(1); + } + } + } + + private static void clearSeededRows(Connection connection, long tsCode, List segments) throws SQLException { + for (YearSegment segment : segments) { + try (PreparedStatement statement = connection.prepareStatement( + "delete from at_tsv_" + segment.year + " where ts_code = ?")) { + statement.setLong(1, tsCode); + statement.executeUpdate(); + } + } + try (PreparedStatement statement = connection.prepareStatement( + "delete from at_ts_extents where ts_code = ?")) { + statement.setLong(1, tsCode); + statement.executeUpdate(); + } + } + + private static void insertSeededRows(Connection connection, long tsCode, List segments) throws SQLException { + for (YearSegment segment : segments) { + String sql = "insert /*+ APPEND */ into at_tsv_" + segment.year + + " (ts_code, date_time, version_date, data_entry_date, value, quality_code, dest_flag) " + + "select ?, to_date(?, 'yyyy-mm-dd hh24:mi:ss') + numtodsinterval(level - 1, 'MINUTE'), " + + NON_VERSIONED_DATE_SQL + ", systimestamp, ? + level - 1, 0, 0 " + + "from dual connect by level <= ?"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setLong(1, tsCode); + statement.setString(2, ORACLE_DATE_TIME_FORMAT.format(segment.startTime)); + statement.setLong(3, segment.valueStart); + statement.setInt(4, segment.count); + statement.executeUpdate(); + } + } + } + + private static void updateTsExtents(Connection connection, long tsCode) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "begin cwms_ts.update_ts_extents(?, " + NON_VERSIONED_DATE_SQL + "); end;")) { + statement.setLong(1, tsCode); + statement.execute(); + } + } + + private static long getSeededPointCount(BenchmarkConfig config) throws SQLException { + CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); + return database.connection(connection -> { + try (PreparedStatement statement = connection.prepareStatement( + "select count(*) from av_tsv v " + + "join at_cwms_ts_id t on t.ts_code = v.ts_code " + + "where t.db_office_id = ? and t.cwms_ts_id = ?")) { + statement.setString(1, config.office); + statement.setString(2, config.seriesId); + try (ResultSet resultSet = statement.executeQuery()) { + resultSet.next(); + return resultSet.getLong(1); + } + } catch (SQLException e) { + throw new RuntimeException("Unable to count seeded rows for " + config.seriesId, e); + } + }, "cwms_20"); + } + + private static List buildYearSegments(Instant startTime, int pointCount) { + List segments = new ArrayList<>(); + Instant cursor = startTime; + int remaining = pointCount; + long valueStart = 1L; + while (remaining > 0) { + Instant nextYear = cursor.atOffset(ZoneOffset.UTC) + .withDayOfYear(1) + .withHour(0) + .withMinute(0) + .withSecond(0) + .withNano(0) + .plusYears(1) + .toInstant(); + long minutesUntilNextYear = Math.max(1L, Duration.between(cursor, nextYear).toMinutes()); + int segmentCount = (int) Math.min(remaining, minutesUntilNextYear); + segments.add(new YearSegment(cursor.atOffset(ZoneOffset.UTC).getYear(), cursor, segmentCount, valueStart)); + cursor = cursor.plusSeconds(segmentCount * 60L); + valueStart += segmentCount; + remaining -= segmentCount; + } + return segments; + } + + private static void waitForCdaReady(BenchmarkConfig config) throws Exception { + HttpClient client = HttpClient.newHttpClient(); + URI readinessUri = URI.create(config.resolvedBaseUrl() + "/offices/" + urlEncode(config.office)); + for (int attempt = 0; attempt < 30; attempt++) { + HttpRequest request = HttpRequest.newBuilder(readinessUri) + .header("Accept", ACCEPT_JSON_V2) + .GET() + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream ignored = response.body()) { + if (response.statusCode() == 200) { + return; + } + } + Thread.sleep(1000L); + } + throw new IllegalStateException("CDA did not become ready at " + readinessUri); + } + + private static BenchmarkRun executeRun(BenchmarkConfig config, int runIndex) throws Exception { + Path responseFile = config.responsesDir.resolve("timeseries-read-run-" + runIndex + ".json"); + RequestResult requestResult = executeRequest(config, responseFile); + ResponseSummary responseSummary = summarizeResponse(responseFile); + String responseFileValue = responseFile.toAbsolutePath().toString(); + if (!config.keepResponses && requestResult.httpCode == 200) { + Files.deleteIfExists(responseFile); + responseFileValue = null; + } + return new BenchmarkRun( + runIndex, + requestResult.httpCode, + roundSeconds(requestResult.timeTotalNanos), + responseSummary.responseBytes, + responseSummary.reportedTotal, + responseSummary.reportedPageSize, + responseSummary.firstTimestamp, + responseSummary.lastTimestamp, + requestResult.httpCode == 200 ? null : Files.readString(responseFile), + responseFileValue + ); + } + + private static RequestResult executeRequest(BenchmarkConfig config, Path responseFile) throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder(config.requestUrl()) + .header("Accept", ACCEPT_JSON_V2) + .GET() + .build(); + long startNanos = System.nanoTime(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofFile(responseFile)); + long endNanos = System.nanoTime(); + return new RequestResult(response.statusCode(), endNanos - startNanos); + } + + private static ResponseSummary summarizeResponse(Path responseFile) throws IOException { + Integer reportedTotal = null; + Integer reportedPageSize = null; + Long firstTimestamp = null; + Long lastTimestamp = null; + + try (InputStream inputStream = Files.newInputStream(responseFile); + JsonParser parser = JSON_FACTORY.createParser(inputStream)) { + while (parser.nextToken() != null) { + if (parser.currentToken() != JsonToken.FIELD_NAME) { + continue; + } + String fieldName = parser.currentName(); + JsonToken valueToken = parser.nextToken(); + if ("total".equals(fieldName) && valueToken != JsonToken.VALUE_NULL) { + reportedTotal = parser.getIntValue(); + } else if ("page-size".equals(fieldName) && valueToken != JsonToken.VALUE_NULL) { + reportedPageSize = parser.getIntValue(); + } else if ("values".equals(fieldName) && valueToken == JsonToken.START_ARRAY) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_ARRAY) { + parser.skipChildren(); + continue; + } + parser.nextToken(); + long timestamp = parser.getLongValue(); + if (firstTimestamp == null) { + firstTimestamp = timestamp; + } + lastTimestamp = timestamp; + while (parser.nextToken() != JsonToken.END_ARRAY) { + parser.skipChildren(); + } + } + } else { + parser.skipChildren(); + } + } + } + + return new ResponseSummary( + Files.size(responseFile), + reportedTotal, + reportedPageSize, + firstTimestamp, + lastTimestamp + ); + } + + private static String resolveGitValue(String... command) { + ProcessBuilder processBuilder = new ProcessBuilder(command); + processBuilder.redirectErrorStream(true); + try { + Process process = processBuilder.start(); + byte[] outputBytes = process.getInputStream().readAllBytes(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + return null; + } + String value = new String(outputBytes, StandardCharsets.UTF_8).trim(); + return value.isEmpty() ? null : value; + } catch (IOException e) { + return null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } + + private static double roundSeconds(long nanos) { + return Math.round((nanos / 1_000_000_000.0d) * 1_000_000.0d) / 1_000_000.0d; + } + + private static String urlEncode(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + + private static void shutdownFixtures() throws Exception { + Exception failure = null; + try { + CwmsDataApiSetupCallback.shutdown(); + } catch (Exception e) { + failure = e; + } + + try { + MinIOExtension.shutdown(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + + try { + KeyCloakExtension.shutdown(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + + if (failure != null) { + throw failure; + } + } + + private static final class BenchmarkConfig { + private final String office; + private final String locationId; + private final String seriesId; + private final String units; + private final String baseUrl; + private final Instant startTime; + private final Instant endTime; + private final int pointCount; + private final int pageSize; + private final int runs; + private final boolean warmup; + private final boolean skipSeed; + private final boolean forceReseed; + private final boolean keepResponses; + private final Path resultsDir; + private final Path responsesDir; + + private BenchmarkConfig(String office, String locationId, String seriesId, String units, String baseUrl, + Instant startTime, int pointCount, int pageSize, int runs, boolean warmup, + boolean skipSeed, boolean forceReseed, boolean keepResponses, Path resultsDir, + Path responsesDir) { + this.office = office; + this.locationId = locationId; + this.seriesId = seriesId; + this.units = units; + this.baseUrl = baseUrl; + this.startTime = startTime; + this.endTime = startTime.plusSeconds(Math.max(0L, pointCount - 1L) * 60L); + this.pointCount = pointCount; + this.pageSize = pageSize; + this.runs = runs; + this.warmup = warmup; + this.skipSeed = skipSeed; + this.forceReseed = forceReseed; + this.keepResponses = keepResponses; + this.resultsDir = resultsDir; + this.responsesDir = responsesDir; + } + + private static BenchmarkConfig fromSystemProperties() { + String office = System.getProperty("benchmark.office", "SPK"); + String locationId = System.getProperty("benchmark.locationId", "PERF1MREAD"); + String seriesId = System.getProperty("benchmark.seriesId", "PERF1MREAD.Stage.Inst.1Minute.0.BENCH"); + String units = System.getProperty("benchmark.units", "ft"); + String baseUrl = System.getProperty("benchmark.baseUrl"); + Instant startTime = Instant.parse(System.getProperty("benchmark.startTime", "2024-01-01T00:00:00Z")); + int pointCount = Integer.parseInt(System.getProperty("benchmark.pointCount", "1000000")); + int pageSize = Integer.parseInt(System.getProperty("benchmark.pageSize", String.valueOf(pointCount))); + int runs = Integer.parseInt(System.getProperty("benchmark.runs", "1")); + boolean warmup = Boolean.parseBoolean(System.getProperty("benchmark.warmup", "false")); + boolean skipSeed = Boolean.parseBoolean(System.getProperty("benchmark.skipSeed", "false")); + boolean forceReseed = Boolean.parseBoolean(System.getProperty("benchmark.forceReseed", "false")); + boolean keepResponses = Boolean.parseBoolean(System.getProperty("benchmark.keepResponses", "false")); + Path resultsDir = Paths.get(System.getProperty("benchmark.resultsDir", + "..\\load_data\\performance\\results")).normalize().toAbsolutePath(); + Path responsesDir = Paths.get(System.getProperty("benchmark.responsesDir", + "..\\load_data\\performance\\responses")).normalize().toAbsolutePath(); + return new BenchmarkConfig(office, locationId, seriesId, units, baseUrl, startTime, pointCount, + pageSize, runs, warmup, skipSeed, forceReseed, keepResponses, resultsDir, responsesDir); + } + + private URI requestUrl() { + StringBuilder builder = new StringBuilder(resolvedBaseUrl()); + builder.append("/timeseries?office=").append(urlEncode(office)); + builder.append("&name=").append(urlEncode(seriesId)); + builder.append("&units=").append(urlEncode(units)); + builder.append("&begin=").append(urlEncode(REQUEST_TIME_FORMAT.format(startTime))); + builder.append("&end=").append(urlEncode(REQUEST_TIME_FORMAT.format(endTime))); + builder.append("&page-size=").append(pageSize); + return URI.create(builder.toString()); + } + + private String resolvedBaseUrl() { + if (baseUrl != null && !baseUrl.isBlank()) { + return baseUrl; + } + return CwmsDataApiSetupCallback.httpUrl() + ":" + CwmsDataApiSetupCallback.httpPort() + + System.getProperty("warContext"); + } + } + + private static final class YearSegment { + private final int year; + private final Instant startTime; + private final int count; + private final long valueStart; + + private YearSegment(int year, Instant startTime, int count, long valueStart) { + this.year = year; + this.startTime = startTime; + this.count = count; + this.valueStart = valueStart; + } + } + + private static final class RequestResult { + private final int httpCode; + private final long timeTotalNanos; + + private RequestResult(int httpCode, long timeTotalNanos) { + this.httpCode = httpCode; + this.timeTotalNanos = timeTotalNanos; + } + } + + private static final class ResponseSummary { + private final long responseBytes; + private final Integer reportedTotal; + private final Integer reportedPageSize; + private final Long firstTimestamp; + private final Long lastTimestamp; + + private ResponseSummary(long responseBytes, Integer reportedTotal, Integer reportedPageSize, + Long firstTimestamp, Long lastTimestamp) { + this.responseBytes = responseBytes; + this.reportedTotal = reportedTotal; + this.reportedPageSize = reportedPageSize; + this.firstTimestamp = firstTimestamp; + this.lastTimestamp = lastTimestamp; + } + } + + public static final class SeedInfo { + public final boolean seeded; + public final long pointCount; + + private SeedInfo(boolean seeded, long pointCount) { + this.seeded = seeded; + this.pointCount = pointCount; + } + } + + public static final class BenchmarkSummary { + public final int successfulRuns; + public final Double averageTimeTotalSeconds; + public final Double minTimeTotalSeconds; + public final Double maxTimeTotalSeconds; + + private BenchmarkSummary(int successfulRuns, Double averageTimeTotalSeconds, + Double minTimeTotalSeconds, Double maxTimeTotalSeconds) { + this.successfulRuns = successfulRuns; + this.averageTimeTotalSeconds = averageTimeTotalSeconds; + this.minTimeTotalSeconds = minTimeTotalSeconds; + this.maxTimeTotalSeconds = maxTimeTotalSeconds; + } + + private static BenchmarkSummary fromRuns(List runs) { + List successfulRuns = new ArrayList<>(); + for (BenchmarkRun run : runs) { + if (run.httpCode == 200) { + successfulRuns.add(run); + } + } + if (successfulRuns.isEmpty()) { + return new BenchmarkSummary(0, null, null, null); + } + + double total = 0.0d; + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for (BenchmarkRun run : successfulRuns) { + total += run.timeTotalSeconds; + min = Math.min(min, run.timeTotalSeconds); + max = Math.max(max, run.timeTotalSeconds); + } + return new BenchmarkSummary( + successfulRuns.size(), + Math.round((total / successfulRuns.size()) * 1_000_000.0d) / 1_000_000.0d, + Math.round(min * 1_000_000.0d) / 1_000_000.0d, + Math.round(max * 1_000_000.0d) / 1_000_000.0d + ); + } + } + + public static final class BenchmarkRun { + public final int run; + public final int httpCode; + public final double timeTotalSeconds; + public final long responseBytesOnDisk; + public final Integer reportedTotal; + public final Integer reportedPageSize; + public final Long firstTimestamp; + public final Long lastTimestamp; + public final String errorBody; + public final String responseFile; + + private BenchmarkRun(int run, int httpCode, double timeTotalSeconds, long responseBytesOnDisk, + Integer reportedTotal, Integer reportedPageSize, Long firstTimestamp, + Long lastTimestamp, String errorBody, String responseFile) { + this.run = run; + this.httpCode = httpCode; + this.timeTotalSeconds = timeTotalSeconds; + this.responseBytesOnDisk = responseBytesOnDisk; + this.reportedTotal = reportedTotal; + this.reportedPageSize = reportedPageSize; + this.firstTimestamp = firstTimestamp; + this.lastTimestamp = lastTimestamp; + this.errorBody = errorBody; + this.responseFile = responseFile; + } + } + + public static final class BenchmarkReport { + public final String benchmark; + public final String generatedAt; + public final String gitBranch; + public final String gitCommit; + public final String office; + public final String locationId; + public final String seriesId; + public final String units; + public final String startTimeUtc; + public final String endTimeUtc; + public final int pointCount; + public final int pageSize; + public final String requestUrl; + public final SeedInfo seed; + public final BenchmarkSummary summary; + public final List runs; + + private BenchmarkReport(String benchmark, String generatedAt, String gitBranch, String gitCommit, + String office, String locationId, String seriesId, String units, + String startTimeUtc, String endTimeUtc, int pointCount, int pageSize, + String requestUrl, SeedInfo seed, BenchmarkSummary summary, + List runs) { + this.benchmark = benchmark; + this.generatedAt = generatedAt; + this.gitBranch = gitBranch; + this.gitCommit = gitCommit; + this.office = office; + this.locationId = locationId; + this.seriesId = seriesId; + this.units = units; + this.startTimeUtc = startTimeUtc; + this.endTimeUtc = endTimeUtc; + this.pointCount = pointCount; + this.pageSize = pageSize; + this.requestUrl = requestUrl; + this.seed = seed; + this.summary = summary; + this.runs = runs; + } + } +} diff --git a/load_data/performance/.gitignore b/load_data/performance/.gitignore new file mode 100644 index 0000000000..ddbb6df966 --- /dev/null +++ b/load_data/performance/.gitignore @@ -0,0 +1,2 @@ +results/ +responses/