From 439e40e1191a7d11ba1e15fd49acc7052c940a22 Mon Sep 17 00:00:00 2001 From: Nathan Wand Date: Wed, 26 Nov 2025 17:17:54 -0800 Subject: [PATCH 01/51] add partition timing metrics to LeaderOnlyTokenCrawler Signed-off-by: Nathan Wand --- .../base/LeaderOnlyTokenCrawler.java | 9 +++- .../base/LeaderOnlyTokenCrawlerTest.java | 53 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawler.java index 18a0d9f70f..23bd5aad21 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawler.java @@ -39,6 +39,8 @@ public class LeaderOnlyTokenCrawler implements Crawler client.executePartition(state, buffer, acknowledgementSet)); } private List collectBatch(Iterator iterator) { diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawlerTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawlerTest.java index 1c7fb4f5bc..ca597d0347 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawlerTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/LeaderOnlyTokenCrawlerTest.java @@ -22,6 +22,8 @@ import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.TokenPaginationCrawlerLeaderProgressState; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; import org.opensearch.dataprepper.plugins.source.source_crawler.model.TestItemInfo; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.Counter; import java.time.Duration; import java.time.Instant; @@ -42,6 +44,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.doNothing; import static org.mockito.internal.verification.VerificationModeFactory.times; @ExtendWith(MockitoExtension.class) @@ -62,6 +67,8 @@ class LeaderOnlyTokenCrawlerTest { private AcknowledgementSet acknowledgementSet; @Mock private Buffer> buffer; + @Mock + private PaginationCrawlerWorkerProgressState workerState; private LeaderOnlyTokenCrawler crawler; private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("CrawlerTest", "crawler"); @@ -241,6 +248,50 @@ void testAcknowledgmentTimeout() { verify(coordinator).createPartition(any()); } + @Test + public void testExecutePartitionMetrics() { + reset(leaderPartition); + + // mock timers and counters + Timer mockCrawlingTimer = mock(Timer.class); + Timer partitionWaitTimeTimer = mock(Timer.class); + Timer partitionProcessLatencyTimer = mock(Timer.class); + Timer mockBufferWriteTimer = mock(Timer.class); + Counter mockBatchesFailedCounter = mock(Counter.class); + Counter mockAcknowledgementSetSuccesses = mock(Counter.class); + Counter mockAcknowledgementSetFailures = mock(Counter.class); + + // setup mock plugin metrics + PluginMetrics mockPluginMetrics = mock(PluginMetrics.class); + when(mockPluginMetrics.timer("crawlingTime")).thenReturn(mockCrawlingTimer); + when(mockPluginMetrics.timer("WorkerPartitionWaitTime")).thenReturn(partitionWaitTimeTimer); + when(mockPluginMetrics.timer("WorkerPartitionProcessLatency")).thenReturn(partitionProcessLatencyTimer); + when(mockPluginMetrics.timer("bufferWriteTime")).thenReturn(mockBufferWriteTimer); + when(mockPluginMetrics.counter("batchesFailed")).thenReturn(mockBatchesFailedCounter); + when(mockPluginMetrics.counter("acknowledgementSetSuccesses")).thenReturn(mockAcknowledgementSetSuccesses); + when(mockPluginMetrics.counter("acknowledgementSetFailures")).thenReturn(mockAcknowledgementSetFailures); + + LeaderOnlyTokenCrawler testCrawler = new LeaderOnlyTokenCrawler(client, mockPluginMetrics); + + // test executePartition with metrics + when(workerState.getExportStartTime()).thenReturn(Instant.now().minusSeconds(1)); + + // make latency timer execute the runnable so client.executePartition() gets called + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return null; + }).when(partitionProcessLatencyTimer).record(any(Runnable.class)); + + doNothing().when(partitionWaitTimeTimer).record(any(Duration.class)); + + testCrawler.executePartition(workerState, buffer, acknowledgementSet); + + // verify metrics are recorded + verify(partitionProcessLatencyTimer).record(any(Runnable.class)); + verify(partitionWaitTimeTimer).record(any(Duration.class)); + verify(client).executePartition(workerState, buffer, acknowledgementSet); + } private List createTestItems(int count) { List items = new ArrayList<>(); @@ -249,4 +300,4 @@ private List createTestItems(int count) { } return items; } -} \ No newline at end of file +} From ba3d374060933211949ee557e3f9ad2518caef68 Mon Sep 17 00:00:00 2001 From: Nathan Wand Date: Mon, 22 Dec 2025 14:35:09 -0800 Subject: [PATCH 02/51] Change dimensional time slice crawler and worker metrics to camel case Convert WorkerPartitionWaitTime and WorkerPartitionProcessLatency metrics to camelCase format (workerPartitionWaitTime and workerPartitionProcessLatency) in TokenPaginationCrawler and DimensionalTimeSliceCrawler for consistency with LeaderOnlyTokenCrawler metrics. Signed-off-by: Nathan Wand --- .../source_crawler/base/DimensionalTimeSliceCrawler.java | 4 ++-- .../source/source_crawler/base/LeaderOnlyTokenCrawler.java | 4 ++-- .../source/source_crawler/base/TokenPaginationCrawler.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java index ffeeb2a0a1..1495e55459 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java @@ -40,8 +40,8 @@ public class DimensionalTimeSliceCrawler implements Crawler Date: Thu, 8 Jan 2026 12:24:01 -0800 Subject: [PATCH 03/51] enforced camel case naming and fixed naming in tests Signed-off-by: Nathan Wand --- .../source_crawler/base/DimensionalTimeSliceCrawler.java | 2 +- .../source_crawler/base/DimensionalTimeSliceCrawlerTest.java | 4 ++-- .../source_crawler/base/LeaderOnlyTokenCrawlerTest.java | 4 ++-- .../source_crawler/base/TokenPaginationCrawlerTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java index 1495e55459..cd8b742d4c 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/DimensionalTimeSliceCrawler.java @@ -39,7 +39,7 @@ public class DimensionalTimeSliceCrawler implements Crawler Date: Wed, 3 Dec 2025 15:37:11 +0100 Subject: [PATCH 04/51] Filesource compression support (#5255) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for compressed files in FileSource Signed-off-by: Joël Marty Signed-off-by: Joël Marty <134835+joelmarty@users.noreply.github.com> Signed-off-by: Nathan Wand --- data-prepper-plugins/common/README.md | 2 + .../plugins/source/file/FileSource.java | 19 ++++--- .../plugins/source/file/FileSourceConfig.java | 8 +++ .../plugins/source/file/FileSourceTests.java | 52 ++++++++++++++----- 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index 96e5e560a2..c774d72e95 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -35,6 +35,8 @@ A source plugin to read input data from the specified file path. The file source Temporarily, `type` can either be `event` or `string`. If you would like to use the file source for log analytics use cases like grok, change this to `event`. +* `compression` (String): The source file compression format, if any. Valid options are `none`, `gzip` and `snappy`. Default is `none`. + ## `file` (sink) A sink plugin to write output data to the specified file path. diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSource.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSource.java index a0da7461f1..9698144097 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSource.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSource.java @@ -12,6 +12,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.codec.DecompressionEngine; import org.opensearch.dataprepper.model.codec.InputCodec; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -23,11 +24,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.BufferedReader; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -48,6 +52,7 @@ public class FileSource implements Source> { private final FileSourceConfig fileSourceConfig; private final FileStrategy fileStrategy; private final EventFactory eventFactory; + private final DecompressionEngine decompressionEngine; private Thread readThread; @@ -63,6 +68,7 @@ public FileSource( this.fileSourceConfig = fileSourceConfig; this.isStopRequested = false; this.writeTimeout = FileSourceConfig.DEFAULT_TIMEOUT; + this.decompressionEngine = fileSourceConfig.getCompression().getDecompressionEngine(); if(fileSourceConfig.getCodec() != null) { fileStrategy = new CodecFileStrategy(pluginFactory); @@ -104,7 +110,8 @@ private interface FileStrategy { private class ClassicFileStrategy implements FileStrategy { @Override public void start(Buffer> buffer) { - try (BufferedReader reader = Files.newBufferedReader(Paths.get(fileSourceConfig.getFilePathToRead()), StandardCharsets.UTF_8)) { + Path filePath = Paths.get(fileSourceConfig.getFilePathToRead()); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(decompressionEngine.createInputStream(Files.newInputStream(filePath)), StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null && !isStopRequested) { writeLineAsEventOrString(line, buffer); @@ -166,13 +173,13 @@ private class CodecFileStrategy implements FileStrategy { final PluginModel codecConfiguration = fileSourceConfig.getCodec(); final PluginSetting codecPluginSettings = new PluginSetting(codecConfiguration.getPluginName(), codecConfiguration.getPluginSettings()); codec = pluginFactory.loadPlugin(InputCodec.class, codecPluginSettings); - } @Override public void start(final Buffer> buffer) { - try { - codec.parse(new FileInputStream(fileSourceConfig.getFilePathToRead()), eventRecord -> { + Path filePath = Paths.get(fileSourceConfig.getFilePathToRead()); + try(InputStream is = decompressionEngine.createInputStream(Files.newInputStream(filePath))) { + codec.parse(is, eventRecord -> { try { buffer.write((Record) eventRecord, writeTimeout); } catch (TimeoutException e) { @@ -186,4 +193,4 @@ public void start(final Buffer> buffer) { } } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSourceConfig.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSourceConfig.java index 255857a4bb..9eb8dd961d 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSourceConfig.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/source/file/FileSourceConfig.java @@ -10,6 +10,7 @@ import com.google.common.base.Preconditions; import jakarta.validation.constraints.AssertTrue; import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; import java.util.Objects; @@ -35,6 +36,9 @@ public class FileSourceConfig { @JsonProperty("codec") private PluginModel codec; + @JsonProperty("compression") + private CompressionOption compression = CompressionOption.NONE; + public String getFilePathToRead() { return filePathToRead; } @@ -52,6 +56,10 @@ public PluginModel getCodec() { return codec; } + public CompressionOption getCompression() { + return compression; + } + void validate() { Objects.requireNonNull(filePathToRead, "File path is required"); Preconditions.checkArgument(recordType.equals(EVENT_TYPE) || recordType.equals(DEFAULT_TYPE), "Invalid type: must be either [event] or [string]"); diff --git a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/source/file/FileSourceTests.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/source/file/FileSourceTests.java index 346548e4c8..1a6b20a23a 100644 --- a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/source/file/FileSourceTests.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/source/file/FileSourceTests.java @@ -18,8 +18,10 @@ import org.opensearch.dataprepper.event.TestEventFactory; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.codec.DecompressionEngine; import org.opensearch.dataprepper.model.codec.InputCodec; import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventBuilder; @@ -27,6 +29,7 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBufferConfig; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,6 +93,31 @@ private FileSource createObjectUnderTest() { return new FileSource(fileSourceConfig, pluginMetrics, pluginFactory, TestEventFactory.getTestEventFactory()); } + /** + * Variant of creatgeObjectUnderTest that uses mocks for the configuration instead of object mapper, so we can + * pass concrete mocks to the FileSource through the FileSourceConfig. + * @param codec the codec to use in the configuration + * @param engine the {@link DecompressionEngine} to use in the configuration + * @return + */ + private FileSource createObjectUnderTest(PluginModel codec, DecompressionEngine engine) { + FileSourceConfig fileSourceConfig = mock(FileSourceConfig.class); + + when(fileSourceConfig.getFilePathToRead()).thenReturn(TEST_FILE_PATH_PLAIN); + + if (codec != null) { + when(fileSourceConfig.getCodec()).thenReturn(codec); + } + + if (engine != null) { + CompressionOption compressionOption = mock(CompressionOption.class); + when(compressionOption.getDecompressionEngine()).thenReturn(engine); + when(fileSourceConfig.getCompression()).thenReturn(compressionOption); + } + + return new FileSource(fileSourceConfig, pluginMetrics, pluginFactory, TestEventFactory.getTestEventFactory()); + } + @Nested class WithRecord { private static final String TEST_PIPELINE_NAME = "pipeline"; @@ -285,6 +313,9 @@ class WithCodec { @Mock private Buffer buffer; + @Mock + private DecompressionEngine decompressionEngine; + @BeforeEach void setUp() { Map codecConfiguration = Map.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); @@ -297,21 +328,18 @@ void setUp() { @Test void start_will_parse_codec_with_correct_inputStream() throws IOException { - createObjectUnderTest().start(buffer); + final FileInputStream decompressedStream = new FileInputStream(TEST_FILE_PATH_PLAIN); + DecompressionEngine mockEngine = mock(DecompressionEngine.class); + when(mockEngine.createInputStream(any(InputStream.class))).thenReturn(decompressedStream); - final ArgumentCaptor inputStreamArgumentCaptor = ArgumentCaptor.forClass(InputStream.class); + PluginModel fakeCodec = mock(PluginModel.class); + when(fakeCodec.getPluginName()).thenReturn("fake_codec"); + when(fakeCodec.getPluginSettings()).thenReturn(Map.of()); - await().atMost(2, TimeUnit.SECONDS) - .untilAsserted(() -> verify(inputCodec).parse(any(InputStream.class), any(Consumer.class))); - verify(inputCodec).parse(inputStreamArgumentCaptor.capture(), any(Consumer.class)); - - final InputStream actualInputStream = inputStreamArgumentCaptor.getValue(); + createObjectUnderTest(fakeCodec, mockEngine).start(buffer); - final byte[] actualBytes = actualInputStream.readAllBytes(); - final FileInputStream fileInputStream = new FileInputStream(TEST_FILE_PATH_PLAIN); - final byte[] expectedBytes = fileInputStream.readAllBytes(); - - assertThat(actualBytes, equalTo(expectedBytes)); + await().atMost(2, TimeUnit.SECONDS) + .untilAsserted(() -> verify(inputCodec).parse(eq(decompressedStream), any(Consumer.class))); } @Test From 088b2b3bbb8fbbd15c63c9860b890bbefc25bebb Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 3 Dec 2025 14:07:59 -0600 Subject: [PATCH 05/51] Increase acknowledgment set timeout for opensearch source (#6291) Signed-off-by: Taylor Gray Signed-off-by: Nathan Wand --- .../source/opensearch/worker/WorkerCommonUtils.java | 8 ++++++-- .../plugins/source/opensearch/worker/PitWorkerTest.java | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java index 9075c2f8b0..a05c0a68e5 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/WorkerCommonUtils.java @@ -28,7 +28,11 @@ public class WorkerCommonUtils { static final Duration BACKOFF_ON_EXCEPTION = Duration.ofSeconds(60); static final long DEFAULT_CHECKPOINT_INTERVAL_MILLS = 5 * 60_000; - static final Duration ACKNOWLEDGEMENT_SET_TIMEOUT = Duration.ofMinutes(20); + + // Set acknowledgment timeout to very high value to handle large indexes. + // In case of failure the retries will be handled by source coordination + static final Duration ACKNOWLEDGEMENT_SET_TIMEOUT = Duration.ofHours(1000); + static final Duration OWNERSHIP_TIMEOUT = Duration.ofMinutes(30); static final Duration STARTING_BACKOFF = Duration.ofMillis(500); static final Duration MAX_BACKOFF = Duration.ofSeconds(60); static final int BACKOFF_RATE = 2; @@ -64,7 +68,7 @@ static void completeIndexPartition(final OpenSearchSourceConfiguration openSearc final SourcePartition indexPartition, final SourceCoordinator sourceCoordinator) { if (openSearchSourceConfiguration.isAcknowledgmentsEnabled()) { - sourceCoordinator.updatePartitionForAcknowledgmentWait(indexPartition.getPartitionKey(), ACKNOWLEDGEMENT_SET_TIMEOUT); + sourceCoordinator.updatePartitionForAcknowledgmentWait(indexPartition.getPartitionKey(), OWNERSHIP_TIMEOUT); acknowledgementSet.complete(); } else { sourceCoordinator.closePartition( diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java index e715d54563..29b1bea4b0 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorkerTest.java @@ -69,6 +69,7 @@ import static org.opensearch.dataprepper.plugins.source.opensearch.worker.PitWorker.EXTEND_KEEP_ALIVE_TIME; import static org.opensearch.dataprepper.plugins.source.opensearch.worker.PitWorker.STARTING_KEEP_ALIVE; import static org.opensearch.dataprepper.plugins.source.opensearch.worker.WorkerCommonUtils.ACKNOWLEDGEMENT_SET_TIMEOUT; +import static org.opensearch.dataprepper.plugins.source.opensearch.worker.WorkerCommonUtils.OWNERSHIP_TIMEOUT; @ExtendWith(MockitoExtension.class) public class PitWorkerTest { @@ -316,7 +317,7 @@ void run_with_acknowledgments_enabled_creates_and_deletes_pit_and_closes_that_pa when(schedulingParameterConfiguration.getInterval()).thenReturn(Duration.ZERO); when(openSearchSourceConfiguration.getSchedulingParameterConfiguration()).thenReturn(schedulingParameterConfiguration); - doNothing().when(sourceCoordinator).updatePartitionForAcknowledgmentWait(partitionKey, ACKNOWLEDGEMENT_SET_TIMEOUT); + doNothing().when(sourceCoordinator).updatePartitionForAcknowledgmentWait(partitionKey, OWNERSHIP_TIMEOUT); doNothing().when(sourceCoordinator).closePartition(partitionKey, Duration.ZERO, 1, true); From f9fea79a981b74172459101cde6d43a303a3bce1 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Wed, 3 Dec 2025 22:18:59 -0800 Subject: [PATCH 06/51] PrometheusTimeSeries performance fixes (#6316) * PrometheusTimeSeries performance fixes Signed-off-by: Kondaka * Addressed review comments Signed-off-by: Kondaka * Fixed checkStyle error Signed-off-by: Kondaka --------- Signed-off-by: Kondaka Signed-off-by: Nathan Wand --- .../sink/prometheus/PrometheusSinkAMPIT.java | 29 +- .../sink/prometheus/PrometheusHttpSender.java | 6 + .../prometheus/PrometheusSigV4Signer.java | 6 + .../service/PrometheusTimeSeries.java | 295 +++++++++--------- 4 files changed, 182 insertions(+), 154 deletions(-) diff --git a/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java b/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java index ac0ddf5d49..9fd75fa5e8 100644 --- a/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java +++ b/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java @@ -76,6 +76,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -119,6 +120,8 @@ public class PrometheusSinkAMPIT { @Mock private AwsCredentialsSupplier awsCredentialsSupplier; @Mock + private AwsCredentialsSupplier awsQueryCredentialsSupplier; + @Mock private Counter metricsSuccessCounter; @Mock private Counter metricsFailedCounter; @@ -161,6 +164,8 @@ void setUp() { .build(); + awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); + awsQueryCredentialsSupplier = mock(AwsCredentialsSupplier.class); eventHandle = mock(EventHandle.class); pipelineDescription = mock(PipelineDescription.class); awsCredentialsProvider = DefaultCredentialsProvider.create(); @@ -204,6 +209,7 @@ void setUp() { String remoteWriteUrl = url + "api/v1/remote_write"; queryUrl = url + "api/v1/query"; when(awsCredentialsSupplier.getProvider(any())).thenAnswer(options -> DefaultCredentialsProvider.create()); + lenient().when(awsQueryCredentialsSupplier.getProvider(any())).thenAnswer(options -> DefaultCredentialsProvider.create()); thresholdConfig = mock(PrometheusSinkThresholdConfig.class); when(thresholdConfig.getMaxEvents()).thenReturn(NUM_RECORDS); when(thresholdConfig.getMaxRequestSizeBytes()).thenReturn(100000L); @@ -254,7 +260,7 @@ private void getMetricsFromAMP(final String metricName, final String qs) throws String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); String getUrlQuery = "query=" + query + "&start="+testStartTime+"&end="+endTime+"&step=1s"; String getUrl = queryRangeUrl+"?query=" + encodedQuery + "&start="+testStartTime+"&end="+endTime+"&step=1s"; - PrometheusSigV4Signer signer = new PrometheusSigV4Signer(awsCredentialsSupplier, prometheusSinkConfig, baseUrl + queryRangeUrl); + PrometheusSigV4Signer signer = new PrometheusSigV4Signer(awsQueryCredentialsSupplier, prometheusSinkConfig, baseUrl + queryRangeUrl); final SdkHttpFullRequest signedRequest = signer.signQueryRequest(getUrlQuery); final RequestHeadersBuilder headersBuilder = RequestHeaders.builder() @@ -911,4 +917,25 @@ private Collection> getExponentialHistogramRecordList(int numberOf return records; } + @Test + void testToVerifyLackOfCredentialsResultInFailure() throws Exception { + + AwsCredentialsProvider provider = mock(AwsCredentialsProvider.class); + when(awsCredentialsSupplier.getProvider(any())).thenReturn(provider); + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(1L); + when(thresholdConfig.getMaxEvents()).thenReturn(1); + PrometheusSink sink = createObjectUnderTest(); + Collection> records = getHistogramRecordList(NUM_RECORDS); + sink.doOutput(records); + + long startTimeSeconds = testStartTime.getEpochSecond(); + assertThrows( org.awaitility.core.ConditionTimeoutException.class, () -> await().atMost(Duration.ofSeconds(2)) + .untilAsserted(() -> { + metricsInAMP = 0; + getMetricsFromAMP(histogramMetricName, "histogram"); + assertThat(metricsInAMP, greaterThanOrEqualTo(1)); + })); + + verify(metricsSuccessCounter, times(0)).increment(NUM_RECORDS); + } } diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusHttpSender.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusHttpSender.java index 6ac73307a2..800666e785 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusHttpSender.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusHttpSender.java @@ -127,6 +127,9 @@ public PrometheusPushResult pushToEndpoint(final byte[] payload) { final byte[] compressedBufferData = compressionEngine.compress(payload); final HttpRequest request = buildHttpRequest(compressedBufferData); + if (request == null) { + return new PrometheusPushResult(false, 0); + } final long startTime = System.currentTimeMillis(); // Execute request and wait for completion @@ -170,6 +173,9 @@ private HttpRequest buildHttpRequest(final byte[] payload) { SdkHttpFullRequest sdkHttpRequest = createSdkHttpRequest(config.getUrl(), payload); if (signer != null) { sdkHttpRequest = signer.signRequest(sdkHttpRequest); + if (sdkHttpRequest == null) { + return null; + } } final RequestHeadersBuilder headersBuilder = RequestHeaders.builder() diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSigV4Signer.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSigV4Signer.java index c450fdf9ea..46edf9a6de 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSigV4Signer.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSigV4Signer.java @@ -72,6 +72,9 @@ private static AwsCredentialsOptions convertToCredentialOptions(final AwsConfig } SdkHttpFullRequest signQueryRequest(final String query) { + if (credentialsProvider == null || credentialsProvider.resolveCredentials() == null) { + return null; + } SdkHttpFullRequest unsignedRequest = SdkHttpFullRequest.builder() .method(SdkHttpMethod.POST) .uri(endpointUri) @@ -93,6 +96,9 @@ SdkHttpFullRequest signQueryRequest(final String query) { * @return A signed {@link SdkHttpFullRequest} ready for transmission to the AWS OTLP endpoint */ SdkHttpFullRequest signRequest(final SdkHttpFullRequest unsignedRequest) { + if (credentialsProvider == null || credentialsProvider.resolveCredentials() == null) { + return null; + } return signer.sign(unsignedRequest, Aws4SignerParams.builder() .signingRegion(region) .signingName(SERVICE_NAME) diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java index c932ae1d9d..a8ba7cc23e 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java @@ -1,12 +1,12 @@ - /* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ package org.opensearch.dataprepper.plugins.sink.prometheus.service; @@ -27,21 +27,31 @@ import java.time.Instant; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.commons.lang3.StringUtils.stripEnd; -import static org.apache.commons.lang3.StringUtils.stripStart; - - public class PrometheusTimeSeries { +public class PrometheusTimeSeries { private static final Logger LOG = LoggerFactory.getLogger(PrometheusTimeSeries.class); - private static final Character UNDERSCORE = '_'; + private static final int APPROXIMATE_PROTOBUF_LABEL_OVERHEAD = 8; + private static final int APPROXIMATE_PROTOBUF_SAMPLE_OVERHEAD = 2; + private static final String UNDERSCORE = "_"; + private static final String TOTAL_SUFFIX = "_total"; + private static final String RATIO_SUFFIX = "_ratio"; + private static final String COUNT_SUFFIX = "_count"; + private static final String SUM_SUFFIX = "_sum"; + private static final String MIN_SUFFIX = "_min"; + private static final String MAX_SUFFIX = "_max"; + private static final String BUCKET_SUFFIX = "_bucket"; + private static final String ZERO_COUNT_SUFFIX = "_zero_count"; + private static final String ZERO_THRESHOLD_SUFFIX = "_zero_threshold"; + private static final String NAME_LABEL = "__name__"; + private static final String QUANTILE_LABEL = "quantile"; + private static final String LE_LABEL = "le"; + private static final String GE_LABEL = "ge"; + private static final String PLUS_INF = "+Inf"; + private static final String RESOURCE_PREFIX = "resource_"; + private static final String SCOPE_PREFIX = "scope_"; - /** - * @see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/prometheus - * for the following map and metric/label sanitization rules - */ private static final Map otelToPrometheusUnitsMap = Map.ofEntries( Map.entry("d", "days"), Map.entry("h", "hours"), @@ -78,41 +88,48 @@ public class PrometheusTimeSeries { public PrometheusTimeSeries(Metric metric, final boolean sanitizeNames) throws Exception { this.sanitizeNames = sanitizeNames; - metricName = sanitizeNames ? sanitizeMetricName(metric) : metric.getName(); + this.metricName = sanitizeNames ? sanitizeMetricName(metric) : metric.getName(); + String time = metric.getTime(); String startTime = metric.getStartTime(); - timestamp = (time != null) ? getTimeStampVal(time) : getTimeStampVal(startTime); - timeSeriesList = new ArrayList<>(); - labels = new ArrayList<>(); - Map attributesMap = metric.getAttributes(); - Map flattenedAttributeMap = flattenMap(attributesMap); - for (Map.Entry entry : flattenedAttributeMap.entrySet()) { - addLabel(entry.getKey(), entry.getValue()); + this.timestamp = (time != null) ? getTimestampVal(time) : getTimestampVal(startTime); + + this.timeSeriesList = new ArrayList<>(); + this.labels = new ArrayList<>(); + + // Process all attributes in one pass + processAttributes(metric.getAttributes(), ""); + processResourceAndScopeAttributes(metric); + } + + private void processAttributes(Map attributesMap, String prefix) { + if (attributesMap == null) return; + + for (Map.Entry entry : attributesMap.entrySet()) { + String key = prefix.isEmpty() ? entry.getKey() : prefix + entry.getKey(); + Object value = entry.getValue(); + + if (value instanceof Map) { + processAttributes((Map) value, key + UNDERSCORE); + } else { + addLabel(key, value); + } } - Map resourceAttributesMap = null; - Map scopeAttributesMap = null; + } + + private void processResourceAndScopeAttributes(Metric metric) { try { if (metric.getResource() != null) { - resourceAttributesMap = (Map) metric.getResource().get("attributes"); + Map resourceAttributes = (Map) metric.getResource().get("attributes"); + processAttributes(resourceAttributes, RESOURCE_PREFIX); } if (metric.getScope() != null) { - scopeAttributesMap = (Map) metric.getScope().get("attributes"); + Map scopeAttributes = (Map) metric.getScope().get("attributes"); + processAttributes(scopeAttributes, SCOPE_PREFIX); } } catch (Exception e) { LOG.warn("Failed to get resource/scope attributes", e); } - if (resourceAttributesMap != null) { - flattenedAttributeMap = flattenMap(resourceAttributesMap); - for (Map.Entry entry : flattenedAttributeMap.entrySet()) { - addLabel("resource_"+entry.getKey(), entry.getValue()); - } - } - if (scopeAttributesMap != null) { - flattenedAttributeMap = flattenMap(scopeAttributesMap); - for (Map.Entry entry : flattenedAttributeMap.entrySet()) { - addLabel("scope_"+entry.getKey(), entry.getValue()); - } - } } private void addLabel(String name, final Object value) { @@ -123,231 +140,203 @@ private void addLabel(String name, final Object value) { } private void addLabelSanitized(final String name, final Object value) { - String valueStr; - if (value instanceof String) { - valueStr = (String)value; - } else { - valueStr = String.valueOf(value); - } + String valueStr = (value instanceof String) ? (String) value : String.valueOf(value); Label label = Label.newBuilder().setName(name).setValue(valueStr).build(); labels.add(label); - size += label.toByteArray().length; + size += estimateLabelSize(name, valueStr); } - public List getTimeSeriesList() { - return timeSeriesList; + private int estimateLabelSize(String name, String value) { + return name.length() + value.length() + APPROXIMATE_PROTOBUF_LABEL_OVERHEAD; } private void addTimeSeries(final String labelName, final String labelValue, final Double sampleValue) { - // labelName here is a constant string without any invalid characters - Label label = Label.newBuilder().setName(labelName).setValue(labelValue).build(); - size += label.toByteArray().length; - Sample sample = Sample.newBuilder().setValue(sampleValue).setTimestamp(timestamp).build(); - size += sample.toByteArray().length; + size += estimateLabelSize(labelName, labelValue) + APPROXIMATE_PROTOBUF_SAMPLE_OVERHEAD; timeSeriesList.add(TimeSeries.newBuilder() .addAllLabels(labels) - .addLabels(label) - .addSamples(sample) + .addLabels(Label.newBuilder().setName(labelName).setValue(labelValue).build()) + .addSamples(Sample.newBuilder().setValue(sampleValue).setTimestamp(timestamp).build()) .build()); } private void addTimeSeries(final String metricName, final String labelName, - final String labelValue, final Double sampleValue) { - Label label1 = Label.newBuilder().setName("__name__").setValue(metricName).build(); - // labelName here is a constant string without any invalid characters - Label label2 = Label.newBuilder().setName(labelName).setValue(labelValue).build(); - size += label1.toByteArray().length + label2.toByteArray().length; - Sample sample = Sample.newBuilder().setValue(sampleValue).setTimestamp(timestamp).build(); - size += sample.toByteArray().length; + final String labelValue, final Double sampleValue) { + size += estimateLabelSize(NAME_LABEL, metricName) + estimateLabelSize(labelName, labelValue) + APPROXIMATE_PROTOBUF_SAMPLE_OVERHEAD; timeSeriesList.add(TimeSeries.newBuilder() .addAllLabels(labels) - .addLabels(label1) - .addLabels(label2) - .addSamples(sample) + .addLabels(Label.newBuilder().setName(NAME_LABEL).setValue(metricName).build()) + .addLabels(Label.newBuilder().setName(labelName).setValue(labelValue).build()) + .addSamples(Sample.newBuilder().setValue(sampleValue).setTimestamp(timestamp).build()) .build()); } - - public long getTimeStamp() { - return timestamp; - } + public List getTimeSeriesList() { return timeSeriesList; } + public long getTimestamp() { return timestamp; } + public int getSize() { return size; } public void addSumMetric(Sum sum) { - addTimeSeries("__name__", metricName, sum.getValue()); + addTimeSeries(NAME_LABEL, metricName, sum.getValue()); } public void addGaugeMetric(Gauge gauge) { - addTimeSeries("__name__", metricName, gauge.getValue()); + addTimeSeries(NAME_LABEL, metricName, gauge.getValue()); } public void addSummaryMetric(Summary summary) { List quantiles = summary.getQuantiles(); - for (int i = 0; i < quantiles.size(); i++) { - Quantile quantile = quantiles.get(i); - addTimeSeries(metricName, "quantile", quantile.getQuantile().toString(), (double)quantile.getValue()); - + for (Quantile quantile : quantiles) { + addTimeSeries(metricName, QUANTILE_LABEL, quantile.getQuantile().toString(), (double) quantile.getValue()); } } public void addHistogramMetric(Histogram histogram) { - addTimeSeries("__name__", metricName + "_count", (double)histogram.getCount()); - addTimeSeries("__name__", metricName + "_sum", (double)histogram.getSum()); + addTimeSeries(NAME_LABEL, metricName + COUNT_SUFFIX, (double) histogram.getCount()); + addTimeSeries(NAME_LABEL, metricName + SUM_SUFFIX, (double) histogram.getSum()); Double min = histogram.getMin(); if (min != null) { - addTimeSeries("__name__", metricName + "_min", (double)min); + addTimeSeries(NAME_LABEL, metricName + MIN_SUFFIX, min); } Double max = histogram.getMax(); if (max != null) { - addTimeSeries("__name__", metricName + "_max", (double)max); + addTimeSeries(NAME_LABEL, metricName + MAX_SUFFIX, max); } + List explicitBounds = histogram.getExplicitBoundsList(); List bucketCounts = histogram.getBucketCountsList(); if (explicitBounds != null && bucketCounts != null) { - addLabelSanitized("__name__", metricName+"_bucket"); + addLabelSanitized(NAME_LABEL, metricName + BUCKET_SUFFIX); + int lastIndex = bucketCounts.size() - 1; for (int i = 0; i < bucketCounts.size(); i++) { - final String labelValue = (i == bucketCounts.size()-1) ? "+Inf" : explicitBounds.get(i).toString(); - addTimeSeries("le", labelValue, (double)bucketCounts.get(i)); + String labelValue = (i == lastIndex) ? PLUS_INF : explicitBounds.get(i).toString(); + addTimeSeries(LE_LABEL, labelValue, (double) bucketCounts.get(i)); } } } public void addExponentialHistogramMetric(ExponentialHistogram histogram) { - addTimeSeries("__name__", metricName + "_count", (double)histogram.getCount()); - addTimeSeries("__name__", metricName + "_sum", (double)histogram.getSum()); + addTimeSeries(NAME_LABEL, metricName + COUNT_SUFFIX, (double) histogram.getCount()); + addTimeSeries(NAME_LABEL, metricName + SUM_SUFFIX, (double) histogram.getSum()); + Long zeroCount = histogram.getZeroCount(); if (zeroCount != null) { - addTimeSeries("__name__", metricName + "_zero_count", (double)zeroCount); + addTimeSeries(NAME_LABEL, metricName + ZERO_COUNT_SUFFIX, (double) zeroCount); } Double zeroThreshold = histogram.getZeroThreshold(); if (zeroThreshold != null) { - addTimeSeries("__name__", metricName + "_zero_threshold", (double)zeroThreshold); + addTimeSeries(NAME_LABEL, metricName + ZERO_THRESHOLD_SUFFIX, zeroThreshold); } Integer scale = histogram.getScale(); if (scale != null) { - addTimeSeries("__name__", metricName + "_zero_threshold", (double)scale); + addTimeSeries(NAME_LABEL, metricName + ZERO_THRESHOLD_SUFFIX, (double) scale); } + List positiveBucketCounts = histogram.getPositive(); Integer positiveOffset = histogram.getPositiveOffset(); List negativeBucketCounts = histogram.getNegative(); Integer negativeOffset = histogram.getPositiveOffset(); + boolean positiveBucketsPresent = (positiveBucketCounts != null) && (positiveOffset != null); boolean negativeBucketsPresent = (negativeBucketCounts != null) && (negativeOffset != null); if (positiveBucketsPresent || negativeBucketsPresent) { - addLabelSanitized("__name__", metricName+"_bucket"); + addLabelSanitized(NAME_LABEL, metricName + BUCKET_SUFFIX); + if (positiveBucketsPresent) { for (int i = 0; i < positiveBucketCounts.size(); i++) { double bound = calculateBucketBound(i + positiveOffset + 1, scale); - addTimeSeries("le", Double.toString(bound), (double)positiveBucketCounts.get(i)); + addTimeSeries(LE_LABEL, Double.toString(bound), (double) positiveBucketCounts.get(i)); } } if (negativeBucketsPresent) { for (int i = 0; i < negativeBucketCounts.size(); i++) { - double bound = -1 * calculateBucketBound(i + negativeOffset + 1, scale); - addTimeSeries("ge", Double.toString(bound), (double)negativeBucketCounts.get(i)); + double bound = -calculateBucketBound(i + negativeOffset + 1, scale); + addTimeSeries(GE_LABEL, Double.toString(bound), (double) negativeBucketCounts.get(i)); } } } } - private double calculateBucketBound(int index, int scale) { + private static double calculateBucketBound(int index, int scale) { return Math.pow(2, index * Math.pow(2, -scale)); } - private static void flattenHelper(Map map, String prefix, Map flatMap) { - for (Map.Entry entry : map.entrySet()) { - String key = prefix.isEmpty() ? entry.getKey() : prefix + "_" + entry.getKey(); - Object value = entry.getValue(); - - if (value instanceof Map) { - flattenHelper((Map) value, key, flatMap); - } else { - flatMap.put(key, value.toString()); - } - } - } - - // See https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/prometheus static String sanitizeMetricName(final Metric metric) { - final String name = metric.getName(); final String unit = metric.getUnit(); final boolean isGauge = metric.getKind().equals(Metric.KIND.GAUGE.toString()); final boolean isCounter = metric.getKind().equals(Metric.KIND.SUM.toString()) && - ((Sum)metric).isMonotonic() && - ((Sum)metric).getAggregationTemporality().equals("AGGREGATION_TEMPORALITY_CUMULATIVE"); + ((Sum) metric).isMonotonic() && + ((Sum) metric).getAggregationTemporality().equals("AGGREGATION_TEMPORALITY_CUMULATIVE"); + + StringBuilder metricNameBuilder = new StringBuilder(sanitizeName(name, true, false)); + String suffix = isCounter ? TOTAL_SUFFIX : ""; - String metricName = sanitizeName(name, true, false); // metric names allow colon - String suffix = isCounter ? (UNDERSCORE+"total") : ""; if (unit.startsWith("{")) { - return metricName+suffix; + return metricNameBuilder.append(suffix).toString(); } - if (unit.equals("1") && isGauge) { - return metricName+UNDERSCORE+"ratio"; + if ("1".equals(unit) && isGauge) { + return metricNameBuilder.append(RATIO_SUFFIX).toString(); } - String val = otelToPrometheusUnitsMap.get(unit); - if (val != null) { - return metricName+UNDERSCORE+val+suffix; + + String mappedUnit = otelToPrometheusUnitsMap.get(unit); + if (mappedUnit != null) { + return metricNameBuilder.append(UNDERSCORE).append(mappedUnit).append(suffix).toString(); } + if (unit.contains("/")) { - String[] unitSplit = unit.split("/"); + String[] unitSplit = unit.split("/", 2); if (unitSplit.length == 2) { String unit1 = otelToPrometheusUnitsMap.get(unitSplit[0]); String unit2 = otelToPrometheusUnitsMap.get(unitSplit[1]); if (unit1 != null && unit2 != null) { - return metricName + UNDERSCORE + unit1 + UNDERSCORE + unit2+suffix; + return metricNameBuilder.append(UNDERSCORE).append(unit1) + .append(UNDERSCORE).append(unit2).append(suffix).toString(); } } } - return unit.equals("1") ? metricName+suffix : metricName+UNDERSCORE+unit+suffix; + + if (!"1".equals(unit)) { + metricNameBuilder.append(UNDERSCORE).append(unit); + } + return metricNameBuilder.append(suffix).toString(); } static String sanitizeLabelName(final String name) { - return sanitizeName(name, false, true); // label names do NOT allow colon + return sanitizeName(name, false, true); } static String sanitizeName(final String name, final boolean allowColon, final boolean isLabel) { StringBuilder sb = new StringBuilder(name.length()); - Character prevChar = null; + char prevChar = 0; + for (int i = 0; i < name.length(); i++) { - Character curChar = sanitizeChar(name.charAt(i), i == 0, allowColon); - if (isLabel || (curChar != UNDERSCORE || prevChar != UNDERSCORE)) { + char curChar = sanitizeChar(name.charAt(i), i == 0, allowColon); + if (isLabel || (curChar != '_' || prevChar != '_')) { sb.append(curChar); } prevChar = curChar; } - return isLabel ? sb.toString() : stripEnd(stripStart(sb.toString(), "_"), "_"); - } - private static char sanitizeChar(char c, boolean isFirst, boolean allowColon) { - if (allowColon && c == ':') { - return c; - } - if (isFirst) { - return (Character.isLetter(c)) ? c : '_'; - } else { - return (Character.isLetterOrDigit(c)) ? c : '_'; + String result = sb.toString(); + if (!isLabel) { + // Strip leading and trailing underscores + int start = 0, end = result.length(); + while (start < end && result.charAt(start) == '_') start++; + while (end > start && result.charAt(end - 1) == '_') end--; + result = result.substring(start, end); } + return result; } - - private static Map flattenMap(Map map) { - Map flatMap = new HashMap<>(); - flattenHelper(map, "", flatMap); - return flatMap; - } - - private static long getTimeStampVal(final String time) throws Exception { - long timeStampVal = 0; - timeStampVal = Instant.parse(time).toEpochMilli(); - return timeStampVal; + private static char sanitizeChar(char c, boolean isFirst, boolean allowColon) { + if (allowColon && c == ':') return c; + if (isFirst) return Character.isLetter(c) ? c : '_'; + return Character.isLetterOrDigit(c) ? c : '_'; } - public int getSize() { - return size; + private static long getTimestampVal(final String time) throws Exception { + return Instant.parse(time).toEpochMilli(); } - } - From 59bbaa95617d83d4b17fef540ff231e1cc847dfd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 06:44:35 -0800 Subject: [PATCH 07/51] Bump software.amazon.awssdk:auth in /performance-test (#6315) Bumps software.amazon.awssdk:auth from 2.32.13 to 2.39.5. --- updated-dependencies: - dependency-name: software.amazon.awssdk:auth dependency-version: 2.39.5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Nathan Wand --- performance-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance-test/build.gradle b/performance-test/build.gradle index 60d7f963d6..3cb3109bce 100644 --- a/performance-test/build.gradle +++ b/performance-test/build.gradle @@ -16,7 +16,7 @@ configurations.all { group 'org.opensearch.dataprepper.test.performance' dependencies { - gatlingImplementation 'software.amazon.awssdk:auth:2.32.13' + gatlingImplementation 'software.amazon.awssdk:auth:2.39.5' implementation 'com.fasterxml.jackson.core:jackson-core' testRuntimeOnly testLibs.junit.engine From 905773b7c6cd2f55c982f04f19cb4c82f8906828 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 06:46:07 -0800 Subject: [PATCH 08/51] Bump commons-validator:commons-validator in /data-prepper-core (#6310) Bumps [commons-validator:commons-validator](https://github.com/apache/commons-validator) from 1.10.0 to 1.10.1. - [Changelog](https://github.com/apache/commons-validator/blob/master/RELEASE-NOTES.txt) - [Commits](https://github.com/apache/commons-validator/compare/rel/commons-validator-1.10.0...rel/commons-validator-1.10.1) --- updated-dependencies: - dependency-name: commons-validator:commons-validator dependency-version: 1.10.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Nathan Wand --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 8da0442736..b18dea8ecf 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -62,7 +62,7 @@ dependencies { implementation 'software.amazon.awssdk:acm' implementation 'software.amazon.awssdk:s3' implementation 'software.amazon.awssdk:apache-client' - implementation 'commons-validator:commons-validator:1.10.0' + implementation 'commons-validator:commons-validator:1.10.1' implementation 'software.amazon.awssdk:servicediscovery' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' testImplementation testLibs.junit.vintage From 6a9be4625e8c08fe96021dc29a56f1847b61b2ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 06:49:10 -0800 Subject: [PATCH 09/51] Bump org.wiremock:wiremock in /data-prepper-plugins/opensearch (#6308) Bumps [org.wiremock:wiremock](https://github.com/wiremock/wiremock) from 3.10.0 to 3.13.2. - [Release notes](https://github.com/wiremock/wiremock/releases) - [Commits](https://github.com/wiremock/wiremock/compare/3.10.0...3.13.2) --- updated-dependencies: - dependency-name: org.wiremock:wiremock dependency-version: 3.13.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Nathan Wand --- data-prepper-plugins/opensearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 8affad0e79..8f2ab00a5c 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -47,7 +47,7 @@ dependencies { testImplementation 'net.bytebuddy:byte-buddy-agent:1.17.8' testImplementation testLibs.slf4j.simple testImplementation project(path: ':data-prepper-test:test-common') - testImplementation 'org.wiremock:wiremock:3.10.0' + testImplementation 'org.wiremock:wiremock:3.13.2' } sourceSets { From 1eca5af78b4c7440768ed6dae48ed89f0f46d1a6 Mon Sep 17 00:00:00 2001 From: Xun Zhang Date: Thu, 4 Dec 2025 13:25:45 -0800 Subject: [PATCH 10/51] set retry time interval configurable, increase the http client read timeout (#6320) * set retry time interval configurable and increase the http client read timeout Signed-off-by: Xun Zhang * address comments Signed-off-by: Xun Zhang --------- Signed-off-by: Xun Zhang Signed-off-by: Nathan Wand --- .../ml-inference-processor/build.gradle | 1 + .../processor/MLProcessorConfig.java | 16 +++ .../common/BedrockBatchJobCreator.java | 27 +++++ .../processor/util/SdkHttpClientExecutor.java | 21 +++- .../common/BedrockBatchJobCreatorTest.java | 100 ++++++++++++++++++ 5 files changed, 162 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/ml-inference-processor/build.gradle b/data-prepper-plugins/ml-inference-processor/build.gradle index 9f9c208e0c..40f94c5317 100644 --- a/data-prepper-plugins/ml-inference-processor/build.gradle +++ b/data-prepper-plugins/ml-inference-processor/build.gradle @@ -13,6 +13,7 @@ dependencies { implementation 'org.json:json' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'org.projectlombok:lombok:1.18.22' + implementation 'org.hibernate.validator:hibernate-validator:8.0.2.Final' annotationProcessor 'org.projectlombok:lombok:1.18.20' implementation 'software.amazon.awssdk:s3' testImplementation project(':data-prepper-test:test-event') diff --git a/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/MLProcessorConfig.java b/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/MLProcessorConfig.java index cb4a7ed5e6..d755116b09 100644 --- a/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/MLProcessorConfig.java +++ b/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/MLProcessorConfig.java @@ -9,6 +9,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import org.hibernate.validator.constraints.time.DurationMax; +import org.hibernate.validator.constraints.time.DurationMin; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import lombok.Getter; @@ -33,6 +35,7 @@ public class MLProcessorConfig { private static final int DEFAULT_MAX_BATCH_SIZE = 100; public static final Duration DEFAULT_RETRY_WINDOW = Duration.ofMinutes(10); + public static final int DEFAULT_RETRY_INTERVAL_SECONDS = 60; // default retry interval is 1 minute @JsonProperty("aws") @NotNull @@ -89,6 +92,19 @@ public class MLProcessorConfig { @JsonProperty("retry_time_window") private Duration retryTimeWindow = DEFAULT_RETRY_WINDOW; + @JsonPropertyDescription("The retry interval for throttled records. " + + "Supports ISO_8601 duration notation (\"PT1M\", \"PT30S\") and simple notation (\"60s\", \"2m\"). " + + "Valid range: 3 seconds to 5 minutes. Default is 60 seconds.") + @ExampleValues({ + @ExampleValues.Example(value = "\"PT1M\"", description = "ISO-8601 format for 1 minute"), + @ExampleValues.Example(value = "\"60s\"", description = "Simple format for 60 seconds"), + @ExampleValues.Example(value = "\"2m\"", description = "Simple format for 2 minutes") + }) + @JsonProperty("retry_interval") + @DurationMin(seconds = 3) + @DurationMax(seconds = 300) + private Duration retryInterval = Duration.ofSeconds(DEFAULT_RETRY_INTERVAL_SECONDS); + @JsonProperty("dlq") private PluginModel dlq; diff --git a/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreator.java b/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreator.java index 5cb0b8786d..fb8a29ccf2 100644 --- a/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreator.java +++ b/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreator.java @@ -38,6 +38,8 @@ public class BedrockBatchJobCreator extends AbstractBatchJobCreator { @Getter private final ConcurrentLinkedQueue throttledRecords = new ConcurrentLinkedQueue<>(); private final Lock processingLock; + private final long retryIntervalMillis; + private volatile long lastRetryTimestamp; private static final String BEDROCK_PAYLOAD_TEMPLATE = "{\"parameters\": {\"inputDataConfig\": {\"s3InputDataConfig\": {\"s3Uri\": \"s3://\"}}," + "\"jobName\": \"\", \"outputDataConfig\": {\"s3OutputDataConfig\": {\"s3Uri\": \"s3://\"}}}}"; @@ -46,6 +48,8 @@ public BedrockBatchJobCreator(final MLProcessorConfig mlProcessorConfig, final A super(mlProcessorConfig, awsCredentialsSupplier, pluginMetrics, dlqPushHandler); this.awsCredentialsSupplier = awsCredentialsSupplier; this.processingLock = new ReentrantLock(); + this.retryIntervalMillis = mlProcessorConfig.getRetryInterval().toMillis(); + this.lastRetryTimestamp = System.currentTimeMillis(); } @Override @@ -150,16 +154,39 @@ private void processRecord(Record record, List> resultRecor @Override public void addProcessedBatchRecordsToResults(List> resultRecords) { + if (throttledRecords.isEmpty()) { + return; + } + + long currentTime = System.currentTimeMillis(); + long timeSinceLastRetry = currentTime - lastRetryTimestamp; + + if (timeSinceLastRetry < retryIntervalMillis) { + LOG.debug("Skipping retry processing. Only {}ms passed since last retry (need {}ms)", + timeSinceLastRetry, retryIntervalMillis); + return; + } + if (!processingLock.tryLock()) { LOG.debug("Another thread is currently processing results, skipping this attempt"); return; } try { + if (throttledRecords.isEmpty()) { + LOG.debug("Queue became empty after acquiring lock, skipping timestamp update"); + return; + } + + LOG.info(NOISY, "Processing {} throttled records ({}s since last retry)", + throttledRecords.size(), timeSinceLastRetry / 1000); + processThrottledRecords(resultRecords); } catch (Exception e) { LOG.error("Error in batch processing throttled records. Error: {}", e.getMessage()); } finally { + // Always update timestamp after a retry attempt (success or failure) + lastRetryTimestamp = currentTime; processingLock.unlock(); } } diff --git a/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/util/SdkHttpClientExecutor.java b/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/util/SdkHttpClientExecutor.java index 0b75985091..22ca4e32e7 100644 --- a/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/util/SdkHttpClientExecutor.java +++ b/data-prepper-plugins/ml-inference-processor/src/main/java/org/opensearch/dataprepper/plugins/ml_inference/processor/util/SdkHttpClientExecutor.java @@ -17,11 +17,26 @@ public class SdkHttpClientExecutor implements HttpClientExecutor { private final SdkHttpClient httpClient; + // Configuration constants + private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(30); + private static final Duration DEFAULT_READ_TIMEOUT = Duration.ofSeconds(30); + private static final int DEFAULT_MAX_CONNECTIONS = 10; + public SdkHttpClientExecutor() { + this(DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT, DEFAULT_MAX_CONNECTIONS); + } + + /** + * Constructor with configurable timeouts for flexibility + * @param connectionTimeout timeout for establishing connection + * @param readTimeout timeout for reading response data + * @param maxConnections maximum number of connections + */ + public SdkHttpClientExecutor(Duration connectionTimeout, Duration readTimeout, int maxConnections) { AttributeMap attributeMap = AttributeMap.builder() - .put(SdkHttpConfigurationOption.CONNECTION_TIMEOUT, Duration.ofMillis(30000)) - .put(SdkHttpConfigurationOption.READ_TIMEOUT, Duration.ofMillis(3000)) - .put(SdkHttpConfigurationOption.MAX_CONNECTIONS, 10) + .put(SdkHttpConfigurationOption.CONNECTION_TIMEOUT, connectionTimeout) + .put(SdkHttpConfigurationOption.READ_TIMEOUT, readTimeout) + .put(SdkHttpConfigurationOption.MAX_CONNECTIONS, maxConnections) .build(); this.httpClient = new DefaultSdkHttpClientBuilder().buildWithDefaults(attributeMap); } diff --git a/data-prepper-plugins/ml-inference-processor/src/test/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreatorTest.java b/data-prepper-plugins/ml-inference-processor/src/test/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreatorTest.java index 19d5971d58..1aac6f6eee 100644 --- a/data-prepper-plugins/ml-inference-processor/src/test/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreatorTest.java +++ b/data-prepper-plugins/ml-inference-processor/src/test/java/org/opensearch/dataprepper/plugins/ml_inference/processor/common/BedrockBatchJobCreatorTest.java @@ -21,6 +21,7 @@ import org.opensearch.dataprepper.plugins.ml_inference.processor.dlq.DlqPushHandler; import org.opensearch.dataprepper.plugins.ml_inference.processor.exception.MLBatchJobException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -36,6 +37,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.ml_inference.processor.MLProcessorConfig.DEFAULT_RETRY_INTERVAL_SECONDS; import static org.opensearch.dataprepper.plugins.ml_inference.processor.MLProcessorConfig.DEFAULT_RETRY_WINDOW; import static org.opensearch.dataprepper.plugins.ml_inference.processor.common.AbstractBatchJobCreator.NUMBER_OF_FAILED_BATCH_JOBS_CREATION; import static org.opensearch.dataprepper.plugins.ml_inference.processor.common.AbstractBatchJobCreator.NUMBER_OF_RECORDS_FAILED_IN_BATCH_JOB; @@ -280,4 +282,102 @@ void testCreateMLBatchJob_ThrottledThenSuccess() { verify(bedrockBatchJobCreator, times(1)).incrementSuccessCounter(); } } + + @Test + void testRetryInterval_SkipsRetryBeforeIntervalElapses() throws InterruptedException { + // Mock retry interval BEFORE creating the object + when(mlProcessorConfig.getRetryInterval()).thenReturn(Duration.ofSeconds(DEFAULT_RETRY_INTERVAL_SECONDS)); // 1 second for testing + + // Create object with mocked config + bedrockBatchJobCreator = spy(new BedrockBatchJobCreator(mlProcessorConfig, awsCredentialsSupplier, pluginMetrics, dlqPushHandler)); + + Event event = mock(Event.class); + Record record = new Record<>(event); + + when(event.getJsonNode()).thenReturn(OBJECT_MAPPER.createObjectNode() + .put("bucket", "test-bucket") + .put("key", "input.jsonl")); + + try (MockedStatic mockedStatic = mockStatic(RetryUtil.class)) { + // First attempt - gets throttled + mockedStatic.when(() -> RetryUtil.retryWithBackoffWithResult(any(Runnable.class), any())) + .thenReturn(new RetryUtil.RetryResult(false, new MLBatchJobException(429, "throttled"), 1)); + + List> resultRecords = new ArrayList<>(); + + // First attempt - gets throttled + bedrockBatchJobCreator.createMLBatchJob(Arrays.asList(record), resultRecords); + assertEquals(1, bedrockBatchJobCreator.getThrottledRecords().size()); + + // Try to process immediately (should skip due to retry interval) + bedrockBatchJobCreator.addProcessedBatchRecordsToResults(resultRecords); + + // Verify record is still in queue (not processed due to retry interval) + assertEquals(1, bedrockBatchJobCreator.getThrottledRecords().size()); + BedrockBatchJobCreator.RetryRecord throttledRecord = bedrockBatchJobCreator.getThrottledRecords().peek(); + assertNotNull(throttledRecord); + assertEquals(0, throttledRecord.getRetryCount()); // Not incremented because retry was skipped + assertTrue(resultRecords.isEmpty()); + } + } + + @Test + void testRetryInterval_ProcessesAfterIntervalElapses() throws InterruptedException { + // Mock retry interval BEFORE creating the object + when(mlProcessorConfig.getRetryInterval()).thenReturn(Duration.ofSeconds(1)); // 1 second for testing + + // Create object with mocked config + bedrockBatchJobCreator = spy(new BedrockBatchJobCreator(mlProcessorConfig, awsCredentialsSupplier, pluginMetrics, dlqPushHandler)); + + Event event = mock(Event.class); + Record record = new Record<>(event); + + when(event.getJsonNode()).thenReturn(OBJECT_MAPPER.createObjectNode() + .put("bucket", "test-bucket") + .put("key", "input.jsonl")); + + try (MockedStatic mockedStatic = mockStatic(RetryUtil.class)) { + // First throttled, then success + mockedStatic.when(() -> RetryUtil.retryWithBackoffWithResult(any(Runnable.class), any())) + .thenReturn(new RetryUtil.RetryResult(false, new MLBatchJobException(429, "throttled"), 1)) + .thenReturn(new RetryUtil.RetryResult(true, null, 1)); + + List> resultRecords = new ArrayList<>(); + + // First attempt - gets throttled + bedrockBatchJobCreator.createMLBatchJob(Arrays.asList(record), resultRecords); + assertEquals(1, bedrockBatchJobCreator.getThrottledRecords().size()); + + // Wait for retry interval to elapse + Thread.sleep(1100); // Wait 1.1 seconds + + // Now retry should proceed + bedrockBatchJobCreator.addProcessedBatchRecordsToResults(resultRecords); + + // Verify record was processed successfully + assertTrue(bedrockBatchJobCreator.getThrottledRecords().isEmpty()); + assertEquals(1, resultRecords.size()); + verify(bedrockBatchJobCreator, times(1)).incrementSuccessCounter(); + } + } + + @Test + void testRetryInterval_EmptyQueueDoesNotUpdateTimestamp() throws Exception { + List> resultRecords = new ArrayList<>(); + + // Try to process with empty queue + long timestampBefore = getLastRetryTimestamp(bedrockBatchJobCreator); + bedrockBatchJobCreator.addProcessedBatchRecordsToResults(resultRecords); + long timestampAfter = getLastRetryTimestamp(bedrockBatchJobCreator); + + // Verify timestamp was not updated (queue was empty) + assertEquals(timestampBefore, timestampAfter); + } + + // Helper method to access private lastRetryTimestamp field using reflection + private long getLastRetryTimestamp(BedrockBatchJobCreator creator) throws Exception { + java.lang.reflect.Field field = BedrockBatchJobCreator.class.getDeclaredField("lastRetryTimestamp"); + field.setAccessible(true); + return (long) field.get(creator); + } } From 3c980ccb872ed0cb006b4af7f8d26b8c0d282850 Mon Sep 17 00:00:00 2001 From: chrisale000 Date: Tue, 9 Dec 2025 12:30:04 -0800 Subject: [PATCH 11/51] Centralize Metrics, Create MetricHelper Unit Tests, and Add M365 Logging (#6338) Signed-off-by: Alexander Christensen Co-authored-by: Alexander Christensen Signed-off-by: Nathan Wand --- .../Office365RestClient.java | 88 ++-- .../Office365RestClientTest.java | 269 +---------- .../source_crawler/utils/MetricsHelper.java | 199 +++++++- .../utils/MetricsHelperTest.java | 441 ++++++++++++++++++ 4 files changed, 698 insertions(+), 299 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java index c58462cf4c..451cdc6983 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java @@ -10,7 +10,6 @@ package org.opensearch.dataprepper.plugins.source.microsoft_office365; import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; @@ -28,7 +27,6 @@ import org.springframework.web.client.RestTemplate; import javax.inject.Named; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.List; import java.util.Map; @@ -37,6 +35,12 @@ import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.CONTENT_TYPES; import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.getErrorTypeMetricCounterMap; import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishErrorTypeMetricCounter; +import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishGetResponseSizeMetricInBytes; +import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishGetRequestsSuccessMetric; +import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.provideGetRequestsFailureCounter; +import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishSearchResponseSizeMetricInBytes; +import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishSearchRequestsSuccessMetric; +import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.provideSearchRequestFailureCounter; /** * REST client for interacting with Office 365 Management API. @@ -46,15 +50,9 @@ @Named public class Office365RestClient { private static final String AUDIT_LOG_FETCH_LATENCY = "auditLogFetchLatency"; - private static final String AUDIT_LOG_RESPONSE_SIZE = "auditLogResponseSizeBytes"; - private static final String AUDIT_LOG_REQUESTS_FAILED = "auditLogRequestsFailed"; - private static final String AUDIT_LOG_REQUESTS_SUCCESS = "auditLogRequestsSuccess"; private static final String API_CALLS = "apiCalls"; private static final String AUDIT_LOGS_REQUESTED = "auditLogsRequested"; private static final String SEARCH_CALL_LATENCY = "searchCallLatency"; - private static final String SEARCH_RESPONSE_SIZE = "searchResponseSizeBytes"; - private static final String SEARCH_REQUESTS_SUCCESS = "searchRequestsSuccess"; - private static final String SEARCH_REQUESTS_FAILED = "searchRequestsFailed"; private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/"; @@ -63,13 +61,8 @@ public class Office365RestClient { private final Timer auditLogFetchLatencyTimer; private final Timer searchCallLatencyTimer; private final Counter auditLogsRequestedCounter; - private final Counter auditLogRequestsFailedCounter; - private final Counter auditLogRequestsSuccessCounter; - private final Counter searchRequestsFailedCounter; - private final Counter searchRequestsSuccessCounter; private final Counter apiCallsCounter; - private final DistributionSummary auditLogResponseSizeSummary; - private final DistributionSummary searchResponseSizeSummary; + private final PluginMetrics pluginMetrics; private Map errorTypeMetricCounterMap; @@ -77,17 +70,11 @@ public Office365RestClient(final Office365AuthenticationInterface authConfig, final PluginMetrics pluginMetrics) { // TODO: Abstract into a Office365PluginMetrics this.authConfig = authConfig; + this.pluginMetrics = pluginMetrics; this.auditLogFetchLatencyTimer = pluginMetrics.timer(AUDIT_LOG_FETCH_LATENCY); this.searchCallLatencyTimer = pluginMetrics.timer(SEARCH_CALL_LATENCY); this.auditLogsRequestedCounter = pluginMetrics.counter(AUDIT_LOGS_REQUESTED); - this.auditLogRequestsFailedCounter = pluginMetrics.counter(AUDIT_LOG_REQUESTS_FAILED); - this.auditLogRequestsSuccessCounter = pluginMetrics.counter(AUDIT_LOG_REQUESTS_SUCCESS); - this.searchRequestsFailedCounter = pluginMetrics.counter(SEARCH_REQUESTS_FAILED); - this.searchRequestsSuccessCounter = pluginMetrics.counter(SEARCH_REQUESTS_SUCCESS); this.apiCallsCounter = pluginMetrics.counter(API_CALLS); - this.auditLogResponseSizeSummary = pluginMetrics.summary(AUDIT_LOG_RESPONSE_SIZE); - this.searchResponseSizeSummary = pluginMetrics.summary(SEARCH_RESPONSE_SIZE); - this.errorTypeMetricCounterMap = getErrorTypeMetricCounterMap(pluginMetrics); } @@ -176,6 +163,7 @@ public AuditLogsResponse searchAuditLogs(final String contentType, startTime.toString(), endTime.toString()); + log.debug("Searching audit logs with URL: {}", url); final HttpHeaders headers = new HttpHeaders(); return searchCallLatencyTimer.record(() -> { @@ -191,8 +179,27 @@ public AuditLogsResponse searchAuditLogs(final String contentType, new HttpEntity<>(headers), new ParameterizedTypeReference<>() {} ); - // Record search request size. - searchResponseSizeSummary.record(response.getHeaders().getContentLength()); + + // Log response details + List> responseBody = response.getBody(); + if (responseBody == null) { + log.debug("Search audit logs response is null for URL: {}", url); + } else { + log.debug("Search audit logs response received {} entries for URL: {}", + responseBody.size(), url); + String responseStr = responseBody.toString(); + // Size protection for log limits. + if (responseStr.length() > 10000) { + log.debug("Search audit logs response body (truncated to first 10000 chars): {}", + responseStr.substring(0, 10000) + "... [TRUNCATED - total length: " + responseStr.length() + "]"); + } else { + log.debug("Search audit logs response body: {}", responseBody); + } + } + + // Publish centralized search metrics + publishSearchResponseSizeMetricInBytes(pluginMetrics, response); + publishSearchRequestsSuccessMetric(pluginMetrics); // Extract NextPageUri from response headers List nextPageHeaders = response.getHeaders().get("NextPageUri"); @@ -203,15 +210,15 @@ public AuditLogsResponse searchAuditLogs(final String contentType, log.debug("Next page URI found: {}", nextPageUri); } - searchRequestsSuccessCounter.increment(); return new AuditLogsResponse(response.getBody(), nextPageUri); }, authConfig::renewCredentials, - searchRequestsFailedCounter + provideSearchRequestFailureCounter(pluginMetrics) ); } catch (Exception e) { publishErrorTypeMetricCounter(e, this.errorTypeMetricCounterMap); - log.error(NOISY, "Error while fetching audit logs for content type {}", contentType, e); + log.error(NOISY, "Error while fetching audit logs for content type {} from URL: {}", + contentType, url, e); throw new SaaSCrawlerException("Failed to fetch audit logs", e, true); } }); @@ -228,6 +235,8 @@ public String getAuditLog(String contentUri) { if (!contentUri.startsWith(MANAGEMENT_API_BASE_URL)) { throw new SaaSCrawlerException("ContentUri must be from Office365 Management API: " + contentUri, false); } + + log.debug("Getting audit log from content URI: {}", contentUri); auditLogsRequestedCounter.increment(); final HttpHeaders headers = new HttpHeaders(); @@ -243,15 +252,28 @@ public String getAuditLog(String contentUri) { String.class ); - // Record audit log request size from response body - String responseBody = responseEntity.getBody(); - if (responseBody != null) { - auditLogResponseSizeSummary.record(responseBody.getBytes(StandardCharsets.UTF_8).length); + return responseEntity.getBody(); + }, authConfig::renewCredentials, provideGetRequestsFailureCounter(pluginMetrics)); + + // Log response details + if (response == null) { + log.debug("Get audit log response is null for content URI: {}", contentUri); + } else { + log.debug("Get audit log response received {} characters for content URI: {}", + response.length(), contentUri); + // Size protection for log limits. + if (response.length() > 10000) { + log.debug("Get audit log response content (truncated to first 10000 chars): {}", + response.substring(0, 10000) + "... [TRUNCATED - total length: " + response.length() + "]"); + } else { + log.debug("Get audit log response content: {}", response); } + } + + // Publish centralized GET request metrics + publishGetResponseSizeMetricInBytes(pluginMetrics, response); + publishGetRequestsSuccessMetric(pluginMetrics); - return responseBody; - }, authConfig::renewCredentials, auditLogRequestsFailedCounter); - auditLogRequestsSuccessCounter.increment(); return response; } catch (Exception e) { publishErrorTypeMetricCounter(e, this.errorTypeMetricCounterMap); diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java index a4a993c7f1..949a8d75d7 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java @@ -308,13 +308,10 @@ void testTokenRenewal() { void testSearchAuditLogsFailureCounterIncrementsOnEachRetry() throws Exception { Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); Instant endTime = Instant.now(); - + when(authConfig.getTenantId()).thenReturn("test-tenant-id"); when(authConfig.getAccessToken()).thenReturn("test-access-token"); - Counter mockCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "searchRequestsFailedCounter", mockCounter); - when(restTemplate.exchange( anyString(), eq(HttpMethod.GET), @@ -322,7 +319,7 @@ void testSearchAuditLogsFailureCounterIncrementsOnEachRetry() throws Exception { any(ParameterizedTypeReference.class) )).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); - assertThrows(SaaSCrawlerException.class, () -> + assertThrows(SaaSCrawlerException.class, () -> office365RestClient.searchAuditLogs( "Audit.AzureActiveDirectory", startTime, @@ -330,19 +327,13 @@ void testSearchAuditLogsFailureCounterIncrementsOnEachRetry() throws Exception { null ) ); - - // Verify counter.increment() was called exactly 6 times (once for each retry attempt) - verify(mockCounter, times(6)).increment(); } @Test void testGetAuditLogFailureCounterIncrementsOnEachRetry() throws Exception { String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - Counter mockCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogRequestsFailedCounter", mockCounter); + when(authConfig.getAccessToken()).thenReturn("test-access-token"); when(restTemplate.exchange( eq(contentUri), @@ -351,12 +342,9 @@ void testGetAuditLogFailureCounterIncrementsOnEachRetry() throws Exception { eq(String.class) )).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); - assertThrows(SaaSCrawlerException.class, () -> + assertThrows(SaaSCrawlerException.class, () -> office365RestClient.getAuditLog(contentUri) ); - - // Verify counter.increment() was called exactly 6 times (once for each retry attempt) - verify(mockCounter, times(6)).increment(); } @Test @@ -365,60 +353,36 @@ void testMetricsInitialization() { // inside RetryHandler.executeWithRetry() static method calls, which would require complex static mocking // to test for invocation. Testing initialization ensures the metrics infrastructure is properly set up. - // Mock all required timers and counters for Office365RestClient constructor + // Mock only the local timers and counters for Office365RestClient constructor PluginMetrics mockPluginMetrics = org.mockito.Mockito.mock(PluginMetrics.class); Timer mockAuditLogFetchLatencyTimer = org.mockito.Mockito.mock(Timer.class); Timer mockSearchCallLatencyTimer = org.mockito.Mockito.mock(Timer.class); Counter mockAuditLogsRequestedCounter = org.mockito.Mockito.mock(Counter.class); - Counter mockAuditLogRequestsFailedCounter = org.mockito.Mockito.mock(Counter.class); - Counter mockAuditLogRequestsSuccessCounter = org.mockito.Mockito.mock(Counter.class); - Counter mockSearchRequestsFailedCounter = org.mockito.Mockito.mock(Counter.class); - Counter mockSearchRequestsSuccessCounter = org.mockito.Mockito.mock(Counter.class); Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - DistributionSummary mockAuditLogRequestSizeSummary = org.mockito.Mockito.mock(DistributionSummary.class); - DistributionSummary mockSearchRequestSizeSummary = org.mockito.Mockito.mock(DistributionSummary.class); when(mockPluginMetrics.timer("auditLogFetchLatency")).thenReturn(mockAuditLogFetchLatencyTimer); when(mockPluginMetrics.timer("searchCallLatency")).thenReturn(mockSearchCallLatencyTimer); when(mockPluginMetrics.counter("auditLogsRequested")).thenReturn(mockAuditLogsRequestedCounter); - when(mockPluginMetrics.counter("auditLogRequestsFailed")).thenReturn(mockAuditLogRequestsFailedCounter); - when(mockPluginMetrics.counter("auditLogRequestsSuccess")).thenReturn(mockAuditLogRequestsSuccessCounter); - when(mockPluginMetrics.counter("searchRequestsFailed")).thenReturn(mockSearchRequestsFailedCounter); - when(mockPluginMetrics.counter("searchRequestsSuccess")).thenReturn(mockSearchRequestsSuccessCounter); when(mockPluginMetrics.counter("apiCalls")).thenReturn(mockApiCallsCounter); - when(mockPluginMetrics.summary("auditLogResponseSizeBytes")).thenReturn(mockAuditLogRequestSizeSummary); - when(mockPluginMetrics.summary("searchResponseSizeBytes")).thenReturn(mockSearchRequestSizeSummary); // Create Office365RestClient with mocked metrics Office365RestClient testClient = new Office365RestClient(authConfig, mockPluginMetrics); - // Verify all metrics were requested during construction + // Verify only local metrics were requested during construction verify(mockPluginMetrics).timer("auditLogFetchLatency"); verify(mockPluginMetrics).timer("searchCallLatency"); verify(mockPluginMetrics).counter("auditLogsRequested"); - verify(mockPluginMetrics).counter("auditLogRequestsFailed"); - verify(mockPluginMetrics).counter("auditLogRequestsSuccess"); - verify(mockPluginMetrics).counter("searchRequestsFailed"); - verify(mockPluginMetrics).counter("searchRequestsSuccess"); verify(mockPluginMetrics).counter("apiCalls"); - verify(mockPluginMetrics).summary("auditLogResponseSizeBytes"); - verify(mockPluginMetrics).summary("searchResponseSizeBytes"); } @Test void testGetAuditLogMetricsInvocation() throws NoSuchFieldException, IllegalAccessException { - // Test metrics for getAuditLog() method - both success and failure scenarios - - // Create mock metrics and inject them + // Create mock metrics for only local fields that still exist Counter mockAuditLogsRequestedCounter = org.mockito.Mockito.mock(Counter.class); - Counter mockAuditLogRequestsFailedCounter = org.mockito.Mockito.mock(Counter.class); Timer mockAuditLogFetchLatencyTimer = org.mockito.Mockito.mock(Timer.class); - DistributionSummary mockAuditLogRequestSizeSummary = org.mockito.Mockito.mock(DistributionSummary.class); ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogsRequestedCounter", mockAuditLogsRequestedCounter); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogRequestsFailedCounter", mockAuditLogRequestsFailedCounter); ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogFetchLatencyTimer", mockAuditLogFetchLatencyTimer); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogResponseSizeSummary", mockAuditLogRequestSizeSummary); // Mock timer.record() to execute the lambda when(mockAuditLogFetchLatencyTimer.record(any(java.util.function.Supplier.class))).thenAnswer(invocation -> { @@ -436,34 +400,17 @@ void testGetAuditLogMetricsInvocation() throws NoSuchFieldException, IllegalAcce office365RestClient.getAuditLog(contentUri); - // Verify success metrics + // Verify only local metrics that still exist verify(mockAuditLogsRequestedCounter).increment(); // Called directly before RetryHandler verify(mockAuditLogFetchLatencyTimer).record(any(java.util.function.Supplier.class)); // Timer wrapper - verify(mockAuditLogRequestSizeSummary).record(mockAuditLog.getBytes(java.nio.charset.StandardCharsets.UTF_8).length); // Size metric inside RetryHandler - - // Test failure scenario - when(restTemplate.exchange(eq(contentUri), eq(HttpMethod.GET), any(), eq(String.class))) - .thenThrow(new HttpClientErrorException(HttpStatus.INTERNAL_SERVER_ERROR)); - - assertThrows(SaaSCrawlerException.class, () -> office365RestClient.getAuditLog(contentUri)); - - // Verify failure metrics - verify(mockAuditLogsRequestedCounter, times(2)).increment(); // Called again before retry - verify(mockAuditLogRequestsFailedCounter, times(6)).increment(); // Called 6 times (once for each retry attempt) } @Test void testSearchAuditLogsMetricsInvocation() throws NoSuchFieldException, IllegalAccessException { - // Test metrics for searchAuditLogs() method - both success and failure scenarios - - // Create mock metrics and inject them - Counter mockSearchRequestsFailedCounter = org.mockito.Mockito.mock(Counter.class); + // Create mock metrics for only local fields that still exist Timer mockSearchCallLatencyTimer = org.mockito.Mockito.mock(Timer.class); - DistributionSummary mockSearchRequestSizeSummary = org.mockito.Mockito.mock(DistributionSummary.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "searchRequestsFailedCounter", mockSearchRequestsFailedCounter); ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "searchCallLatencyTimer", mockSearchCallLatencyTimer); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "searchResponseSizeSummary", mockSearchRequestSizeSummary); // Mock timer.record() to execute the lambda when(mockSearchCallLatencyTimer.record(any(java.util.function.Supplier.class))).thenAnswer(invocation -> { @@ -486,206 +433,10 @@ void testSearchAuditLogsMetricsInvocation() throws NoSuchFieldException, Illegal null ); - // Verify success metrics + // Verify only local metrics that still exist verify(mockSearchCallLatencyTimer).record(any(java.util.function.Supplier.class)); // Timer wrapper - verify(mockSearchRequestSizeSummary).record(1024L); // Size metric inside RetryHandler - - // Test failure scenario - when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), any(), any(ParameterizedTypeReference.class))) - .thenThrow(new HttpClientErrorException(HttpStatus.INTERNAL_SERVER_ERROR)); - - assertThrows(RuntimeException.class, () -> office365RestClient.searchAuditLogs( - "Audit.AzureActiveDirectory", - Instant.now().minus(1, ChronoUnit.HOURS), - Instant.now(), - null - )); - - // Verify failure metrics - verify(mockSearchRequestsFailedCounter, times(6)).increment(); // Called 6 times (once for each retry attempt) - } - - @ParameterizedTest - @CsvSource({ - "FORBIDDEN, true", - "UNAUTHORIZED, true", - "TOO_MANY_REQUESTS, true", - "NOT_FOUND, true", - "INTERNAL_SERVER_ERROR, false", - "BAD_GATEWAY, false" - }) - void testPublishErrorTypeMetricCounterForGetAuditLog(HttpStatus status, boolean shouldIncrementCounter) throws NoSuchFieldException, IllegalAccessException { - // Mock error type counter map - Map mockErrorTypeMetricCounterMap = new HashMap<>(); - Counter mockCounter = org.mockito.Mockito.mock(Counter.class); - mockErrorTypeMetricCounterMap.put(HttpStatus.FORBIDDEN.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.UNAUTHORIZED.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.TOO_MANY_REQUESTS.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.NOT_FOUND.getReasonPhrase(), mockCounter); - - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, - "errorTypeMetricCounterMap", mockErrorTypeMetricCounterMap); - - // Mock REST call to throw FORBIDDEN error - String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - when(restTemplate.exchange( - eq(contentUri), - eq(HttpMethod.GET), - any(), - eq(String.class) - )).thenThrow(new HttpClientErrorException(status)); - - // Execute and verify exception - SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, - () -> office365RestClient.getAuditLog(contentUri)); - assertEquals("Failed to fetch audit log", exception.getMessage()); - assertTrue(exception.isRetryable()); - - // Verify counter increment - if (shouldIncrementCounter) { - verify(mockCounter).increment(); - } else { - verify(mockCounter, never()).increment(); - } - } - - @Test - void testPublishErrorTypeMetricCounterWithUnmappedError() throws NoSuchFieldException, IllegalAccessException { - // Mock error type counter map with no matching error type - Map mockErrorTypeMetricCounterMap = new HashMap<>(); - - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, - "errorTypeMetricCounterMap", mockErrorTypeMetricCounterMap); - - // Mock REST call to throw unmapped error - String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - when(restTemplate.exchange( - eq(contentUri), - eq(HttpMethod.GET), - any(), - eq(String.class) - )).thenThrow(new HttpClientErrorException(HttpStatus.BAD_GATEWAY)); - - // Execute and verify exception - SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, - () -> office365RestClient.getAuditLog(contentUri)); - assertEquals("Failed to fetch audit log", exception.getMessage()); - assertTrue(exception.isRetryable()); - - // Verify no counters were incremented - assertTrue(mockErrorTypeMetricCounterMap.isEmpty()); } - @ParameterizedTest - @CsvSource({ - "FORBIDDEN, true", - "UNAUTHORIZED, true", - "TOO_MANY_REQUESTS, true", - "NOT_FOUND, true", - "INTERNAL_SERVER_ERROR, false", - "BAD_GATEWAY, false" - }) - void testPublishErrorTypeMetricCounterForSearchAuditLogs(HttpStatus status, boolean shouldIncrementCounter) - throws NoSuchFieldException, IllegalAccessException { - // Mock error type counter map - Map mockErrorTypeMetricCounterMap = new HashMap<>(); - Counter mockCounter = org.mockito.Mockito.mock(Counter.class); - mockErrorTypeMetricCounterMap.put(HttpStatus.FORBIDDEN.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.UNAUTHORIZED.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.TOO_MANY_REQUESTS.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.NOT_FOUND.getReasonPhrase(), mockCounter); - - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, - "errorTypeMetricCounterMap", mockErrorTypeMetricCounterMap); - - // Set up test parameters - Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); - Instant endTime = Instant.now(); - String contentType = "Audit.AzureActiveDirectory"; - - // Mock auth config - when(authConfig.getTenantId()).thenReturn("test-tenant-id"); - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock REST call to throw error - when(restTemplate.exchange( - anyString(), - eq(HttpMethod.GET), - any(), - any(ParameterizedTypeReference.class) - )).thenThrow(new HttpClientErrorException(status)); - - // Execute and verify exception - SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, - () -> office365RestClient.searchAuditLogs( - contentType, - startTime, - endTime, - null - )); - assertEquals("Failed to fetch audit logs", exception.getMessage()); - assertTrue(exception.isRetryable()); - - // Verify counter increment - if (shouldIncrementCounter) { - verify(mockCounter).increment(); - } else { - verify(mockCounter, never()).increment(); - } - } - - @ParameterizedTest - @CsvSource({ - "FORBIDDEN, true", - "UNAUTHORIZED, true", - "TOO_MANY_REQUESTS, true", - "NOT_FOUND, true", - "INTERNAL_SERVER_ERROR, false", - "BAD_GATEWAY, false" - }) - void testPublishErrorTypeMetricCounterForStartSubscriptions(HttpStatus status, boolean shouldIncrementCounter) - throws NoSuchFieldException, IllegalAccessException { - // Mock error type counter map - Map mockErrorTypeMetricCounterMap = new HashMap<>(); - Counter mockCounter = org.mockito.Mockito.mock(Counter.class); - mockErrorTypeMetricCounterMap.put(HttpStatus.FORBIDDEN.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.UNAUTHORIZED.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.TOO_MANY_REQUESTS.getReasonPhrase(), mockCounter); - mockErrorTypeMetricCounterMap.put(HttpStatus.NOT_FOUND.getReasonPhrase(), mockCounter); - - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, - "errorTypeMetricCounterMap", mockErrorTypeMetricCounterMap); - - // Mock auth config - when(authConfig.getTenantId()).thenReturn("test-tenant-id"); - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock REST call to throw error - when(restTemplate.exchange( - anyString(), - eq(HttpMethod.POST), - any(), - eq(String.class) - )).thenThrow(new HttpClientErrorException(status)); - - // Execute and verify exception - SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, - () -> office365RestClient.startSubscriptions()); - if (status == HttpStatus.FORBIDDEN) { - assertEquals("Failed to initialize subscriptions: Access forbidden: 403 FORBIDDEN", - exception.getMessage()); - } else { - assertEquals("Failed to initialize subscriptions: " + status.toString(), - exception.getMessage()); - } - assertTrue(exception.isRetryable()); - - if (shouldIncrementCounter) { - verify(mockCounter).increment(); - } else { - verify(mockCounter, never()).increment(); - } - } @Test void testApiCallsCounterIncrementForStartSubscriptions() throws NoSuchFieldException, IllegalAccessException { diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java index 19a5bb98bf..a348e6f658 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java @@ -11,7 +11,11 @@ package org.opensearch.dataprepper.plugins.source.source_crawler.utils; import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.HttpServerErrorException; @@ -24,11 +28,22 @@ */ public class MetricsHelper { + private static final Logger LOG = LoggerFactory.getLogger(MetricsHelper.class); + // specific retryable/non-retryable metric names private static final String REQUEST_ACCESS_DENIED = "requestAccessDenied"; private static final String REQUEST_THROTTLED = "requestThrottled"; private static final String RESOURCE_NOT_FOUND = "resourceNotFound"; + // specific API metric names + private static final String GET_REQUESTS_FAILED = "getRequestsFailed"; + private static final String GET_REQUESTS_SUCCESS = "getRequestsSuccess"; + private static final String GET_RESPONSE_SIZE = "getResponseSizeBytes"; + private static final String SEARCH_REQUESTS_FAILED = "searchRequestsFailed"; + private static final String SEARCH_REQUESTS_SUCCESS = "searchRequestsSuccess"; + private static final String SEARCH_RESPONSE_SIZE = "searchResponseSizeBytes"; + + // other errors in crawlerClient public static final String REQUEST_ERRORS = "requestErrors"; @@ -38,9 +53,9 @@ public class MetricsHelper { * TOO_MANY_REQUESTS = requestThrottled * NOT_FOUND = resourceNotFound * @param pluginMetrics - metric object class to initialize metric counters - * - * @return errorTypeMetricCounterMap - */ + * + * @return errorTypeMetricCounterMap + */ public static Map getErrorTypeMetricCounterMap(PluginMetrics pluginMetrics) { Map errorTypeMetricCounterMap = new HashMap<>(); errorTypeMetricCounterMap.put(HttpStatus.FORBIDDEN.getReasonPhrase(), pluginMetrics.counter(REQUEST_ACCESS_DENIED)); @@ -52,14 +67,14 @@ public static Map getErrorTypeMetricCounterMap(PluginMetrics pl /** * Increment the errorType metric if it exists in errorTypeMetricCounterMap - * Should only be the following: + * Should only be the following: * FORBIDDEN/UNAUTHORIZED = requestAccessDenied * TOO_MANY_REQUESTS = requestThrottled * NOT_FOUND = resourceNotFound - * + * * @param ex - exception from RestClient * @param errorTypeMetricCounterMap - the map of errorType to metric counter - */ + */ public static void publishErrorTypeMetricCounter(Exception ex, Map errorTypeMetricCounterMap) { Optional statusCode = Optional.empty(); if (ex instanceof HttpClientErrorException) { @@ -80,4 +95,174 @@ public static void publishErrorTypeMetricCounter(Exception ex, Map response){ + DistributionSummary summary = pluginMetrics.summary(SEARCH_RESPONSE_SIZE); + if(response != null && response.getBody() != null){ + summary.record(response.getHeaders().getContentLength()); + } else { + LOG.error("Response or response body is null when recording API response size metric"); + summary.record(-1L); + } + } + + /** + * Records the size of an API response String in bytes as a distribution summary metric. + * + * This overloaded method calculates the size in bytes of the response string content + * and records it as a metric to help monitor API response sizes for performance analysis + * and capacity planning. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the distribution summary + * @param response the response string content from the API call + * + * @implNote If the response is null, a value of -1 is recorded for consistency. + * The size is calculated using UTF-8 encoding to get accurate byte count. + * + * @see DistributionSummary#record(double) for how the metric value is recorded + */ + public static void publishSearchResponseSizeMetricInBytes(PluginMetrics pluginMetrics, String response){ + DistributionSummary summary = pluginMetrics.summary(SEARCH_RESPONSE_SIZE); + if(response != null){ + summary.record(response.getBytes(java.nio.charset.StandardCharsets.UTF_8).length); + } else { + LOG.error("Response is null when recording API response size metric"); + summary.record(-1L); + } + } + + /** + * Records a successful search request by incrementing the predefined success counter. + * + * This method is used to track successful search API calls for monitoring API health, + * success rates, and overall system performance metrics. Uses the hardcoded metric + * name "searchRequestsSuccess" for consistency across the application. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the counter + * + * @see Counter#increment() for how the metric value is incremented + */ + public static void publishSearchRequestsSuccessMetric(PluginMetrics pluginMetrics) { + Counter successCounter = pluginMetrics.counter(SEARCH_REQUESTS_SUCCESS); + successCounter.increment(); + } + + /** + * Provides a failure counter for search requests to be incremented by the caller. + * + * This method is used to get the failure counter for search API calls for monitoring API health, + * failure rates, and identifying system issues. Uses the hardcoded metric name + * "searchRequestsFailed" for consistency. The caller (such as RetryHandler) is responsible + * for incrementing the counter when appropriate. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the counter + * @return the failure counter to be incremented by caller + */ + public static Counter provideSearchRequestFailureCounter(PluginMetrics pluginMetrics) { + Counter failureCounter = pluginMetrics.counter(SEARCH_REQUESTS_FAILED); + return failureCounter; + } + + + + /** + * Records the size of an individual GET request response in bytes as a distribution summary metric. + * + * This overloaded method handles generic ResponseEntity types (e.g., ResponseEntity<List<Map<String, Object>>>) + * by using the Content-Length header from the HTTP response. Designed to work with any ResponseEntity type. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the distribution summary + * @param response the HTTP response entity containing headers from the GET API call + * + * @implNote If the response or response body is null, a value of -1 is recorded to match + * the pattern used by HttpEntity.getContentLength() which returns -1 when the + * content length is not known. This allows for consistent tracking of responses + * where size information is unavailable. + * + * @see DistributionSummary#record(double) for how the metric value is recorded + */ + public static void publishGetResponseSizeMetricInBytes(PluginMetrics pluginMetrics, ResponseEntity response) { + DistributionSummary summary = pluginMetrics.summary(GET_RESPONSE_SIZE); + if(response != null && response.getBody() != null){ + summary.record(response.getHeaders().getContentLength()); + } else { + LOG.error("Response or response body is null when recording GET request response size metric"); + summary.record(-1L); + } + } + + /** + * Records the size of an individual GET request response String in bytes as a distribution summary metric. + * + * This overloaded method calculates the size in bytes of the response string content + * and records it as a metric to help monitor individual GET request response sizes for performance analysis + * and capacity planning. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the distribution summary + * @param response the response string content from the GET API call + * + * @implNote If the response is null, a value of -1 is recorded for consistency. + * The size is calculated using UTF-8 encoding to get accurate byte count. + * + * @see DistributionSummary#record(double) for how the metric value is recorded + */ + public static void publishGetResponseSizeMetricInBytes(PluginMetrics pluginMetrics, String response){ + DistributionSummary summary = pluginMetrics.summary(GET_RESPONSE_SIZE); + if(response != null){ + summary.record(response.getBytes(java.nio.charset.StandardCharsets.UTF_8).length); + } else { + LOG.error("Response is null when recording GET request response size metric"); + summary.record(-1L); + } + } + + /** + * Records a successful individual GET request by incrementing the predefined success counter. + * + * This method is used to track successful individual GET API calls for monitoring API health, + * success rates, and overall system performance metrics. Uses the hardcoded metric + * name "getRequestsSuccess" for consistency across the application. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the counter + * + * @see Counter#increment() for how the metric value is incremented + */ + public static void publishGetRequestsSuccessMetric(PluginMetrics pluginMetrics) { + Counter successCounter = pluginMetrics.counter(GET_REQUESTS_SUCCESS); + successCounter.increment(); + } + + /** + * Provides a failure counter for GET requests to be incremented by the caller. + * + * This method is used to get the failure counter for individual GET API calls for monitoring API health, + * failure rates, and identifying system issues. Uses the hardcoded metric name + * "getRequestsFailed" for consistency. The caller (such as RetryHandler) is responsible + * for incrementing the counter when appropriate. + * + * @param pluginMetrics the PluginMetrics instance used to create and manage the counter + * @return the failure counter to be incremented by caller + */ + public static Counter provideGetRequestsFailureCounter(PluginMetrics pluginMetrics) { + Counter failureCounter = pluginMetrics.counter(GET_REQUESTS_FAILED); + return failureCounter; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java new file mode 100644 index 0000000000..a762275a83 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java @@ -0,0 +1,441 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import static org.mockito.Mockito.lenient; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class MetricsHelperTest { + + @Mock + private PluginMetrics pluginMetrics; + + @Mock + private Counter mockCounter; + + @Mock + private DistributionSummary mockDistributionSummary; + + @BeforeEach + void setUp() { + lenient().when(pluginMetrics.counter(anyString())).thenReturn(mockCounter); + lenient().when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + } + + @Test + void testGetErrorTypeMetricCounterMap() { + Map result = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + + assertNotNull(result); + assertEquals(4, result.size()); + + // Verify expected HTTP status mappings + assertTrue(result.containsKey(HttpStatus.FORBIDDEN.getReasonPhrase())); + assertTrue(result.containsKey(HttpStatus.UNAUTHORIZED.getReasonPhrase())); + assertTrue(result.containsKey(HttpStatus.TOO_MANY_REQUESTS.getReasonPhrase())); + assertTrue(result.containsKey(HttpStatus.NOT_FOUND.getReasonPhrase())); + + result.values().forEach(counter -> assertEquals(mockCounter, counter)); + + // requestAccessDenied is called twice for FORBIDDEN and UNAUTHORIZED + verify(pluginMetrics, times(2)).counter("requestAccessDenied"); + verify(pluginMetrics).counter("requestThrottled"); + verify(pluginMetrics).counter("resourceNotFound"); + } + + @Test + void testPublishErrorTypeMetricCounterWithHttpClientErrorException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.FORBIDDEN); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + verify(mockCounter).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithHttpServerErrorException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + HttpServerErrorException exception = new HttpServerErrorException(HttpStatus.INTERNAL_SERVER_ERROR); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + // Should not increment because INTERNAL_SERVER_ERROR is not in the supported error map + verify(mockCounter, never()).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithSecurityException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + SecurityException exception = new SecurityException("Access denied"); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + verify(mockCounter).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithUnauthorizedException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.UNAUTHORIZED); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + verify(mockCounter).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithTooManyRequestsException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + verify(mockCounter).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithNotFoundException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.NOT_FOUND); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + verify(mockCounter).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithUnsupportedException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + RuntimeException exception = new RuntimeException("Unsupported exception"); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + // Should not increment because RuntimeException is not handled + verify(mockCounter, never()).increment(); + } + + @Test + void testPublishErrorTypeMetricCounterWithNullMap() { + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.FORBIDDEN); + + // Should not throw exception when map is null + assertDoesNotThrow(() -> + MetricsHelper.publishErrorTypeMetricCounter(exception, null) + ); + } + + @Test + void testPublishErrorTypeMetricCounterWithBadRequestException() { + Map errorTypeMetricCounterMap = MetricsHelper.getErrorTypeMetricCounterMap(pluginMetrics); + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.BAD_REQUEST); + + MetricsHelper.publishErrorTypeMetricCounter(exception, errorTypeMetricCounterMap); + + // Should not increment because BAD_REQUEST is not in the supported error map + verify(mockCounter, never()).increment(); + } + + // Data providers for parameterized tests + static Stream responseEntityTestCases() { + return Stream.of( + Arguments.of("ValidResponse", 1024L, "test body", 1024L), + Arguments.of("NullResponse", null, null, -1L), + Arguments.of("NullBody", 1024L, null, -1L), + Arguments.of("ZeroContentLength", 0L, "", 0L), + Arguments.of("NoContentLength", -1L, "test body", -1L), + Arguments.of("LargeContentLength", 10_485_760L, "large body", 10_485_760L) + ); + } + + static Stream stringTestCases() { + return Stream.of( + Arguments.of("ValidString", "test response content"), + Arguments.of("NullString", null), + Arguments.of("EmptyString", ""), + Arguments.of("Utf8String", "test with unicode: 测试 and émojis 🎉") + ); + } + + static Stream metricMethods() { + return Stream.of( + Arguments.of("search", "searchResponseSizeBytes", + (BiConsumer>) MetricsHelper::publishSearchResponseSizeMetricInBytes, + (BiConsumer) MetricsHelper::publishSearchResponseSizeMetricInBytes), + Arguments.of("get", "getResponseSizeBytes", + (BiConsumer>) MetricsHelper::publishGetResponseSizeMetricInBytes, + (BiConsumer) MetricsHelper::publishGetResponseSizeMetricInBytes) + ); + } + + @ParameterizedTest(name = "{0} ResponseEntity test: {1}") + @MethodSource("responseEntityTestCases") + void testResponseSizeMetricWithResponseEntity(String testName, Long contentLength, String body, long expectedRecord) { + metricMethods().forEach(args -> { + String methodType = (String) args.get()[0]; + String metricName = (String) args.get()[1]; + BiConsumer> method = + (BiConsumer>) args.get()[2]; + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + ResponseEntity response = null; + if (contentLength != null) { + HttpHeaders headers = new HttpHeaders(); + if (contentLength >= 0) { + headers.setContentLength(contentLength); + } + response = new ResponseEntity<>(body, headers, HttpStatus.OK); + } + + method.accept(pluginMetrics, response); + + verify(pluginMetrics).summary(metricName); + verify(mockDistributionSummary).record(expectedRecord); + }); + } + + @ParameterizedTest(name = "{0} String test: {1}") + @MethodSource("stringTestCases") + void testResponseSizeMetricWithString(String testName, String responseBody) { + metricMethods().forEach(args -> { + String methodType = (String) args.get()[0]; + String metricName = (String) args.get()[1]; + BiConsumer method = + (BiConsumer) args.get()[3]; + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + method.accept(pluginMetrics, responseBody); + + long expectedSize = responseBody != null ? + responseBody.getBytes(java.nio.charset.StandardCharsets.UTF_8).length : -1L; + + verify(pluginMetrics).summary(metricName); + verify(mockDistributionSummary).record(expectedSize); + }); + } + + + @ParameterizedTest(name = "Success metric for {0} requests") + @MethodSource({"org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelperTest#successMetricMethods"}) + void testSuccessMetrics(String methodType, String expectedMetricName) { + if ("search".equals(methodType)) { + MetricsHelper.publishSearchRequestsSuccessMetric(pluginMetrics); + } else { + MetricsHelper.publishGetRequestsSuccessMetric(pluginMetrics); + } + + verify(pluginMetrics).counter(expectedMetricName); + verify(mockCounter).increment(); + } + + static Stream successMetricMethods() { + return Stream.of( + Arguments.of("search", "searchRequestsSuccess"), + Arguments.of("get", "getRequestsSuccess") + ); + } + + @ParameterizedTest(name = "Failure counter provider for {0} requests") + @MethodSource({"org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelperTest#failureCounterMethods"}) + void testFailureCounterProviders(String methodType, String expectedMetricName) { + Counter result; + if ("search".equals(methodType)) { + result = MetricsHelper.provideSearchRequestFailureCounter(pluginMetrics); + } else { + result = MetricsHelper.provideGetRequestsFailureCounter(pluginMetrics); + } + + verify(pluginMetrics).counter(expectedMetricName); + verify(mockCounter, never()).increment(); // Should NOT increment - RetryHandler's responsibility + assertEquals(mockCounter, result); + } + + static Stream failureCounterMethods() { + return Stream.of( + Arguments.of("search", "searchRequestsFailed"), + Arguments.of("get", "getRequestsFailed") + ); + } + + @Test + void testResponseEntityCompatibilityWithVariousResponseTypes() { + // Test that ResponseEntity methods work with different ResponseEntity types + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + String testContent = "test content for ResponseEntity compatibility"; + + // Test search methods with ResponseEntity + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength(1000L); + ResponseEntity responseEntity = new ResponseEntity<>(testContent, headers, HttpStatus.OK); + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, responseEntity); + + // Test get methods with ResponseEntity + headers.setContentLength(2000L); + ResponseEntity getResponseEntity = new ResponseEntity<>(testContent, headers, HttpStatus.OK); + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, getResponseEntity); + + // Verify both method types use their respective metric names + verify(pluginMetrics).summary("searchResponseSizeBytes"); + verify(pluginMetrics).summary("getResponseSizeBytes"); + + // Test with different ResponseEntity generic types + HttpHeaders genericHeaders = new HttpHeaders(); + genericHeaders.setContentLength(500L); + ResponseEntity stringEntityResponse = new ResponseEntity<>("test string", genericHeaders, HttpStatus.OK); + + assertDoesNotThrow(() -> { + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, stringEntityResponse); + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, stringEntityResponse); + }); + } + + @Test + void testSearchResponseSizeMetricWithGenericResponseEntity() { + // Test the new ResponseEntity overload that handles ResponseEntity>> + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + // Test with ResponseEntity>> like used in OktaSSORestClient + java.util.List> eventsList = java.util.Arrays.asList( + java.util.Map.of("id", "123", "message", "test event"), + java.util.Map.of("id", "456", "message", "another event") + ); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength(3000L); + ResponseEntity>> genericResponse = + new ResponseEntity<>(eventsList, headers, HttpStatus.OK); + + // This should work with the new ResponseEntity overload + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, genericResponse); + + verify(pluginMetrics).summary("searchResponseSizeBytes"); + verify(mockDistributionSummary).record(3000L); + } + + @Test + void testGetResponseSizeMetricWithGenericResponseEntity() { + // Test the new ResponseEntity overload for GET requests + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + // Test with ResponseEntity>> like used in OktaSSORestClient + java.util.List> eventsList = java.util.Arrays.asList( + java.util.Map.of("id", "789", "content", "single event detail") + ); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength(1500L); + ResponseEntity>> genericResponse = + new ResponseEntity<>(eventsList, headers, HttpStatus.OK); + + // This should work with the new ResponseEntity overload + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, genericResponse); + + verify(pluginMetrics).summary("getResponseSizeBytes"); + verify(mockDistributionSummary).record(1500L); + } + + @Test + void testGenericResponseEntityOverloadsWithNullValues() { + // Test the new generic ResponseEntity overloads with null scenarios + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + // Test null response + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, (ResponseEntity) null); + verify(pluginMetrics).summary("searchResponseSizeBytes"); + verify(mockDistributionSummary).record(-1L); + + reset(mockDistributionSummary); + + // Test null body + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength(2000L); + ResponseEntity>> responseWithNullBody = + new ResponseEntity<>(null, headers, HttpStatus.OK); + + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, responseWithNullBody); + verify(mockDistributionSummary).record(-1L); + + reset(mockDistributionSummary); + + // Test GET request with null + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, (ResponseEntity) null); + verify(pluginMetrics).summary("getResponseSizeBytes"); + verify(mockDistributionSummary).record(-1L); + } + + @Test + void testGenericResponseEntityCompatibilityWithVariousTypes() { + // Test that the generic ResponseEntity works with various response types + + reset(pluginMetrics, mockDistributionSummary); + when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentLength(4000L); + + // Test with different generic types to ensure wildcard compatibility + ResponseEntity stringResponse = new ResponseEntity<>("test", headers, HttpStatus.OK); + ResponseEntity integerResponse = new ResponseEntity<>(42, headers, HttpStatus.OK); + ResponseEntity> mapResponse = + new ResponseEntity<>(java.util.Map.of("key", "value"), headers, HttpStatus.OK); + + // All should work with the generic ResponseEntity overloads + assertDoesNotThrow(() -> { + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, stringResponse); + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, integerResponse); + MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, mapResponse); + + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, stringResponse); + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, integerResponse); + MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, mapResponse); + }); + + // Verify all calls recorded the same Content-Length header value + verify(mockDistributionSummary, times(6)).record(4000L); + } +} From a37b576bc12297626b3313bee5b30d5416bc85eb Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 10 Dec 2025 10:24:12 -0600 Subject: [PATCH 12/51] Use Eclipse Temurin by default in the tarball smoke test. Updates to the documentation for running smoke tests to reference Eclipse Temurin. (#6296) Signed-off-by: David Venable Signed-off-by: Nathan Wand --- release/smoke-tests/README.md | 4 ++-- release/smoke-tests/run-tarball-files-smoke-tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/release/smoke-tests/README.md b/release/smoke-tests/README.md index eea887aca0..43e523d262 100644 --- a/release/smoke-tests/README.md +++ b/release/smoke-tests/README.md @@ -38,8 +38,8 @@ You can also customize what it tests against. The `-i` parameter specifies a bas The values for `-t` are `opensearch-data-prepper` or `opensearch-data-prepper-jdk`. ```shell -./release/smoke-tests/run-tarball-files-smoke-tests.sh -i openjdk:11 -t opensearch-data-prepper -./release/smoke-tests/run-tarball-files-smoke-tests.sh -i openjdk:17 -t opensearch-data-prepper +./release/smoke-tests/run-tarball-files-smoke-tests.sh -i eclipse-temurin:11 -t opensearch-data-prepper +./release/smoke-tests/run-tarball-files-smoke-tests.sh -i eclipse-temurin:17 -t opensearch-data-prepper ./release/smoke-tests/run-tarball-files-smoke-tests.sh -i ubuntu:latest -t opensearch-data-prepper-jdk ``` diff --git a/release/smoke-tests/run-tarball-files-smoke-tests.sh b/release/smoke-tests/run-tarball-files-smoke-tests.sh index 272705ef33..c7ab7db822 100755 --- a/release/smoke-tests/run-tarball-files-smoke-tests.sh +++ b/release/smoke-tests/run-tarball-files-smoke-tests.sh @@ -193,7 +193,7 @@ fi if ! is_defined "${FROM_IMAGE}" then - export FROM_IMAGE="openjdk:11" + export FROM_IMAGE="eclipse-temurin:11" fi if ! is_defined "${TAR_NAME}" From 4631ceb1b81e9804beba70b4d083fe6bbc26f891 Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 10 Dec 2025 13:11:44 -0600 Subject: [PATCH 13/51] Enable cross-region writes in the S3 sink. (#6323) Signed-off-by: David Venable Signed-off-by: Nathan Wand --- .../plugins/sink/s3/ClientFactory.java | 20 +++---- .../plugins/sink/s3/ClientFactoryTest.java | 52 +++---------------- 2 files changed, 14 insertions(+), 58 deletions(-) diff --git a/data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactory.java b/data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactory.java index f647057af4..3803ffeda9 100644 --- a/data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactory.java +++ b/data-prepper-plugins/s3-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactory.java @@ -1,6 +1,10 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ package org.opensearch.dataprepper.plugins.sink.s3; @@ -16,33 +20,23 @@ import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; -import software.amazon.awssdk.services.s3.S3Client; public final class ClientFactory { private ClientFactory() { } - static S3Client createS3Client(final S3SinkConfig s3SinkConfig, final AwsCredentialsSupplier awsCredentialsSupplier) { - final AwsCredentialsOptions awsCredentialsOptions = convertToCredentialsOptions(s3SinkConfig.getAwsAuthenticationOptions()); - final AwsCredentialsProvider awsCredentialsProvider = awsCredentialsSupplier.getProvider(awsCredentialsOptions); - - return S3Client.builder() - .region(s3SinkConfig.getAwsAuthenticationOptions().getAwsRegion()) - .credentialsProvider(awsCredentialsProvider) - .overrideConfiguration(createOverrideConfiguration(s3SinkConfig)).build(); - } - static S3AsyncClient createS3AsyncClient(final S3SinkConfig s3SinkConfig, final AwsCredentialsSupplier awsCredentialsSupplier) { final AwsCredentialsOptions awsCredentialsOptions = convertToCredentialsOptions(s3SinkConfig.getAwsAuthenticationOptions()); final AwsCredentialsProvider awsCredentialsProvider = awsCredentialsSupplier.getProvider(awsCredentialsOptions); - S3AsyncClientBuilder s3AsyncClientBuilder = S3AsyncClient.builder() + final S3AsyncClientBuilder s3AsyncClientBuilder = S3AsyncClient.builder() .region(s3SinkConfig.getAwsAuthenticationOptions().getAwsRegion()) + .crossRegionAccessEnabled(true) .credentialsProvider(awsCredentialsProvider) .overrideConfiguration(createOverrideConfiguration(s3SinkConfig)); if (s3SinkConfig.getClientOptions() != null) { final ClientOptions clientOptions = s3SinkConfig.getClientOptions(); - SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder() + final SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder() .connectionAcquisitionTimeout(clientOptions.getAcquireTimeout()) .maxConcurrency(clientOptions.getMaxConnections()) .build(); diff --git a/data-prepper-plugins/s3-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactoryTest.java b/data-prepper-plugins/s3-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactoryTest.java index 947bc728e9..b334be8e6d 100644 --- a/data-prepper-plugins/s3-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactoryTest.java +++ b/data-prepper-plugins/s3-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/s3/ClientFactoryTest.java @@ -1,6 +1,10 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ package org.opensearch.dataprepper.plugins.sink.s3; @@ -25,8 +29,6 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; import java.time.Duration; import java.util.Map; @@ -62,14 +64,14 @@ void setUp() { @Test void createS3AsyncClient_with_real_S3AsyncClient() { when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.US_EAST_1); - final S3Client s3Client = ClientFactory.createS3Client(s3SinkConfig, awsCredentialsSupplier); + final S3AsyncClient s3Client = ClientFactory.createS3AsyncClient(s3SinkConfig, awsCredentialsSupplier); assertThat(s3Client, notNullValue()); } @ParameterizedTest @ValueSource(strings = {"us-east-1", "us-west-2", "eu-central-1"}) - void createS3Client_provides_correct_inputs(final String regionString) { + void createS3AsyncClient_with_client_options_returns_expected_client(final String regionString) { final Region region = Region.of(regionString); final String stsRoleArn = UUID.randomUUID().toString(); final String externalId = UUID.randomUUID().toString(); @@ -82,49 +84,9 @@ void createS3Client_provides_correct_inputs(final String regionString) { final AwsCredentialsProvider expectedCredentialsProvider = mock(AwsCredentialsProvider.class); when(awsCredentialsSupplier.getProvider(any())).thenReturn(expectedCredentialsProvider); - final S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - when(s3ClientBuilder.region(region)).thenReturn(s3ClientBuilder); - when(s3ClientBuilder.credentialsProvider(any())).thenReturn(s3ClientBuilder); - when(s3ClientBuilder.overrideConfiguration(any(ClientOverrideConfiguration.class))).thenReturn(s3ClientBuilder); - try(final MockedStatic s3ClientMockedStatic = mockStatic(S3Client.class)) { - s3ClientMockedStatic.when(S3Client::builder) - .thenReturn(s3ClientBuilder); - ClientFactory.createS3Client(s3SinkConfig, awsCredentialsSupplier); - } - - final ArgumentCaptor credentialsProviderArgumentCaptor = ArgumentCaptor.forClass(AwsCredentialsProvider.class); - verify(s3ClientBuilder).credentialsProvider(credentialsProviderArgumentCaptor.capture()); - - final AwsCredentialsProvider actualCredentialsProvider = credentialsProviderArgumentCaptor.getValue(); - - assertThat(actualCredentialsProvider, equalTo(expectedCredentialsProvider)); - - final ArgumentCaptor optionsArgumentCaptor = ArgumentCaptor.forClass(AwsCredentialsOptions.class); - verify(awsCredentialsSupplier).getProvider(optionsArgumentCaptor.capture()); - - final AwsCredentialsOptions actualCredentialsOptions = optionsArgumentCaptor.getValue(); - assertThat(actualCredentialsOptions.getRegion(), equalTo(region)); - assertThat(actualCredentialsOptions.getStsRoleArn(), equalTo(stsRoleArn)); - assertThat(actualCredentialsOptions.getStsExternalId(), equalTo(externalId)); - assertThat(actualCredentialsOptions.getStsHeaderOverrides(), equalTo(stsHeaderOverrides)); - } - - @Test - void createS3AsyncClient_with_client_options_returns_expected_client() { - final Region region = Region.of("us-east-1"); - final String stsRoleArn = UUID.randomUUID().toString(); - final String externalId = UUID.randomUUID().toString(); - final Map stsHeaderOverrides = Map.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - when(awsAuthenticationOptions.getAwsRegion()).thenReturn(region); - when(awsAuthenticationOptions.getAwsStsRoleArn()).thenReturn(stsRoleArn); - when(awsAuthenticationOptions.getAwsStsExternalId()).thenReturn(externalId); - when(awsAuthenticationOptions.getAwsStsHeaderOverrides()).thenReturn(stsHeaderOverrides); - - final AwsCredentialsProvider expectedCredentialsProvider = mock(AwsCredentialsProvider.class); - when(awsCredentialsSupplier.getProvider(any())).thenReturn(expectedCredentialsProvider); - final S3AsyncClientBuilder s3AsyncClientBuilder = mock(S3AsyncClientBuilder.class); when(s3AsyncClientBuilder.region(region)).thenReturn(s3AsyncClientBuilder); + when(s3AsyncClientBuilder.crossRegionAccessEnabled(true)).thenReturn(s3AsyncClientBuilder); when(s3AsyncClientBuilder.credentialsProvider(any())).thenReturn(s3AsyncClientBuilder); when(s3AsyncClientBuilder.overrideConfiguration(any(ClientOverrideConfiguration.class))).thenReturn(s3AsyncClientBuilder); From f9db3da1eb72f509a0f446d034f6c3ca465a16a3 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 10 Dec 2025 11:16:33 -0800 Subject: [PATCH 14/51] Confluence and CloudWatch and multiple other failing tests fix (#6348) Making the tests less flaky. More reliable. Avoiding possible Out of memory issue with large pay load generation. Signed-off-by: Nathan Wand --- .../integration/ProcessorValidationIT.java | 6 ++++ .../client/CloudWatchLogsClientFactory.java | 7 +++- .../client/CloudWatchLogsServiceTest.java | 22 +++++++----- .../processor/date/DateProcessorConfig.java | 6 ++++ .../date/DateProcessorConfigTest.java | 35 +++++++++++-------- .../build.gradle | 1 + .../DynamoDbSourceCoordinationStoreIT.java | 14 ++++++-- .../ConfluenceConfigHelperTest.java | 4 +-- .../source/jira/JiraConfigHelperTest.java | 6 ++-- .../Office365RestClientTest.java | 5 --- .../utils/MetricsHelperTest.java | 11 ++++-- 11 files changed, 78 insertions(+), 39 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorValidationIT.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorValidationIT.java index 79aabb3bd1..ebc3ec8be0 100644 --- a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorValidationIT.java +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorValidationIT.java @@ -211,6 +211,12 @@ private void verifyProcessingResults(String pipelineType, int expectedTotalEvent } private static void verifySingleThreadUsage() { + // Wait for all processor instances to be registered (one per worker) + await().atMost(WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .untilAsserted(() -> assertThat( + SingleThreadEventsTrackingTestProcessor.getProcessors().size(), + equalTo(4))); + List singleThreadProcessors = SingleThreadEventsTrackingTestProcessor.getProcessors(); assertThat(singleThreadProcessors.size(), equalTo(4)); assertAll( diff --git a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsClientFactory.java b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsClientFactory.java index d6875f501a..5542ebb9ec 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsClientFactory.java +++ b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsClientFactory.java @@ -11,6 +11,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClientBuilder; @@ -64,8 +65,12 @@ public static CloudWatchLogsClient createCwlClient(final AwsConfig awsConfig, } private static ClientOverrideConfiguration createOverrideConfiguration(final Map customHeaders) { + final RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(AwsConfig.DEFAULT_CONNECTION_ATTEMPTS) + .build(); + final ClientOverrideConfiguration.Builder configBuilder = ClientOverrideConfiguration.builder() - .retryPolicy(r -> r.numRetries(AwsConfig.DEFAULT_CONNECTION_ATTEMPTS)); + .retryPolicy(retryPolicy); customHeaders.forEach(configBuilder::putHeader); diff --git a/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsServiceTest.java b/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsServiceTest.java index c6b3d80428..5cd122d1a3 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsServiceTest.java +++ b/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsServiceTest.java @@ -12,7 +12,9 @@ import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventHandle; import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.log.JacksonLog; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.dlq.DlqPushHandler; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.buffer.Buffer; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.buffer.InMemoryBuffer; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.buffer.InMemoryBufferFactory; @@ -20,8 +22,6 @@ import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.config.ThresholdConfig; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.utils.CloudWatchLogsLimits; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; -import org.opensearch.dataprepper.plugins.dlq.DlqPushHandler; -import org.opensearch.dataprepper.model.log.JacksonLog; import java.util.ArrayList; import java.util.Collection; @@ -31,13 +31,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.times; -import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.doAnswer; class CloudWatchLogsServiceTest { private static final int LARGE_THREAD_COUNT = 1000; @@ -95,8 +95,10 @@ Collection> getSampleRecordsCollection() { Collection> getSampleRecordsOfLargerSize() { final ArrayList> returnCollection = new ArrayList<>(); + int messageSize = (int) (thresholdConfig.getMaxRequestSizeBytes() / 24); for (int i = 0; i < thresholdConfig.getBatchSize() * 2; i++) { - JacksonEvent mockJacksonEvent = (JacksonEvent) JacksonEvent.fromMessage("a".repeat((int) (thresholdConfig.getMaxRequestSizeBytes()/24))); + JacksonEvent mockJacksonEvent = + (JacksonEvent) JacksonEvent.fromMessage(RandomStringUtils.insecure().nextAlphabetic(messageSize)); returnCollection.add(new Record<>(mockJacksonEvent)); } @@ -105,8 +107,10 @@ Collection> getSampleRecordsOfLargerSize() { Collection> getSampleRecordsOfLimitSize() { final ArrayList> returnCollection = new ArrayList<>(); + int messageSize = (int) thresholdConfig.getMaxEventSizeBytes(); for (int i = 0; i < thresholdConfig.getBatchSize(); i++) { - JacksonEvent mockJacksonEvent = (JacksonEvent) JacksonEvent.fromMessage("testMessage".repeat((int) thresholdConfig.getMaxEventSizeBytes())); + JacksonEvent mockJacksonEvent = + (JacksonEvent) JacksonEvent.fromMessage(RandomStringUtils.insecure().nextAlphabetic(messageSize)); returnCollection.add(new Record<>(mockJacksonEvent)); } @@ -248,8 +252,8 @@ void GIVEN_large_thread_count_WHEN_processing_log_events_THEN_dispatcher_should_ } private Record getLargeRecord(long size) { - final Event event = JacksonLog.builder().withData(Map.of("key", RandomStringUtils.randomAlphabetic((int)size))).withEventHandle(eventHandle).build(); + final Event event = JacksonLog.builder().withData(Map.of("key", RandomStringUtils.insecure().nextAlphabetic((int)size))).withEventHandle(eventHandle).build(); return new Record<>(event); - } + } } diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java index 05deb5aa07..8eea5ee8d7 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java @@ -108,12 +108,18 @@ public boolean isValidPatterns() { } public static boolean isValidPattern(final String pattern) { + // Check for valid epoch patterns first if (pattern.equals("epoch_second") || pattern.equals("epoch_milli") || pattern.equals("epoch_micro") || pattern.equals("epoch_nano")) { return true; } + // Reject any other pattern starting with "epoch_" as invalid + if (pattern.startsWith("epoch_")) { + return false; + } + // Validate as DateTimeFormatter pattern try { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); return true; diff --git a/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfigTest.java b/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfigTest.java index b1dddfa013..efe52d2cb9 100644 --- a/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfigTest.java +++ b/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfigTest.java @@ -69,23 +69,28 @@ void isValidMatchAndFromTimestampReceived_should_return_false_if_from_time_recei assertThat(dateProcessorConfig.isValidMatchAndFromTimestampReceived(), equalTo(false)); } - @Test - void testValidAndInvalidOutputFormats() throws NoSuchFieldException, IllegalAccessException { - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", random); - assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(false)); - - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", "epoch_second"); - assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(true)); - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", "epoch_milli"); - assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(true)); - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", "epoch_nano"); - assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(true)); - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", "epoch_micro"); + @ParameterizedTest + @ValueSource(strings = { + "epoch_second", + "epoch_milli", + "epoch_nano", + "epoch_micro", + "yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnnXXX" + }) + void testValidOutputFormats(String outputFormat) throws NoSuchFieldException, IllegalAccessException { + setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", outputFormat); assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(true)); - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", "epoch_xyz"); + } + + @ParameterizedTest + @ValueSource(strings = { + "invalid[pattern]format", + "epoch_xyz", + "epoch_invalid" + }) + void testInvalidOutputFormats(String outputFormat) throws NoSuchFieldException, IllegalAccessException { + setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", outputFormat); assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(false)); - setField(DateProcessorConfig.class, dateProcessorConfig, "outputFormat", "yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnnXXX"); - assertThat(dateProcessorConfig.isValidOutputFormat(), equalTo(true)); } @Test diff --git a/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle b/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle index 211c4960b0..85223aea8a 100644 --- a/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle +++ b/data-prepper-plugins/dynamodb-source-coordination-store/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation 'software.amazon.awssdk:sts' implementation 'javax.inject:javax.inject:1' testImplementation 'com.amazonaws:DynamoDBLocal:2.2.1' + testImplementation 'org.awaitility:awaitility:4.2.0' } configurations { diff --git a/data-prepper-plugins/dynamodb-source-coordination-store/src/test/java/org/opensearch/dataprepper/plugins/sourcecoordinator/dynamodb/DynamoDbSourceCoordinationStoreIT.java b/data-prepper-plugins/dynamodb-source-coordination-store/src/test/java/org/opensearch/dataprepper/plugins/sourcecoordinator/dynamodb/DynamoDbSourceCoordinationStoreIT.java index 5cf092f66d..28b989ee61 100644 --- a/data-prepper-plugins/dynamodb-source-coordination-store/src/test/java/org/opensearch/dataprepper/plugins/sourcecoordinator/dynamodb/DynamoDbSourceCoordinationStoreIT.java +++ b/data-prepper-plugins/dynamodb-source-coordination-store/src/test/java/org/opensearch/dataprepper/plugins/sourcecoordinator/dynamodb/DynamoDbSourceCoordinationStoreIT.java @@ -36,7 +36,9 @@ import java.util.Optional; import java.util.Random; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -265,8 +267,16 @@ void tryAcquireAvailablePartition_gets_first_unassigned_partition() { objectUnderTest.tryCreatePartitionItem(sourceIdentifier, unassignedPartitionKey3, SourcePartitionStatus.UNASSIGNED, 1L, partitionProgressState, false); - final Optional maybeAcquired = - objectUnderTest.tryAcquireAvailablePartition(sourceIdentifier, ownerId, Duration.ofSeconds(20)); + // Wait for partition to be available in DynamoDB Local before attempting to acquire + final Optional[] maybeAcquiredHolder = new Optional[]{Optional.empty()}; + await().atMost(5, TimeUnit.SECONDS) + .pollInterval(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + maybeAcquiredHolder[0] = objectUnderTest.tryAcquireAvailablePartition(sourceIdentifier, ownerId, Duration.ofSeconds(20)); + assertThat(maybeAcquiredHolder[0].isPresent(), equalTo(true)); + }); + + final Optional maybeAcquired = maybeAcquiredHolder[0]; assertThat(maybeAcquired, notNullValue()); assertThat(maybeAcquired.isPresent(), equalTo(true)); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java index 80528646e4..35c8263727 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -118,7 +118,7 @@ void testValidateConfig() { @Test void testValidateConfigBasic() { - when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://somedomain.atlassian.net"); when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); @@ -137,7 +137,7 @@ void testValidateConfigBasic() { @Test void testValidateConfigOauth2() { - when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://somedomain.atlassian.net"); when(confluenceSourceConfig.getAuthType()).thenReturn(OAUTH2); when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java index 960027c659..4e27fb5528 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java @@ -126,7 +126,7 @@ void testGetProjectNameFilter() { void testValidateConfig() { assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(jiraSourceConfig.getAccountUrl()).thenReturn("https://somedomain.atlassian.net"); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); when(jiraSourceConfig.getAuthType()).thenReturn("fakeType"); @@ -135,7 +135,7 @@ void testValidateConfig() { @Test void testValidateConfigBasic() { - when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(jiraSourceConfig.getAccountUrl()).thenReturn("https://somedomain.atlassian.net"); when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); when(jiraSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); @@ -154,7 +154,7 @@ void testValidateConfigBasic() { @Test void testValidateConfigOauth2() { - when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(jiraSourceConfig.getAccountUrl()).thenReturn("https://somedomain.atlassian.net"); when(jiraSourceConfig.getAuthType()).thenReturn(OAUTH2); when(jiraSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java index 949a8d75d7..8741fc7cb7 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java @@ -10,7 +10,6 @@ package org.opensearch.dataprepper.plugins.source.microsoft_office365; import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Timer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -41,9 +40,6 @@ import java.util.Map; import java.util.ArrayList; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -53,7 +49,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.times; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.CONTENT_TYPES; diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java index a762275a83..96546397c3 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java @@ -32,9 +32,16 @@ import java.util.function.BiConsumer; import java.util.stream.Stream; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class MetricsHelperTest { From 9c46ba51c7ec095da56893383c78c27d00e2ba27 Mon Sep 17 00:00:00 2001 From: David Venable Date: Thu, 11 Dec 2025 12:00:19 -0600 Subject: [PATCH 15/51] Fixes the trace-analytics-sample-app project and updates it. (#6350) Use Gradle 9.2.1, the current latest. Update to Spring Boot 4.0.0. Updated to Java 21. Use a more fixed Docker image when building to avoid future build failures - always Gradle 9 and JDK 21. Also, updates or adds copyright headers. Signed-off-by: David Venable Signed-off-by: Nathan Wand --- .../sample-app/Dockerfile | 15 ++++-- .../sample-app/analytics-service/Dockerfile | 15 ++++-- .../sample-app/analytics-service/build.gradle | 34 +++++--------- .../gradle/wrapper/gradle-wrapper.jar | Bin 60756 -> 45633 bytes .../gradle/wrapper/gradle-wrapper.properties | 4 +- .../sample-app/analytics-service/gradlew | 44 +++++++++++------- .../sample-app/analytics-service/gradlew.bat | 26 ++++++----- 7 files changed, 79 insertions(+), 59 deletions(-) diff --git a/examples/trace-analytics-sample-app/sample-app/Dockerfile b/examples/trace-analytics-sample-app/sample-app/Dockerfile index 3c160cd98f..2ed4f5d06a 100644 --- a/examples/trace-analytics-sample-app/sample-app/Dockerfile +++ b/examples/trace-analytics-sample-app/sample-app/Dockerfile @@ -1,11 +1,20 @@ -FROM gradle:jdk17 as cache +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +FROM gradle:9-jdk21 as cache RUN mkdir -p /home/gradle/cache_home ENV GRADLE_USER_HOME /home/gradle/cache_home COPY analytics-service/build.gradle /home/gradle/src/ WORKDIR /home/gradle/src RUN gradle clean build --daemon -FROM gradle:jdk17 AS build +FROM gradle:9-jdk21 AS build COPY --from=cache /home/gradle/cache_home /home/gradle/.gradle COPY analytics-service /home/gradle/src/ WORKDIR /home/gradle/src @@ -13,7 +22,7 @@ RUN gradle bootJar --daemon RUN wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.30.0/opentelemetry-javaagent.jar -FROM eclipse-temurin:17-jre-jammy +FROM eclipse-temurin:21-jre-jammy RUN apt-get -y update \ && apt-get -y upgrade \ diff --git a/examples/trace-analytics-sample-app/sample-app/analytics-service/Dockerfile b/examples/trace-analytics-sample-app/sample-app/analytics-service/Dockerfile index a8b6cd87a6..5ee6d41497 100644 --- a/examples/trace-analytics-sample-app/sample-app/analytics-service/Dockerfile +++ b/examples/trace-analytics-sample-app/sample-app/analytics-service/Dockerfile @@ -1,11 +1,20 @@ -FROM gradle:jdk17 as cache +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +FROM gradle:9-jdk21 as cache RUN mkdir -p /home/gradle/cache_home ENV GRADLE_USER_HOME /home/gradle/cache_home COPY build.gradle /home/gradle/src/ WORKDIR /home/gradle/src RUN gradle clean build --daemon -FROM gradle:jdk17 AS build +FROM gradle:9-jdk21 AS build COPY --from=cache /home/gradle/cache_home /home/gradle/.gradle COPY . /home/gradle/src/ WORKDIR /home/gradle/src @@ -13,7 +22,7 @@ RUN gradle bootJar --daemon RUN wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.30.0/opentelemetry-javaagent.jar -FROM eclipse-temurin:17-jre-jammy +FROM eclipse-temurin:21-jre-jammy EXPOSE 8087 RUN mkdir /app COPY --from=build /home/gradle/src/build/libs/*.jar /app/spring-boot-application.jar diff --git a/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle b/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle index 5d2f5b7cf8..ffec49939a 100644 --- a/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle +++ b/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle @@ -1,16 +1,19 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. */ plugins { id 'java' - id 'org.springframework.boot' version '3.1.6' - id 'io.spring.dependency-management' version '1.1.4' + id 'org.springframework.boot' version '4.0.0' + id 'io.spring.dependency-management' version '1.1.7' } - -group = 'com.example.restservice' +group = 'org.opensearch.dataprepper.examples.restservice' version = '0.0.1-SNAPSHOT' repositories { @@ -19,30 +22,17 @@ repositories { java { toolchain { - languageVersion = JavaLanguageVersion.of(17) - } -} - -configurations.all { - resolutionStrategy.eachDependency { DependencyResolveDetails details -> - if (details.requested.group == 'org.yaml') { - details.useVersion '2.0' - } else if (details.requested.group == 'org.apache.tomcat.embed') { - details.useVersion '10.1.14' - details.because('Fixes CVE-2023-44487') - } else if (details.requested.group == 'ch.qos.logback') { - details.useVersion '1.4.14' - details.because('Fixes CVE-2023-6378') - } + languageVersion = JavaLanguageVersion.of(21) } } dependencies { - implementation('org.springframework.boot:spring-boot-starter-web') - testImplementation('org.springframework.boot:spring-boot-starter-test') + implementation 'org.springframework.boot:spring-boot-starter' + implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.boot:spring-boot-jackson2' + testImplementation 'org.springframework.boot:spring-boot-starter-test' } bootJar { mainClass = 'com.example.restservice.RestServiceApplication' } - diff --git a/examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.jar b/examples/trace-analytics-sample-app/sample-app/analytics-service/gradle/wrapper/gradle-wrapper.jar index 249e5832f090a2944b7473328c07c9755baa3196..f8e1ee3125fe0768e9a76ee977ac089eb657005e 100644 GIT binary patch literal 45633 zcma&NV|1n6wyqu9PQ|uu+csuwn-$x(T~Woh?Nr6KUD3(A)@l1Yd+oj6Z_U=8`RAE` z#vE6_`?!1WLs1443=Ieh3JM4ai0JG2|2{}S&_HrxszP*9^5P7#QX*pVDq?D?;6T8C z{bWO1$9at%!*8ax*TT&F99vwf1Ls+3lklsb|bC`H`~Q z_w}*E9P=Wq;PYlGYhZ^lt#N97bt5aZ#mQcOr~h^B;R>f-b0gf{y(;VA{noAt`RZzU z7vQWD{%|q!urW2j0Z&%ChtL(^9m` zgaU%|B;V#N_?%iPvu0PVkX=1m9=*SEGt-Lp#&Jh%rz6EJXlV^O5B5YfM5j{PCeElx z8sipzw8d=wVhFK+@mgrWyA)Sv3BJq=+q+cL@=wuH$2;LjY z^{&+X4*HFA0{QvlM_V4PTQjIdd;d|2YuN;s|bi!@<)r-G%TuOCHz$O(_-K z)5in&6uNN<0UfwY=K>d;cL{{WK2FR|NihJMN0Q4X+(1lE)$kY?T$7UWleIU`i zQG#X-&&m-8x^(;n@o}$@vPMYRoq~|FqC~CU3MnoiifD{(CwAGd%X#kFHq#4~%_a!{ zeX{XXDT#(DvX7NtAs7S}2ZuiZ>gtd;tCR7E)3{J^`~#Vd**9qz%~JRFAiZf{zt|Dr zvQw!)n7fNUn_gH`o9?8W8t_%x6~=y*`r46bjj(t{YU*qfqd}J}*mkgUfsXTI>Uxl6 z)Fj>#RMy{`wINIR;{_-!xGLgVaTfNJ2-)%YUfO&X5z&3^E#4?k-_|Yv$`fpgYkvnA%E{CiV zP|-zAf8+1@R`sT{rSE#)-nuU7Pwr-z>0_+CLQT|3vc-R22ExKT4ym@Gj77j$aTVns zp4Kri#Ml?t7*n(;>nkxKdhOU9Qbwz%*#i9_%K<`m4T{3aPbQ?J(Mo`6E5cDdbAk%X z+4bN%E#a(&ZXe{G#V!2Nt+^L$msKVHP z|APpBhq7knz(O2yY)$$VyI_Xg4UIC*$!i7qQG~KEZnO@Q1i89@4ZKW*3^Wh?o?zSkfPxdhnTxlO!3tAqe_ zuEqHVcAk3uQIFTpP~C{d$?>7yt3G3Fo>syXTus>o0tJdFpQWC27hDiwC%O09i|xCq z@H6l|+maB;%CYQIChyhu;PVYz9e&5a@EEQs3$DS6dLIS+;N@I0)V}%B`jdYv;JDck zd|xxp(I?aedivE7*19hesoa-@Xm$^EHbbVmh$2^W-&aTejsyc$i+}A#n2W*&0Qt`5 zJS!2A|LVV;L!(*x2N)GjJC;b1RB_f(#D&g_-};a*|BTRvfdIX}Gau<;uCylMNC;UG zzL((>6KQBQ01wr%7u9qI2HLEDY!>XisIKb#6=F?pAz)!_JX}w|>1V>X^QkMdFi@Jr z`1N*V4xUl{qvECHoF?#lXuO#Dg2#gh|AU$Wc=nuIbmVPBEGd(R#&Z`TP9*o%?%#ob zWN%ByU+55yBNfjMjkJnBjT!cVDi}+PR3N&H(f8$d^Pu;A_WV*{)c2Q{IiE7&LPsd4 z!rvkUf{sco_WNSIdW+btM#O+4n`JiceH6%`7pDV zRqJ@lj=Dt(e-Gkz$b!c2>b)H$lf(fuAPdIsLSe(dZ4E~9+Ge!{3j~>nS%r)eQZ;Iq ztWGpp=2Ptc!LK_TQ8cgJXUlU5mRu|7F2{eu*;a>_5S<;bus=t*IXcfzJRPv4xIs;s zt2<&}OM>KxkTxa=dFMfNr42=DL~I}6+_{`HT_YJBiWkpVZND1Diad~Yr*Fuq{zljr z*_+jXk=qVBdwlQkYuIrB4GG*#voba$?h*u0uRNL+87-?AjzG2X_R9mzQ7BJEawutObr|ey~%in>6k%A`K*`pb-|DF5m})!`b=~osoiW2)IFh?_y9y<3Cix_ znvC=bjBX1J820!%%9FaB@v?hAsd05e@w$^ZAvtUp*=Bi+Owkl?rLa6F#yl{s+?563 zmn2 zV95%gySAJ$L!Vvk4kx!n@mo`3Mfi`2lXUkBmd%)u)7C?Pa;oK~zUQ#p0u{a|&0;zNO#9a4`v^3df90X#~l_k$q7n&L5 z?TszF842~g+}tgUP}UG?ObLCE1(Js_$e>XS7m%o7j@@VdxePtg)w{i5an+xK95r?s zDeEhgMO-2$H?@0{p-!4NJ)}zP+3LzZB?FVap)ObHV6wp}Lrxvz$cjBND1T6ln$EfJ zZRPeR2lP}K0p8x`ahxB??Ud;i7$Y5X!5}qBFS+Zp=P^#)08nQi_HuJcN$0=x;2s53 zwoH}He9BlKT4GdWfWt)@o@$4zN$B@5gVIN~aHtwIhh{O$uHiMgYl=&Vd$w#B2 zRv+xK3>4E{!)+LXA2#*K6H~HpovXAQeXV(^Pd%G_>ro0(4_@`{2Ag(+8{9pqJ>Co$ zRRV(oX;nD+Jel_2^BlNO=cQP8q*G#~R3PTERUxvug_C4T3qwb9MQE|^{5(H*nt`fn z^%*p-RwkAhT6(r>E@5w8FaB)Q<{#`H9fTdc6QBuSr9D-x!Tb9f?wI=M{^$cB5@1;0 z+yLHh?3^c-Qte@JI<SW`$bs5Vv9!yWjJD%oY z8Cdc$a(LLy@tB2)+rUCt&0$&+;&?f~W6+3Xk3g zy9L�|d9Zj^A1Dgv5yzCONAB>8LM`TRL&7v_NKg(bEl#y&Z$py}mu<4DrT@8HHjE zqD@4|aM>vt!Yvc2;9Y#V;KJ8M>vPjiS2ycq52qkxInUK*QqA3$&OJ`jZBo zpzw&PT%w0$D94KD%}VN9c)eCueh1^)utGt2OQ+DP(BXszodfc1kFPWl~BQ5Psy*d`UIf zc}zQ8TVw35jdCSc78)MljC-g3$GX2$<0<3MEQXS&i<(ZFClz9WlL}}?%u>S2hhEk_ zyzfm&@Q%YVB-vw3KH|lU#c_)0aeG^;aDG&!bwfOz_9)6gLe;et;h(?*0d-RV0V)1l zzliq#`b9Y*c`0!*6;*mU@&EFSbW>9>L5xUX+unp%@tCW#kLfz)%3vwN{1<-R*g+B_C^W8)>?n%G z<#+`!wU$L&dn)Pz(9DGGI%RlmM2RpeDy9)31OZV$c2T>-Jl&4$6nul&e7){1u-{nP zE$uZs%gyanu+yBcAb+jTYGy(^<;&EzeLeqveN12Lvv)FQFn0o&*qAaH+gLJ)*xT9y z>`Y`W?M#K7%w26w?Oen>j7=R}EbZ;+jcowV&i}P|IfW^C5GJHt5D;Q~)|=gW3iQ;N zQGl4SQFtz=&~BGon6hO@mRnjpmM79ye^LY_L2no{f_M?j80pr`o3BrI7ice#8#Zt4 zO45G97Hpef+AUEU%jN-dLmPYHY(|t#D)9|IeB^i1X|eEq+ymld_Uj$l^zVAPRilx- z^II$sL4G~{^7?sik2BK7;ZV-VIVhrKjUxBIsf^N&K`)5;PjVg-DTm1Xtw4-tGtElU zJgVTCk4^N4#-kPuX=7p~GMf5Jj5A#>)GX)FIcOqY4lf}Vv2gjrOTuFusB@ERW-&fb zTp=E0E?gXkwzn)AMMY*QCftp%MOL-cbsG{02$0~b?-JD{-nwj58 zBHO1YL~yn~RpnZ6*;XA|MSJeBfX-D?afH*E!2uGjT%k!jtx~OG_jJ`Ln}lMQb7W41 zmTIRd%o$pu;%2}}@2J$x%fg{DZEa-Wxdu6mRP~Ea0zD2+g;Dl*to|%sO-5mUrZ`~C zjJ zUe^**YRgBvlxl<(r0LjxjSQKiTx+E<7$@9VO=RYgL9ldTyKzfqR;Y&gu^ub!fVX7u z3H@;8j#tVgga~EMuXv_#Q8<*uK@R{mGzn92eDYkF1sbxh5!P|M-D)T~Ae*SO`@u$Q z7=5s)HM)w~s2j5{I67cqSn6BLLhCMcn0=OTVE?T7bAmY!T+xZ_N3op~wZ3Oxlm6(a5qB({6KghlvBd9HJ#V6YY_zxbj-zI`%FN|C*Q`DiV z#>?Kk7VbuoE*I9tJaa+}=i7tJnMRn`P+(08 za*0VeuAz!eI7giYTsd26P|d^E2p1f#oF*t{#klPhgaShQ1*J7?#CTD@iDRQIV+Z$@ z>qE^3tR3~MVu=%U%*W(1(waaFG_1i5WE}mvAax;iwZKv^g1g}qXY7lAd;!QQa#5e= z1_8KLHje1@?^|6Wb(A{HQ_krJJP1GgE*|?H0Q$5yPBQJlGi;&Lt<3Qc+W4c}Ih~@* zj8lYvme}hwf@Js%Oj=4BxXm15E}7zS0(dW`7X0|$damJ|gJ6~&qKL>gB_eC7%1&Uh zLtOkf7N0b;B`Qj^9)Bfh-( z0or96!;EwEMnxwp!CphwxxJ+DDdP4y3F0i`zZp-sQ5wxGIHIsZCCQz5>QRetx8gq{ zA33BxQ}8Lpe!_o?^u2s3b!a-$DF$OoL=|9aNa7La{$zI#JTu_tYG{m2ly$k?>Yc); zTA9ckzd+ibu>SE6Rc=Yd&?GA9S5oaQgT~ER-|EwANJIAY74|6 z($#j^GP}EJqi%)^jURCj&i;Zl^-M9{=WE69<*p-cmBIz-400wEewWVEd^21}_@A#^ z2DQMldk_N)6bhFZeo8dDTWD@-IVunEY*nYRON_FYII-1Q@@hzzFe(lTvqm}InfjQ2 zN>>_rUG0Lhaz`s;GRPklV?0 z;~t4S8M)ZBW-ED?#UNbCrsWb=??P># zVc}MW_f80ygG_o~SW+Q6oeIUdFqV2Fzys*7+vxr^ZDeXcZZc;{kqK;(kR-DKL zByDdPnUQgnX^>x?1Tz~^wZ%Flu}ma$Xmgtc7pSmBIH%&H*Tnm=L-{GzCv^UBIrTH5 zaoPO|&G@SB{-N8Xq<+RVaM_{lHo@X-q}`zjeayVZ9)5&u*Y>1!$(wh9Qoe>yWbPgw zt#=gnjCaT_+$}w^*=pgiHD8N$hzqEuY5iVL_!Diw#>NP7mEd?1I@Io+?=$?7cU=yK zdDKk_(h_dB9A?NX+&=%k8g+?-f&`vhAR}&#zP+iG%;s}kq1~c{ac1@tfK4jP65Z&O zXj8Ew>l7c|PMp!cT|&;o+(3+)-|SK&0EVU-0-c&guW?6F$S`=hcKi zpx{Z)UJcyihmN;^E?*;fxjE3kLN4|&X?H&$md+Ege&9en#nUe=m>ep3VW#C?0V=aS zLhL6v)|%$G5AO4x?Jxy8e+?*)YR~<|-qrKO7k7`jlxpl6l5H&!C4sePiVjAT#)b#h zEwhfkpFN9eY%EAqg-h&%N>E0#%`InXY?sHyptcct{roG42Mli5l)sWt66D_nG2ed@ z#4>jF?sor7ME^`pDlPyQ(|?KL9Q88;+$C&3h*UV*B+*g$L<{yT9NG>;C^ZmPbVe(a z09K^qVO2agL`Hy{ISUJ{khPKh@5-)UG|S8Sg%xbJMF)wawbgll3bxk#^WRqmdY7qv zr_bqa3{`}CCbREypKd!>oIh^IUj4yl1I55=^}2mZAAW6z}Kpt3_o1b4__sQ;b zv)1=xHO?gE-1FL}Y$0YdD-N!US;VSH>UXnyKoAS??;T%tya@-u zfFo)@YA&Q#Q^?Mtam19`(PS*DL{PHjEZa(~LV7DNt5yoo1(;KT)?C7%^Mg;F!C)q= z6$>`--hQX4r?!aPEXn;L*bykF1r8JVDZ)x4aykACQy(5~POL;InZPU&s5aZm-w1L< z`crCS5=x>k_88n(*?zn=^w*;0+8>ui2i>t*Kr!4?aA1`yj*GXi#>$h8@#P{S)%8+N zCBeL6%!Ob1YJs5+a*yh{vZ8jH>5qpZhz_>(ph}ozKy9d#>gba1x3}`-s_zi+SqIeR z0NCd7B_Z|Fl+(r$W~l@xbeAPl5{uJ{`chq}Q;y8oUN0sUr4g@1XLZQ31z9h(fE_y( z_iQ(KB39LWd;qwPIzkvNNkL(P(6{Iu{)!#HvBlsbm`g2qy&cTsOsAbwMYOEw8!+75D!>V{9SZ?IP@pR9sFG{T#R*6ez2&BmP8*m^6+H2_ z>%9pg(+R^)*(S21iHjLmdt$fmq6y!B9L!%+;wL5WHc^MZRNjpL9EqbBMaMns2F(@h zN0BEqZ3EWGLjvY&I!8@-WV-o@>biD;nx;D}8DPapQF5ivpHVim8$G%3JrHtvN~U&) zb1;=o*lGfPq#=9Moe$H_UhQPBjzHuYw;&e!iD^U2veY8)!QX_E(X@3hAlPBIc}HoD z*NH1vvCi5xy@NS41F1Q3=Jkfu&G{Syin^RWwWX|JqUIX_`}l;_UIsj&(AFQ)ST*5$ z{G&KmdZcO;jGIoI^+9dsg{#=v5eRuPO41<*Ym!>=zHAXH#=LdeROU-nzj_@T4xr4M zJI+d{Pp_{r=IPWj&?%wfdyo`DG1~|=ef?>=DR@|vTuc)w{LHqNKVz9`Dc{iCOH;@H5T{ zc<$O&s%k_AhP^gCUT=uzrzlEHI3q`Z3em0*qOrPHpfl1v=8Xkp{!f9d2p!4 zL40+eJB4@5IT=JTTawIA=Z%3AFvv=l1A~JX>r6YUMV7GGLTSaIn-PUw| z;9L`a<)`D@Qs(@P(TlafW&-87mcZuwFxo~bpa01_M9;$>;4QYkMQlFPgmWv!eU8Ut zrV2<(`u-@1BTMc$oA*fX;OvklC1T$vQlZWS@&Wl}d!72MiXjOXxmiL8oq;sP{)oBe zS#i5knjf`OfBl}6l;BSHeY31w8c~8G>$sJ9?^^!)Z*Z*Xg zbTbkcbBpgFui(*n32hX~sC7gz{L?nlnOjJBd@ zUC4gd`o&YB4}!T9JGTe9tqo0M!JnEw4KH7WbrmTRsw^Nf z^>RxG?2A33VG3>E?iN|`G6jgr`wCzKo(#+zlOIzp-^E0W0%^a>zO)&f(Gc93WgnJ2p-%H-xhe{MqmO z8Iacz=Qvx$ML>Lhz$O;3wB(UI{yTk1LJHf+KDL2JPQ6#m%^bo>+kTj4-zQ~*YhcqS z2mOX!N!Q$d+KA^P0`EEA^%>c12X(QI-Z}-;2Rr-0CdCUOZ=7QqaxjZPvR%{pzd21HtcUSU>u1nw?)ZCy+ zAaYQGz59lqhNXR4GYONpUwBU+V&<{z+xA}`Q$fajmR86j$@`MeH}@zz*ZFeBV9Ot< ze8BLzuIIDxM&8=dS!1-hxiAB-x-cVmtpN}JcP^`LE#2r9ti-k8>Jnk{?@Gw>-WhL=v+H!*tv*mcNvtwo)-XpMnV#X>U1F z?HM?tn^zY$6#|(|S~|P!BPp6mur58i)tY=Z-9(pM&QIHq+I5?=itn>u1FkXiehCRC zW_3|MNOU)$-zrjKnU~{^@i9V^OvOJMp@(|iNnQ%|iojG2_Snnt`1Cqx2t)`vW&w2l zwb#`XLNY@FsnC-~O&9|#Lpvw7n!$wL9azSk)$O}?ygN@FEY({2%bTl)@F2wevCv`; zZb{`)uMENiwE|mti*q5U4;4puX{VWFJ#QIaa*%IHKyrU*HtjW_=@!3SlL~pqLRs?L zoqi&}JLsaP)yEH!=_)zmV-^xy!*MCtc{n|d%O zRM>N>eMG*Qi_XAxg@82*#zPe+!!f#;xBxS#6T-$ziegN-`dLm z=tTN|xpfCPng06|X^6_1JgN}dM<_;WsuL9lu#zLVt!0{%%D9*$nT2E>5@F(>Fxi%Y zpLHE%4LZSJ1=_qm0;^Wi%x56}k3h2Atro;!Ey}#g&*BpbNXXS}v>|nn=Mi0O(5?=1V7y1^1Bdt5h3}oL@VsG>NAH z1;5?|Sth=0*>dbXSQ%MQKB?eN$LRu?yBy@qQVaUl*f#p+sLy$Jd>*q;(l>brvNUbIF0OCf zk%Q;Zg!#0w0_#l)!t?3iz~`X8A>Yd3!P&A4Ov6&EdZmOixeTd4J`*Wutura(}4w@KV>i#rf(0PYL&v^89QiXBP6sj=N;q8kVxS}hA! z|3QaiYz!w+xQ%9&Zg${JgQ*Ip_bg2rmmG`JkX^}&5gbZF!Z(gDD1s5{QwarPK(li- zW9y-CiQ`5Ug1ceN1w7lCxl=2}7c*8_XH8W7y0AICn19qZ`w}z0iCJ$tJ}NjzQCH90 zc!UzpKvk%3;`XfFi2;F*q2eMQQ5fzO{!`KU1T^J?Z64|2Z}b1b6h80_H%~J)J)kbM0hsj+FV6%@_~$FjK9OG7lY}YA zRzyYxxy18z<+mCBiX?3Q{h{TrNRkHsyF|eGpLo0fKUQ|19Z0BamMNE9sW z?vq)r`Qge{9wN|ezzW=@ojpVQRwp##Q91F|B5c`a0A{HaIcW>AnqQ*0WT$wj^5sWOC1S;Xw7%)n(=%^in zw#N*+9bpt?0)PY$(vnU9SGSwRS&S!rpd`8xbF<1JmD&6fwyzyUqk){#Q9FxL*Z9%#rF$} zf8SsEkE+i91VY8d>Fap#FBacbS{#V&r0|8bQa;)D($^v2R1GdsQ8YUk(_L2;=DEyN%X*3 z;O@fS(pPLRGatI93mApLsX|H9$VL2)o(?EYqlgZMP{8oDYS8)3G#TWE<(LmZ6X{YA zRdvPLLBTatiUG$g@WK9cZzw%s6TT1Chmw#wQF&&opN6^(D`(5p0~ zNG~fjdyRsZv9Y?UCK(&#Q2XLH5G{{$9Y4vgMDutsefKVVPoS__MiT%qQ#_)3UUe=2fK)*36yXbQUp#E98ah(v`E$c3kAce_8a60#pa7rq6ZRtzSx6=I^-~A|D%>Riv{Y`F9n3CUPL>d`MZdRmBzCum2K%}z@Z(b7#K!-$Hb<+R@Rl9J6<~ z4Wo8!!y~j(!4nYsDtxPIaWKp+I*yY(ib`5Pg356Wa7cmM9sG6alwr7WB4IcAS~H3@ zWmYt|TByC?wY7yODHTyXvay9$7#S?gDlC?aS147Ed7zW!&#q$^E^_1sgB7GKfhhYu zOqe*Rojm~)8(;b!gsRgQZ$vl5mN>^LDgWicjGIcK9x4frI?ZR4Z%l1J=Q$0lSd5a9 z@(o?OxC72<>Gun*Y@Z8sq@od{7GGsf8lnBW^kl6sX|j~UA2$>@^~wtceTt^AtqMIx zO6!N}OC#Bh^qdQV+B=9hrwTj>7HvH1hfOQ{^#nf%e+l)*Kgv$|!kL5od^ka#S)BNT z{F(miX_6#U3+3k;KxPyYXE0*0CfL8;hDj!QHM@)sekF9uyBU$DRZkka4ie^-J2N8w z3PK+HEv7kMnJU1Y+>rheEpHdQ3_aTQkM3`0`tC->mpV=VtvU((Cq$^(S^p=+$P|@} zueLA}Us^NTI83TNI-15}vrC7j6s_S`f6T(BH{6Jj{Lt;`C+)d}vwPGx62x7WXOX19 z2mv1;f^p6cG|M`vfxMhHmZxkkmWHRNyu2PDTEpC(iJhH^af+tl7~h?Y(?qNDa`|Ogv{=+T@7?v344o zvge%8Jw?LRgWr7IFf%{-h>9}xlP}Y#GpP_3XM7FeGT?iN;BN-qzy=B# z=r$79U4rd6o4Zdt=$|I3nYy;WwCb^`%oikowOPGRUJ3IzChrX91DUDng5_KvhiEZwXl^y z+E!`Z6>}ijz5kq$nNM8JA|5gf_(J-);?SAn^N-(q2r6w31sQh6vLYp^ z<>+GyGLUe_6eTzX7soWpw{dDbP-*CsyKVw@I|u`kVX&6_h5m!A5&3#=UbYHYJ5GK& zLcq@0`%1;8KjwLiup&i&u&rmt*LqALkIqxh-)Exk&(V)gh9@Fn+WU=6-UG^X2~*Q-hnQ$;;+<&lRZ>g0I`~yuv!#84 zy>27(l&zrfDI!2PgzQyV*R(YFd`C`YwR_oNY+;|79t{NNMN1@fp?EaNjuM2DKuG%W z5749Br2aU6K|b=g4(IR39R8_!|B`uQ)bun^C9wR4!8isr$;w$VOtYk+1L9#CiJ#F) z)L}>^6>;X~0q&CO>>ZBo0}|Ex9$p*Hor@Ej9&75b&AGqzpGpM^dx}b~E^pPKau2i5 zr#tT^S+01mMm}z480>-WjU#q`6-gw4BJMWmW?+VXBZ#JPzPW5QQm@RM#+zbQMpr>M zX$huprL(A?yhv8Y81K}pTD|Gxs#z=K(Wfh+?#!I$js5u8+}vykZh~NcoLO?ofpg0! zlV4E9BAY_$pN~e-!VETD&@v%7J~_jdtS}<_U<4aRqEBa&LDpc?V;n72lTM?pIVG+> z*5cxz_iD@3vIL5f9HdHov{o()HQ@6<+c}hfC?LkpBEZ4xzMME^~AdB8?2F=#6ff!F740l&v7FN!n_ zoc1%OfX(q}cg4LDk-1%|iZ^=`x5Vs{oJYhXufP;BgVd*&@a04pSek6OS@*UH`*dAp z7wY#70IO^kSqLhoh9!qIj)8t4W6*`Kxy!j%Bi%(HKRtASZ2%vA0#2fZ=fHe0zDg8^ zucp;9(vmuO;Zq9tlNH)GIiPufZlt?}>i|y|haP!l#dn)rvm8raz5L?wKj9wTG znpl>V@};D!M{P!IE>evm)RAn|n=z-3M9m5J+-gkZHZ{L1Syyw|vHpP%hB!tMT+rv8 zIQ=keS*PTV%R7142=?#WHFnEJsTMGeG*h)nCH)GpaTT@|DGBJ6t>3A)XO)=jKPO<# zhkrgZtDV6oMy?rW$|*NdJYo#5?e|Nj>OAvCXHg~!MC4R;Q!W5xcMwX#+vXhI+{ywS zGP-+ZNr-yZmpm-A`e|Li#ehuWB{{ul8gB&6c98(k59I%mMN9MzK}i2s>Ejv_zVmcMsnobQLkp z)jmsJo2dwCR~lcUZs@-?3D6iNa z2k@iM#mvemMo^D1bu5HYpRfz(3k*pW)~jt8UrU&;(FDI5ZLE7&|ApGRFLZa{yynWx zEOzd$N20h|=+;~w$%yg>je{MZ!E4p4x05dc#<3^#{Fa5G4ZQDWh~%MPeu*hO-6}2*)t-`@rBMoz&gn0^@c)N>z|Ikj8|7Uvdf5@ng296rq2LiM#7KrWq{Jc7;oJ@djxbC1s6^OE>R6cuCItGJ? z6AA=5i=$b;RoVo7+GqbqKzFk>QKMOf?`_`!!S!6;PSCI~IkcQ?YGxRh_v86Q%go2) zG=snIC&_n9G^|`+KOc$@QwNE$b7wxBY*;g=K1oJnw8+ZR)ye`1Sn<@P&HZm0wDJV* z=rozX4l;bJROR*PEfHHSmFVY3M#_fw=4b_={0@MP<5k4RCa-ZShp|CIGvW^9$f|BM#Z`=3&=+=p zp%*DC-rEH3N;$A(Z>k_9rDGGj2&WPH|}=Pe3(g}v3=+`$+A=C5PLB3UEGUMk92-erU%0^)5FkU z^Yx#?Gjyt*$W>Os^Fjk-r-eu`{0ZJbhlsOsR;hD=`<~eP6ScQ)%8fEGvJ15u9+M0c|LM4@D(tTx!T(sRv zWg?;1n7&)-y0oXR+eBs9O;54ZKg=9eJ4gryudL84MAMsKwGo$85q6&cz+vi)9Y zvg#u>v&pQQ1NfOhD#L@}NNZe+l_~BQ+(xC1j-+({Cg3_jrZ(YpI{3=0F1GZsf+3&f z#+sRf=v7DVwTcYw;SiNxi5As}hE-Tpt)-2+lBmcAO)8cP55d0MXS*A3yI5A!Hq&IN zzb+)*y8d8WTE~Vm3(pgOzy%VI_e4lBx&hJEVBu!!P|g}j(^!S=rNaJ>H=Ef;;{iS$$0k-N(`n#J_K40VJP^8*3YR2S`* zED;iCzkrz@mP_(>i6ol5pMh!mnhrxM-NYm0gxPF<%(&Az*pqoRTpgaeC!~-qYKZHJ z2!g(qL_+hom-fp$7r=1#mU~Dz?(UFkV|g;&XovHh~^6 z1eq4BcKE%*aMm-a?zrj+p;2t>oJxxMgsmJ^Cm%SwDO?odL%v6fXU869KBEMoC0&x>qebmE%y+W z51;V2xca9B=wtmln74g7LcEgJe1z7o>kwc1W=K1X7WAcW%73eGwExo&{SSTnXR+pA zRL)j$LV7?Djn8{-8CVk94n|P>RAw}F9uvp$bpNz<>Yw3PgWVJo?zFYH9jzq zU|S+$C6I?B?Jm>V{P67c9aRvK283bnM(uikbL=``ew5E)AfV$SR4b8&4mPDkKT&M3 zok(sTB}>Gz%RzD{hz|7(AFjB$@#3&PZFF5_Ay&V3?c&mT8O;9(vSgWdwcy?@L-|`( z@@P4$nXBmVE&Xy(PFGHEl*K;31`*ilik77?w@N11G7IW!eL@1cz~XpM^02Z?CRv1R z5&x6kevgJ5Bh74Q8p(-u#_-3`246@>kY~V4!XlYgz|zMe18m7Vs`0+D!LQwTPzh?a zp?X169uBrRvG3p%4U@q_(*^M`uaNY!T6uoKk@>x(29EcJW_eY@I|Un z*d;^-XTsE{Vjde=Pp3`In(n!ohHxqB%V`0vSVMsYsbjN6}N6NC+Ea`Hhv~yo@ z|Ab%QndSEzidwOqoXCaF-%oZ?SFWn`*`1pjc1OIk2G8qSJ$QdrMzd~dev;uoh z>SneEICV>k}mz6&xMqp=Bs_0AW81D{_hqJXl6ZWPRNm@cC#+pF&w z{{TT0=$yGcqkPQL>NN%!#+tn}4H>ct#L#Jsg_I35#t}p)nNQh>j6(dfd6ng#+}x3^ zEH`G#vyM=;7q#SBQzTc%%Dz~faHJK+H;4xaAXn)7;)d(n*@Bv5cUDNTnM#byv)DTG zaD+~o&c-Z<$c;HIOc!sERIR>*&bsB8V_ldq?_>fT!y4X-UMddUmfumowO!^#*pW$- z_&)moxY0q!ypaJva)>Bc&tDs?D=Rta*Wc^n@uBO%dd+mnsCi0aBZ3W%?tz844FkZD zzhl+RuCVk=9Q#k;8EpXtSmR;sZUa5(o>dt+PBe96@6G}h`2)tAx(WKR4TqXy(YHIT z@feU+no42!!>y5*3Iv$!rn-B_%sKf6f4Y{2UpRgGg*dxU)B@IRQ`b{ncLrg9@Q)n$ zOZ7q3%zL99j1{56$!W(Wu{#m|@(6BBb-*zV23M!PmH7nzOD@~);0aK^iixd%>#BwR zyIlVF*t4-Ww*IPTGko3RuyJ*^bo-h}wJ{YkHa2y3mIK%U%>PFunkx0#EeIm{u93PX z4L24jUh+37=~WR47l=ug2cn_}7CLR(kWaIpH8ojFsD}GN3G}v6fI-IMK2sXnpgS5O zHt<|^d9q}_znrbP0~zxoJ-hh6o81y+N;i@6M8%S@#UT)#aKPYdm-xlbL@v*`|^%VS(M$ zMQqxcVVEKe5s~61T77N=9x7ndQ=dzWp^+#cX}v`1bbnH@&{k?%I%zUPTDB(DCWY6( zR`%eblFFkL&C{Q}T6PTF0@lW0JViFzz4s5Qt?P?wep8G8+z3QFAJ{Q8 z9J41|iAs{Um!2i{R7&sV=ESh*k(9`2MM2U#EXF4!WGl(6lI!mg_V%pRenG>dEhJug z^oLZ?bErlIPc@Jo&#@jy@~D<3Xo%x$)(5Si@~}ORyawQ{z^mzNSa$nwLYTh6E%!w_ zUe?c`JJ&RqFh1h18}LE47$L1AwR#xAny*v9NWjK$&6(=e0)H_v^+ZIJ{iVg^e_K-I z|L;t=x>(vU{1+G+P5=i7QzubN=dWIe(bqeBJ2fX85qrBYh5pj*f05=8WxcP7do(_h zkfEQ1Fhf^}%V~vr>ed9*Z2aL&OaYSRhJQFWHtirwJFFkfJdT$gZo;aq70{}E#rx((U`7NMIb~uf>{Y@Fy@-kmo{)ei*VjvpSH7AU zQG&3Eol$C{Upe`034cH43cD*~Fgt?^0R|)r(uoq3ZjaJqfj@tiI~`dQnxfcQIY8o| zx?Ye>NWZK8L1(kkb1S9^8Z8O_(anGZY+b+@QY;|DoLc>{O|aq(@x2=s^G<9MAhc~H z+C1ib(J*&#`+Lg;GpaQ^sWw~f&#%lNQ~GO}O<5{cJ@iXSW4#};tQz2#pIfu71!rQ( z4kCuX$!&s;)cMU9hv?R)rQE?_vV6Kg?&KyIEObikO?6Nay}u#c#`ywL(|Y-0_4B_| zZFZ?lHfgURDmYjMmoR8@i&Z@2Gxs;4uH)`pIv#lZ&^!198Fa^Jm;?}TWtz8sulPrL zKbu$b{{4m1$lv0`@ZWKA|0h5U!uIwqUkm{p7gFZ|dl@!5af*zlF% zpT-i|4JMt%M|0c1qZ$s8LIRgm6_V5}6l6_$cFS# z83cqh6K^W(X|r?V{bTQp14v|DQg;&;fZMu?5QbEN|DizzdZSB~$ZB%UAww;P??AT_-JFKAde%=4c z*WK^Iy5_Y`*IZ+cF`jvkCv~Urz3`nP{hF!UT7Z&e;MlB~LBDvL^hy{%; z7t5+&Ik;KwQ5H^i!;(ly8mfp@O>kH67-aW0cAAT~U)M1u`B>fG=Q2uC8k}6}DEV=% z<0n@WaN%dDBTe*&LIe^r-!r&t`a?#mEwYQuwZ69QU3&}7##(|SIP*4@y+}%v^Gb3# zrJ~68hi~77ya4=W-%{<(XErMm>&kvG`{7*$QxRf(jrz|KGXJN3Hs*8BfBx&9|5sZ1 zpFJ1(B%-bD42(%cOiT@2teyYoUBS`L%<(g;$b6nECbs|ADH5$LYxj?i3+2^#L@d{%E(US^chG<>aL7o>Fg~ zW@9wW@Mb&X;BoMz+kUPUcrDQOImm;-%|nxkXJ8xRz|MlPz5zcJHP<+yvqjB4hJAPE zRv>l{lLznW~SOGRU~u77UcOZyR#kuJrIH_){hzx!6NMX z>(OKAFh@s2V;jk|$k5-Q_ufVe;(KCrD}*^oBx{IZq^AB|7z*bH+g_-tkT~8S$bzdU zhbMY*g?Qb;-m|0`&Jm}A8SEI0twaTfXhIc=no}$>)n5^cc)v!C^YmpxLt=|kf%!%f zp5L$?mnzMt!o(fg7V`O^BLyjG=rNa}=$hiZzYo~0IVX$bp^H-hQn!;9JiFAF<3~nt zVhpABVoLWDQ}2vEEF3-?zzUA(yoYw&$YeHB#WGCXkK+YrG=+t0N~!OmTN;fK*k>^! zJW_v+4Q4n2GP7vgBmK;xHg^7zFqyTTfq|0+1^H2lXhn6PpG#TB*``?1STTC#wcaj3 zG~Q9!XHZ#1oPZo zB6h(BVIW5K+S@JG_HctDLHWb;wobZ0h(3xr6(uUspOSK0WoSHeF$ZLw@)cpoIP|kL zu`GnW>gD$rMt}J0qa9kJzn0s`@JNy1Crkb&;ve|()+_%!x%us>1_Xz|BS>9oQeD3O zy#CHX#(q^~`=@_p$XV6N&RG*~oEH$z96b8S16(6wqH)$vPs=ia!(xPVX5o&5OIYQ%E(-QAR1}CnLTIy zgu1MCqL{_wE)gkj0BAezF|AzPJs=8}H2bHAT-Q@Vuff?0GL=)t3hn{$Le?|+{-2N~`HWe24?!1a^UpC~3nK$(yZ_Gp(EzP~a{qe>xK@fN zEETlwEV_%9d1aWU0&?U>p3%4%>t5Pa@kMrL4&S@ zmSn!Dllj>DIO{6w+0^gt{RO_4fDC)f+Iq4?_cU@t8(B^je`$)eOOJh1Xs)5%u3hf; zjw$47aUJ9%1n1pGWTuBfjeBumDI)#nkldRmBPRW|;l|oDBL@cq1A~Zq`dXwO)hZkI zZ=P7a{Azp06yl(!tREU`!JsmXRps!?Z~zar>ix0-1C+}&t)%ist94(Ty$M}ZKn1sDaiZpcoW{q&ns8aWPf$bRkbMdSgG+=2BSRQ6GG_f%Lu#_F z&DxHu+nKZ!GuDhb>_o^vZn&^Sl8KWHRDV;z#6r*1Vp@QUndqwscd3kK;>7H!_nvYH zUl|agIWw_LPRj95F=+Ex$J05p??T9_#uqc|q>SXS&=+;eTYdcOOCJDhz7peuvzKoZhTAj&^RulU`#c?SktERgU|C$~O)>Q^$T8ippom{6Ze0_44rQB@UpR~wB? zPsL@8C)uCKxH7xrDor zeNvVfLLATsB!DD{STl{Fn3}6{tRWwG8*@a2OTysNQz2!b6Q2)r*|tZwIovIK9Ik#- z0k=RUmu97T$+6Lz%WQYdmL*MNII&MI^0WWWGKTTi&~H&*Ay7&^6Bpm!0yoVNlSvkB z;!l3U21sJyqc`dt)82)oXA5p>P_irU*EyG72iH%fEpUkm1K$?1^#-^$$Sb=c8_? zOWxxguW7$&-qzSI=Z{}sRGAqzy3J-%QYz2Cffj6SOU|{CshhHx z6?5L$V_QIUbI)HZ9pwP9S15 zXc%$`dxETq+S3_jrfmi$k=)YO5iUeuQ&uX}rCFvz&ubO?u)tv|^-G_`h$pb+8vn@f z7@eQe#Kx|8^37a4d0GulYIUAW|@I5|NIh%=OqHU{(>(UhKvJ}i_X*>!Geb+Rs0MWf66Lf z-cQ(4QOENSbTX$6w_9w4{5eR?14#?)Jqf2UCk5US4bnz8!e>vFduH6(cZZ=5*_!M# zUTZ_b<4v@}dSQOcH@wt-s;3JhkVDct$6k9!ETdi-tplkaxl^qF=p}Q8KMVm+ zeIa2q?RYr}nM0d_W2YWv%JKyCrGSePj8GrRN)<$Nsq8l$X=>`W;?>0eME3|8t&d$~ zH`XG45lBh>-te_f0Mh0??)=Ee0~zESx=sZPv<#!sAVv$0qTn@CmCUNJU<#=`GC)&P z9zuV~9*3_n2*ZQBUh)2xIi;0yo)9XXJxM-VB*6xpyz{Rx2ZCvFnF$2aPcYFG( zyXkO(B30?mt;5GW&{m^w3?!P`#_o;Y%P2z^A`|4%Bt2@3G?C2dcSPNy1#HMXZ>{+L z3BE#xvqR@Ub}uKfzGC=RO|W%dJpUK#m8p&Dk|6Ub8S+dN3qxf9dJ_|WFdM9CSNQv~ zjaFxIX`xx-($#Fq+EI76uB@kK=B4FS0k=9(c8UQnr(nLQxa2qWbuJyD7%`zuqH|eF zNrpM@SIBy@lKb%*$uLeRJQ->ko3yaG~8&}9|f z*KE`oMHQ(HdHlb&)jIzj5~&z8r}w?IM1KSdR=|GFYzDwbn8-uUfu+^h?80e*-9h%Nr;@)Q-TI#dN1V zQPT2;!Wk)DP`kiY<{o7*{on%It(j0&qSv=fNfg3qeNjT@CW{WT<)1Eig!g9lAGx6& zk9_Zrp2I+w_f!LRFsgxKA}gO=xSPSY``kn=c~orU4+0|^K762LWuk_~oK{!-4N8p8 zUDVu0ZhvoD0fN8!3RD~9Bz5GNEn%0~#+E-Js}NTBX;JXE@29MdGln$Aoa3Nzd@%Z= z^zuGY4xk?r(ax7i4RfxA?IPe27s87(e-2Z_KJ(~YI!7bhMQvfN4QX{!68nj@lz^-& z1Zwf=V5ir;j*30AT$nKSfB;K9(inDFwbI^%ohwEDOglz}2l}0!#LsdS3IW43= zBR#E@135bu#VExrtj?)RH^PM(K4B`d=Z6^kix`8$C1&q)w1<&?bAS?70}9fZwZU7R z5RYFo?2Q>e3RW2dl&3E^!&twE<~Lk+apY?#4PM5GWJb2xuWyZs6aAH-9gqg${<1?M zoK&n+$ZyGIi=hakHqRu{^8T4h@$xl?9OM46t;~1_mPs9}jV58E-sp!_CPH4<^A|Q5 zedUHmiyxTc2zgdxU?4PyQ{ON@r+Ucn1kjWSOsh6WzLV~Bv&vWLaj#Xz4VSDs*F#@M>#e^ixNCQ-J|iC=LcB*M4WUb>?v6C z14^8h9Ktd1>XhO$kb-rRL}SFTH)kSu+Dwds$oed7qL)Jbd zhQys4$Uw~yj03)6Kq+K-BsEDftLgjDZk@qLjAyrb5UMeuO^>D43g%0GoKJ~TO0o!D z9E$WfxEDFTT?~sT?|!7aYY*mpt`}i;WTgY|Cb4{Cscrmzb(?UE+nz1wC3#QSjbg>N zleu?7MGaQ&FtejK#?07Uq$vIZX5FqR*a=(zUm`Fq$VUl){GQ{2MA)_j4H$U8FZ`=A z&GU_an)?g%ULunbBq4EUT7uT=vI6~uapKC|H6uz1#Rqt$G(!hE7|c8_#JH%wp9+F? zX`ZigNe9GzC(|Nr8GlmwPre3*Nfu+ zF=SHtv_g@vvoVpev$Jxs|F7CH`X5#HAI=ke(>G6DQQ=h^U8>*J=t5Z3Fi>eH9}1|6 znwv3k>D=kufcp= zAyK#v05qERJxS_ts79QVns}M?sIf(hCO0Q9hKe49a@PzvqzZXTAde6a)iZLw|8V-) ziK`-s)d(oQSejO?eJki$UtP0ped)5T1b)uVFQJq*`7w8liL4TX*#K`hdS!pY9aLD+ zLt=c$c_wt^$Wp~N^!_nT(HiDVibxyq2oM^dw-jC~+3m-#=n!`h^8JYkDTP2fqcVC& zA`VWy*eJC$Eo7qIe@KK;HyTYo0c{Po-_yp=>J(1h#)aH5nV8WGT(oSP)LPgusH%N$?o%U%2I@Ftso10xd z)Tx(jT_vrmTQJDx0QI%9BRI1i!wMNy(LzFXM_wucgJGRBUefc413a9+)}~*UzvNI{KL# z_t4U&srNV|0+ZqwL(<}<%8QtjUD8kSB&p$v^y}vuEC2wyW{aXp2{LTi$EBEHjVnS# z+4=G$GUllsjw&hTbh6z%D2j=cG>gkNVlh|24QUfD*-x9OMzTO93n*pE(U7Vz7BaL% z@(c!GbEjK~fH}sqbB1JNI!~b+AYb5le<-qxDA9&r2o)|epl9@5Ya7}yVkcM)yW6KY7QOX_0-N=)+M!A$NpG? z6BvZ8Tb}Pw(i9f7S00=KbWmNvJGL(-MsAz3@aR~PM$Z>t)%AiCZu?A|?P*~UdhhFT`;Nb)MxIg*0QlkYVX+46( zSd%WoWR@kYToK7)(J=#qUD-ss;4M&27w#03y6$gk6X<-VL8AJM@NFTx#Z!n)F5T357%njjKyjro(yW8ceP{!%;*Y>DN`&_18p(z2Hg$%K zohbgJcp%+ux%q6F?(sc_mYJ<$;DxgkTEi?yjT6Du@+n(KsKtFHcO%7O z=AsfLSTdE2>7a@0^`;)?Fg|s2XOPV&fo<%Q)Izaw4s&RvrX0^+aPNq|yE?oSa7 zsnNs!+vGcTM4yM|$9so*2Nv;ngDD}b0MjH6i4e|l^O`lzCRj)-qa6f%|afJpmf(S1J2k7Nt^!;Q}0 z4ejPF?^M~Sv+@LYn&IFUk2;1h?kb8lfrT`oMm=JBm{fo5N|HY~yQQ`T*e2?!tF%*t zf+ncx15$NdF82GXrpP5rJ7!PVE3>u`ME$9Hw5RlP zUh+s#pg{9kEOsAhvu2pry#@dvbB3Lti+9VkLxPZSl;fNr9}wv1cTahUw_Py7%Xp;C zaz__|kz*ydKiYbsqK{?cXhqR(!1KMoV-+!mz>3S8S`Va4kD#(aKyqecGXB^nF*>mS z1gG>fKZc?R~Tye>%x+43D8=e zf0eKr-)>VEu7^I{%T}BT-WaGXO3+x<2w2jwnXePdc2#BdofU6wbE)ZWHsyj=_NT3o z)kySji#CTEnx8*-n=88Ld+TuNy;x$+vDpZ)=XwCr_Gx-+N=;=LCE7CqKX9 zQ-0{jIr zktqqWCgBa3PYK*qQqd=BO70DfM#|JvuW*0%zmTE{mBI$55J=Y2b2UoZ)Yk z3M%rrX7!nwk#@CXTr5=J__(3cI-8~*MC+>R);Z)0Zkj2kpsifdJeH)2uhA|9^B;S$ z4lT3;_fF@g%#qFotZ#|r-IB*zSo;fokxbsmMrfNfJEU&&TF%|!+YuN=#8jFS4^f*m zazCA-2krJ-;Tkufh!-urx#z*imYo|n6+NDGT#*EH355(vRfrGnr*x z5PWMD7>3IwEh=lO^V>O>iLP~S!GjrvI5lx<7oOg(d;6uEFqo5>IwptBQz;`>zx`n$ zjZQ#Hb)qJdQy#ML&qcfmb$KT+f_1#uYNo7HHDY}7xAw8qbl;9LWO-cndfI=5$%jBw zb}K3U%88Fg^|&0Vc~99bKl|$3JzdawRZ|`7%1S<8B7>9*rWAT0U<@mHDfnL1`~1U| zDw7m@<@}C|zqeHM(OK@di6~sKHiJvk^I0^S<LBe^_xZsUOzVkYSE)Bxn*NekQYbyTn5SRt!n{EseOo-$u)vjM(PV%6cIG3Kv$>dd}HUyXi;_Lv>}OyUj38dPe8+1Pr?{LXnIBCoTnocD60@vhsz+GG5lJB9ncgP8T6@LwuzZ)J zKETBS~AvzGE!{u^+Rd-|Gn!rc@UUnioP0{@_j_>tg8YI#?y zL-H$=&xXkCJ2Qe7&exbI!z`OyPxBp|4_ zZrrc;OAb%T4Ze%7E}FBB`8t$QN0sA3vpwU>?7QAmE%-ethXdCtby$Qm3v$lNxB2a7 ze6F5eEWV`={#W(G)Va}7?$D65WF|f0nmfZT;?=LE6Yz{{W3CV2h^Ma+LXdZ(HMVKZ z!YXJ*34lo!FA>)jSo@*!Hs_)IwmTo6pBr3c^j2u_amZ~g;&Z2jZIw!}v@w8DtZz7|A%rFksD4^HYB!xFAqX;u0HxPeG!3Z(z z4}+^N5-nckKf2YSR5R_}PD+2?Wq#BOiON74#{`u=4f59WKdy_77EYq~_|X6cNtno{ zZ?WLwbV57Z6uI|uY_;vzv~~`eiiOl($Au7C*X<&MY5v0b`KEu-GW}{2UNfmmrP!^Y zAOczy!}TIJsom=}kxH)9W`&Rp&rR6T7y&~5nXbut;wcs@M?aa^9j{ZDtx=1?P8TV{ zee2kKf%CE$mogyKKT=xQQ#)OCl9bjc)}{p2X$}aG`^B0w0yi-rI!d4e-u9uR$kJK3 zhqBG9Wx<-3DFw5olJ6neF@hB;8o(r(GB_;p1i>}cjN`JNEZg-dlxtLL=8~gfLrBy_ z1~bGh{I>_xqh(}?%bCf1U6~K@+N*i}bTi+pUAW)oM0`D*PeJq=S(-|Plxe9OqxBRg zM((r)xkSH@j!8@+=cA4US0fDL&O?W~x=Mlu>7zvHO2sy7D5_7ulP+YMecP~}F0b*K z3oO2j{o&WHd<&UWcyA(&6hvBJv}qUZ!@R<(mwKB^;y3zeE1>LzbDWSkRD1|5MZPx( zxd=&MsQi1eE@@6W+4N`cF?yh!3R5JlAV--&RONWQ#?SbrQ95<@ag>C{jQmGXpQX{) z1dbFg1_`qLxuDZnX#PKfCW*Jl3F&^7@gO&{>Nb8um$VBcF1!AL=N6`A%BFj=`QaPI z+m^`n+{o)KLif;Gt|7aQ(XXRP@x)jJt}s{&S`I3}jPTY>$@W0BD3Oif^ehs~!H7T1FUSWxLS&W;0q6+azjbWn?3!q$ z9qbmdr4H4Y)p^NOACJ^L>u}NS8T0_5hW)G z%Hv}dAqM}d@t;|hf8>+NHHPi*xePsRlqr46njzhiXXZti7i5+GTKcrlxA->OJ9*Pna`02EIA5~(SMV`T@H6F2VtwwP1$tYujbC1^VE$Yd&I`WSwB^1( zT7NP3|85z#R%&wktjwY_i*n_$RRZPM^ota{LPV%*>=>sAv%fn*cnkCIX{^SJRmwZv z!?f@T&D%Lz@*!mNYTGp{J|7)~PR*ib`;l^E)rQw@)Qn0ECnB8W1S_SbLZWdqcmo?V zX5g0_3qhn4TrN27^x#Qdq*4*G1L|)I^b8GuP_8O{p|M`uvZO6McXa>OSQRW|kQTNPZ#Zyj~SZ<`6B)Y+}jxpn+YT>MhZ!Rxyd@rU>N zP>MkDBLX|<)SJaO?Ge=!D>i+Wq&PgneO?ZXUq4IQuTq z+V{ZGkuw77o~o$!b>4ov`6CKJ)$cf=S6%1ZQyYU!kz_qiuNxY2*Bh;K9J6o_YV6xQ znW|>x+#Mymu&wF9P|3wP*(ZjwE+ou|{eFqMv}d_iEyH zQ?NSf3VX+EpbrIKmp|oD-t_rh(D#e)fp)dYbG{=yPj-3-#l+iu7r+~#w|(#wv@G0` z38`Yhf5CznhyDEhD;jzaz7fc8L?(n-m zR#|5hqq#yRoeTm+h^9J42mnB>BY>HSu&&O-Hxo6j!dqck)dGS&odS@Hsk2-*Z~x z0!%{@gT645S5DeF@JZeE$DFl*nJB8Z|JKvs%7d`KjbJ*AsA_=fEZ&V9=*+K{(TF^( ztjjYr(7@fV^tDs9c*#=8)ZRKO17A5Z`8v*)U+?hS>3sEfgh3`#vFO^7n}&&adV?}n zdy&BY1h|I@eBm=l*kqiJn>vNkOH4l$Op5Hw3K_w8lF!6T@-H)S2W|Km#6!-X#NqLJ zsiVDrc%*@I3^Gen$)6O0C_qw;8{aucF;}U^1%YE`?AYTtb`Z$B$vfhcHQF`VCB(Pf z_G#fV*Colv-k!O+=^nDNe(03?m+RTu&28d%>JrrwFNb{ND&?Ad(=DP@voz$usk1|w z&#gTB7F)#*LtY6@pIb(g72*LcnXRlTPQAD?)ZFnB*EsZqxM&Uk_KGXnR{4}K`I6i- zU9}R>tiO0De1Hx=kAy>7O+nKO@kGQEYOai&S9&WTY+flvR?uhI695W-xZnq4aRMh8 zwfp)+KYWVB#r=5AwwlSdM4@x7-R_{2;1iqz2lXL$7iu1>5W*+I)jlkMs>60=LN)Y= zbPw;;%U+%p_&{2Obemh$BLmbpDd31YxJ8#TpH3~3B8QLUMvx1X5Vl48hWSNN*UTlO zQgQyZbmyjGC-s$3tnB z0mfKUu2+_c`ZVvDVwUy#j3W*l^BSXXQ%=r6Z}C73jx8DAk!t7k{dK^udpHIcUejp# zyx}og$Hr+f>9kaZvno*Om`d|VTUce9tHM=R8thoG!a=NT$s;g@n_rAN%cp7nnLuav z6}j56TSSfPL$p#y#!5TVyqa3zTzi7@#IoeR=E6CdS`JrR+@i2DwZ?T*bh+(k5!a)0 zgRdF93z8XJ|5?>hDN!YAW5cK=+BwDLNT_+otd zqC@*{S0hCKZ+TnN*2&qx+WP;ZjHA`yytPcwKl~)uy)sQ}Q*0-&3X|YFYAjmolaciq zxS$r5^fxICetD*Dw78M9leVvhAOZ$=;SP7L!Vs?+0f1h*YCuTXIt03iAf)0=0KEvZ zB69o-zg`0C#hQ>`4`}1g=a~EID(j9HbjJG^tV-zumR-+fahTPveA{%0u2uQwMZ%}5 zwY!|}i0oTd&>^QSRhIKU+cMC#|C3f>|647?v1B(wH)EWb{vuJEJh~!#|J7%=h!x3| zCH6m}wg;>Q&?@5Ct1%n`lj%*>9a52d@wmvE`=aQjtz$sWj3V;fDns5<7d2*``)u1( zh!Ub>!#N0m=Vz1n1=El zwb2IVRw$6NIFRpGyUoM0iqc$IPehcmm7<0s7F*Yv+zq?_%pf*SS~~}s0M`m(rMbx% zi?|Wjr6fJN`_J8&B2$4+V+iO~m>s~Zr2T3Y3HGREFQ%%pEoU0N));AeSVM#gYQ>l} z0`RhgS`R^pJH31YQ~eTeJiI}g$&^|nv{!h?8mJK{{XDt+sG8D`7)$jvM#hjPI(5sS zfFW4s7wao%Lo| z#pJRC?iZOai;57ANs|vm6%}rPlGo}}Aso1t#xJn}%VW@~1WSjh(@JTgM$0x6ZQ)gB zdiox3f>kqGZY}+R<;wlNoWJ8#X-v)1;wRD*ec*wnvsN06Q@cZuD`deT-Bu&G;2fBC z0FE1%pG@{Yo2O87&dE;w???%`9s1gs=3GpM8xx_}=AB$K9y=cD);^iE*p4;T1RU%B zBPr)yqOBX<2}xt%g9qr>;z&|?4vhhw7@$a}Uy2b%_^VdB^VfzrebKUPnq;hliCNU% zVt3R5EHkhN^Pv`REF+npA@#HdCQN9IbQbqSDs^+zt(A6;rLwN+@Em}WrV5vPEo!w^ zSCd3RZ8{7a@d9@|IF&&G%irS7FHle?@49LctrtTt=rP$W)se*#RkFmyf)D1^U6EYI zfh+N?uH?-))O$9zM19VsuGn8?o~5`scXU?!P@_cWP&1U4PQqGus=sQzrX+YvKG%XBL3nt6!&M<#}wqA;Mo(}qrq<1lNkpQD-T#-y>grt|E+JNU) z2j+g+QPcA9VEFc0k;H(hSNOpp$I+!$ z&d&W6kBM9+c{X%vr_X0}tdB5dvEDyk5H2*T(QW8Yz-#tjvF?up=^Kfym``^!&O-X! z@HdfpHn;}_)y$Xjb-5cR$Q#-XdhKpmJG5pl>h*Q2(u*gt_4(>6?kG)%T3*&TT0qI( zL!aR~4HiJiaHlgdNcOQP6xx1f3AWx&8}(NEps|G!cO>J^rE2@&-t#_Jb7GYgnLnML~1ze1D$?~BwbgA^=pr55tC|d7w42vN11_8bS75u z_MRKqE7Xik8fk>6(VE5{qT}6rSzd|o}Zb>*aI*Bwg%ccE$_ytH;g2H z^i3qY!+aE*&s^BMH9TI6GLm&9c`D6)3{-+?2Pon+040Yuv$2(LqV*krKhTg5CHOj* zquacxc1&~=S(O@gR8aI#?R%)meONmw1rub9E2QzeM$pBBm2wbPNR3tab{op53<oFwaUbARdD5jSA_6zmKX7!VicEP1m)rYnk{P- zruRj;4c8S29Rd#Baf|fq_pA^r3K#qRHS;($XNoLI*`puZjM?bA0tH>FDiVc9qR*|3 zGn#nhqxkvqFwRfCB~2yA0pxWapfjCdAem$utuon-`*6}mUP?l%$CE(FjAwL%Oe7GQbu7*+&q>*(cAofJr^gg>xw>hx-SO7Lx2)I} zJ)tV1XKbkE4sS&La#-smSq>S9gBzGLH%v?KVezdGv%Xs}kDJZJi{lDl(FpLZupBta z3iDlkd6LlkRro}+El?GIObw06D%NTXpL{W}Ve*%u#{wTC=+VHS%o`sAez&cYz|Tn` zcK_~pvN%cd^8FlFypCjTjw9@ulLoJ^!QAK*++^wC2~}CFeoY;q6y~r&f^+0>LR6)n z$hSev@GzzGgDc>)#u5_;{T9^5y5I?m=z7=J!eVId8p6R5>NV8)h|bA}#3KUufq4CPGiWYvGj%0=H@Q66);F)#cDMND4 zX|?rg>Bb28q*a!_sgVF(A=OeC&je$C4>$0%yy;Fla-hl(|9Ww4!@Q#E2hpJMMxpQ2L+R;+ZMpS+|j*F`Fh}p)`a_*<`AaeFzNEq^- zlF$7BFKD%p@K+3$Vx%N{QOayKKWU#JOAwXiLO62cA6=|DiDG_Z=ef;f&gQ5-?+Pb+ z)4NsyEZXCdjq5tgDN39V9!6#w25+R1;PD7ss;hFvQn}Hnl3^3h<`ylzJdVEL>|Jj0 zg>=Pscwx&;pWEzMn`ld**$1F-nhqlMuX;G{lWrT<<4$7MZ^*4a2hAMf)3eYiT$lRz&9({j<=%DWIRpgu zoOns@gF}AQ_6Y5RhySg7yMtJcYQap6^hgy{`zX1Zv26q4<)g@t%aIi|-lmcySuRN8*5f*$aEFi8o#kMKRCMnrAY~l`= zez#50^@Qo+6r508>iKfAbbc3JwCnjnmw;~=mlMG`(H8EJz7W6mh@mdinO&)#zHX=| z&|fo@s`;njVkkCMczSnp+TnW8YPU4w2&QmzEh1}orF~KlT=V+`!!rH|PtULCcL!P*m0EaN0Ad2qBw%Gs40jfu=%`N*k@z2-p?&B?Yum-p+h?7(!D^ z&f2Bn_#t!4HM2y^*1GN;U+_x8T$Z2>U9Yx;p_9Qf=ww z2hxO^*{%p9-CwMKz}C4mTi8xvqhivltE|}Kgq5MK@f6tBT&`@RYzsFFi>*eMZ0Z6Y zKBl`GOh!U%C+PXJ|7PF)V*~#8eS80D@v-NL2U&;i62W}k+vJAC+7xF`eq%c0b?{PVTcqiDr%6jLBdkVcTwLJSd313SP)1r=;2`cORbMzrhqZxMWcTWru5-l_H8;f|?{^M%%7>sU zGx2{fX*t;7SewS|NvPR-6F5p(ji7d}CK#%7y}jsPkgj%F5cUbQ?b7uWpYks^|DL*n zau%X$^(%wXMS3c;C4=p*#q>ahmLH5woLsn-YcZP~mH-rGnRyl#KU4MsLu+G3z90+q zM$HCWgZYR`8_I%8)SYuBltP$sN`-6hcjnzhDsVl+Y}yqMN*4MWsJX_6R>Cyw8cHGQ z1>r%vkDxxc#ACA4+-ZO|QBMUz`YHrS{l-*$> zi(n_;4{Gn+d2gn)TA<9) zibWdKJv#s_f5K}vM=d0NaYrd;5A+Fy^=+WgKC`@bS>!P5@K4fzE#VYfMcNdbbvLPY zeR~!f3xU>|pfq-LOsoF=t94x%K!8>#8tR4KQ2G3Yr?Cb98^KL*+G8``rHMpNUN}-T z5HGAkiLh{WR;N$Nk3X_2^3pW=vOFTOb(LS0Wu)0)I{8sZj>}5ZGtD=va-72l&5`L= zhyzBWie2UrC|?(sTcuk$OwvV4oVlxc3ncXPj|cD%%*6(hoKMd5wzPQs^6g)B0xK#d zemOodB7D(!@v!|eYqMfx@M#b+D)PwAuvimOW#13i-xAR5)Ai; zXNX(A@M*y&+TVZI zGHo$F*Ipg~Rnp`KlMNAl2o86}r%Yv9#!O-oo`pe`880;-Y28tR)b4H%nqXXHxN9m0 zI&#!(XhT=T3$WS$)K4#Y=ceN`MsP0v1X{nIoQ14S2^--MnUp21=V3&Uv8|y}^}7Vl zI5tRbOp#?@ay6uncZFE0hg}kt(k%piw^M8;0yynsK_!l~uP??IqzmKJMUqAW^GG{~ z7Fg)Q&zBlp z%Tj8jOUpuR>YHP6zYsX?)aJ`)_pRwu+Tn8I;brOW_`v$u$`$9T)cO*O$j=?mg>dW$ zw=&3=v||fqCr`-$okN*$S9(Nyrs}+Lu#IwDg2xSBz_VfU*?A&26vwv>&>*U_TT7-7 zS~X}fT%9+q(Xvc0qzOG^8gmMcZE9izi5feqvY(aY=%reP+wVZ&cRd`^y6}-gJ&_6n zR%Wdl3vQ4DOt!X9ry7j%=+7pLPdus*@7dZMBo0_WKZPD1(o{=;D> zyc9_WFI3{URv=d6EXcnOG0$(J(R#8Oz$kmuSFQ{-Y20}1027!FkodTU!fouSybwqn zRO-$2BH(w4)$wiPo<1w-4*p=Q0@YKRm^cgiA>~ho)U8^e>SBk*!@xvr0CdvnLHS#CACVuQfgzF>8qV znqf{oO1}RWhiZ3g!Tx9sk!JfLqcP`>Ksx#vZuLg-DC6h4mT!vlU zqw0`0CzZgY!EN0*{sQnDNFn;T<+e_x$zY|n;p0@d^hK*n!S!=#^;P{*D^6~h!T7r6 zoiMxtovMo-dj*{qZPy*c3gaMBEDQDkINU%d8HeBZVlRuzkCId9rx{?L= z-dLlk$w&JX5wn+8`mtqCpKnx+w+$@6DEUI}8P%xN$MEsw%S1-$9PM6r^jP-@?cS<# zhg$wl0X=s3{8EZ2U9(};p{X_b1@jJuGgx`gDK{6MpF|XON_=Rv%-<Ee1cuuy?nl9xVDa~x=+8ppnOQ9 zN$53qi4QQ!co(;f!#YJ8(=Z>_9UF#(QOVjS7T!g2)*Oecrf-R^)tFugBkQsMVNua# zS;1V^#fJS{h+!O+FgS%0=Pd9;lMa0QHn?-n(<0b2$<|@r>fjiyw6u*UoGmU$ayJM@ zfp;c4@{$b*Z_v9?8ZEp{m6Q(mDHW<``n?jg-ZN)Hhvxn*l=O1f*K%{5s77WCt!ugS?*2oG5-Q)JEJd0+W5=doeD$Wh?U$ZRg)K$v8cmQ{hba9jw_mF&X zi-dV?WITgIz!!0uB~jE?(t`&qo{WGyUspX| zc6+F2K4l5$LqxERF#`I&k^^opVIMZjGhsJ^vI0c%kV+|&_k>~}ueTtj;^Dfb@xHs` z)-39elzVA~D~n_aoyBQ1>Qd2!;E!G*pZM&RX`r*y)b`yxvP2;#vM*;CQGPg|gni)} z47`Log3PUyVfdmJ2zvHBhg7T#D-H=myzkeUa$@);WC(yB4k^*$wda3=S-UH5Q1Hx6 zPcGxMP&kXBa+4$s#Sw3-V?mlHj^8&bLpIN~GkYj;!;M!$ZxvtQY4j&Ngz_mxuQRqx zYTbN6epx@-!0jRV5yiSIJ<^mCZ<|;&x2~a)t+(eAVB!1XpCZok*Z2C5P7&>z-Oy?t zf@F(_FLsSrfCus61+Vt~svP%(u<4pzT5{w*0XqfPV%~|=%aq^$=*U+_trGQaoUxbt zBV#Yqx+ULku8yPJs4gGcC?+3iRt_6)Oi0DNLxdb(!n!cup_XUZ3eDe(!DChZ!IG&L?_;T-1GB!R;;Sk;l3Y*JQ!I|l20_f}ZyC;4D7R@6F z>%z~wV;Bj1b(*kp26Ed!Y-OKxNbt3%t))xxOrazWsmwvW;uaSaJ0ou+{01vXvU>_V z6Ha@+;giVaiyg`J8ENQf)Pq>!Nf22>XFHnXTNk84&jp-^YwmlUqnOll8)5mzlO$o! z#fSMwH8Pn+Fy7O5M5#ZGr$cKfaGf8g;XN)<*TrQjMk<}_oRf&b6qZoR38Q{Zxo{V; zby+J_hCZT1>`4~jnQxo|ji%BQ0=BLzC6c!1=B(jS5+fcp%q)JI)=c3{D|=k5;0&c2 zrbRE|qxkNqah2nvextOvjYA{T43n1c6eO7B9DH)tLqB46E7;0xKM=%#wx-*-+*OY{ zQ#7gMStz%I&2&rbo>#T20OD_#g`WYbt9+!MC08%zSMhqMoRk)7VOk%~`sD%(U6zzO zdmSC9@x0GCv2_)umYc5@#%efP0_cu+=f^}k$H9$N_>piA_(5UM_o{++8+Yf8SJ)?C zDd3l=GGm3EEy;&Z6N=+XP@IM0L=uW^ooyYQYyx1vwFR?@U~BAtAqTu%Mi2 zTCQh$K=UZA{P`Cw0I$xAh_f?fq-Goe`7I38{3L8?K3`lRhSAyB)tHT@4c!Y;bJAAS z3u>Q7qx>9SJs4$EB=hxh)u`W5jp?>^g1s_MV7<1zN zXt{FSt?Mt&8aCy67<)b@eg@h0iCW@%+pF-V>p${fyEk6_Gvp|ms{Whi-9eNId?xzZ zm|MI>F;JSuaUnQp#|}k3o&ddCZEeTI608txuU4~7K(wg9 zg%+}(7h2@(%>LI1F*puF(h$ZD`Q+ar!VoVajPY0-XS$>6F_F?sc6Mr7>SL-&{pC;2 zKx@2{@ULz7RCpaKg$iu2rcY+y*~qaPo0}^7T1K$_(NPS<1;V zTj8-xC%WvgDI_YYEG{bySvyO3M>XKY)oXgGG*eB{yDgNQ3s3)A~@n>!O#lNh0! z(-dqW#_z&mMfq#2+u61N`L^({4UoU8wE5`4c}{SGFzKb(BK8hM%cf_zj_HmC48)M& z398ICVJTGzBaz7K{L+Ew=;z^0xA``wbtPs`r+Wrb^_vzzhukq{;A`t&-ktzb zbqy`Z0#D6fdVAiodjF3J+qI*vu#=OCjiL4bIIXEf4?zmN7(H|+<+WfR7@7jrMx7FY z5*0X1enhay-q^M?j}3Pd^|U9(C3#CQU3=hlc~@y9@NQD{UZNfC^5?Cuuuu{ebn_<7 zEzudv*b@QP%)N^5jP;86nQGb<*SOytCM5wmf-=rH#K{Wd$2(X#S$jF}XIxZC1)zir zU2Wq>hIB44nCTqx2x<{_wiVzLSJR}L%P!Y|lFHtA_=bDj=OqvmmSZ}ffuqPge#V-f zZDk|XX0RK}=73LxL`H%OXxK*^I2!fp&kxatErK~&tM3@j1a(Yrq$z)R()i?}p|0^Y zhW&8!IpRA1jJ3e!p66ZY=eBmEA+$A`!%s+{Cz!s$IA`{_Dh0^jt!vn;+Nw}hx019Q z_Wg=#-G-~&@>l=&H~48$L8`LX)!Bcq%(DFa2Loc91u@WcwlHzJwo{cdur>bQ;{fr_ z`rC5QRQ_)`8EadJzz-{K&sUI~>NX>P|c4l)fKS0gkuGe_P ziaQy!%CK(CtAwj-J8&#kyU=G(k%3y`!gS9dU&1xIrGRL|!&aVMEaezUIpopoET~xE zp`%~`LZfn!Lu^+00?>v4UOfM!HeeQoLZP<#o`^9oi69|$0BM?n17R~tGpY)eJiv@$ zTV-~ZZ*}C1J{a}p`>l$Bx8qRBq91;dLdmp84auzmcd|XzJG%I|r z^E-8Tm~jRn_>as(R=@~z3I2E3<=#hXn>A=0`wfOGIxiP)N2%!cG?&^w=E#TR z`lSY@Mm36zu4p3}+S#67MpL$d{gf@dnP%*ZMW=gCXK-%0E(xAC!^+b7hCSMF$m;Rn zCTErbBK#;a)>kHX5}w6PRmnw(!Gy>m_g*2opfklHyx>eb1bu|_lwJdf!ogxhk}X^v zc+^L;F7ta!8+i%6?M}XvQn4b%aOSCpDW+4#JDDG(wvXC*9%9(XBhbv4LX3R5G&(+@ z)nbdivYRQ5pW;9~@YGf{h~Rm(@MfV8Tj&T@EejO6(C#(+z7FVNBR`@j!#wScHM5ki%j+^GykUJ2m zYgpwm;#Q)~LoozUSV($?r3vQ~#ZU_}ggl~J%z*1dYt_^4K6e7o&qs_ORz{km+D+^a zqDdUO)d}|)v9h(Zz3}#DLWyRVCY!=PMCO{=PA)Upb@)1j?c)||l{6&pI=;U#bS#Jk zOOiwVH3FM!SuJDIPnN$|ZKz5fQwHmzn8f^?B+T2ew%~PSE#X_jk`Wu;a{4}9%AHg7 zZm8^bAee$bdpwklIE`$fV15=pI+tgJpll4uQjIM;Q!gvISFc_{@=lUSc-lABE%U?+ zHW$;!NcH1&F;AS~7RH=n<=!NTKnm3t`B@YeL?8d2{WGrmSjG;yBbY*9$N&DT^e?l2 z|1A2482Or7n7KF_TpRn|nmqD}`-=?QJ0z5q$C9Td^sML&aN7OGi+W$uYjDXKJg+0W@S=FoQP2dBI=48|FH>p2mh zFrdu!AwoG$NkvnZp_KT8HEo=RNNJ4IxucGXLr2N*I5Ao>Efb+pNOm9Zw0_7_s|9ac zS6}W##>$W*cBmksip;43p#a4&iTpM)8(gRGekW+AKm5zb)xpUFT>~b+FOH`Zs!$RDgpSCE z>;CL8Uu|EWeR~TvgDX@K=mtReFed;FZ!M2SjzW35i;UqfyemM?rq5yZS#hK5Y~|wt z2#^`Q6$b~uGT_++C3+B~#(oFHdSL&hh`Z8{t5#=ZkoaWVJoLm)3vT_@5HOnZGa;s~ z;4=E`3Eo@=$BxFjS`Iu|8SALB`<#TPTeE%h(dol+#CzJ=Zb&EHpw*=0H*~8x6 z`G`b<@>L2(AS*J!NVp`DN{g!8R#h(~URslf zC8PwGM$5V}+$WcoT*C~*$WmCpS6Gis&sZo|9OfRiwjX$f*&25Gjv6$YPde1smwGw( zb@y=gbl1!8>hm-il3&~zFca0~aJN!?b97+$E>2$Gn$31OR&UnE=Tm= zH44$Dx2HNN1lrCGjfuwo@+(m2j85w-oxre9FopupEV+6HACFyTbt}s-`lCCJ8om5RIE~T#Yg_DWu1u zyAp%jp;3&%D4;CRaR6g=f*ZvPqw2BadP=*ZYy_~CV3@wFx5YA(E8)jfqx z8tjEkMf>msMqi)zaY2fWrMq`lZzZdiMcluc(@(yxK(4hPEFk0~HO3^CUZk3;?Tv3` ze-rjZ8@hBrVPzA$^4hW?<33{d2)h7Jw?$t%V6(C_m+bNhXl9vXCJcBWmMeQoLDm5b zt9|A5pDHY#Y@(rlEo_WzXila!uaZE*WVc`=IM)SSc`#liZ2Wt*~fHgm9uH^ISX2d@)XGZ)_$qnbx6?J<14_=SS(ITs#LPDk03a&%x;bAuGz=P ze^<4p@tD@J|M;88;~IsEOPpB+&3C4!3q;}Kk2tb*WuuE z2u(BE$1(2AwbbBrmU-YLI4>#K((6&QZ~m2Yp;I14x0N8hos}{uoQuMG)Wy?ogaNayqmc&`I=8y6&dPf{Fky#B7 z#F=Xy213s`NFxjKuMqH3+ibWsFRi=QtH*j$9^)Zy8F|^vSmgj~l5<04MiU;BNyAn) zlM+c20Y#%@>WgdY>5kx}H)7*!D~BZJdg8d5iHx|>(jj=!MEmr)-$kH8?A#;DyBone(uz;e^|=9nIwfuWY?yw; zC|H`;8#O$vTPm5AW1Gg-Up&#Ca$<@!JZkAUDbmd*?X}QSA5$(*c+FZ|l+}F%*L1OH z{ck}P=j@=7>6ga#cqzj|ODXHD>ckIBmOd9Fh=~>?C7$uII_3rEX%UKdywsInR~{t- zg|t`~l=L1P_QPkZN53Q>!^A*QDZ zK(f;%VVQo)n1bsy)LWL#?&|wN`hL~Rnxhd3d-bOvlRQAiybH&=i;SlnwP$3P-!%x3^o)t6aoT-zXU}ARq-l^bOW-zg$@b|19Aua zF+k$V!uO;fNwCUEi;6!|5?4_MKtTq}|C`2gXh8EhWP1bTgZ)DqHZ&-x|E2*6Ka!RZ zS5jsHN&IW7%g1yUln@bn$cO!hR2b+`P~1-3dFIx!6EltRa{a z6Z@Y$_ug)~d%u)K$+?LYfc<87}bupdiK(3|m%hiA$Pc>zKNP0hqBj{X*L0rm@j(0s(f>>t{1L0?w#rS+#E)IdBKcF5|Dq-S zZ*-X3x;NeSuOSxS<3Q%uy1zwQ+?Kj&)Ou~-|2+&J{Zi^T=lx9+&+B^K_lQ;hY2H6D zeZ9T!H&;?$+kt+MLCs%i{8QEVi8<(Pft!mFt`}r~k5Y%93jAjQ!fgoD?Zh|Vi~q5A z27G^+_!lc1Zfo3}625-J{(B@p`IW|R4(!c|yX*Pn?*SA0)3iUGUB11uH>ab1{F$$g z|7q4=O#$9cezU54J)`wKI1_%J{14{0Zj0P3wEcKU`%-=?@(1PW+Zs0qGuI`%??IID dD~*3C;60WFKt@K_BOwYX49GZ$DDV2e{|AYb(KrAA literal 60756 zcmb5WV{~QRw(p$^Dz@00IL3?^hro$gg*4VI_WAaTyVM5Foj~O|-84 z$;06hMwt*rV;^8iB z1~&0XWpYJmG?Ts^K9PC62H*`G}xom%S%yq|xvG~FIfP=9*f zZoDRJBm*Y0aId=qJ?7dyb)6)JGWGwe)MHeNSzhi)Ko6J<-m@v=a%NsP537lHe0R* z`If4$aaBA#S=w!2z&m>{lpTy^Lm^mg*3?M&7HFv}7K6x*cukLIGX;bQG|QWdn{%_6 zHnwBKr84#B7Z+AnBXa16a?or^R?+>$4`}{*a_>IhbjvyTtWkHw)|ay)ahWUd-qq$~ zMbh6roVsj;_qnC-R{G+Cy6bApVOinSU-;(DxUEl!i2)1EeQ9`hrfqj(nKI7?Z>Xur zoJz-a`PxkYit1HEbv|jy%~DO^13J-ut986EEG=66S}D3!L}Efp;Bez~7tNq{QsUMm zh9~(HYg1pA*=37C0}n4g&bFbQ+?-h-W}onYeE{q;cIy%eZK9wZjSwGvT+&Cgv z?~{9p(;bY_1+k|wkt_|N!@J~aoY@|U_RGoWX<;p{Nu*D*&_phw`8jYkMNpRTWx1H* z>J-Mi_!`M468#5Aix$$u1M@rJEIOc?k^QBc?T(#=n&*5eS#u*Y)?L8Ha$9wRWdH^3D4|Ps)Y?m0q~SiKiSfEkJ!=^`lJ(%W3o|CZ zSrZL-Xxc{OrmsQD&s~zPfNJOpSZUl%V8tdG%ei}lQkM+z@-4etFPR>GOH9+Y_F<3=~SXln9Kb-o~f>2a6Xz@AS3cn^;c_>lUwlK(n>z?A>NbC z`Ud8^aQy>wy=$)w;JZzA)_*Y$Z5hU=KAG&htLw1Uh00yE!|Nu{EZkch zY9O6x7Y??>!7pUNME*d!=R#s)ghr|R#41l!c?~=3CS8&zr6*aA7n9*)*PWBV2w+&I zpW1-9fr3j{VTcls1>ua}F*bbju_Xq%^v;-W~paSqlf zolj*dt`BBjHI)H9{zrkBo=B%>8}4jeBO~kWqO!~Thi!I1H(in=n^fS%nuL=X2+s!p}HfTU#NBGiwEBF^^tKU zbhhv+0dE-sbK$>J#t-J!B$TMgN@Wh5wTtK2BG}4BGfsZOoRUS#G8Cxv|6EI*n&Xxq zt{&OxCC+BNqz$9b0WM7_PyBJEVObHFh%%`~!@MNZlo*oXDCwDcFwT~Rls!aApL<)^ zbBftGKKBRhB!{?fX@l2_y~%ygNFfF(XJzHh#?`WlSL{1lKT*gJM zs>bd^H9NCxqxn(IOky5k-wALFowQr(gw%|`0991u#9jXQh?4l|l>pd6a&rx|v=fPJ z1mutj{YzpJ_gsClbWFk(G}bSlFi-6@mwoQh-XeD*j@~huW4(8ub%^I|azA)h2t#yG z7e_V_<4jlM3D(I+qX}yEtqj)cpzN*oCdYHa!nm%0t^wHm)EmFP*|FMw!tb@&`G-u~ zK)=Sf6z+BiTAI}}i{*_Ac$ffr*Wrv$F7_0gJkjx;@)XjYSh`RjAgrCck`x!zP>Ifu z&%he4P|S)H*(9oB4uvH67^0}I-_ye_!w)u3v2+EY>eD3#8QR24<;7?*hj8k~rS)~7 zSXs5ww)T(0eHSp$hEIBnW|Iun<_i`}VE0Nc$|-R}wlSIs5pV{g_Dar(Zz<4X3`W?K z6&CAIl4U(Qk-tTcK{|zYF6QG5ArrEB!;5s?tW7 zrE3hcFY&k)+)e{+YOJ0X2uDE_hd2{|m_dC}kgEKqiE9Q^A-+>2UonB+L@v3$9?AYw zVQv?X*pK;X4Ovc6Ev5Gbg{{Eu*7{N3#0@9oMI~}KnObQE#Y{&3mM4`w%wN+xrKYgD zB-ay0Q}m{QI;iY`s1Z^NqIkjrTlf`B)B#MajZ#9u41oRBC1oM1vq0i|F59> z#StM@bHt|#`2)cpl_rWB($DNJ3Lap}QM-+A$3pe}NyP(@+i1>o^fe-oxX#Bt`mcQc zb?pD4W%#ep|3%CHAYnr*^M6Czg>~L4?l16H1OozM{P*en298b+`i4$|w$|4AHbzqB zHpYUsHZET$Z0ztC;U+0*+amF!@PI%^oUIZy{`L{%O^i{Xk}X0&nl)n~tVEpcAJSJ} zverw15zP1P-O8h9nd!&hj$zuwjg?DoxYIw{jWM zW5_pj+wFy8Tsa9g<7Qa21WaV&;ejoYflRKcz?#fSH_)@*QVlN2l4(QNk| z4aPnv&mrS&0|6NHq05XQw$J^RR9T{3SOcMKCXIR1iSf+xJ0E_Wv?jEc*I#ZPzyJN2 zUG0UOXHl+PikM*&g$U@g+KbG-RY>uaIl&DEtw_Q=FYq?etc!;hEC_}UX{eyh%dw2V zTTSlap&5>PY{6I#(6`j-9`D&I#|YPP8a;(sOzgeKDWsLa!i-$frD>zr-oid!Hf&yS z!i^cr&7tN}OOGmX2)`8k?Tn!!4=tz~3hCTq_9CdiV!NIblUDxHh(FJ$zs)B2(t5@u z-`^RA1ShrLCkg0)OhfoM;4Z{&oZmAec$qV@ zGQ(7(!CBk<5;Ar%DLJ0p0!ResC#U<+3i<|vib1?{5gCebG7$F7URKZXuX-2WgF>YJ^i zMhHDBsh9PDU8dlZ$yJKtc6JA#y!y$57%sE>4Nt+wF1lfNIWyA`=hF=9Gj%sRwi@vd z%2eVV3y&dvAgyuJ=eNJR+*080dbO_t@BFJO<@&#yqTK&+xc|FRR;p;KVk@J3$S{p` zGaMj6isho#%m)?pOG^G0mzOAw0z?!AEMsv=0T>WWcE>??WS=fII$t$(^PDPMU(P>o z_*0s^W#|x)%tx8jIgZY~A2yG;US0m2ZOQt6yJqW@XNY_>_R7(Nxb8Ged6BdYW6{prd!|zuX$@Q2o6Ona8zzYC1u!+2!Y$Jc9a;wy+pXt}o6~Bu1oF1c zp7Y|SBTNi@=I(K%A60PMjM#sfH$y*c{xUgeSpi#HB`?|`!Tb&-qJ3;vxS!TIzuTZs-&%#bAkAyw9m4PJgvey zM5?up*b}eDEY+#@tKec)-c(#QF0P?MRlD1+7%Yk*jW;)`f;0a-ZJ6CQA?E%>i2Dt7T9?s|9ZF|KP4;CNWvaVKZ+Qeut;Jith_y{v*Ny6Co6!8MZx;Wgo z=qAi%&S;8J{iyD&>3CLCQdTX*$+Rx1AwA*D_J^0>suTgBMBb=*hefV+Ars#mmr+YsI3#!F@Xc1t4F-gB@6aoyT+5O(qMz*zG<9Qq*f0w^V!03rpr*-WLH}; zfM{xSPJeu6D(%8HU%0GEa%waFHE$G?FH^kMS-&I3)ycx|iv{T6Wx}9$$D&6{%1N_8 z_CLw)_9+O4&u94##vI9b-HHm_95m)fa??q07`DniVjAy`t7;)4NpeyAY(aAk(+T_O z1om+b5K2g_B&b2DCTK<>SE$Ode1DopAi)xaJjU>**AJK3hZrnhEQ9E`2=|HHe<^tv z63e(bn#fMWuz>4erc47}!J>U58%<&N<6AOAewyzNTqi7hJc|X{782&cM zHZYclNbBwU6673=!ClmxMfkC$(CykGR@10F!zN1Se83LR&a~$Ht&>~43OX22mt7tcZUpa;9@q}KDX3O&Ugp6< zLZLfIMO5;pTee1vNyVC$FGxzK2f>0Z-6hM82zKg44nWo|n}$Zk6&;5ry3`(JFEX$q zK&KivAe${e^5ZGc3a9hOt|!UOE&OocpVryE$Y4sPcs4rJ>>Kbi2_subQ9($2VN(3o zb~tEzMsHaBmBtaHAyES+d3A(qURgiskSSwUc9CfJ@99&MKp2sooSYZu+-0t0+L*!I zYagjOlPgx|lep9tiU%ts&McF6b0VE57%E0Ho%2oi?=Ks+5%aj#au^OBwNwhec zta6QAeQI^V!dF1C)>RHAmB`HnxyqWx?td@4sd15zPd*Fc9hpDXP23kbBenBxGeD$k z;%0VBQEJ-C)&dTAw_yW@k0u?IUk*NrkJ)(XEeI z9Y>6Vel>#s_v@=@0<{4A{pl=9cQ&Iah0iD0H`q)7NeCIRz8zx;! z^OO;1+IqoQNak&pV`qKW+K0^Hqp!~gSohcyS)?^P`JNZXw@gc6{A3OLZ?@1Uc^I2v z+X!^R*HCm3{7JPq{8*Tn>5;B|X7n4QQ0Bs79uTU%nbqOJh`nX(BVj!#f;#J+WZxx4 z_yM&1Y`2XzhfqkIMO7tB3raJKQS+H5F%o83bM+hxbQ zeeJm=Dvix$2j|b4?mDacb67v-1^lTp${z=jc1=j~QD>7c*@+1?py>%Kj%Ejp7Y-!? z8iYRUlGVrQPandAaxFfks53@2EC#0)%mrnmGRn&>=$H$S8q|kE_iWko4`^vCS2aWg z#!`RHUGyOt*k?bBYu3*j3u0gB#v(3tsije zgIuNNWNtrOkx@Pzs;A9un+2LX!zw+p3_NX^Sh09HZAf>m8l@O*rXy_82aWT$Q>iyy zqO7Of)D=wcSn!0+467&!Hl))eff=$aneB?R!YykdKW@k^_uR!+Q1tR)+IJb`-6=jj zymzA>Sv4>Z&g&WWu#|~GcP7qP&m*w-S$)7Xr;(duqCTe7p8H3k5>Y-n8438+%^9~K z3r^LIT_K{i7DgEJjIocw_6d0!<;wKT`X;&vv+&msmhAAnIe!OTdybPctzcEzBy88_ zWO{6i4YT%e4^WQZB)KHCvA(0tS zHu_Bg+6Ko%a9~$EjRB90`P(2~6uI@SFibxct{H#o&y40MdiXblu@VFXbhz>Nko;7R z70Ntmm-FePqhb%9gL+7U8@(ch|JfH5Fm)5${8|`Lef>LttM_iww6LW2X61ldBmG0z zax3y)njFe>j*T{i0s8D4=L>X^j0)({R5lMGVS#7(2C9@AxL&C-lZQx~czI7Iv+{%1 z2hEG>RzX4S8x3v#9sgGAnPzptM)g&LB}@%E>fy0vGSa(&q0ch|=ncKjNrK z`jA~jObJhrJ^ri|-)J^HUyeZXz~XkBp$VhcTEcTdc#a2EUOGVX?@mYx#Vy*!qO$Jv zQ4rgOJ~M*o-_Wptam=~krnmG*p^j!JAqoQ%+YsDFW7Cc9M%YPiBOrVcD^RY>m9Pd< zu}#9M?K{+;UIO!D9qOpq9yxUquQRmQNMo0pT`@$pVt=rMvyX)ph(-CCJLvUJy71DI zBk7oc7)-%ngdj~s@76Yse3L^gV0 z2==qfp&Q~L(+%RHP0n}+xH#k(hPRx(!AdBM$JCfJ5*C=K3ts>P?@@SZ_+{U2qFZb>4kZ{Go37{# zSQc+-dq*a-Vy4?taS&{Ht|MLRiS)Sn14JOONyXqPNnpq&2y~)6wEG0oNy>qvod$FF z`9o&?&6uZjhZ4_*5qWVrEfu(>_n2Xi2{@Gz9MZ8!YmjYvIMasE9yVQL10NBrTCczq zcTY1q^PF2l!Eraguf{+PtHV3=2A?Cu&NN&a8V(y;q(^_mFc6)%Yfn&X&~Pq zU1?qCj^LF(EQB1F`8NxNjyV%fde}dEa(Hx=r7$~ts2dzDwyi6ByBAIx$NllB4%K=O z$AHz1<2bTUb>(MCVPpK(E9wlLElo(aSd(Os)^Raum`d(g9Vd_+Bf&V;l=@mM=cC>) z)9b0enb)u_7V!!E_bl>u5nf&Rl|2r=2F3rHMdb7y9E}}F82^$Rf+P8%dKnOeKh1vs zhH^P*4Ydr^$)$h@4KVzxrHyy#cKmWEa9P5DJ|- zG;!Qi35Tp7XNj60=$!S6U#!(${6hyh7d4q=pF{`0t|N^|L^d8pD{O9@tF~W;#Je*P z&ah%W!KOIN;SyAEhAeTafJ4uEL`(RtnovM+cb(O#>xQnk?dzAjG^~4$dFn^<@-Na3 z395;wBnS{t*H;Jef2eE!2}u5Ns{AHj>WYZDgQJt8v%x?9{MXqJsGP|l%OiZqQ1aB! z%E=*Ig`(!tHh>}4_z5IMpg{49UvD*Pp9!pxt_gdAW%sIf3k6CTycOT1McPl=_#0?8 zVjz8Hj*Vy9c5-krd-{BQ{6Xy|P$6LJvMuX$* zA+@I_66_ET5l2&gk9n4$1M3LN8(yEViRx&mtd#LD}AqEs?RW=xKC(OCWH;~>(X6h!uDxXIPH06xh z*`F4cVlbDP`A)-fzf>MuScYsmq&1LUMGaQ3bRm6i7OsJ|%uhTDT zlvZA1M}nz*SalJWNT|`dBm1$xlaA>CCiQ zK`xD-RuEn>-`Z?M{1%@wewf#8?F|(@1e0+T4>nmlSRrNK5f)BJ2H*$q(H>zGD0>eL zQ!tl_Wk)k*e6v^m*{~A;@6+JGeWU-q9>?+L_#UNT%G?4&BnOgvm9@o7l?ov~XL+et zbGT)|G7)KAeqb=wHSPk+J1bdg7N3$vp(ekjI1D9V$G5Cj!=R2w=3*4!z*J-r-cyeb zd(i2KmX!|Lhey!snRw z?#$Gu%S^SQEKt&kep)up#j&9}e+3=JJBS(s>MH+|=R(`8xK{mmndWo_r`-w1#SeRD&YtAJ#GiVI*TkQZ}&aq<+bU2+coU3!jCI6E+Ad_xFW*ghnZ$q zAoF*i&3n1j#?B8x;kjSJD${1jdRB;)R*)Ao!9bd|C7{;iqDo|T&>KSh6*hCD!rwv= zyK#F@2+cv3=|S1Kef(E6Niv8kyLVLX&e=U;{0x{$tDfShqkjUME>f8d(5nzSkY6@! z^-0>DM)wa&%m#UF1F?zR`8Y3X#tA!*7Q$P3lZJ%*KNlrk_uaPkxw~ zxZ1qlE;Zo;nb@!SMazSjM>;34ROOoygo%SF);LL>rRonWwR>bmSd1XD^~sGSu$Gg# zFZ`|yKU0%!v07dz^v(tY%;So(e`o{ZYTX`hm;@b0%8|H>VW`*cr8R%3n|ehw2`(9B+V72`>SY}9^8oh$En80mZK9T4abVG*to;E z1_S6bgDOW?!Oy1LwYy=w3q~KKdbNtyH#d24PFjX)KYMY93{3-mPP-H>@M-_>N~DDu zENh~reh?JBAK=TFN-SfDfT^=+{w4ea2KNWXq2Y<;?(gf(FgVp8Zp-oEjKzB%2Iqj;48GmY3h=bcdYJ}~&4tS`Q1sb=^emaW$IC$|R+r-8V- zf0$gGE(CS_n4s>oicVk)MfvVg#I>iDvf~Ov8bk}sSxluG!6#^Z_zhB&U^`eIi1@j( z^CK$z^stBHtaDDHxn+R;3u+>Lil^}fj?7eaGB z&5nl^STqcaBxI@v>%zG|j))G(rVa4aY=B@^2{TFkW~YP!8!9TG#(-nOf^^X-%m9{Z zCC?iC`G-^RcBSCuk=Z`(FaUUe?hf3{0C>>$?Vs z`2Uud9M+T&KB6o4o9kvdi^Q=Bw!asPdxbe#W-Oaa#_NP(qpyF@bVxv5D5))srkU#m zj_KA+#7sqDn*Ipf!F5Byco4HOSd!Ui$l94|IbW%Ny(s1>f4|Mv^#NfB31N~kya9!k zWCGL-$0ZQztBate^fd>R!hXY_N9ZjYp3V~4_V z#eB)Kjr8yW=+oG)BuNdZG?jaZlw+l_ma8aET(s+-x+=F-t#Qoiuu1i`^x8Sj>b^U} zs^z<()YMFP7CmjUC@M=&lA5W7t&cxTlzJAts*%PBDAPuqcV5o7HEnqjif_7xGt)F% zGx2b4w{@!tE)$p=l3&?Bf#`+!-RLOleeRk3 z7#pF|w@6_sBmn1nECqdunmG^}pr5(ZJQVvAt$6p3H(16~;vO>?sTE`Y+mq5YP&PBo zvq!7#W$Gewy`;%6o^!Dtjz~x)T}Bdk*BS#=EY=ODD&B=V6TD2z^hj1m5^d6s)D*wk zu$z~D7QuZ2b?5`p)E8e2_L38v3WE{V`bVk;6fl#o2`) z99JsWhh?$oVRn@$S#)uK&8DL8>An0&S<%V8hnGD7Z^;Y(%6;^9!7kDQ5bjR_V+~wp zfx4m3z6CWmmZ<8gDGUyg3>t8wgJ5NkkiEm^(sedCicP^&3D%}6LtIUq>mXCAt{9eF zNXL$kGcoUTf_Lhm`t;hD-SE)m=iBnxRU(NyL}f6~1uH)`K!hmYZjLI%H}AmEF5RZt z06$wn63GHnApHXZZJ}s^s)j9(BM6e*7IBK6Bq(!)d~zR#rbxK9NVIlgquoMq z=eGZ9NR!SEqP6=9UQg#@!rtbbSBUM#ynF);zKX+|!Zm}*{H z+j=d?aZ2!?@EL7C~%B?6ouCKLnO$uWn;Y6Xz zX8dSwj732u(o*U3F$F=7xwxm>E-B+SVZH;O-4XPuPkLSt_?S0)lb7EEg)Mglk0#eS z9@jl(OnH4juMxY+*r03VDfPx_IM!Lmc(5hOI;`?d37f>jPP$?9jQQIQU@i4vuG6MagEoJrQ=RD7xt@8E;c zeGV*+Pt+t$@pt!|McETOE$9k=_C!70uhwRS9X#b%ZK z%q(TIUXSS^F0`4Cx?Rk07C6wI4!UVPeI~-fxY6`YH$kABdOuiRtl73MqG|~AzZ@iL&^s?24iS;RK_pdlWkhcF z@Wv-Om(Aealfg)D^adlXh9Nvf~Uf@y;g3Y)i(YP zEXDnb1V}1pJT5ZWyw=1i+0fni9yINurD=EqH^ciOwLUGi)C%Da)tyt=zq2P7pV5-G zR7!oq28-Fgn5pW|nlu^b!S1Z#r7!Wtr{5J5PQ>pd+2P7RSD?>(U7-|Y z7ZQ5lhYIl_IF<9?T9^IPK<(Hp;l5bl5tF9>X-zG14_7PfsA>6<$~A338iYRT{a@r_ zuXBaT=`T5x3=s&3=RYx6NgG>No4?5KFBVjE(swfcivcIpPQFx5l+O;fiGsOrl5teR z_Cm+;PW}O0Dwe_(4Z@XZ)O0W-v2X><&L*<~*q3dg;bQW3g7)a#3KiQP>+qj|qo*Hk z?57>f2?f@`=Fj^nkDKeRkN2d$Z@2eNKpHo}ksj-$`QKb6n?*$^*%Fb3_Kbf1(*W9K>{L$mud2WHJ=j0^=g30Xhg8$#g^?36`p1fm;;1@0Lrx+8t`?vN0ZorM zSW?rhjCE8$C|@p^sXdx z|NOHHg+fL;HIlqyLp~SSdIF`TnSHehNCU9t89yr@)FY<~hu+X`tjg(aSVae$wDG*C zq$nY(Y494R)hD!i1|IIyP*&PD_c2FPgeY)&mX1qujB1VHPG9`yFQpLFVQ0>EKS@Bp zAfP5`C(sWGLI?AC{XEjLKR4FVNw(4+9b?kba95ukgR1H?w<8F7)G+6&(zUhIE5Ef% z=fFkL3QKA~M@h{nzjRq!Y_t!%U66#L8!(2-GgFxkD1=JRRqk=n%G(yHKn%^&$dW>; zSjAcjETMz1%205se$iH_)ZCpfg_LwvnsZQAUCS#^FExp8O4CrJb6>JquNV@qPq~3A zZ<6dOU#6|8+fcgiA#~MDmcpIEaUO02L5#T$HV0$EMD94HT_eXLZ2Zi&(! z&5E>%&|FZ`)CN10tM%tLSPD*~r#--K(H-CZqIOb99_;m|D5wdgJ<1iOJz@h2Zkq?} z%8_KXb&hf=2Wza(Wgc;3v3TN*;HTU*q2?#z&tLn_U0Nt!y>Oo>+2T)He6%XuP;fgn z-G!#h$Y2`9>Jtf}hbVrm6D70|ERzLAU>3zoWhJmjWfgM^))T+2u$~5>HF9jQDkrXR z=IzX36)V75PrFjkQ%TO+iqKGCQ-DDXbaE;C#}!-CoWQx&v*vHfyI>$HNRbpvm<`O( zlx9NBWD6_e&J%Ous4yp~s6)Ghni!I6)0W;9(9$y1wWu`$gs<$9Mcf$L*piP zPR0Av*2%ul`W;?-1_-5Zy0~}?`e@Y5A&0H!^ApyVTT}BiOm4GeFo$_oPlDEyeGBbh z1h3q&Dx~GmUS|3@4V36&$2uO8!Yp&^pD7J5&TN{?xphf*-js1fP?B|`>p_K>lh{ij zP(?H%e}AIP?_i^f&Li=FDSQ`2_NWxL+BB=nQr=$ zHojMlXNGauvvwPU>ZLq!`bX-5F4jBJ&So{kE5+ms9UEYD{66!|k~3vsP+mE}x!>%P za98bAU0!h0&ka4EoiDvBM#CP#dRNdXJcb*(%=<(g+M@<)DZ!@v1V>;54En?igcHR2 zhubQMq}VSOK)onqHfczM7YA@s=9*ow;k;8)&?J3@0JiGcP! zP#00KZ1t)GyZeRJ=f0^gc+58lc4Qh*S7RqPIC6GugG1gXe$LIQMRCo8cHf^qXgAa2 z`}t>u2Cq1CbSEpLr~E=c7~=Qkc9-vLE%(v9N*&HF`(d~(0`iukl5aQ9u4rUvc8%m) zr2GwZN4!s;{SB87lJB;veebPmqE}tSpT>+`t?<457Q9iV$th%i__Z1kOMAswFldD6 ztbOvO337S5o#ZZgN2G99_AVqPv!?Gmt3pzgD+Hp3QPQ`9qJ(g=kjvD+fUSS3upJn! zqoG7acIKEFRX~S}3|{EWT$kdz#zrDlJU(rPkxjws_iyLKU8+v|*oS_W*-guAb&Pj1 z35Z`3z<&Jb@2Mwz=KXucNYdY#SNO$tcVFr9KdKm|%^e-TXzs6M`PBper%ajkrIyUe zp$vVxVs9*>Vp4_1NC~Zg)WOCPmOxI1V34QlG4!aSFOH{QqSVq1^1)- z0P!Z?tT&E-ll(pwf0?=F=yOzik=@nh1Clxr9}Vij89z)ePDSCYAqw?lVI?v?+&*zH z)p$CScFI8rrwId~`}9YWPFu0cW1Sf@vRELs&cbntRU6QfPK-SO*mqu|u~}8AJ!Q$z znzu}50O=YbjwKCuSVBs6&CZR#0FTu)3{}qJJYX(>QPr4$RqWiwX3NT~;>cLn*_&1H zaKpIW)JVJ>b{uo2oq>oQt3y=zJjb%fU@wLqM{SyaC6x2snMx-}ivfU<1- znu1Lh;i$3Tf$Kh5Uk))G!D1UhE8pvx&nO~w^fG)BC&L!_hQk%^p`Kp@F{cz>80W&T ziOK=Sq3fdRu*V0=S53rcIfWFazI}Twj63CG(jOB;$*b`*#B9uEnBM`hDk*EwSRdwP8?5T?xGUKs=5N83XsR*)a4|ijz|c{4tIU+4j^A5C<#5 z*$c_d=5ml~%pGxw#?*q9N7aRwPux5EyqHVkdJO=5J>84!X6P>DS8PTTz>7C#FO?k#edkntG+fJk8ZMn?pmJSO@`x-QHq;7^h6GEXLXo1TCNhH z8ZDH{*NLAjo3WM`xeb=X{((uv3H(8&r8fJJg_uSs_%hOH%JDD?hu*2NvWGYD+j)&` zz#_1%O1wF^o5ryt?O0n;`lHbzp0wQ?rcbW(F1+h7_EZZ9{>rePvLAPVZ_R|n@;b$;UchU=0j<6k8G9QuQf@76oiE*4 zXOLQ&n3$NR#p4<5NJMVC*S);5x2)eRbaAM%VxWu9ohlT;pGEk7;002enCbQ>2r-us z3#bpXP9g|mE`65VrN`+3mC)M(eMj~~eOf)do<@l+fMiTR)XO}422*1SL{wyY(%oMpBgJagtiDf zz>O6(m;};>Hi=t8o{DVC@YigqS(Qh+ix3Rwa9aliH}a}IlOCW1@?%h_bRbq-W{KHF z%Vo?-j@{Xi@=~Lz5uZP27==UGE15|g^0gzD|3x)SCEXrx`*MP^FDLl%pOi~~Il;dc z^hrwp9sYeT7iZ)-ajKy@{a`kr0-5*_!XfBpXwEcFGJ;%kV$0Nx;apKrur zJN2J~CAv{Zjj%FolyurtW8RaFmpn&zKJWL>(0;;+q(%(Hx!GMW4AcfP0YJ*Vz!F4g z!ZhMyj$BdXL@MlF%KeInmPCt~9&A!;cRw)W!Hi@0DY(GD_f?jeV{=s=cJ6e}JktJw zQORnxxj3mBxfrH=x{`_^Z1ddDh}L#V7i}$njUFRVwOX?qOTKjfPMBO4y(WiU<)epb zvB9L=%jW#*SL|Nd_G?E*_h1^M-$PG6Pc_&QqF0O-FIOpa4)PAEPsyvB)GKasmBoEt z?_Q2~QCYGH+hW31x-B=@5_AN870vY#KB~3a*&{I=f);3Kv7q4Q7s)0)gVYx2#Iz9g(F2;=+Iy4 z6KI^8GJ6D@%tpS^8boU}zpi=+(5GfIR)35PzrbuXeL1Y1N%JK7PG|^2k3qIqHfX;G zQ}~JZ-UWx|60P5?d1e;AHx!_;#PG%d=^X(AR%i`l0jSpYOpXoKFW~7ip7|xvN;2^? zsYC9fanpO7rO=V7+KXqVc;Q5z%Bj})xHVrgoR04sA2 zl~DAwv=!(()DvH*=lyhIlU^hBkA0$e*7&fJpB0|oB7)rqGK#5##2T`@_I^|O2x4GO z;xh6ROcV<9>?e0)MI(y++$-ksV;G;Xe`lh76T#Htuia+(UrIXrf9?

L(tZ$0BqX1>24?V$S+&kLZ`AodQ4_)P#Q3*4xg8}lMV-FLwC*cN$< zt65Rf%7z41u^i=P*qO8>JqXPrinQFapR7qHAtp~&RZ85$>ob|Js;GS^y;S{XnGiBc zGa4IGvDl?x%gY`vNhv8wgZnP#UYI-w*^4YCZnxkF85@ldepk$&$#3EAhrJY0U)lR{F6sM3SONV^+$;Zx8BD&Eku3K zKNLZyBni3)pGzU0;n(X@1fX8wYGKYMpLmCu{N5-}epPDxClPFK#A@02WM3!myN%bkF z|GJ4GZ}3sL{3{qXemy+#Uk{4>Kf8v11;f8I&c76+B&AQ8udd<8gU7+BeWC`akUU~U zgXoxie>MS@rBoyY8O8Tc&8id!w+_ooxcr!1?#rc$-|SBBtH6S?)1e#P#S?jFZ8u-Bs&k`yLqW|{j+%c#A4AQ>+tj$Y z^CZajspu$F%73E68Lw5q7IVREED9r1Ijsg#@DzH>wKseye>hjsk^{n0g?3+gs@7`i zHx+-!sjLx^fS;fY!ERBU+Q zVJ!e0hJH%P)z!y%1^ZyG0>PN@5W~SV%f>}c?$H8r;Sy-ui>aruVTY=bHe}$e zi&Q4&XK!qT7-XjCrDaufT@>ieQ&4G(SShUob0Q>Gznep9fR783jGuUynAqc6$pYX; z7*O@@JW>O6lKIk0G00xsm|=*UVTQBB`u1f=6wGAj%nHK_;Aqmfa!eAykDmi-@u%6~ z;*c!pS1@V8r@IX9j&rW&d*}wpNs96O2Ute>%yt{yv>k!6zfT6pru{F1M3P z2WN1JDYqoTB#(`kE{H676QOoX`cnqHl1Yaru)>8Ky~VU{)r#{&s86Vz5X)v15ULHA zAZDb{99+s~qI6;-dQ5DBjHJP@GYTwn;Dv&9kE<0R!d z8tf1oq$kO`_sV(NHOSbMwr=To4r^X$`sBW4$gWUov|WY?xccQJN}1DOL|GEaD_!@& z15p?Pj+>7d`@LvNIu9*^hPN)pwcv|akvYYq)ks%`G>!+!pW{-iXPZsRp8 z35LR;DhseQKWYSD`%gO&k$Dj6_6q#vjWA}rZcWtQr=Xn*)kJ9kacA=esi*I<)1>w^ zO_+E>QvjP)qiSZg9M|GNeLtO2D7xT6vsj`88sd!94j^AqxFLi}@w9!Y*?nwWARE0P znuI_7A-saQ+%?MFA$gttMV-NAR^#tjl_e{R$N8t2NbOlX373>e7Ox=l=;y#;M7asp zRCz*CLnrm$esvSb5{T<$6CjY zmZ(i{Rs_<#pWW>(HPaaYj`%YqBra=Ey3R21O7vUbzOkJJO?V`4-D*u4$Me0Bx$K(lYo`JO}gnC zx`V}a7m-hLU9Xvb@K2ymioF)vj12<*^oAqRuG_4u%(ah?+go%$kOpfb`T96P+L$4> zQ#S+sA%VbH&mD1k5Ak7^^dZoC>`1L%i>ZXmooA!%GI)b+$D&ziKrb)a=-ds9xk#~& z7)3iem6I|r5+ZrTRe_W861x8JpD`DDIYZNm{$baw+$)X^Jtjnl0xlBgdnNY}x%5za zkQ8E6T<^$sKBPtL4(1zi_Rd(tVth*3Xs!ulflX+70?gb&jRTnI8l+*Aj9{|d%qLZ+ z>~V9Z;)`8-lds*Zgs~z1?Fg?Po7|FDl(Ce<*c^2=lFQ~ahwh6rqSjtM5+$GT>3WZW zj;u~w9xwAhOc<kF}~`CJ68 z?(S5vNJa;kriPlim33{N5`C{9?NWhzsna_~^|K2k4xz1`xcui*LXL-1#Y}Hi9`Oo!zQ>x-kgAX4LrPz63uZ+?uG*84@PKq-KgQlMNRwz=6Yes) zY}>YN+qP}nwr$(CZQFjUOI=-6J$2^XGvC~EZ+vrqWaOXB$k?%Suf5k=4>AveC1aJ! ziaW4IS%F$_Babi)kA8Y&u4F7E%99OPtm=vzw$$ zEz#9rvn`Iot_z-r3MtV>k)YvErZ<^Oa${`2>MYYODSr6?QZu+be-~MBjwPGdMvGd!b!elsdi4% z`37W*8+OGulab8YM?`KjJ8e+jM(tqLKSS@=jimq3)Ea2EB%88L8CaM+aG7;27b?5` z4zuUWBr)f)k2o&xg{iZ$IQkJ+SK>lpq4GEacu~eOW4yNFLU!Kgc{w4&D$4ecm0f}~ zTTzquRW@`f0}|IILl`!1P+;69g^upiPA6F{)U8)muWHzexRenBU$E^9X-uIY2%&1w z_=#5*(nmxJ9zF%styBwivi)?#KMG96-H@hD-H_&EZiRNsfk7mjBq{L%!E;Sqn!mVX*}kXhwH6eh;b42eD!*~upVG@ z#smUqz$ICm!Y8wY53gJeS|Iuard0=;k5i5Z_hSIs6tr)R4n*r*rE`>38Pw&lkv{_r!jNN=;#?WbMj|l>cU(9trCq; z%nN~r^y7!kH^GPOf3R}?dDhO=v^3BeP5hF|%4GNQYBSwz;x({21i4OQY->1G=KFyu z&6d`f2tT9Yl_Z8YACZaJ#v#-(gcyeqXMhYGXb=t>)M@fFa8tHp2x;ODX=Ap@a5I=U z0G80^$N0G4=U(>W%mrrThl0DjyQ-_I>+1Tdd_AuB3qpYAqY54upwa3}owa|x5iQ^1 zEf|iTZxKNGRpI>34EwkIQ2zHDEZ=(J@lRaOH>F|2Z%V_t56Km$PUYu^xA5#5Uj4I4RGqHD56xT%H{+P8Ag>e_3pN$4m8n>i%OyJFPNWaEnJ4McUZPa1QmOh?t8~n& z&RulPCors8wUaqMHECG=IhB(-tU2XvHP6#NrLVyKG%Ee*mQ5Ps%wW?mcnriTVRc4J`2YVM>$ixSF2Xi+Wn(RUZnV?mJ?GRdw%lhZ+t&3s7g!~g{%m&i<6 z5{ib-<==DYG93I(yhyv4jp*y3#*WNuDUf6`vTM%c&hiayf(%=x@4$kJ!W4MtYcE#1 zHM?3xw63;L%x3drtd?jot!8u3qeqctceX3m;tWetK+>~q7Be$h>n6riK(5@ujLgRS zvOym)k+VAtyV^mF)$29Y`nw&ijdg~jYpkx%*^ z8dz`C*g=I?;clyi5|!27e2AuSa$&%UyR(J3W!A=ZgHF9OuKA34I-1U~pyD!KuRkjA zbkN!?MfQOeN>DUPBxoy5IX}@vw`EEB->q!)8fRl_mqUVuRu|C@KD-;yl=yKc=ZT0% zB$fMwcC|HE*0f8+PVlWHi>M`zfsA(NQFET?LrM^pPcw`cK+Mo0%8*x8@65=CS_^$cG{GZQ#xv($7J z??R$P)nPLodI;P!IC3eEYEHh7TV@opr#*)6A-;EU2XuogHvC;;k1aI8asq7ovoP!* z?x%UoPrZjj<&&aWpsbr>J$Er-7!E(BmOyEv!-mbGQGeJm-U2J>74>o5x`1l;)+P&~ z>}f^=Rx(ZQ2bm+YE0u=ZYrAV@apyt=v1wb?R@`i_g64YyAwcOUl=C!i>=Lzb$`tjv zOO-P#A+)t-JbbotGMT}arNhJmmGl-lyUpMn=2UacVZxmiG!s!6H39@~&uVokS zG=5qWhfW-WOI9g4!R$n7!|ViL!|v3G?GN6HR0Pt_L5*>D#FEj5wM1DScz4Jv@Sxnl zB@MPPmdI{(2D?;*wd>3#tjAirmUnQoZrVv`xM3hARuJksF(Q)wd4P$88fGYOT1p6U z`AHSN!`St}}UMBT9o7i|G`r$ zrB=s$qV3d6$W9@?L!pl0lf%)xs%1ko^=QY$ty-57=55PvP(^6E7cc zGJ*>m2=;fOj?F~yBf@K@9qwX0hA803Xw+b0m}+#a(>RyR8}*Y<4b+kpp|OS+!whP( zH`v{%s>jsQI9rd$*vm)EkwOm#W_-rLTHcZRek)>AtF+~<(did)*oR1|&~1|e36d-d zgtm5cv1O0oqgWC%Et@P4Vhm}Ndl(Y#C^MD03g#PH-TFy+7!Osv1z^UWS9@%JhswEq~6kSr2DITo59+; ze=ZC}i2Q?CJ~Iyu?vn|=9iKV>4j8KbxhE4&!@SQ^dVa-gK@YfS9xT(0kpW*EDjYUkoj! zE49{7H&E}k%5(>sM4uGY)Q*&3>{aitqdNnRJkbOmD5Mp5rv-hxzOn80QsG=HJ_atI-EaP69cacR)Uvh{G5dTpYG7d zbtmRMq@Sexey)||UpnZ?;g_KMZq4IDCy5}@u!5&B^-=6yyY{}e4Hh3ee!ZWtL*s?G zxG(A!<9o!CL+q?u_utltPMk+hn?N2@?}xU0KlYg?Jco{Yf@|mSGC<(Zj^yHCvhmyx z?OxOYoxbptDK()tsJ42VzXdINAMWL$0Gcw?G(g8TMB)Khw_|v9`_ql#pRd2i*?CZl z7k1b!jQB=9-V@h%;Cnl7EKi;Y^&NhU0mWEcj8B|3L30Ku#-9389Q+(Yet0r$F=+3p z6AKOMAIi|OHyzlHZtOm73}|ntKtFaXF2Fy|M!gOh^L4^62kGUoWS1i{9gsds_GWBc zLw|TaLP64z3z9?=R2|T6Xh2W4_F*$cq>MtXMOy&=IPIJ`;!Tw?PqvI2b*U1)25^<2 zU_ZPoxg_V0tngA0J+mm?3;OYw{i2Zb4x}NedZug!>EoN3DC{1i)Z{Z4m*(y{ov2%- zk(w>+scOO}MN!exSc`TN)!B=NUX`zThWO~M*ohqq;J2hx9h9}|s#?@eR!=F{QTrq~ zTcY|>azkCe$|Q0XFUdpFT=lTcyW##i;-e{}ORB4D?t@SfqGo_cS z->?^rh$<&n9DL!CF+h?LMZRi)qju!meugvxX*&jfD!^1XB3?E?HnwHP8$;uX{Rvp# zh|)hM>XDv$ZGg=$1{+_bA~u-vXqlw6NH=nkpyWE0u}LQjF-3NhATL@9rRxMnpO%f7 z)EhZf{PF|mKIMFxnC?*78(}{Y)}iztV12}_OXffJ;ta!fcFIVjdchyHxH=t%ci`Xd zX2AUB?%?poD6Zv*&BA!6c5S#|xn~DK01#XvjT!w!;&`lDXSJT4_j$}!qSPrb37vc{ z9^NfC%QvPu@vlxaZ;mIbn-VHA6miwi8qJ~V;pTZkKqqOii<1Cs}0i?uUIss;hM4dKq^1O35y?Yp=l4i zf{M!@QHH~rJ&X~8uATV><23zZUbs-J^3}$IvV_ANLS08>k`Td7aU_S1sLsfi*C-m1 z-e#S%UGs4E!;CeBT@9}aaI)qR-6NU@kvS#0r`g&UWg?fC7|b^_HyCE!8}nyh^~o@< zpm7PDFs9yxp+byMS(JWm$NeL?DNrMCNE!I^ko-*csB+dsf4GAq{=6sfyf4wb>?v1v zmb`F*bN1KUx-`ra1+TJ37bXNP%`-Fd`vVQFTwWpX@;s(%nDQa#oWhgk#mYlY*!d>( zE&!|ySF!mIyfING+#%RDY3IBH_fW$}6~1%!G`suHub1kP@&DoAd5~7J55;5_noPI6eLf{t;@9Kf<{aO0`1WNKd?<)C-|?C?)3s z>wEq@8=I$Wc~Mt$o;g++5qR+(6wt9GI~pyrDJ%c?gPZe)owvy^J2S=+M^ z&WhIE`g;;J^xQLVeCtf7b%Dg#Z2gq9hp_%g)-%_`y*zb; zn9`f`mUPN-Ts&fFo(aNTsXPA|J!TJ{0hZp0^;MYHLOcD=r_~~^ymS8KLCSeU3;^QzJNqS z5{5rEAv#l(X?bvwxpU;2%pQftF`YFgrD1jt2^~Mt^~G>T*}A$yZc@(k9orlCGv&|1 zWWvVgiJsCAtamuAYT~nzs?TQFt<1LSEx!@e0~@yd6$b5!Zm(FpBl;(Cn>2vF?k zOm#TTjFwd2D-CyA!mqR^?#Uwm{NBemP>(pHmM}9;;8`c&+_o3#E5m)JzfwN?(f-a4 zyd%xZc^oQx3XT?vcCqCX&Qrk~nu;fxs@JUoyVoi5fqpi&bUhQ2y!Ok2pzsFR(M(|U zw3E+kH_zmTRQ9dUMZWRE%Zakiwc+lgv7Z%|YO9YxAy`y28`Aw;WU6HXBgU7fl@dnt z-fFBV)}H-gqP!1;V@Je$WcbYre|dRdp{xt!7sL3Eoa%IA`5CAA%;Wq8PktwPdULo! z8!sB}Qt8#jH9Sh}QiUtEPZ6H0b*7qEKGJ%ITZ|vH)5Q^2m<7o3#Z>AKc%z7_u`rXA zqrCy{-{8;9>dfllLu$^M5L z-hXs))h*qz%~ActwkIA(qOVBZl2v4lwbM>9l70Y`+T*elINFqt#>OaVWoja8RMsep z6Or3f=oBnA3vDbn*+HNZP?8LsH2MY)x%c13@(XfuGR}R?Nu<|07{$+Lc3$Uv^I!MQ z>6qWgd-=aG2Y^24g4{Bw9ueOR)(9h`scImD=86dD+MnSN4$6 z^U*o_mE-6Rk~Dp!ANp#5RE9n*LG(Vg`1)g6!(XtDzsov$Dvz|Gv1WU68J$CkshQhS zCrc|cdkW~UK}5NeaWj^F4MSgFM+@fJd{|LLM)}_O<{rj z+?*Lm?owq?IzC%U%9EBga~h-cJbIu=#C}XuWN>OLrc%M@Gu~kFEYUi4EC6l#PR2JS zQUkGKrrS#6H7}2l0F@S11DP`@pih0WRkRJl#F;u{c&ZC{^$Z+_*lB)r)-bPgRFE;* zl)@hK4`tEP=P=il02x7-C7p%l=B`vkYjw?YhdJU9!P!jcmY$OtC^12w?vy3<<=tlY zUwHJ_0lgWN9vf>1%WACBD{UT)1qHQSE2%z|JHvP{#INr13jM}oYv_5#xsnv9`)UAO zuwgyV4YZ;O)eSc3(mka6=aRohi!HH@I#xq7kng?Acdg7S4vDJb6cI5fw?2z%3yR+| zU5v@Hm}vy;${cBp&@D=HQ9j7NcFaOYL zj-wV=eYF{|XTkFNM2uz&T8uH~;)^Zo!=KP)EVyH6s9l1~4m}N%XzPpduPg|h-&lL` zAXspR0YMOKd2yO)eMFFJ4?sQ&!`dF&!|niH*!^*Ml##o0M(0*uK9&yzekFi$+mP9s z>W9d%Jb)PtVi&-Ha!o~Iyh@KRuKpQ@)I~L*d`{O8!kRObjO7=n+Gp36fe!66neh+7 zW*l^0tTKjLLzr`x4`_8&on?mjW-PzheTNox8Hg7Nt@*SbE-%kP2hWYmHu#Fn@Q^J(SsPUz*|EgOoZ6byg3ew88UGdZ>9B2Tq=jF72ZaR=4u%1A6Vm{O#?@dD!(#tmR;eP(Fu z{$0O%=Vmua7=Gjr8nY%>ul?w=FJ76O2js&17W_iq2*tb!i{pt#`qZB#im9Rl>?t?0c zicIC}et_4d+CpVPx)i4~$u6N-QX3H77ez z?ZdvXifFk|*F8~L(W$OWM~r`pSk5}#F?j_5u$Obu9lDWIknO^AGu+Blk7!9Sb;NjS zncZA?qtASdNtzQ>z7N871IsPAk^CC?iIL}+{K|F@BuG2>qQ;_RUYV#>hHO(HUPpk@ z(bn~4|F_jiZi}Sad;_7`#4}EmD<1EiIxa48QjUuR?rC}^HRocq`OQPM@aHVKP9E#q zy%6bmHygCpIddPjE}q_DPC`VH_2m;Eey&ZH)E6xGeStOK7H)#+9y!%-Hm|QF6w#A( zIC0Yw%9j$s-#odxG~C*^MZ?M<+&WJ+@?B_QPUyTg9DJGtQN#NIC&-XddRsf3n^AL6 zT@P|H;PvN;ZpL0iv$bRb7|J{0o!Hq+S>_NrH4@coZtBJu#g8#CbR7|#?6uxi8d+$g z87apN>EciJZ`%Zv2**_uiET9Vk{pny&My;+WfGDw4EVL#B!Wiw&M|A8f1A@ z(yFQS6jfbH{b8Z-S7D2?Ixl`j0{+ZnpT=;KzVMLW{B$`N?Gw^Fl0H6lT61%T2AU**!sX0u?|I(yoy&Xveg7XBL&+>n6jd1##6d>TxE*Vj=8lWiG$4=u{1UbAa5QD>5_ z;Te^42v7K6Mmu4IWT6Rnm>oxrl~b<~^e3vbj-GCdHLIB_>59}Ya+~OF68NiH=?}2o zP(X7EN=quQn&)fK>M&kqF|<_*H`}c zk=+x)GU>{Af#vx&s?`UKUsz})g^Pc&?Ka@t5$n$bqf6{r1>#mWx6Ep>9|A}VmWRnowVo`OyCr^fHsf# zQjQ3Ttp7y#iQY8l`zEUW)(@gGQdt(~rkxlkefskT(t%@i8=|p1Y9Dc5bc+z#n$s13 zGJk|V0+&Ekh(F};PJzQKKo+FG@KV8a<$gmNSD;7rd_nRdc%?9)p!|B-@P~kxQG}~B zi|{0}@}zKC(rlFUYp*dO1RuvPC^DQOkX4<+EwvBAC{IZQdYxoq1Za!MW7%p7gGr=j zzWnAq%)^O2$eItftC#TTSArUyL$U54-O7e|)4_7%Q^2tZ^0-d&3J1}qCzR4dWX!)4 zzIEKjgnYgMus^>6uw4Jm8ga6>GBtMjpNRJ6CP~W=37~||gMo_p@GA@#-3)+cVYnU> zE5=Y4kzl+EbEh%dhQokB{gqNDqx%5*qBusWV%!iprn$S!;oN_6E3?0+umADVs4ako z?P+t?m?};gev9JXQ#Q&KBpzkHPde_CGu-y z<{}RRAx=xlv#mVi+Ibrgx~ujW$h{?zPfhz)Kp7kmYS&_|97b&H&1;J-mzrBWAvY} zh8-I8hl_RK2+nnf&}!W0P+>5?#?7>npshe<1~&l_xqKd0_>dl_^RMRq@-Myz&|TKZBj1=Q()) zF{dBjv5)h=&Z)Aevx}+i|7=R9rG^Di!sa)sZCl&ctX4&LScQ-kMncgO(9o6W6)yd< z@Rk!vkja*X_N3H=BavGoR0@u0<}m-7|2v!0+2h~S2Q&a=lTH91OJsvms2MT~ zY=c@LO5i`mLpBd(vh|)I&^A3TQLtr>w=zoyzTd=^f@TPu&+*2MtqE$Avf>l>}V|3-8Fp2hzo3y<)hr_|NO(&oSD z!vEjTWBxbKTiShVl-U{n*B3#)3a8$`{~Pk}J@elZ=>Pqp|MQ}jrGv7KrNcjW%TN_< zZz8kG{#}XoeWf7qY?D)L)8?Q-b@Na&>i=)(@uNo zr;cH98T3$Iau8Hn*@vXi{A@YehxDE2zX~o+RY`)6-X{8~hMpc#C`|8y> zU8Mnv5A0dNCf{Ims*|l-^ z(MRp{qoGohB34|ggDI*p!Aw|MFyJ|v+<+E3brfrI)|+l3W~CQLPbnF@G0)P~Ly!1TJLp}xh8uW`Q+RB-v`MRYZ9Gam3cM%{ zb4Cb*f)0deR~wtNb*8w-LlIF>kc7DAv>T0D(a3@l`k4TFnrO+g9XH7;nYOHxjc4lq zMmaW6qpgAgy)MckYMhl?>sq;-1E)-1llUneeA!ya9KM$)DaNGu57Z5aE>=VST$#vb zFo=uRHr$0M{-ha>h(D_boS4zId;3B|Tpqo|?B?Z@I?G(?&Iei+-{9L_A9=h=Qfn-U z1wIUnQe9!z%_j$F_{rf&`ZFSott09gY~qrf@g3O=Y>vzAnXCyL!@(BqWa)Zqt!#_k zfZHuwS52|&&)aK;CHq9V-t9qt0au{$#6c*R#e5n3rje0hic7c7m{kW$p(_`wB=Gw7 z4k`1Hi;Mc@yA7dp@r~?@rfw)TkjAW++|pkfOG}0N|2guek}j8Zen(!+@7?qt_7ndX zB=BG6WJ31#F3#Vk3=aQr8T)3`{=p9nBHlKzE0I@v`{vJ}h8pd6vby&VgFhzH|q;=aonunAXL6G2y(X^CtAhWr*jI zGjpY@raZDQkg*aMq}Ni6cRF z{oWv}5`nhSAv>usX}m^GHt`f(t8@zHc?K|y5Zi=4G*UG1Sza{$Dpj%X8 zzEXaKT5N6F5j4J|w#qlZP!zS7BT)9b+!ZSJdToqJts1c!)fwih4d31vfb{}W)EgcA zH2pZ^8_k$9+WD2n`6q5XbOy8>3pcYH9 z07eUB+p}YD@AH!}p!iKv><2QF-Y^&xx^PAc1F13A{nUeCDg&{hnix#FiO!fe(^&%Qcux!h znu*S!s$&nnkeotYsDthh1dq(iQrE|#f_=xVgfiiL&-5eAcC-> z5L0l|DVEM$#ulf{bj+Y~7iD)j<~O8CYM8GW)dQGq)!mck)FqoL^X zwNdZb3->hFrbHFm?hLvut-*uK?zXn3q1z|UX{RZ;-WiLoOjnle!xs+W0-8D)kjU#R z+S|A^HkRg$Ij%N4v~k`jyHffKaC~=wg=9)V5h=|kLQ@;^W!o2^K+xG&2n`XCd>OY5Ydi= zgHH=lgy++erK8&+YeTl7VNyVm9-GfONlSlVb3)V9NW5tT!cJ8d7X)!b-$fb!s76{t z@d=Vg-5K_sqHA@Zx-L_}wVnc@L@GL9_K~Zl(h5@AR#FAiKad8~KeWCo@mgXIQ#~u{ zgYFwNz}2b6Vu@CP0XoqJ+dm8px(5W5-Jpis97F`+KM)TuP*X8H@zwiVKDKGVp59pI zifNHZr|B+PG|7|Y<*tqap0CvG7tbR1R>jn70t1X`XJixiMVcHf%Ez*=xm1(CrTSDt z0cle!+{8*Ja&EOZ4@$qhBuKQ$U95Q%rc7tg$VRhk?3=pE&n+T3upZg^ZJc9~c2es% zh7>+|mrmA-p&v}|OtxqmHIBgUxL~^0+cpfkSK2mhh+4b=^F1Xgd2)}U*Yp+H?ls#z zrLxWg_hm}AfK2XYWr!rzW4g;+^^&bW%LmbtRai9f3PjU${r@n`JThy-cphbcwn)rq9{A$Ht`lmYKxOacy z6v2R(?gHhD5@&kB-Eg?4!hAoD7~(h>(R!s1c1Hx#s9vGPePUR|of32bS`J5U5w{F) z>0<^ktO2UHg<0{oxkdOQ;}coZDQph8p6ruj*_?uqURCMTac;>T#v+l1Tc~%^k-Vd@ zkc5y35jVNc49vZpZx;gG$h{%yslDI%Lqga1&&;mN{Ush1c7p>7e-(zp}6E7f-XmJb4nhk zb8zS+{IVbL$QVF8pf8}~kQ|dHJAEATmmnrb_wLG}-yHe>W|A&Y|;muy-d^t^<&)g5SJfaTH@P1%euONny=mxo+C z4N&w#biWY41r8k~468tvuYVh&XN&d#%QtIf9;iVXfWY)#j=l`&B~lqDT@28+Y!0E+MkfC}}H*#(WKKdJJq=O$vNYCb(ZG@p{fJgu;h z21oHQ(14?LeT>n5)s;uD@5&ohU!@wX8w*lB6i@GEH0pM>YTG+RAIWZD;4#F1&F%Jp zXZUml2sH0!lYJT?&sA!qwez6cXzJEd(1ZC~kT5kZSp7(@=H2$Azb_*W&6aA|9iwCL zdX7Q=42;@dspHDwYE?miGX#L^3xD&%BI&fN9^;`v4OjQXPBaBmOF1;#C)8XA(WFlH zycro;DS2?(G&6wkr6rqC>rqDv3nfGw3hmN_9Al>TgvmGsL8_hXx09};l9Ow@)F5@y z#VH5WigLDwZE4nh^7&@g{1FV^UZ%_LJ-s<{HN*2R$OPg@R~Z`c-ET*2}XB@9xvAjrK&hS=f|R8Gr9 zr|0TGOsI7RD+4+2{ZiwdVD@2zmg~g@^D--YL;6UYGSM8i$NbQr4!c7T9rg!8;TM0E zT#@?&S=t>GQm)*ua|?TLT2ktj#`|R<_*FAkOu2Pz$wEc%-=Y9V*$&dg+wIei3b*O8 z2|m$!jJG!J!ZGbbIa!(Af~oSyZV+~M1qGvelMzPNE_%5?c2>;MeeG2^N?JDKjFYCy z7SbPWH-$cWF9~fX%9~v99L!G(wi!PFp>rB!9xj7=Cv|F+7CsGNwY0Q_J%FID%C^CBZQfJ9K(HK%k31j~e#&?hQ zNuD6gRkVckU)v+53-fc} z7ZCzYN-5RG4H7;>>Hg?LU9&5_aua?A0)0dpew1#MMlu)LHe(M;OHjHIUl7|%%)YPo z0cBk;AOY00%Fe6heoN*$(b<)Cd#^8Iu;-2v@>cE-OB$icUF9EEoaC&q8z9}jMTT2I z8`9;jT%z0;dy4!8U;GW{i`)3!c6&oWY`J3669C!tM<5nQFFrFRglU8f)5Op$GtR-3 zn!+SPCw|04sv?%YZ(a7#L?vsdr7ss@WKAw&A*}-1S|9~cL%uA+E~>N6QklFE>8W|% zyX-qAUGTY1hQ-+um`2|&ji0cY*(qN!zp{YpDO-r>jPk*yuVSay<)cUt`t@&FPF_&$ zcHwu1(SQ`I-l8~vYyUxm@D1UEdFJ$f5Sw^HPH7b!9 zzYT3gKMF((N(v0#4f_jPfVZ=ApN^jQJe-X$`A?X+vWjLn_%31KXE*}5_}d8 zw_B1+a#6T1?>M{ronLbHIlEsMf93muJ7AH5h%;i99<~JX^;EAgEB1uHralD*!aJ@F zV2ruuFe9i2Q1C?^^kmVy921eb=tLDD43@-AgL^rQ3IO9%+vi_&R2^dpr}x{bCVPej z7G0-0o64uyWNtr*loIvslyo0%)KSDDKjfThe0hcqs)(C-MH1>bNGBDRTW~scy_{w} zp^aq8Qb!h9Lwielq%C1b8=?Z=&U)ST&PHbS)8Xzjh2DF?d{iAv)Eh)wsUnf>UtXN( zL7=$%YrZ#|^c{MYmhn!zV#t*(jdmYdCpwqpZ{v&L8KIuKn`@IIZfp!uo}c;7J57N` zAxyZ-uA4=Gzl~Ovycz%MW9ZL7N+nRo&1cfNn9(1H5eM;V_4Z_qVann7F>5f>%{rf= zPBZFaV@_Sobl?Fy&KXyzFDV*FIdhS5`Uc~S^Gjo)aiTHgn#<0C=9o-a-}@}xDor;D zZyZ|fvf;+=3MZd>SR1F^F`RJEZo+|MdyJYQAEauKu%WDol~ayrGU3zzbHKsnHKZ*z zFiwUkL@DZ>!*x05ql&EBq@_Vqv83&?@~q5?lVmffQZ+V-=qL+!u4Xs2Z2zdCQ3U7B&QR9_Iggy} z(om{Y9eU;IPe`+p1ifLx-XWh?wI)xU9ik+m#g&pGdB5Bi<`PR*?92lE0+TkRuXI)z z5LP!N2+tTc%cB6B1F-!fj#}>S!vnpgVU~3!*U1ej^)vjUH4s-bd^%B=ItQqDCGbrEzNQi(dJ`J}-U=2{7-d zK8k^Rlq2N#0G?9&1?HSle2vlkj^KWSBYTwx`2?9TU_DX#J+f+qLiZCqY1TXHFxXZqYMuD@RU$TgcnCC{_(vwZ-*uX)~go#%PK z@}2Km_5aQ~(<3cXeJN6|F8X_1@L%@xTzs}$_*E|a^_URF_qcF;Pfhoe?FTFwvjm1o z8onf@OY@jC2tVcMaZS;|T!Ks(wOgPpRzRnFS-^RZ4E!9dsnj9sFt609a|jJbb1Dt@ z<=Gal2jDEupxUSwWu6zp<<&RnAA;d&4gKVG0iu6g(DsST(4)z6R)zDpfaQ}v{5ARt zyhwvMtF%b-YazR5XLz+oh=mn;y-Mf2a8>7?2v8qX;19y?b>Z5laGHvzH;Nu9S`B8} zI)qN$GbXIQ1VL3lnof^6TS~rvPVg4V?Dl2Bb*K2z4E{5vy<(@@K_cN@U>R!>aUIRnb zL*)=787*cs#zb31zBC49x$`=fkQbMAef)L2$dR{)6BAz!t5U_B#1zZG`^neKSS22oJ#5B=gl%U=WeqL9REF2g zZnfCb0?quf?Ztj$VXvDSWoK`0L=Zxem2q}!XWLoT-kYMOx)!7fcgT35uC~0pySEme z`{wGWTkGr7>+Kb^n;W?BZH6ZP(9tQX%-7zF>vc2}LuWDI(9kh1G#7B99r4x6;_-V+k&c{nPUrR zAXJGRiMe~aup{0qzmLNjS_BC4cB#sXjckx{%_c&^xy{M61xEb>KW_AG5VFXUOjAG4 z^>Qlm9A#1N{4snY=(AmWzatb!ngqiqPbBZ7>Uhb3)dTkSGcL#&SH>iMO-IJBPua`u zo)LWZ>=NZLr758j{%(|uQuZ)pXq_4c!!>s|aDM9#`~1bzK3J1^^D#<2bNCccH7~-X}Ggi!pIIF>uFx%aPARGQsnC8ZQc8lrQ5o~smqOg>Ti^GNme94*w z)JZy{_{#$jxGQ&`M z!OMvZMHR>8*^>eS%o*6hJwn!l8VOOjZQJvh)@tnHVW&*GYPuxqXw}%M!(f-SQf`=L z5;=5w2;%82VMH6Xi&-K3W)o&K^+vJCepWZ-rW%+Dc6X3(){z$@4zjYxQ|}8UIojeC zYZpQ1dU{fy=oTr<4VX?$q)LP}IUmpiez^O&N3E_qPpchGTi5ZM6-2ScWlQq%V&R2Euz zO|Q0Hx>lY1Q1cW5xHv5!0OGU~PVEqSuy#fD72d#O`N!C;o=m+YioGu-wH2k6!t<~K zSr`E=W9)!g==~x9VV~-8{4ZN9{~-A9zJpRe%NGg$+MDuI-dH|b@BD)~>pPCGUNNzY zMDg||0@XGQgw`YCt5C&A{_+J}mvV9Wg{6V%2n#YSRN{AP#PY?1FF1#|vO_%e+#`|2*~wGAJaeRX6=IzFNeWhz6gJc8+(03Ph4y6ELAm=AkN7TOgMUEw*N{= z_)EIDQx5q22oUR+_b*tazu9+pX|n1c*IB-}{DqIj z-?E|ks{o3AGRNb;+iKcHkZvYJvFsW&83RAPs1Oh@IWy%l#5x2oUP6ZCtv+b|q>jsf zZ_9XO;V!>n`UxH1LvH8)L4?8raIvasEhkpQoJ`%!5rBs!0Tu(s_D{`4opB;57)pkX z4$A^8CsD3U5*!|bHIEqsn~{q+Ddj$ME@Gq4JXtgVz&7l{Ok!@?EA{B3P~NAqb9)4? zkQo30A^EbHfQ@87G5&EQTd`frrwL)&Yw?%-W@uy^Gn23%j?Y!Iea2xw<-f;esq zf%w5WN@E1}zyXtYv}}`U^B>W`>XPmdLj%4{P298|SisrE;7HvXX;A}Ffi8B#3Lr;1 zHt6zVb`8{#+e$*k?w8|O{Uh|&AG}|DG1PFo1i?Y*cQm$ZwtGcVgMwtBUDa{~L1KT-{jET4w60>{KZ27vXrHJ;fW{6| z=|Y4!&UX020wU1>1iRgB@Q#m~1^Z^9CG1LqDhYBrnx%IEdIty z!46iOoKlKs)c}newDG)rWUikD%j`)p z_w9Ph&e40=(2eBy;T!}*1p1f1SAUDP9iWy^u^Ubdj21Kn{46;GR+hwLO=4D11@c~V zI8x&(D({K~Df2E)Nx_yQvYfh4;MbMJ@Z}=Dt3_>iim~QZ*hZIlEs0mEb z_54+&*?wMD`2#vsQRN3KvoT>hWofI_Vf(^C1ff-Ike@h@saEf7g}<9T`W;HAne-Nd z>RR+&SP35w)xKn8^U$7))PsM!jKwYZ*RzEcG-OlTrX3}9a{q%#Un5E5W{{hp>w~;` zGky+3(vJvQyGwBo`tCpmo0mo((?nM8vf9aXrrY1Ve}~TuVkB(zeds^jEfI}xGBCM2 zL1|#tycSaWCurP+0MiActG3LCas@_@tao@(R1ANlwB$4K53egNE_;!&(%@Qo$>h`^1S_!hN6 z)vZtG$8fN!|BXBJ=SI>e(LAU(y(i*PHvgQ2llulxS8>qsimv7yL}0q_E5WiAz7)(f zC(ahFvG8&HN9+6^jGyLHM~$)7auppeWh_^zKk&C_MQ~8;N??OlyH~azgz5fe^>~7F zl3HnPN3z-kN)I$4@`CLCMQx3sG~V8hPS^}XDXZrQA>}mQPw%7&!sd(Pp^P=tgp-s^ zjl}1-KRPNWXgV_K^HkP__SR`S-|OF0bR-N5>I%ODj&1JUeAQ3$9i;B~$S6}*^tK?= z**%aCiH7y?xdY?{LgVP}S0HOh%0%LI$wRx;$T|~Y8R)Vdwa}kGWv8?SJVm^>r6+%I z#lj1aR94{@MP;t-scEYQWc#xFA30^}?|BeX*W#9OL;Q9#WqaaM546j5j29((^_8Nu z4uq}ESLr~r*O7E7$D{!k9W>`!SLoyA53i9QwRB{!pHe8um|aDE`Cg0O*{jmor)^t)3`>V>SWN-2VJcFmj^1?~tT=JrP`fVh*t zXHarp=8HEcR#vFe+1a%XXuK+)oFs`GDD}#Z+TJ}Ri`FvKO@ek2ayn}yaOi%(8p%2$ zpEu)v0Jym@f}U|-;}CbR=9{#<^z28PzkkTNvyKvJDZe+^VS2bES3N@Jq!-*}{oQlz z@8bgC_KnDnT4}d#&Cpr!%Yb?E!brx0!eVOw~;lLwUoz#Np%d$o%9scc3&zPm`%G((Le|6o1 zM(VhOw)!f84zG^)tZ1?Egv)d8cdNi+T${=5kV+j;Wf%2{3g@FHp^Gf*qO0q!u$=m9 zCaY`4mRqJ;FTH5`a$affE5dJrk~k`HTP_7nGTY@B9o9vvnbytaID;^b=Tzp7Q#DmD zC(XEN)Ktn39z5|G!wsVNnHi) z%^q94!lL|hF`IijA^9NR0F$@h7k5R^ljOW(;Td9grRN0Mb)l_l7##{2nPQ@?;VjXv zaLZG}yuf$r$<79rVPpXg?6iiieX|r#&`p#Con2i%S8*8F}(E) zI5E6c3tG*<;m~6>!&H!GJ6zEuhH7mkAzovdhLy;)q z{H2*8I^Pb}xC4s^6Y}6bJvMu=8>g&I)7!N!5QG$xseeU#CC?ZM-TbjsHwHgDGrsD= z{%f;@Sod+Ch66Ko2WF~;Ty)v>&x^aovCbCbD7>qF*!?BXmOV3(s|nxsb*Lx_2lpB7 zokUnzrk;P=T-&kUHO}td+Zdj!3n&NR?K~cRU zAXU!DCp?51{J4w^`cV#ye}(`SQhGQkkMu}O3M*BWt4UsC^jCFUy;wTINYmhD$AT;4 z?Xd{HaJjP`raZ39qAm;%beDbrLpbRf(mkKbANan7XsL>_pE2oo^$TgdidjRP!5-`% zv0d!|iKN$c0(T|L0C~XD0aS8t{*&#LnhE;1Kb<9&=c2B+9JeLvJr*AyyRh%@jHej=AetOMSlz^=!kxX>>B{2B1uIrQyfd8KjJ+DBy!h)~*(!|&L4^Q_07SQ~E zcemVP`{9CwFvPFu7pyVGCLhH?LhEVb2{7U+Z_>o25#+3<|8%1T^5dh}*4(kfJGry} zm%r#hU+__Z;;*4fMrX=Bkc@7|v^*B;HAl0((IBPPii%X9+u3DDF6%bI&6?Eu$8&aWVqHIM7mK6?Uvq$1|(-T|)IV<>e?!(rY zqkmO1MRaLeTR=)io(0GVtQT@s6rN%C6;nS3@eu;P#ry4q;^O@1ZKCJyp_Jo)Ty^QW z+vweTx_DLm{P-XSBj~Sl<%_b^$=}odJ!S2wAcxenmzFGX1t&Qp8Vxz2VT`uQsQYtdn&_0xVivIcxZ_hnrRtwq4cZSj1c-SG9 z7vHBCA=fd0O1<4*=lu$6pn~_pVKyL@ztw1swbZi0B?spLo56ZKu5;7ZeUml1Ws1?u zqMf1p{5myAzeX$lAi{jIUqo1g4!zWLMm9cfWcnw`k6*BR^?$2(&yW?>w;G$EmTA@a z6?y#K$C~ZT8+v{87n5Dm&H6Pb_EQ@V0IWmG9cG=O;(;5aMWWrIPzz4Q`mhK;qQp~a z+BbQrEQ+w{SeiuG-~Po5f=^EvlouB@_|4xQXH@A~KgpFHrwu%dwuCR)=B&C(y6J4J zvoGk9;lLs9%iA-IJGU#RgnZZR+@{5lYl8(e1h6&>Vc_mvg0d@);X zji4T|n#lB!>pfL|8tQYkw?U2bD`W{na&;*|znjmalA&f;*U++_aBYerq;&C8Kw7mI z7tsG*?7*5j&dU)Lje;^{D_h`%(dK|pB*A*1(Jj)w^mZ9HB|vGLkF1GEFhu&rH=r=8 zMxO42e{Si6$m+Zj`_mXb&w5Q(i|Yxyg?juUrY}78uo@~3v84|8dfgbPd0iQJRdMj< zncCNGdMEcsxu#o#B5+XD{tsg*;j-eF8`mp~K8O1J!Z0+>0=7O=4M}E?)H)ENE;P*F z$Ox?ril_^p0g7xhDUf(q652l|562VFlC8^r8?lQv;TMvn+*8I}&+hIQYh2 z1}uQQaag&!-+DZ@|C+C$bN6W;S-Z@)d1|en+XGvjbOxCa-qAF*LA=6s(Jg+g;82f$ z(Vb)8I)AH@cdjGFAR5Rqd0wiNCu!xtqWbcTx&5kslzTb^7A78~Xzw1($UV6S^VWiP zFd{Rimd-0CZC_Bu(WxBFW7+k{cOW7DxBBkJdJ;VsJ4Z@lERQr%3eVv&$%)b%<~ zCl^Y4NgO}js@u{|o~KTgH}>!* z_iDNqX2(As7T0xivMH|3SC1ivm8Q}6Ffcd7owUKN5lHAtzMM4<0v+ykUT!QiowO;`@%JGv+K$bBx@*S7C8GJVqQ_K>12}M`f_Ys=S zKFh}HM9#6Izb$Y{wYzItTy+l5U2oL%boCJn?R3?jP@n$zSIwlmyGq30Cw4QBO|14` zW5c);AN*J3&eMFAk$SR~2k|&+&Bc$e>s%c{`?d~85S-UWjA>DS5+;UKZ}5oVa5O(N zqqc@>)nee)+4MUjH?FGv%hm2{IlIF-QX}ym-7ok4Z9{V+ZHVZQl$A*x!(q%<2~iVv znUa+BX35&lCb#9VE-~Y^W_f;Xhl%vgjwdjzMy$FsSIj&ok}L+X`4>J=9BkN&nu^E*gbhj3(+D>C4E z@Fwq_=N)^bKFSHTzZk?-gNU$@l}r}dwGyh_fNi=9b|n}J>&;G!lzilbWF4B}BBq4f zYIOl?b)PSh#XTPp4IS5ZR_2C!E)Z`zH0OW%4;&~z7UAyA-X|sh9@~>cQW^COA9hV4 zXcA6qUo9P{bW1_2`eo6%hgbN%(G-F1xTvq!sc?4wN6Q4`e9Hku zFwvlAcRY?6h^Fj$R8zCNEDq8`=uZB8D-xn)tA<^bFFy}4$vA}Xq0jAsv1&5!h!yRA zU()KLJya5MQ`q&LKdH#fwq&(bNFS{sKlEh_{N%{XCGO+po#(+WCLmKW6&5iOHny>g z3*VFN?mx!16V5{zyuMWDVP8U*|BGT$(%IO|)?EF|OI*sq&RovH!N%=>i_c?K*A>>k zyg1+~++zY4Q)J;VWN0axhoIKx;l&G$gvj(#go^pZskEVj8^}is3Jw26LzYYVos0HX zRPvmK$dVxM8(Tc?pHFe0Z3uq){{#OK3i-ra#@+;*=ui8)y6hsRv z4Fxx1c1+fr!VI{L3DFMwXKrfl#Q8hfP@ajgEau&QMCxd{g#!T^;ATXW)nUg&$-n25 zruy3V!!;{?OTobo|0GAxe`Acn3GV@W=&n;~&9 zQM>NWW~R@OYORkJAo+eq1!4vzmf9K%plR4(tB@TR&FSbDoRgJ8qVcH#;7lQub*nq&?Z>7WM=oeEVjkaG zT#f)=o!M2DO5hLR+op>t0CixJCIeXH*+z{-XS|%jx)y(j&}Wo|3!l7{o)HU3m7LYyhv*xF&tq z%IN7N;D4raue&&hm0xM=`qv`+TK@;_xAcGKuK(2|75~ar2Yw)geNLSmVxV@x89bQu zpViVKKnlkwjS&&c|-X6`~xdnh}Ps)Hs z4VbUL^{XNLf7_|Oi>tA%?SG5zax}esF*FH3d(JH^Gvr7Rp*n=t7frH!U;!y1gJB^i zY_M$KL_}mW&XKaDEi9K-wZR|q*L32&m+2n_8lq$xRznJ7p8}V>w+d@?uB!eS3#u<} zIaqi!b!w}a2;_BfUUhGMy#4dPx>)_>yZ`ai?Rk`}d0>~ce-PfY-b?Csd(28yX22L% zI7XI>OjIHYTk_@Xk;Gu^F52^Gn6E1&+?4MxDS2G_#PQ&yXPXP^<-p|2nLTb@AAQEY zI*UQ9Pmm{Kat}wuazpjSyXCdnrD&|C1c5DIb1TnzF}f4KIV6D)CJ!?&l&{T)e4U%3HTSYqsQ zo@zWB1o}ceQSV)<4G<)jM|@@YpL+XHuWsr5AYh^Q{K=wSV99D~4RRU52FufmMBMmd z_H}L#qe(}|I9ZyPRD6kT>Ivj&2Y?qVZq<4bG_co_DP`sE*_Xw8D;+7QR$Uq(rr+u> z8bHUWbV19i#)@@G4bCco@Xb<8u~wVDz9S`#k@ciJtlu@uP1U0X?yov8v9U3VOig2t zL9?n$P3=1U_Emi$#slR>N5wH-=J&T=EdUHA}_Z zZIl3nvMP*AZS9{cDqFanrA~S5BqxtNm9tlu;^`)3X&V4tMAkJ4gEIPl= zoV!Gyx0N{3DpD@)pv^iS*dl2FwANu;1;%EDl}JQ7MbxLMAp>)UwNwe{=V}O-5C*>F zu?Ny+F64jZn<+fKjF01}8h5H_3pey|;%bI;SFg$w8;IC<8l|3#Lz2;mNNik6sVTG3 z+Su^rIE#40C4a-587$U~%KedEEw1%r6wdvoMwpmlXH$xPnNQN#f%Z7|p)nC>WsuO= z4zyqapLS<8(UJ~Qi9d|dQijb_xhA2)v>la)<1md5s^R1N&PiuA$^k|A<+2C?OiHbj z>Bn$~t)>Y(Zb`8hW7q9xQ=s>Rv81V+UiuZJc<23HplI88isqRCId89fb`Kt|CxVIg znWcwprwXnotO>3s&Oypkte^9yJjlUVVxSe%_xlzmje|mYOVPH^vjA=?6xd0vaj0Oz zwJ4OJNiFdnHJX3rw&inskjryukl`*fRQ#SMod5J|KroJRsVXa5_$q7whSQ{gOi*s0 z1LeCy|JBWRsDPn7jCb4s(p|JZiZ8+*ExC@Vj)MF|*Vp{B(ziccSn`G1Br9bV(v!C2 z6#?eqpJBc9o@lJ#^p-`-=`4i&wFe>2)nlPK1p9yPFzJCzBQbpkcR>={YtamIw)3nt z(QEF;+)4`>8^_LU)_Q3 zC5_7lgi_6y>U%m)m@}Ku4C}=l^J=<<7c;99ec3p{aR+v=diuJR7uZi%aQv$oP?dn?@6Yu_+*^>T0ptf(oobdL;6)N-I!TO`zg^Xbv3#L0I~sn@WGk-^SmPh5>W+LB<+1PU}AKa?FCWF|qMNELOgdxR{ zbqE7@jVe+FklzdcD$!(A$&}}H*HQFTJ+AOrJYnhh}Yvta(B zQ_bW4Rr;R~&6PAKwgLWXS{Bnln(vUI+~g#kl{r+_zbngT`Y3`^Qf=!PxN4IYX#iW4 zucW7@LLJA9Zh3(rj~&SyN_pjO8H&)|(v%!BnMWySBJV=eSkB3YSTCyIeJ{i;(oc%_hk{$_l;v>nWSB)oVeg+blh=HB5JSlG_r7@P z3q;aFoZjD_qS@zygYqCn=;Zxjo!?NK!%J$ z52lOP`8G3feEj+HTp@Tnn9X~nG=;tS+z}u{mQX_J0kxtr)O30YD%oo)L@wy`jpQYM z@M>Me=95k1p*FW~rHiV1CIfVc{K8r|#Kt(ApkXKsDG$_>76UGNhHExFCw#Ky9*B-z zNq2ga*xax!HMf_|Vp-86r{;~YgQKqu7%szk8$hpvi_2I`OVbG1doP(`gn}=W<8%Gn z%81#&WjkH4GV;4u43EtSW>K_Ta3Zj!XF?;SO3V#q=<=>Tc^@?A`i;&`-cYj|;^ zEo#Jl5zSr~_V-4}y8pnufXLa80vZY4z2ko7fj>DR)#z=wWuS1$$W!L?(y}YC+yQ|G z@L&`2upy3f>~*IquAjkVNU>}c10(fq#HdbK$~Q3l6|=@-eBbo>B9(6xV`*)sae58*f zym~RRVx;xoCG3`JV`xo z!lFw)=t2Hy)e!IFs?0~7osWk(d%^wxq&>_XD4+U#y&-VF%4z?XH^i4w`TxpF{`XhZ z%G}iEzf!T(l>g;W9<~K+)$g!{UvhW{E0Lis(S^%I8OF&%kr!gJ&fMOpM=&=Aj@wuL zBX?*6i51Qb$uhkwkFYkaD_UDE+)rh1c;(&Y=B$3)J&iJfQSx!1NGgPtK!$c9OtJuu zX(pV$bfuJpRR|K(dp@^j}i&HeJOh@|7lWo8^$*o~Xqo z5Sb+!EtJ&e@6F+h&+_1ETbg7LfP5GZjvIUIN3ibCOldAv z)>YdO|NH$x7AC8dr=<2ekiY1%fN*r~e5h6Yaw<{XIErujKV~tiyrvV_DV0AzEknC- zR^xKM3i<1UkvqBj3C{wDvytOd+YtDSGu!gEMg+!&|8BQrT*|p)(dwQLEy+ zMtMzij3zo40)CA!BKZF~yWg?#lWhqD3@qR)gh~D{uZaJO;{OWV8XZ_)J@r3=)T|kt zUS1pXr6-`!Z}w2QR7nP%d?ecf90;K_7C3d!UZ`N(TZoWNN^Q~RjVhQG{Y<%E1PpV^4 z-m-K+$A~-+VDABs^Q@U*)YvhY4Znn2^w>732H?NRK(5QSS$V@D7yz2BVX4)f5A04~$WbxGOam22>t&uD)JB8-~yiQW6ik;FGblY_I>SvB_z2?PS z*Qm&qbKI{H1V@YGWzpx`!v)WeLT02};JJo*#f$a*FH?IIad-^(;9XC#YTWN6;Z6+S zm4O1KH=#V@FJw7Pha0!9Vb%ZIM$)a`VRMoiN&C|$YA3~ZC*8ayZRY^fyuP6$n%2IU z$#XceYZeqLTXw(m$_z|33I$B4k~NZO>pP6)H_}R{E$i%USGy{l{-jOE;%CloYPEU+ zRFxOn4;7lIOh!7abb23YKD+_-?O z0FP9otcAh+oSj;=f#$&*ExUHpd&e#bSF%#8*&ItcL2H$Sa)?pt0Xtf+t)z$_u^wZi z44oE}r4kIZGy3!Mc8q$B&6JqtnHZ>Znn!Zh@6rgIu|yU+zG8q`q9%B18|T|oN3zMq z`l&D;U!OL~%>vo&q0>Y==~zLiCZk4v%s_7!9DxQ~id1LLE93gf*gg&2$|hB#j8;?3 z5v4S;oM6rT{Y;I+#FdmNw z){d%tNM<<#GN%n9ox7B=3#;u7unZ~tLB_vRZ52a&2=IM)2VkXm=L+Iqq~uk#Dug|x z>S84e+A7EiOY5lj*!q?6HDkNh~0g;0Jy(al!ZHHDtur9T$y-~)94HelX1NHjXWIM7UAe}$?jiz z9?P4`I0JM=G5K{3_%2jPLC^_Mlw?-kYYgb7`qGa3@dn|^1fRMwiyM@Ch z;CB&o7&&?c5e>h`IM;Wnha0QKnEp=$hA8TJgR-07N~U5(>9vJzeoFsSRBkDq=x(YgEMpb=l4TDD`2 zwVJpWGTA_u7}?ecW7s6%rUs&NXD3+n;jB86`X?8(l3MBo6)PdakI6V6a}22{)8ilT zM~T*mU}__xSy|6XSrJ^%lDAR3Lft%+yxC|ZUvSO_nqMX!_ul3;R#*{~4DA=h$bP)%8Yv9X zyp><|e8=_ttI}ZAwOd#dlnSjck#6%273{E$kJuCGu=I@O)&6ID{nWF5@gLb16sj|&Sb~+du4e4O_%_o`Ix4NRrAsyr1_}MuP94s>de8cH-OUkVPk3+K z&jW)It9QiU-ti~AuJkL`XMca8Oh4$SyJ=`-5WU<{cIh+XVH#e4d&zive_UHC!pN>W z3TB;Mn5i)9Qn)#6@lo4QpI3jFYc0~+jS)4AFz8fVC;lD^+idw^S~Qhq>Tg(!3$yLD zzktzoFrU@6s4wwCMz}edpF5i5Q1IMmEJQHzp(LAt)pgN3&O!&d?3W@6U4)I^2V{;- z6A(?zd93hS*uQmnh4T)nHnE{wVhh(=MMD(h(P4+^p83Om6t<*cUW>l(qJzr%5vp@K zN27ka(L{JX=1~e2^)F^i=TYj&;<7jyUUR2Bek^A8+3Up*&Xwc{)1nRR5CT8vG>ExV zHnF3UqXJOAno_?bnhCX-&kwI~Ti8t4`n0%Up>!U`ZvK^w2+0Cs-b9%w%4`$+To|k= zKtgc&l}P`*8IS>8DOe?EB84^kx4BQp3<7P{Pq}&p%xF_81pg!l2|u=&I{AuUgmF5n zJQCTLv}%}xbFGYtKfbba{CBo)lWW%Z>i(_NvLhoQZ*5-@2l&x>e+I~0Nld3UI9tdL zRzu8}i;X!h8LHVvN?C+|M81e>Jr38%&*9LYQec9Ax>?NN+9(_>XSRv&6hlCYB`>Qm z1&ygi{Y()OU4@D_jd_-7vDILR{>o|7-k)Sjdxkjgvi{@S>6GqiF|o`*Otr;P)kLHN zZkpts;0zw_6;?f(@4S1FN=m!4^mv~W+lJA`&7RH%2$)49z0A+8@0BCHtj|yH--AEL z0tW6G%X-+J+5a{5*WKaM0QDznf;V?L5&uQw+yegDNDP`hA;0XPYc6e0;Xv6|i|^F2WB)Z$LR|HR4 zTQsRAby9(^Z@yATyOgcfQw7cKyr^3Tz7lc7+JEwwzA7)|2x+PtEb>nD(tpxJQm)Kn zW9K_*r!L%~N*vS8<5T=iv|o!zTe9k_2jC_j*7ik^M_ zaf%k{WX{-;0*`t`G!&`eW;gChVXnJ-Rn)To8vW-?>>a%QU1v`ZC=U)f8iA@%JG0mZ zDqH;~mgBnrCP~1II<=V9;EBL)J+xzCoiRBaeH&J6rL!{4zIY8tZka?_FBeQeNO3q6 zyG_alW54Ba&wQf{&F1v-r1R6ID)PTsqjIBc+5MHkcW5Fnvi~{-FjKe)t1bl}Y;z@< z=!%zvpRua>>t_x}^}z0<7MI!H2v6|XAyR9!t50q-A)xk0nflgF4*OQlCGK==4S|wc zRMsSscNhRzHMBU8TdcHN!q^I}x0iXJ%uehac|Zs_B$p@CnF)HeXPpB_Za}F{<@6-4 zl%kml@}kHQ(ypD8FsPJ2=14xXJE|b20RUIgs!2|R3>LUMGF6X*B_I|$`Qg=;zm7C z{mEDy9dTmPbued7mlO@phdmAmJ7p@GR1bjCkMw6*G7#4+`k>fk1czdJUB!e@Q(~6# zwo%@p@V5RL0ABU2LH7Asq^quDUho@H>eTZH9f*no9fY0T zD_-9px3e}A!>>kv5wk91%C9R1J_Nh!*&Kk$J3KNxC}c_@zlgpJZ+5L)Nw|^p=2ue}CJtm;uj*Iqr)K})kA$xtNUEvX;4!Px*^&9T_`IN{D z{6~QY=Nau6EzpvufB^hflc#XIsSq0Y9(nf$d~6ZwK}fal92)fr%T3=q{0mP-EyP_G z)UR5h@IX}3Qll2b0oCAcBF>b*@Etu*aTLPU<%C>KoOrk=x?pN!#f_Og-w+;xbFgjQ zXp`et%lDBBh~OcFnMKMUoox0YwBNy`N0q~bSPh@+enQ=4RUw1) zpovN`QoV>vZ#5LvC;cl|6jPr}O5tu!Ipoyib8iXqy}TeJ;4+_7r<1kV0v5?Kv>fYp zg>9L`;XwXa&W7-jf|9~uP2iyF5`5AJ`Q~p4eBU$MCC00`rcSF>`&0fbd^_eqR+}mK z4n*PMMa&FOcc)vTUR zlDUAn-mh`ahi_`f`=39JYTNVjsTa_Y3b1GOIi)6dY)D}xeshB0T8Eov5%UhWd1)u}kjEQ|LDo{tqKKrYIfVz~@dp!! zMOnah@vp)%_-jDTUG09l+;{CkDCH|Q{NqX*uHa1YxFShy*1+;J`gywKaz|2Q{lG8x zP?KBur`}r`!WLKXY_K;C8$EWG>jY3UIh{+BLv0=2)KH%P}6xE2kg)%(-uA6lC?u8}{K(#P*c zE9C8t*u%j2r_{;Rpe1A{9nNXU;b_N0vNgyK!EZVut~}+R2rcbsHilqsOviYh-pYX= zHw@53nlmwYI5W5KP>&`dBZe0Jn?nAdC^HY1wlR6$u^PbpB#AS&5L6zqrXN&7*N2Q` z+Rae1EwS)H=aVSIkr8Ek^1jy2iS2o7mqm~Mr&g5=jjt7VxwglQ^`h#Mx+x2v|9ZAwE$i_9918MjJxTMr?n!bZ6n$}y11u8I9COTU`Z$Fi z!AeAQLMw^gp_{+0QTEJrhL424pVDp%wpku~XRlD3iv{vQ!lAf!_jyqd_h}+Tr1XG| z`*FT*NbPqvHCUsYAkFnM`@l4u_QH&bszpUK#M~XLJt{%?00GXY?u_{gj3Hvs!=N(I z(=AuWPijyoU!r?aFTsa8pLB&cx}$*%;K$e*XqF{~*rA-qn)h^!(-;e}O#B$|S~c+U zN4vyOK0vmtx$5K!?g*+J@G1NmlEI=pyZXZ69tAv=@`t%ag_Hk{LP~OH9iE)I= zaJ69b4kuCkV0V zo(M0#>phpQ_)@j;h%m{-a*LGi(72TP)ws2w*@4|C-3+;=5DmC4s7Lp95%n%@Ko zfdr3-a7m*dys9iIci$A=4NPJ`HfJ;hujLgU)ZRuJI`n;Pw|yksu!#LQnJ#dJysgNb z@@qwR^wrk(jbq4H?d!lNyy72~Dnn87KxsgQ!)|*m(DRM+eC$wh7KnS-mho3|KE)7h zK3k;qZ;K1Lj6uEXLYUYi)1FN}F@-xJ z@@3Hb84sl|j{4$3J}aTY@cbX@pzB_qM~APljrjju6P0tY{C@ zpUCOz_NFmALMv1*blCcwUD3?U6tYs+N%cmJ98D%3)%)Xu^uvzF zS5O!sc#X6?EwsYkvPo6A%O8&y8sCCQH<%f2togVwW&{M;PR!a(ZT_A+jVAbf{@5kL zB@Z(hb$3U{T_}SKA_CoQVU-;j>2J=L#lZ~aQCFg-d<9rzs$_gO&d5N6eFSc z1ml8)P*FSi+k@!^M9nDWR5e@ATD8oxtDu=36Iv2!;dZzidIS(PCtEuXAtlBb1;H%Z zwnC^Ek*D)EX4#Q>R$$WA2sxC_t(!!6Tr?C#@{3}n{<^o;9id1RA&-Pig1e-2B1XpG zliNjgmd3c&%A}s>qf{_j#!Z`fu0xIwm4L0)OF=u(OEmp;bLCIaZX$&J_^Z%4Sq4GZ zPn6sV_#+6pJmDN_lx@1;Zw6Md_p0w9h6mHtzpuIEwNn>OnuRSC2=>fP^Hqgc)xu^4 z<3!s`cORHJh#?!nKI`Et7{3C27+EuH)Gw1f)aoP|B3y?fuVfvpYYmmukx0ya-)TQX zR{ggy5cNf4X|g)nl#jC9p>7|09_S7>1D2GTRBUTW zAkQ=JMRogZqG#v;^=11O6@rPPwvJkr{bW-Qg8`q8GoD#K`&Y+S#%&B>SGRL>;ZunM@49!}Uy zN|bBCJ%sO;@3wl0>0gbl3L@1^O60ONObz8ZI7nder>(udj-jt`;yj^nTQ$L9`OU9W zX4alF#$|GiR47%x@s&LV>2Sz2R6?;2R~5k6V>)nz!o_*1Y!$p>BC5&?hJg_MiE6UBy>RkVZj`9UWbRkN-Hk!S`=BS3t3uyX6)7SF#)71*}`~Ogz z1rap5H6~dhBJ83;q-Y<5V35C2&F^JI-it(=5D#v!fAi9p#UwV~2tZQI+W(Dv?1t9? zfh*xpxxO{-(VGB>!Q&0%^YW_F!@aZS#ucP|YaD#>wd1Fv&Z*SR&mc;asi}1G) z_H>`!akh-Zxq9#io(7%;a$)w+{QH)Y$?UK1Dt^4)up!Szcxnu}kn$0afcfJL#IL+S z5gF_Y30j;{lNrG6m~$Ay?)*V9fZuU@3=kd40=LhazjFrau>(Y>SJNtOz>8x_X-BlA zIpl{i>OarVGj1v(4?^1`R}aQB&WCRQzS~;7R{tDZG=HhgrW@B`W|#cdyj%YBky)P= zpxuOZkW>S6%q7U{VsB#G(^FMsH5QuGXhb(sY+!-R8Bmv6Sx3WzSW<1MPPN1!&PurYky(@`bP9tz z52}LH9Q?+FF5jR6-;|+GVdRA!qtd;}*-h&iIw3Tq3qF9sDIb1FFxGbo&fbG5n8$3F zyY&PWL{ys^dTO}oZ#@sIX^BKW*bon=;te9j5k+T%wJ zNJtoN1~YVj4~YRrlZl)b&kJqp+Z`DqT!la$x&&IxgOQw#yZd-nBP3!7FijBXD|IsU8Zl^ zc6?MKpJQ+7ka|tZQLfchD$PD|;K(9FiLE|eUZX#EZxhG!S-63C$jWX1Yd!6-Yxi-u zjULIr|0-Q%D9jz}IF~S%>0(jOqZ(Ln<$9PxiySr&2Oic7vb<8q=46)Ln%Z|<*z5&> z3f~Zw@m;vR(bESB<=Jqkxn(=#hQw42l(7)h`vMQQTttz9XW6^|^8EK7qhju4r_c*b zJIi`)MB$w@9epwdIfnEBR+?~);yd6C(LeMC& zn&&N*?-g&BBJcV;8&UoZi4Lmxcj16ojlxR~zMrf=O_^i1wGb9X-0@6_rpjPYemIin zmJb+;lHe;Yp=8G)Q(L1bzH*}I>}uAqhj4;g)PlvD9_e_ScR{Ipq|$8NvAvLD8MYr}xl=bU~)f%B3E>r3Bu9_t|ThF3C5~BdOve zEbk^r&r#PT&?^V1cb{72yEWH}TXEE}w>t!cY~rA+hNOTK8FAtIEoszp!qqptS&;r$ zaYV-NX96-h$6aR@1xz6_E0^N49mU)-v#bwtGJm)ibygzJ8!7|WIrcb`$XH~^!a#s& z{Db-0IOTFq#9!^j!n_F}#Z_nX{YzBK8XLPVmc&X`fT7!@$U-@2KM9soGbmOSAmqV z{nr$L^MBo_u^Joyf0E^=eo{Rt0{{e$IFA(#*kP@SQd6lWT2-#>` zP1)7_@IO!9lk>Zt?#CU?cuhiLF&)+XEM9B)cS(gvQT!X3`wL*{fArTS;Ak`J<84du zALKPz4}3nlG8Fo^MH0L|oK2-4xIY!~Oux~1sw!+It)&D3p;+N8AgqKI`ld6v71wy8I!eP0o~=RVcFQR2Gr(eP_JbSytoQ$Yt}l*4r@A8Me94y z8cTDWhqlq^qoAhbOzGBXv^Wa4vUz$(7B!mX`T=x_ueKRRDfg&Uc-e1+z4x$jyW_Pm zp?U;-R#xt^Z8Ev~`m`iL4*c#65Nn)q#=Y0l1AuD&+{|8-Gsij3LUZXpM0Bx0u7WWm zH|%yE@-#XEph2}-$-thl+S;__ciBxSSzHveP%~v}5I%u!z_l_KoW{KRx2=eB33umE zIYFtu^5=wGU`Jab8#}cnYry@9p5UE#U|VVvx_4l49JQ;jQdp(uw=$^A$EA$LM%vmE zvdEOaIcp5qX8wX{mYf0;#51~imYYPn4=k&#DsKTxo{_Mg*;S495?OBY?#gv=edYC* z^O@-sd-qa+U24xvcbL0@C7_6o!$`)sVr-jSJE4XQUQ$?L7}2(}Eixqv;L8AdJAVqc zq}RPgpnDb@E_;?6K58r3h4-!4rT4Ab#rLHLX?eMOfluJk=3i1@Gt1i#iA=O`M0@x! z(HtJP9BMHXEzuD93m|B&woj0g6T?f#^)>J>|I4C5?Gam>n9!8CT%~aT;=oco5d6U8 zMXl(=W;$ND_8+DD*?|5bJ!;8ebESXMUKBAf7YBwNVJibGaJ*(2G`F%wx)grqVPjudiaq^Kl&g$8A2 zWMxMr@_$c}d+;_B`#kUX-t|4VKH&_f^^EP0&=DPLW)H)UzBG%%Tra*5 z%$kyZe3I&S#gfie^z5)!twG={3Cuh)FdeA!Kj<-9** zvT*5%Tb`|QbE!iW-XcOuy39>D3oe6x{>&<#E$o8Ac|j)wq#kQzz|ATd=Z0K!p2$QE zPu?jL8Lb^y3_CQE{*}sTDe!2!dtlFjq&YLY@2#4>XS`}v#PLrpvc4*@q^O{mmnr5D zmyJq~t?8>FWU5vZdE(%4cuZuao0GNjp3~Dt*SLaxI#g_u>hu@k&9Ho*#CZP~lFJHj z(e!SYlLigyc?&5-YxlE{uuk$9b&l6d`uIlpg_z15dPo*iU&|Khx2*A5Fp;8iK_bdP z?T6|^7@lcx2j0T@x>X7|kuuBSB7<^zeY~R~4McconTxA2flHC0_jFxmSTv-~?zVT| zG_|yDqa9lkF*B6_{j=T>=M8r<0s;@z#h)3BQ4NLl@`Xr__o7;~M&dL3J8fP&zLfDfy z);ckcTev{@OUlZ`bCo(-3? z1u1xD`PKgSg?RqeVVsF<1SLF;XYA@Bsa&cY!I48ZJn1V<3d!?s=St?TLo zC0cNr`qD*M#s6f~X>SCNVkva^9A2ZP>CoJ9bvgXe_c}WdX-)pHM5m7O zrHt#g$F0AO+nGA;7dSJ?)|Mo~cf{z2L)Rz!`fpi73Zv)H=a5K)*$5sf_IZypi($P5 zsPwUc4~P-J1@^3C6-r9{V-u0Z&Sl7vNfmuMY4yy*cL>_)BmQF!8Om9Dej%cHxbIzA zhtV0d{=%cr?;bpBPjt@4w=#<>k5ee=TiWAXM2~tUGfm z$s&!Dm0R^V$}fOR*B^kGaipi~rx~A2cS0;t&khV1a4u38*XRUP~f za!rZMtay8bsLt6yFYl@>-y^31(*P!L^^s@mslZy(SMsv9bVoX`O#yBgEcjCmGpyc* zeH$Dw6vB5P*;jor+JOX@;6K#+xc)Z9B8M=x2a@Wx-{snPGpRmOC$zpsqW*JCh@M2Y z#K+M(>=#d^>Of9C`))h<=Bsy)6zaMJ&x-t%&+UcpLjV`jo4R2025 zXaG8EA!0lQa)|dx-@{O)qP6`$rhCkoQqZ`^SW8g-kOwrwsK8 z3ms*AIcyj}-1x&A&vSq{r=QMyp3CHdWH35!sad#!Sm>^|-|afB+Q;|Iq@LFgqIp#Z zD1%H+3I?6RGnk&IFo|u+E0dCxXz4yI^1i!QTu7uvIEH>i3rR{srcST`LIRwdV1P;W z+%AN1NIf@xxvVLiSX`8ILA8MzNqE&7>%jMzGt9wm78bo9<;h*W84i29^w!>V>{N+S zd`5Zmz^G;f=icvoOZfK5#1ctx*~UwD=ab4DGQXehQ!XYnak*dee%YN$_ZPL%KZuz$ zD;$PpT;HM^$KwtQm@7uvT`i6>Hae1CoRVM2)NL<2-k2PiX=eAx+-6j#JI?M}(tuBW zkF%jjLR)O`gI2fcPBxF^HeI|DWwQWHVR!;;{BXXHskxh8F@BMDn`oEi-NHt;CLymW z=KSv5)3dyzec0T5B*`g-MQ<;gz=nIWKUi9ko<|4I(-E0k$QncH>E4l z**1w&#={&zv4Tvhgz#c29`m|;lU-jmaXFMC11 z*dlXDMEOG>VoLMc>!rApwOu2prKSi*!w%`yzGmS+k(zm*CsLK*wv{S_0WX^8A-rKy zbk^Gf_92^7iB_uUF)EE+ET4d|X|>d&mdN?x@vxKAQk`O+r4Qdu>XGy(a(19g;=jU} zFX{O*_NG>!$@jh!U369Lnc+D~qch3uT+_Amyi}*k#LAAwh}k8IPK5a-WZ81ufD>l> z$4cF}GSz>ce`3FAic}6W4Z7m9KGO?(eWqi@L|5Hq0@L|&2flN1PVl}XgQ2q*_n2s3 zt5KtowNkTYB5b;SVuoXA@i5irXO)A&%7?V`1@HGCB&)Wgk+l|^XXChq;u(nyPB}b3 zY>m5jkxpZgi)zfbgv&ec4Zqdvm+D<?Im*mXweS9H+V>)zF#Zp3)bhl$PbISY{5=_z!8&*Jv~NYtI-g!>fDs zmvL5O^U%!^VaKA9gvKw|5?-jk>~%CVGvctKmP$kpnpfN{D8@X*Aazi$txfa%vd-|E z>kYmV66W!lNekJPom29LdZ%(I+ZLZYTXzTg*to~m?7vp%{V<~>H+2}PQ?PPAq`36R z<%wR8v6UkS>Wt#hzGk#44W<%9S=nBfB);6clKwnxY}T*w21Qc3_?IJ@4gYzC7s;WP zVQNI(M=S=JT#xsZy7G`cR(BP9*je0bfeN8JN5~zY(DDs0t{LpHOIbN);?T-69Pf3R zSNe*&p2%AwXHL>__g+xd4Hlc_vu<25H?(`nafS%)3UPP7_4;gk-9ckt8SJRTv5v0M z_Hww`qPudL?ajIR&X*;$y-`<)6dxx1U~5eGS13CB!lX;3w7n&lDDiArbAhSycd}+b zya_3p@A`$kQy;|NJZ~s44Hqo7Hwt}X86NK=(ey>lgWTtGL6k@Gy;PbO!M%1~Wcn2k zUFP|*5d>t-X*RU8g%>|(wwj*~#l4z^Aatf^DWd1Wj#Q*AY0D^V@sC`M zjJc6qXu0I7Y*2;;gGu!plAFzG=J;1%eIOdn zQA>J&e05UN*7I5@yRhK|lbBSfJ+5Uq;!&HV@xfPZrgD}kE*1DSq^=%{o%|LChhl#0 zlMb<^a6ixzpd{kNZr|3jTGeEzuo}-eLT-)Q$#b{!vKx8Tg}swCni>{#%vDY$Ww$84 zew3c9BBovqb}_&BRo#^!G(1Eg((BScRZ}C)Oz?y`T5wOrv);)b^4XR8 zhJo7+<^7)qB>I;46!GySzdneZ>n_E1oWZY;kf94#)s)kWjuJN1c+wbVoNQcmnv}{> zN0pF+Sl3E}UQ$}slSZeLJrwT>Sr}#V(dVaezCQl2|4LN`7L7v&siYR|r7M(*JYfR$ zst3=YaDw$FSc{g}KHO&QiKxuhEzF{f%RJLKe3p*7=oo`WNP)M(9X1zIQPP0XHhY3c znrP{$4#Ol$A0s|4S7Gx2L23dv*Gv2o;h((XVn+9+$qvm}s%zi6nI-_s6?mG! zj{DV;qesJb&owKeEK?=J>UcAlYckA7Sl+I&IN=yasrZOkejir*kE@SN`fk<8Fgx*$ zy&fE6?}G)d_N`){P~U@1jRVA|2*69)KSe_}!~?+`Yb{Y=O~_+@!j<&oVQQMnhoIRU zA0CyF1OFfkK44n*JD~!2!SCPM;PRSk%1XL=0&rz00wxPs&-_eapJy#$h!eqY%nS0{ z!aGg58JIJPF3_ci%n)QSVpa2H`vIe$RD43;#IRfDV&Ibit z+?>HW4{2wOfC6Fw)}4x}i1maDxcE1qi@BS*qcxD2gE@h3#4cgU*D-&3z7D|tVZWt= z-Cy2+*Cm@P4GN_TPUtaVyVesbVDazF@)j8VJ4>XZv!f%}&eO1SvIgr}4`A*3#vat< z_MoByL(qW6L7SFZ#|Gc1fFN)L2PxY+{B8tJp+pxRyz*87)vXR}*=&ahXjBlQKguuf zX6x<<6fQulE^C*KH8~W%ptpaC0l?b=_{~*U4?5Vt;dgM4t_{&UZ1C2j?b>b+5}{IF_CUyvz-@QZPMlJ)r_tS$9kH%RPv#2_nMb zRLj5;chJ72*U`Z@Dqt4$@_+k$%|8m(HqLG!qT4P^DdfvGf&){gKnGCX#H0!;W=AGP zbA&Z`-__a)VTS}kKFjWGk z%|>yE?t*EJ!qeQ%dPk$;xIQ+P0;()PCBDgjJm6Buj{f^awNoVx+9<|lg3%-$G(*f) zll6oOkN|yamn1uyl2*N-lnqRI1cvs_JxLTeahEK=THV$Sz*gQhKNb*p0fNoda#-&F zB-qJgW^g}!TtM|0bS2QZekW7_tKu%GcJ!4?lObt0z_$mZ4rbQ0o=^curCs3bJK6sq z9fu-aW-l#>z~ca(B;4yv;2RZ?tGYAU)^)Kz{L|4oPj zdOf_?de|#yS)p2v8-N||+XL=O*%3+y)oI(HbM)Ds?q8~HPzIP(vs*G`iddbWq}! z(2!VjP&{Z1w+%eUq^ /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -133,22 +132,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -165,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -193,16 +198,19 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/examples/trace-analytics-sample-app/sample-app/analytics-service/gradlew.bat b/examples/trace-analytics-sample-app/sample-app/analytics-service/gradlew.bat index f127cfd49d..c4bdd3ab8e 100644 --- a/examples/trace-analytics-sample-app/sample-app/analytics-service/gradlew.bat +++ b/examples/trace-analytics-sample-app/sample-app/analytics-service/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @@ -26,6 +28,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -42,11 +45,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,22 +59,21 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell From c1739b0484ad51c43c4a2df53b47306590720b2f Mon Sep 17 00:00:00 2001 From: "mend-for-github-com[bot]" <50673670+mend-for-github-com[bot]@users.noreply.github.com> Date: Thu, 11 Dec 2025 16:59:59 -0800 Subject: [PATCH 16/51] Update dependency urllib3 to v2.6.0 (#6345) Co-authored-by: mend-for-github-com[bot] <50673670+mend-for-github-com[bot]@users.noreply.github.com> Signed-off-by: Nathan Wand --- examples/trace-analytics-sample-app/sample-app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/trace-analytics-sample-app/sample-app/requirements.txt b/examples/trace-analytics-sample-app/sample-app/requirements.txt index 6bcba62ec4..48d510e652 100644 --- a/examples/trace-analytics-sample-app/sample-app/requirements.txt +++ b/examples/trace-analytics-sample-app/sample-app/requirements.txt @@ -8,6 +8,6 @@ opentelemetry-sdk==1.25.0 protobuf==4.25.8 requests==2.32.4 setuptools==78.1.1 -urllib3==2.5.0 +urllib3==2.6.0 werkzeug==3.0.6 zipp==3.19.1 From 2aed31eddc52c2780e2b8186225c82d6c677097d Mon Sep 17 00:00:00 2001 From: David Venable Date: Fri, 12 Dec 2025 14:14:39 -0600 Subject: [PATCH 17/51] Adds Kiro and Visual Studio Code directories to the .gitignore file. Some reorganization of this file. (#6353) Signed-off-by: David Venable Signed-off-by: Nathan Wand --- .gitignore | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 4247753be3..418934fe46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,6 @@ -# Ignore Gradle project-specific cache directory -.DS_Store -.idea -*.iml -.gradle - -# Ignore Gradle build output directory +# Gradle directories build - -# Ignore things downloaded for gradle es plugin +.gradle gradle/tools # Ignore config file generated by test @@ -19,3 +12,10 @@ data-prepper-main/src/test/resources/logstash-conf/logstash-filter.yaml # output folder created when we run test cases **/out/ + +# Development tools +.DS_Store +.idea +*.iml +.kiro +.vscode From 2d9026fc8a54c0942bad669e66fa533832481a21 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Mon, 15 Dec 2025 08:44:05 -0600 Subject: [PATCH 18/51] Do not clear offsets after failure to commit offsets due to rebalance exception (#6346) Signed-off-by: Taylor Gray Signed-off-by: Nathan Wand --- .../kafka/consumer/KafkaCustomConsumer.java | 7 +- .../consumer/KafkaCustomConsumerTest.java | 85 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java b/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java index 7324abed11..f51c5cec2a 100644 --- a/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java +++ b/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java @@ -17,6 +17,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.errors.RebalanceInProgressException; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.TopicPartition; @@ -354,11 +355,15 @@ private void commitOffsets(boolean forceCommit) { offsetsToCommit.forEach(((partition, offset) -> updateCommitCountMetric(partition, offset))); try { consumer.commitSync(offsetsToCommit); + lastCommitTime = currentTimeMillis; + } catch (final RebalanceInProgressException ex) { + LOG.error("Failed to commit offsets in topic {} due to rebalance in progress", topicName, ex); + return; } catch (Exception e) { LOG.error("Failed to commit offsets in topic {}", topicName, e); } + offsetsToCommit.clear(); - lastCommitTime = currentTimeMillis; } } diff --git a/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java b/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java index 1bbc60ecdb..702fd26849 100644 --- a/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java +++ b/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java @@ -16,6 +16,7 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.RebalanceInProgressException; import org.apache.kafka.common.errors.RecordDeserializationException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -60,6 +61,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -589,6 +592,88 @@ public void testAwsGlueErrorWithAcknowledgements() throws Exception { }); } + @Test + public void testCommitOffsets_RebalanceInProgressException_DoesNotClearOffsets() throws Exception { + String topic = topicConfig.getName(); + TopicPartition topicPartition = new TopicPartition(topic, testPartition); + + when(topicConfig.getSerdeFormat()).thenReturn(MessageFormat.PLAINTEXT); + when(topicConfig.getAutoCommit()).thenReturn(false); + when(topicConfig.getCommitInterval()).thenReturn(Duration.ofMillis(0)); + + consumer = createObjectUnderTest("plaintext", false); + consumer.onPartitionsAssigned(List.of(topicPartition)); + + consumerRecords = createPlainTextRecords(topic, 100L); + when(kafkaConsumer.poll(any(Duration.class))).thenReturn(consumerRecords); + + doThrow(new RebalanceInProgressException("Rebalance in progress")) + .when(kafkaConsumer).commitSync(anyMap()); + + consumer.consumeRecords(); + + Map offsetsBeforeCommit = new HashMap<>(consumer.getOffsetsToCommit()); + Assertions.assertFalse(offsetsBeforeCommit.isEmpty(), "Offsets should be populated after consuming records"); + Assertions.assertEquals(102L, offsetsBeforeCommit.get(topicPartition).offset()); + + Thread testThread = new Thread(() -> { + try { + java.lang.reflect.Method method = consumer.getClass().getDeclaredMethod("commitOffsets", boolean.class); + method.setAccessible(true); + method.invoke(consumer, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + testThread.start(); + testThread.join(5000); + + Map offsetsAfterFailedCommit = consumer.getOffsetsToCommit(); + Assertions.assertFalse(offsetsAfterFailedCommit.isEmpty(), + "Offsets should NOT be cleared after RebalanceInProgressException"); + Assertions.assertEquals(offsetsBeforeCommit.get(topicPartition).offset(), + offsetsAfterFailedCommit.get(topicPartition).offset(), + "Offset value should remain unchanged for retry after rebalance completes"); + } + + @Test + public void testCommitOffsets_OtherException_ClearsOffsets() throws Exception { + String topic = topicConfig.getName(); + TopicPartition topicPartition = new TopicPartition(topic, testPartition); + + when(topicConfig.getAutoCommit()).thenReturn(false); + when(topicConfig.getCommitInterval()).thenReturn(Duration.ofMillis(0)); + + consumer = createObjectUnderTest("plaintext", false); + consumer.onPartitionsAssigned(List.of(topicPartition)); + + consumerRecords = createPlainTextRecords(topic, 100L); + when(kafkaConsumer.poll(any(Duration.class))).thenReturn(consumerRecords); + + doThrow(new RuntimeException("Generic commit failure")) + .when(kafkaConsumer).commitSync(anyMap()); + + consumer.consumeRecords(); + + Assertions.assertFalse(consumer.getOffsetsToCommit().isEmpty(), + "Offsets should be populated after consuming records"); + + Thread testThread = new Thread(() -> { + try { + java.lang.reflect.Method method = consumer.getClass().getDeclaredMethod("commitOffsets", boolean.class); + method.setAccessible(true); + method.invoke(consumer, true); + } catch (Exception e) { + } + }); + testThread.start(); + testThread.join(5000); + + Map offsetsAfterFailedCommit = consumer.getOffsetsToCommit(); + Assertions.assertTrue(offsetsAfterFailedCommit.isEmpty(), + "Offsets should be cleared after non-rebalance exception"); + } + private ConsumerRecords createPlainTextRecords(String topic, final long startOffset) { Map> records = new HashMap<>(); ConsumerRecord record1 = new ConsumerRecord<>(topic, testPartition, startOffset, testKey1, testValue1); From 34ad197f6bbddab6aefffe945e4c035fad4ed921 Mon Sep 17 00:00:00 2001 From: Vecheka Date: Mon, 15 Dec 2025 11:17:46 -0800 Subject: [PATCH 19/51] Remove experimental lable for M365 (#6351) Signed-off-by: Vecheka Chhourn Signed-off-by: Nathan Wand --- .../plugins/source/microsoft_office365/Office365Source.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365Source.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365Source.java index 5964764169..1a270fba61 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365Source.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365Source.java @@ -13,7 +13,6 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; -import org.opensearch.dataprepper.model.annotations.Experimental; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.PluginFactory; @@ -37,7 +36,6 @@ import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.PLUGIN_NAME; -@Experimental @DataPrepperPlugin(name = PLUGIN_NAME, pluginType = Source.class, pluginConfigurationType = Office365SourceConfig.class, From 9d189ee353a9b1e72262ca950bf469ba5003541a Mon Sep 17 00:00:00 2001 From: eatulban Date: Tue, 16 Dec 2025 04:43:36 +0530 Subject: [PATCH 20/51] Refactor Retry Handler To Move Into Source Crawler Package (#6275) Signed-off-by: eatulban Signed-off-by: Nathan Wand --- .../Office365RestClient.java | 18 +- .../microsoft_office365/RetryHandler.java | 90 ---- .../auth/Office365AuthenticationProvider.java | 10 +- .../utils/retry/DefaultRetryStrategy.java | 95 ++++ .../utils/retry/DefaultStatusCodeHandler.java | 73 +++ .../utils/retry/RetryAfterHeaderStrategy.java | 151 +++++++ .../utils/retry/RetryDecision.java | 41 ++ .../utils/retry/RetryHandler.java | 117 +++++ .../utils/retry/RetryStrategy.java | 53 +++ .../utils/retry/StatusCodeHandler.java | 27 ++ .../utils/retry/DefaultRetryStrategyTest.java | 219 +++++++++ .../retry/DefaultStatusCodeHandlerTest.java | 313 +++++++++++++ .../retry/RetryAfterHeaderStrategyTest.java | 418 ++++++++++++++++++ .../utils/retry/RetryDecisionTest.java | 148 +++++++ .../utils/retry/RetryHandlerTest.java | 344 ++++++++++++++ 15 files changed, 2020 insertions(+), 97 deletions(-) delete mode 100644 data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/RetryHandler.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategy.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandler.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategy.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecision.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryStrategy.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/StatusCodeHandler.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategyTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandlerTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategyTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecisionTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java index 451cdc6983..7fc4644b79 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java @@ -16,6 +16,9 @@ import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface; import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException; import org.opensearch.dataprepper.plugins.source.microsoft_office365.models.AuditLogsResponse; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -30,6 +33,7 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.CONTENT_TYPES; @@ -57,6 +61,7 @@ public class Office365RestClient { private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/"; private final RestTemplate restTemplate = new RestTemplate(); + private final RetryHandler retryHandler; private final Office365AuthenticationInterface authConfig; private final Timer auditLogFetchLatencyTimer; private final Timer searchCallLatencyTimer; @@ -76,6 +81,9 @@ public Office365RestClient(final Office365AuthenticationInterface authConfig, this.auditLogsRequestedCounter = pluginMetrics.counter(AUDIT_LOGS_REQUESTED); this.apiCallsCounter = pluginMetrics.counter(API_CALLS); this.errorTypeMetricCounterMap = getErrorTypeMetricCounterMap(pluginMetrics); + this.retryHandler = new RetryHandler( + new DefaultRetryStrategy(), + new DefaultStatusCodeHandler()); } /** @@ -111,7 +119,7 @@ public void startSubscriptions() { authConfig.getTenantId(), contentType); - RetryHandler.executeWithRetry(() -> { + retryHandler.executeWithRetry(() -> { try { headers.setBearerAuth(authConfig.getAccessToken()); apiCallsCounter.increment(); @@ -168,7 +176,7 @@ public AuditLogsResponse searchAuditLogs(final String contentType, return searchCallLatencyTimer.record(() -> { try { - return RetryHandler.executeWithRetry( + return retryHandler.executeWithRetry( () -> { headers.setBearerAuth(authConfig.getAccessToken()); apiCallsCounter.increment(); @@ -213,7 +221,7 @@ public AuditLogsResponse searchAuditLogs(final String contentType, return new AuditLogsResponse(response.getBody(), nextPageUri); }, authConfig::renewCredentials, - provideSearchRequestFailureCounter(pluginMetrics) + Optional.of(provideSearchRequestFailureCounter(pluginMetrics)) ); } catch (Exception e) { publishErrorTypeMetricCounter(e, this.errorTypeMetricCounterMap); @@ -242,7 +250,7 @@ public String getAuditLog(String contentUri) { return auditLogFetchLatencyTimer.record(() -> { try { - String response = RetryHandler.executeWithRetry(() -> { + String response = retryHandler.executeWithRetry(() -> { headers.setBearerAuth(authConfig.getAccessToken()); apiCallsCounter.increment(); ResponseEntity responseEntity = restTemplate.exchange( @@ -253,7 +261,7 @@ public String getAuditLog(String contentUri) { ); return responseEntity.getBody(); - }, authConfig::renewCredentials, provideGetRequestsFailureCounter(pluginMetrics)); + }, authConfig::renewCredentials, Optional.of(provideGetRequestsFailureCounter(pluginMetrics))); // Log response details if (response == null) { diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/RetryHandler.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/RetryHandler.java deleted file mode 100644 index 4bc8df179c..0000000000 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/RetryHandler.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.dataprepper.plugins.source.microsoft_office365; - -import io.micrometer.core.instrument.Counter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.http.HttpStatus; -import org.springframework.web.client.HttpClientErrorException; -import org.springframework.web.client.HttpServerErrorException; - -import java.util.List; -import java.util.function.Supplier; - -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; - -@Slf4j -public class RetryHandler { - public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); - private static final int MAX_RETRIES = 6; - private static final int SLEEP_TIME_MULTIPLIER = 1000; - - public static T executeWithRetry(Supplier operation, Runnable credentialRenewal) { - return executeWithRetry(operation, credentialRenewal, null); - } - - public static T executeWithRetry(Supplier operation, Runnable credentialRenewal, Counter failureCounter) { - int retryCount = 0; - while (retryCount < MAX_RETRIES) { - boolean operationSucceeded = false; - try { - T result = operation.get(); - operationSucceeded = true; - return result; - } catch (HttpClientErrorException | HttpServerErrorException ex) { - HttpStatus statusCode = ex.getStatusCode(); - String statusMessage = ex.getMessage(); - - switch (statusCode) { - case UNAUTHORIZED: - log.error(NOISY, "Token expired. Attempting to renew credentials.", ex); - credentialRenewal.run(); - break; - case FORBIDDEN: - log.error(NOISY, "Access forbidden: {}", statusMessage, ex); - throw new SecurityException("Access forbidden: " + statusMessage); - case TOO_MANY_REQUESTS: - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); - break; - case SERVICE_UNAVAILABLE: - log.error(NOISY, "Service is unavailable. Will retry after backing off.", ex); - break; - default: - if (ex.getStatusCode().is4xxClientError()) { - log.error(NOISY, "Client error: {}. Will not retry.", statusCode, ex); - throw ex; - } else if (ex.getStatusCode().is5xxServerError()) { - log.error(NOISY, "Server error: {}. Will retry after backing off.", statusCode, ex); - } else { - throw ex; - } - } - - if (retryCount == MAX_RETRIES - 1) { - log.error(NOISY, "Exceeded maximum retry attempts.", ex); - throw ex; - } - - try { - Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * SLEEP_TIME_MULTIPLIER); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Retry interrupted", ie); - } - } finally { - if (!operationSucceeded && failureCounter != null) { - failureCounter.increment(); - } - } - retryCount++; - } - throw new RuntimeException("Exceeded max retry attempts"); - } -} diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/auth/Office365AuthenticationProvider.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/auth/Office365AuthenticationProvider.java index 1bc3944203..34f36d3e0b 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/auth/Office365AuthenticationProvider.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/auth/Office365AuthenticationProvider.java @@ -11,7 +11,9 @@ import lombok.Getter; import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365SourceConfig; -import org.opensearch.dataprepper.plugins.source.microsoft_office365.RetryHandler; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; @@ -40,6 +42,7 @@ public class Office365AuthenticationProvider implements Office365AuthenticationI "&scope=%s"; private final RestTemplate restTemplate = new RestTemplate(); + private final RetryHandler retryHandler; private final String tenantId; private final Office365SourceConfig office365SourceConfig; private String accessToken; @@ -53,6 +56,9 @@ public class Office365AuthenticationProvider implements Office365AuthenticationI public Office365AuthenticationProvider(Office365SourceConfig config) { this.tenantId = config.getTenantId(); this.office365SourceConfig = config; + this.retryHandler = new RetryHandler( + new DefaultRetryStrategy(), + new DefaultStatusCodeHandler()); } @Override @@ -76,7 +82,7 @@ public void renewCredentials() { HttpEntity entity = new HttpEntity<>(payload, headers); String tokenEndpoint = String.format(TOKEN_URL, office365SourceConfig.getTenantId()); - ResponseEntity response = RetryHandler.executeWithRetry( + ResponseEntity response = retryHandler.executeWithRetry( () -> restTemplate.postForEntity(tokenEndpoint, entity, Map.class), () -> { } // No credential renewal for authentication endpoint diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategy.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategy.java new file mode 100644 index 0000000000..f7962f962f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategy.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +/** + * Default retry strategy with fixed backoff times + */ +@Slf4j +public class DefaultRetryStrategy implements RetryStrategy { + private static final List DEFAULT_RATE_LIMIT_STATUS_CODES = Arrays.asList(HttpStatus.TOO_MANY_REQUESTS); + + private final List retryAttemptSleepTime; + private final List rateLimitRetrySleepTime; + private final List rateLimitStatusCodes; + private final int maxRetries; + + /** + * Constructor with default sleep times + */ + public DefaultRetryStrategy() { + this.retryAttemptSleepTime = RetryStrategy.DEFAULT_RETRY_ATTEMPT_SLEEP_TIME; + this.rateLimitRetrySleepTime = RetryStrategy.DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME; + this.rateLimitStatusCodes = DEFAULT_RATE_LIMIT_STATUS_CODES; + this.maxRetries = RetryStrategy.MAX_RETRIES; + } + + /** + * Constructor with custom max retries + * + * @param maxRetries Maximum number of retries + */ + public DefaultRetryStrategy(final int maxRetries) { + this.retryAttemptSleepTime = RetryStrategy.DEFAULT_RETRY_ATTEMPT_SLEEP_TIME; + this.rateLimitRetrySleepTime = RetryStrategy.DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME; + this.rateLimitStatusCodes = DEFAULT_RATE_LIMIT_STATUS_CODES; + this.maxRetries = maxRetries; + } + + /** + * Constructor with Custom sleep times for rate limit retries and custom rate limit status codes + * + * @param rateLimitRetrySleepTime Custom sleep times for rate limit retries (in + * seconds) + * @param rateLimitStatusCodes List of status codes that are considered rate limited + */ + public DefaultRetryStrategy(List rateLimitRetrySleepTime, List rateLimitStatusCodes) { + this.retryAttemptSleepTime = RetryStrategy.DEFAULT_RETRY_ATTEMPT_SLEEP_TIME; + this.rateLimitRetrySleepTime = rateLimitRetrySleepTime != null + ? rateLimitRetrySleepTime + : RetryStrategy.DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME; + this.rateLimitStatusCodes = rateLimitStatusCodes != null + ? rateLimitStatusCodes + : DEFAULT_RATE_LIMIT_STATUS_CODES; + this.maxRetries = this.rateLimitRetrySleepTime.size(); + } + + @Override + public long calculateSleepTime(Exception ex, int retryCount) { + Optional statusCode = RetryStrategy.getStatusCode(ex); + + List sleepTimes = (statusCode.isPresent() && rateLimitStatusCodes.contains(statusCode.get())) + ? rateLimitRetrySleepTime + : retryAttemptSleepTime; + + int sleepTimeSeconds = (retryCount < sleepTimes.size()) + ? sleepTimes.get(retryCount) + : sleepTimes.get(sleepTimes.size() - 1); + + log.debug("Retrying in {} seconds (attempt {}/{})", + sleepTimeSeconds, retryCount + 1, getMaxRetries()); + + return sleepTimeSeconds * RetryStrategy.SLEEP_TIME_MULTIPLIER_MS; + } + + @Override + public int getMaxRetries() { + return maxRetries; + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandler.java new file mode 100644 index 0000000000..5c11b36503 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandler.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import java.util.Optional; + +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; + +/** + * Default status code handling - covers common HTTP scenarios + */ +@Slf4j +public class DefaultStatusCodeHandler implements StatusCodeHandler { + + @Override + public RetryDecision handleStatusCode(Exception ex, int retryCount, + Runnable credentialRenewal) { + Optional statusCode = RetryStrategy.getStatusCode(ex); + String statusMessage = ex.getMessage(); + + if (statusCode.isEmpty()) { + return RetryDecision.stop(); + } + + switch (statusCode.get()) { + case UNAUTHORIZED: + log.error(NOISY, "Token expired. Attempting to renew credentials.", ex); + credentialRenewal.run(); + return RetryDecision.retry(); + + case FORBIDDEN: + log.error(NOISY, "Access forbidden: {}", statusMessage, ex); + return RetryDecision.stopWithException( + new SecurityException("Access forbidden: " + statusMessage)); + + case NOT_FOUND: + log.warn(NOISY, "Resource not found (404): {}. " + + "This is expected for deleted/expired resources.", statusMessage); + return RetryDecision.stop(); + + case TOO_MANY_REQUESTS: + log.error(NOISY, "Hitting API rate limit. Backing off.", ex); + return RetryDecision.retry(); + + case SERVICE_UNAVAILABLE: + log.error(NOISY, "Service unavailable. Will retry.", ex); + return RetryDecision.retry(); + + default: + if (statusCode.get().is4xxClientError()) { + log.error(NOISY, "Client error: {}. Will not retry.", statusCode, ex); + return RetryDecision.stop(); + } else if (statusCode.get().is5xxServerError()) { + log.error(NOISY, "Server error: {}. Will retry.", statusCode, ex); + return RetryDecision.retry(); + } else { + log.error(NOISY, "Unexpected status code: {}. Will not retry.", + statusCode, ex); + return RetryDecision.stop(); + } + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategy.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategy.java new file mode 100644 index 0000000000..e507a85722 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategy.java @@ -0,0 +1,151 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; + +/** + * Retry strategy that respects retry-after header + */ +@Slf4j +public class RetryAfterHeaderStrategy implements RetryStrategy { + private static final String RATE_LIMIT_REMAINING = "X-RateLimit-Remaining"; + private static final String RATE_LIMIT_RESET = "X-RateLimit-Reset"; + private static final String RETRY_AFTER = "Retry-After"; + private static final List DEFAULT_RATE_LIMIT_STATUS_CODES = Arrays.asList(HttpStatus.TOO_MANY_REQUESTS); + + private final List retryAttemptSleepTime; + private final List rateLimitRetrySleepTime; + private final List rateLimitStatusCodes; + private final int maxRetries; + + /** + * Constructor with default sleep times + */ + public RetryAfterHeaderStrategy() { + this.retryAttemptSleepTime = RetryStrategy.DEFAULT_RETRY_ATTEMPT_SLEEP_TIME; + this.rateLimitRetrySleepTime = RetryStrategy.DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME; + this.rateLimitStatusCodes = DEFAULT_RATE_LIMIT_STATUS_CODES; + this.maxRetries = RetryStrategy.MAX_RETRIES; + } + + /** + * Constructor with custom max retries + * + * @param maxRetries Maximum number of retries + */ + public RetryAfterHeaderStrategy(final int maxRetries) { + this.retryAttemptSleepTime = RetryStrategy.DEFAULT_RETRY_ATTEMPT_SLEEP_TIME; + this.rateLimitRetrySleepTime = RetryStrategy.DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME; + this.rateLimitStatusCodes = DEFAULT_RATE_LIMIT_STATUS_CODES; + this.maxRetries = maxRetries; + } + + /** + * Constructor with Custom sleep times for rate limit retries and custom rate limit status codes + * + * @param rateLimitRetrySleepTime Custom sleep times for rate limit retries (in + * seconds) + * @param rateLimitStatusCodes List of status codes that are considered rate limited + */ + public RetryAfterHeaderStrategy(List rateLimitRetrySleepTime, List rateLimitStatusCodes) { + this.retryAttemptSleepTime = RetryStrategy.DEFAULT_RETRY_ATTEMPT_SLEEP_TIME; + this.rateLimitRetrySleepTime = rateLimitRetrySleepTime != null + ? rateLimitRetrySleepTime + : RetryStrategy.DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME; + this.rateLimitStatusCodes = rateLimitStatusCodes != null + ? rateLimitStatusCodes + : DEFAULT_RATE_LIMIT_STATUS_CODES; + this.maxRetries = this.rateLimitRetrySleepTime.size(); + } + + @Override + public long calculateSleepTime(Exception ex, int retryCount) { + Optional statusCode = RetryStrategy.getStatusCode(ex); + + if (statusCode.isPresent() && isRateLimited(statusCode.get())) { + final Optional retryAfterSeconds = extractRetryAfterHeader(ex); + if (retryAfterSeconds.isPresent()) { + log.info("Using retry-after header value: {} seconds (attempt {}/{})", + retryAfterSeconds.get(), retryCount + 1, getMaxRetries()); + return retryAfterSeconds.get() * RetryStrategy.SLEEP_TIME_MULTIPLIER_MS; + } + } + + // Fallback to fixed backoff + List sleepTimes = (statusCode.isPresent() && isRateLimited(statusCode.get())) + ? rateLimitRetrySleepTime + : retryAttemptSleepTime; + + int sleepTimeSeconds = (retryCount < sleepTimes.size()) + ? sleepTimes.get(retryCount) + : sleepTimes.get(sleepTimes.size() - 1); + + log.debug("Retrying in {} seconds (attempt {}/{})", + sleepTimeSeconds, retryCount + 1, getMaxRetries()); + + return sleepTimeSeconds * RetryStrategy.SLEEP_TIME_MULTIPLIER_MS; + } + + @Override + public int getMaxRetries() { + return maxRetries; + } + + private boolean isRateLimited(final HttpStatus status) { + return rateLimitStatusCodes.contains(status); + } + + private Optional extractRetryAfterHeader(Exception ex) { + try { + HttpHeaders headers = null; + if (ex instanceof HttpClientErrorException) { + headers = ((HttpClientErrorException) ex).getResponseHeaders(); + } else if (ex instanceof HttpServerErrorException) { + headers = ((HttpServerErrorException) ex).getResponseHeaders(); + } + + if (headers != null && headers.containsKey(RETRY_AFTER)) { + String retryAfter = headers.getFirst(RETRY_AFTER); + if (retryAfter != null) { + int seconds = Integer.parseInt(retryAfter); + return Optional.of(Math.max(seconds, 1)); + } + } + if (headers != null && headers.containsKey(RATE_LIMIT_REMAINING) && headers.containsKey(RATE_LIMIT_RESET)) { + String xRateLimitRemaining = headers.getFirst(RATE_LIMIT_REMAINING); + String resetEpoch = headers.getFirst(RATE_LIMIT_RESET); + if (xRateLimitRemaining != null && xRateLimitRemaining.equals("0") && resetEpoch != null + && !resetEpoch.isBlank()) { + long resetSeconds = Long.parseLong(resetEpoch); + long nowSeconds = Instant.now().getEpochSecond(); + long wait = resetSeconds - nowSeconds + 1; + return Optional.of((int) Math.max(wait, 1)); + } + } + } catch (NumberFormatException e) { + log.warn(NOISY, "Failed to parse retry-after header: {}", e.getMessage()); + } + return Optional.empty(); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecision.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecision.java new file mode 100644 index 0000000000..46e4b4791b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecision.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import lombok.Getter; + +import java.util.Optional; + +/** + * Encapsulates the decision of whether to retry or stop + */ +@Getter +public class RetryDecision { + private final boolean shouldStop; + private final Optional exception; + + private RetryDecision(boolean shouldStop, Optional exception) { + this.shouldStop = shouldStop; + this.exception = exception; + } + + public static RetryDecision retry() { + return new RetryDecision(false, Optional.empty()); + } + + public static RetryDecision stop() { + return new RetryDecision(true, Optional.empty()); + } + + public static RetryDecision stopWithException(Exception exception) { + return new RetryDecision(true, Optional.ofNullable(exception)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java new file mode 100644 index 0000000000..48bd0cdc3e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import io.micrometer.core.instrument.Counter; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.util.Optional; +import java.util.function.Supplier; + +@Slf4j +public class RetryHandler { + private final RetryStrategy retryStrategy; + private final StatusCodeHandler statusCodeHandler; + + /** + * Constructor + * + * @param retryStrategy Strategy for determining retry behavior + * @param statusCodeHandler Handler for HTTP status codes + */ + public RetryHandler(RetryStrategy retryStrategy, StatusCodeHandler statusCodeHandler) { + this.retryStrategy = retryStrategy; + this.statusCodeHandler = statusCodeHandler; + } + + /** + * Executes the given operation with retry logic, optional credential renewal, + * and failure counter. + * + * @param operation The operation to execute. + * @param credentialRenewal The action to renew credentials if needed. + * + * @param The return type of the operation. + * @return The result of the operation. + */ + public T executeWithRetry(Supplier operation, Runnable credentialRenewal) { + return executeWithRetry(operation, credentialRenewal, Optional.empty()); + } + + /** + * Executes the given operation with retry logic, optional credential renewal, + * and failure counter. + * + * @param operation The operation to execute. + * @param credentialRenewal The action to renew credentials if needed. + * @param failureCounter The counter to increment on each failed attempt + * (optional). + * @param The return type of the operation. + * @return The result of the operation. + */ + public T executeWithRetry(Supplier operation, Runnable credentialRenewal, Optional failureCounter) { + if (operation == null) { + throw new SaaSCrawlerException("Operation cannot be null", false); + } + if (credentialRenewal == null) { + throw new SaaSCrawlerException("Credential renewal cannot be null", false); + } + + final int maxRetries = retryStrategy.getMaxRetries(); + int retryCount = 0; + + while (retryCount < maxRetries) { + boolean operationSucceeded = false; + try { + T result = operation.get(); + operationSucceeded = true; + return result; + } catch (HttpClientErrorException | HttpServerErrorException ex) { + RetryDecision decision = statusCodeHandler.handleStatusCode( + ex, retryCount, credentialRenewal); + + if (decision.isShouldStop()) { + decision.getException().ifPresent(e -> { + throw new SecurityException("Access forbidden: " + e.getMessage()); + }); + throw ex; + } + + if (retryCount == maxRetries - 1) { + log.error("Exceeded maximum retry attempts ({})", maxRetries, ex); + throw ex; + } + + // Calculate sleep time and wait + long sleepTimeMs = retryStrategy.calculateSleepTime(ex, retryCount); + sleep(sleepTimeMs); + } finally { + if (!operationSucceeded) { + failureCounter.ifPresent(Counter::increment); + } + } + retryCount++; + } + throw new RuntimeException("Exceeded maximum retry attempts (" + maxRetries + ")"); + } + + private void sleep(long milliseconds) { + try { + Thread.sleep(milliseconds); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Retry interrupted", ie); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryStrategy.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryStrategy.java new file mode 100644 index 0000000000..e3a47dc564 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryStrategy.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.util.List; +import java.util.Optional; + +/** + * Strategy for determining how long to wait before retrying + */ +public interface RetryStrategy { + List DEFAULT_RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); + List DEFAULT_RATE_LIMIT_RETRY_SLEEP_TIME = List.of(5, 10, 30, 60, 120, 300); + int SLEEP_TIME_MULTIPLIER_MS = 1000; + int MAX_RETRIES = DEFAULT_RETRY_ATTEMPT_SLEEP_TIME.size(); + + /** + * Calculate sleep time in milliseconds before next retry + * + * @param ex The exception that triggered the retry + * @param retryCount Current retry attempt (0-based) + * @return Sleep time in milliseconds + */ + long calculateSleepTime(Exception ex, int retryCount); + + /** + * Get maximum number of retries allowed + * + * @return Maximum number of retries + */ + int getMaxRetries(); + + static Optional getStatusCode(final Exception ex) { + if (ex instanceof HttpClientErrorException) { + return Optional.of(((HttpClientErrorException) ex).getStatusCode()); + } else if (ex instanceof HttpServerErrorException) { + return Optional.of(((HttpServerErrorException) ex).getStatusCode()); + } + return Optional.empty(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/StatusCodeHandler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/StatusCodeHandler.java new file mode 100644 index 0000000000..313f5978dd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/StatusCodeHandler.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +/** + * Handler for determining retry behavior based on HTTP status codes + */ +public interface StatusCodeHandler { + /** + * Handle an HTTP exception and determine whether to retry + * + * @param ex The HTTP exception + * @param retryCount Current retry attempt + * @param credentialRenewal Runnable to renew credentials + * @return RetryDecision indicating whether to stop/continue and optional + * exception + */ + RetryDecision handleStatusCode(Exception ex, int retryCount, Runnable credentialRenewal); +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategyTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategyTest.java new file mode 100644 index 0000000000..4903b1de6d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultRetryStrategyTest.java @@ -0,0 +1,219 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +class DefaultRetryStrategyTest { + + @Test + void constructor_WithCustomMaxRetries_InitializesSuccessfully() { + final DefaultRetryStrategy strategy = new DefaultRetryStrategy(1); + assertThat(strategy, notNullValue()); + assertThat(strategy.getMaxRetries(), equalTo(1)); + } + + @Test + void constructor_WithCustomRateLimitSleepTime_InitializesSuccessfully() { + final List customSleepTime = Arrays.asList(10); + final DefaultRetryStrategy strategy = new DefaultRetryStrategy(customSleepTime, null); + assertThat(strategy, notNullValue()); + assertThat(strategy.getMaxRetries(), equalTo(1)); + } + + @Test + void constructor_WithNullRateLimitSleepTime_UsesDefaultValues() { + final DefaultRetryStrategy strategy = new DefaultRetryStrategy(null, null); + assertThat(strategy, notNullValue()); + assertThat(strategy.getMaxRetries(), equalTo(6)); + } + + @Test + void getMaxRetries_ReturnsExpectedValue() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + assertThat(defaultRetryStrategy.getMaxRetries(), equalTo(1)); + } + + @ParameterizedTest + @MethodSource("normalRetryArguments") + void calculateSleepTime_WithNormalRetries_ReturnsExpectedTime(final int retryCount, + final long expectedTimeMs) { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, retryCount); + + assertThat(sleepTime, equalTo(expectedTimeMs)); + } + + @ParameterizedTest + @MethodSource("rateLimitRetryArguments") + void calculateSleepTime_WithRateLimitError_ReturnsExpectedTime(final int retryCount, + final long expectedTimeMs) { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, retryCount); + + assertThat(sleepTime, equalTo(expectedTimeMs)); + } + + @Test + void calculateSleepTime_WithRetryCountExceedingList_ReturnsLastValue() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, 10); + + // Last value in default list is 40 seconds + assertThat(sleepTime, equalTo(40000L)); + } + + @Test + void calculateSleepTime_WithRateLimitAndExceedingCount_ReturnsLastValue() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, 10); + + // Last value in rate limit list is 300 seconds + assertThat(sleepTime, equalTo(300000L)); + } + + @Test + void calculateSleepTime_WithHttpClientErrorException_ReturnsCorrectTime() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.BAD_REQUEST); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithHttpServerErrorException_ReturnsCorrectTime() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.SERVICE_UNAVAILABLE); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithGenericException_ReturnsDefaultTime() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final Exception exception = new RuntimeException("Generic error"); + + final long sleepTime = defaultRetryStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithCustomRateLimitSleepTime_UsesCustomValues() { + final List customSleepTime = Arrays.asList(10); + final DefaultRetryStrategy strategy = new DefaultRetryStrategy(customSleepTime, null); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime1 = strategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime1, equalTo(10000L)); + } + + @Test + void calculateSleepTime_WithCustomRateLimitAndExceedingCount_ReturnsLastCustomValue() { + final List customSleepTime = Arrays.asList(10); + final DefaultRetryStrategy strategy = new DefaultRetryStrategy(customSleepTime, null); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime = strategy.calculateSleepTime(exception, 5); + + assertThat(sleepTime, equalTo(10000L)); + } + + @Test + void calculateSleepTime_WithDifferentHttpStatusCodes_CalculatesCorrectly() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpClientErrorException badRequestException = new HttpClientErrorException( + HttpStatus.BAD_REQUEST); + final HttpServerErrorException badGatewayException = new HttpServerErrorException( + HttpStatus.BAD_GATEWAY); + final HttpClientErrorException tooManyRequestsException = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime1 = defaultRetryStrategy.calculateSleepTime(badRequestException, 0); + final long sleepTime2 = defaultRetryStrategy.calculateSleepTime(badGatewayException, 0); + final long sleepTime3 = defaultRetryStrategy.calculateSleepTime(tooManyRequestsException, + 0); + + assertThat(sleepTime1, equalTo(1000L)); + assertThat(sleepTime2, equalTo(1000L)); + assertThat(sleepTime3, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithSequentialRetries_IncreasesBackoffTime() { + final DefaultRetryStrategy defaultRetryStrategy = new DefaultRetryStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime0 = defaultRetryStrategy.calculateSleepTime(exception, 0); + final long sleepTime1 = defaultRetryStrategy.calculateSleepTime(exception, 1); + final long sleepTime2 = defaultRetryStrategy.calculateSleepTime(exception, 2); + + assertThat(sleepTime0, equalTo(1000L)); + assertThat(sleepTime1, equalTo(2000L)); + assertThat(sleepTime2, equalTo(5000L)); + } + + private static Stream normalRetryArguments() { + return Stream.of( + Arguments.of(0, 1000L), + Arguments.of(1, 2000L), + Arguments.of(2, 5000L), + Arguments.of(3, 10000L), + Arguments.of(4, 20000L), + Arguments.of(5, 40000L)); + } + + private static Stream rateLimitRetryArguments() { + return Stream.of( + Arguments.of(0, 5000L), + Arguments.of(1, 10000L), + Arguments.of(2, 30000L), + Arguments.of(3, 60000L), + Arguments.of(4, 120000L), + Arguments.of(5, 300000L)); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandlerTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandlerTest.java new file mode 100644 index 0000000000..e02b2498d6 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/DefaultStatusCodeHandlerTest.java @@ -0,0 +1,313 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.util.Optional; +import java.util.stream.Stream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +@ExtendWith(MockitoExtension.class) +class DefaultStatusCodeHandlerTest { + + @Mock + private Runnable credentialRenewal; + + private DefaultStatusCodeHandler statusCodeHandler; + + @BeforeEach + void setUp() { + statusCodeHandler = new DefaultStatusCodeHandler(); + } + + @Test + void constructor_InitializesSuccessfully() { + final DefaultStatusCodeHandler handler = new DefaultStatusCodeHandler(); + assertThat(handler, notNullValue()); + } + + @Test + void handleStatusCode_WithUnauthorized_RenewsCredentialsAndRetriesOnce() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.UNAUTHORIZED); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + assertThat(decision.getException(), equalTo(Optional.empty())); + verify(credentialRenewal, times(1)).run(); + } + + @Test + void handleStatusCode_WithForbidden_StopsWithSecurityException() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.FORBIDDEN, "Forbidden"); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException().isPresent(), equalTo(true)); + assertThat(decision.getException().get(), instanceOf(SecurityException.class)); + assertThat(decision.getException().get().getMessage(), + equalTo("Access forbidden: 403 Forbidden")); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithNotFound_Stops() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.NOT_FOUND); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException(), equalTo(Optional.empty())); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithTooManyRequests_Retries() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + assertThat(decision.getException(), equalTo(Optional.empty())); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithServiceUnavailable_Retries() { + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.SERVICE_UNAVAILABLE); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + assertThat(decision.getException(), equalTo(Optional.empty())); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithNullStatusCode_Stops() { + final Exception exception = new RuntimeException("Generic error"); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException(), equalTo(Optional.empty())); + verifyNoInteractions(credentialRenewal); + } + + @ParameterizedTest + @MethodSource("clientErrorArguments") + void handleStatusCode_WithClientErrors_Stops(final HttpStatus status) { + final HttpClientErrorException exception = new HttpClientErrorException(status); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + verifyNoInteractions(credentialRenewal); + } + + @ParameterizedTest + @MethodSource("serverErrorArguments") + void handleStatusCode_WithServerErrors_Retries(final HttpStatus status) { + final HttpServerErrorException exception = new HttpServerErrorException(status); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + assertThat(decision.getException(), equalTo(Optional.empty())); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithBadRequest_Stops() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.BAD_REQUEST); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithInternalServerError_Retries() { + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithBadGateway_Retries() { + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.BAD_GATEWAY); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithGatewayTimeout_Retries() { + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.GATEWAY_TIMEOUT); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithConflict_Stops() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.CONFLICT); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithUnprocessableEntity_Stops() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.UNPROCESSABLE_ENTITY); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + verifyNoInteractions(credentialRenewal); + } + + @Test + void handleStatusCode_WithUnauthorizedMultipleTimes_RenewsCredentialsEachTime() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.UNAUTHORIZED); + + statusCodeHandler.handleStatusCode(exception, 0, credentialRenewal); + statusCodeHandler.handleStatusCode(exception, 1, credentialRenewal); + statusCodeHandler.handleStatusCode(exception, 2, credentialRenewal); + + verify(credentialRenewal, times(3)).run(); + } + + @Test + void handleStatusCode_WithDifferentRetryCount_BehavesConsistently() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final RetryDecision decision1 = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + final RetryDecision decision2 = statusCodeHandler.handleStatusCode(exception, 3, + credentialRenewal); + final RetryDecision decision3 = statusCodeHandler.handleStatusCode(exception, 5, + credentialRenewal); + + assertThat(decision1.isShouldStop(), equalTo(false)); + assertThat(decision2.isShouldStop(), equalTo(false)); + assertThat(decision3.isShouldStop(), equalTo(false)); + } + + @Test + void handleStatusCode_WithForbiddenAndCustomMessage_IncludesMessageInException() { + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.FORBIDDEN, "Custom forbidden message"); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException().get().getMessage(), + equalTo("Access forbidden: 403 Custom forbidden message")); + } + + @Test + void handleStatusCode_WithHttpClientErrorExceptionAsGenericException_HandlesCorrectly() { + final Exception exception = new HttpClientErrorException(HttpStatus.BAD_REQUEST); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(true)); + } + + @Test + void handleStatusCode_WithHttpServerErrorExceptionAsGenericException_HandlesCorrectly() { + final Exception exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final RetryDecision decision = statusCodeHandler.handleStatusCode(exception, 0, + credentialRenewal); + + assertThat(decision.isShouldStop(), equalTo(false)); + } + + private static Stream clientErrorArguments() { + return Stream.of( + Arguments.of(HttpStatus.BAD_REQUEST), + Arguments.of(HttpStatus.PAYMENT_REQUIRED), + Arguments.of(HttpStatus.METHOD_NOT_ALLOWED), + Arguments.of(HttpStatus.NOT_ACCEPTABLE), + Arguments.of(HttpStatus.CONFLICT), + Arguments.of(HttpStatus.GONE), + Arguments.of(HttpStatus.UNPROCESSABLE_ENTITY)); + } + + private static Stream serverErrorArguments() { + return Stream.of( + Arguments.of(HttpStatus.INTERNAL_SERVER_ERROR), + Arguments.of(HttpStatus.BAD_GATEWAY), + Arguments.of(HttpStatus.GATEWAY_TIMEOUT), + Arguments.of(HttpStatus.HTTP_VERSION_NOT_SUPPORTED)); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategyTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategyTest.java new file mode 100644 index 0000000000..8d02c8d444 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryAfterHeaderStrategyTest.java @@ -0,0 +1,418 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +class RetryAfterHeaderStrategyTest { + + @Test + void constructor_WithCustomMaxRetries_InitializesSuccessfully() { + final RetryAfterHeaderStrategy strategy = new RetryAfterHeaderStrategy(1); + assertThat(strategy, notNullValue()); + assertThat(strategy.getMaxRetries(), equalTo(1)); + } + + @Test + void constructor_WithCustomRateLimitSleepTime_InitializesSuccessfully() { + final List customSleepTime = Arrays.asList(10); + final RetryAfterHeaderStrategy strategy = new RetryAfterHeaderStrategy(customSleepTime,null); + assertThat(strategy, notNullValue()); + assertThat(strategy.getMaxRetries(), equalTo(1)); + } + + @Test + void constructor_WithNullRateLimitSleepTime_UsesDefaultValues() { + final RetryAfterHeaderStrategy strategy = new RetryAfterHeaderStrategy(null,null); + assertThat(strategy, notNullValue()); + assertThat(strategy.getMaxRetries(), equalTo(6)); + } + + @Test + void getMaxRetries_ReturnsExpectedValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + assertThat(retryAfterHeaderStrategy.getMaxRetries(), equalTo(1)); + } + + @Test + void calculateSleepTime_WithRetryAfterHeader_UsesHeaderValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "15"); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(15000L)); + } + + @Test + void calculateSleepTime_WithInvalidRetryAfterHeader_FallsBackToDefault() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "invalid"); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + // Should fall back to default rate limit sleep time (5 seconds for first retry) + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithMissingRetryAfterHeader_FallsBackToDefault() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithNullHeaders_FallsBackToDefault() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", null, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithNonRateLimitError_UsesDefaultBackoff() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithServerErrorAndRetryAfterHeader_IgnoresHeader() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "15"); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR, "Internal Server Error", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + // Should use default backoff, not retry-after header + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithCustomRateLimitSleepTime_UsesCustomValues() { + final List customSleepTime = Arrays.asList(10); + final RetryAfterHeaderStrategy strategy = new RetryAfterHeaderStrategy(customSleepTime, null); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime = strategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(10000L)); + } + + @Test + void calculateSleepTime_WithRetryCountExceedingList_ReturnsLastValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 10); + + assertThat(sleepTime, equalTo(40000L)); + } + + @Test + void calculateSleepTime_WithRateLimitAndExceedingCount_ReturnsLastValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 10); + + assertThat(sleepTime, equalTo(300000L)); + } + + @Test + void calculateSleepTime_WithGenericException_UsesDefaultBackoff() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final Exception exception = new RuntimeException("Generic error"); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @ParameterizedTest + @MethodSource("normalRetryArguments") + void calculateSleepTime_WithNormalRetries_ReturnsExpectedTime(final int retryCount, + final long expectedTimeMs) { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, retryCount); + + assertThat(sleepTime, equalTo(expectedTimeMs)); + } + + @ParameterizedTest + @MethodSource("rateLimitRetryArguments") + void calculateSleepTime_WithRateLimitError_ReturnsExpectedTime(final int retryCount, + final long expectedTimeMs) { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, retryCount); + + assertThat(sleepTime, equalTo(expectedTimeMs)); + } + + @Test + void calculateSleepTime_WithRetryAfterHeaderOnSecondRetry_UsesHeaderValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "45"); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 3); + + assertThat(sleepTime, equalTo(45000L)); + } + + @Test + void calculateSleepTime_WithZeroRetryAfterHeader_UsesHeaderValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "0"); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithLargeRetryAfterHeader_UsesHeaderValue() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "600"); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(600000L)); + } + + @Test + void calculateSleepTime_WithEmptyRetryAfterHeader_FallsBackToDefault() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", ""); + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithSequentialRetries_IncreasesBackoffTime() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpServerErrorException exception = new HttpServerErrorException( + HttpStatus.INTERNAL_SERVER_ERROR); + + final long sleepTime0 = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + final long sleepTime1 = retryAfterHeaderStrategy.calculateSleepTime(exception, 1); + final long sleepTime2 = retryAfterHeaderStrategy.calculateSleepTime(exception, 2); + + assertThat(sleepTime0, equalTo(1000L)); + assertThat(sleepTime1, equalTo(2000L)); + assertThat(sleepTime2, equalTo(5000L)); + } + + private static Stream normalRetryArguments() { + return Stream.of( + Arguments.of(0, 1000L), + Arguments.of(1, 2000L), + Arguments.of(2, 5000L), + Arguments.of(3, 10000L), + Arguments.of(4, 20000L), + Arguments.of(5, 40000L)); + } + + private static Stream rateLimitRetryArguments() { + return Stream.of( + Arguments.of(0, 5000L), + Arguments.of(1, 10000L), + Arguments.of(2, 30000L), + Arguments.of(3, 60000L), + Arguments.of(4, 120000L), + Arguments.of(5, 300000L)); + } + + @Test + void calculateSleepTime_WithRetryAfterHeaderAsInvalidHttpDate_ShouldFallBackToDefault() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("retry-after", "Invalid Date Format"); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + // Should fall back to default rate limit sleep time + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithXRateLimitRemainingZero_ShouldCalculateWaitTime() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-RateLimit-Remaining", "0"); + final long resetTime = Instant.now().getEpochSecond() + 300; + headers.set("X-RateLimit-Reset", String.valueOf(resetTime)); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Rate limit exceeded", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(301000L)); + } + + @Test + void calculateSleepTime_WithXRateLimitResetInPast_ShouldReturnMinimum() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-RateLimit-Remaining", "0"); + final long pastResetTime = Instant.now().getEpochSecond() - 60; + headers.set("X-RateLimit-Reset", String.valueOf(pastResetTime)); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.FORBIDDEN, "Rate limit expired", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(1000L)); + } + + @Test + void calculateSleepTime_WithXRateLimitButRemainingNotZero_ShouldIgnoreHeaders() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-RateLimit-Remaining", "10"); + final long resetTime = Instant.now().getEpochSecond() + 300; + headers.set("X-RateLimit-Reset", String.valueOf(resetTime)); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Request", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithXRateLimitResetBlank_ShouldIgnoreHeaders() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-RateLimit-Remaining", "0"); + headers.set("X-RateLimit-Reset", ""); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Request", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithXRateLimitResetInvalidNumber_ShouldIgnoreHeaders() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-RateLimit-Remaining", "0"); + headers.set("X-RateLimit-Reset", "invalid-number"); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Request", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(5000L)); + } + + @Test + void calculateSleepTime_WithBothRetryAfterAndXRateLimit_ShouldPreferRetryAfter() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("Retry-After", "60"); + headers.set("X-RateLimit-Remaining", "0"); + final long resetTime = Instant.now().getEpochSecond() + 300; + headers.set("X-RateLimit-Reset", String.valueOf(resetTime)); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Too Many Requests", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(60000L)); + } + + @Test + void calculateSleepTime_WithXRateLimitResetOneSecondAway_ShouldReturnMinimum() { + final RetryAfterHeaderStrategy retryAfterHeaderStrategy = new RetryAfterHeaderStrategy(1); + final HttpHeaders headers = new HttpHeaders(); + headers.set("X-RateLimit-Remaining", "0"); + final long resetTime = Instant.now().getEpochSecond() + 1; + headers.set("X-RateLimit-Reset", String.valueOf(resetTime)); + + final HttpClientErrorException exception = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS, "Rate limit exceeded", headers, null, null); + + final long sleepTime = retryAfterHeaderStrategy.calculateSleepTime(exception, 0); + + assertThat(sleepTime, equalTo(2000L)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecisionTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecisionTest.java new file mode 100644 index 0000000000..81f86772f5 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryDecisionTest.java @@ -0,0 +1,148 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +class RetryDecisionTest { + + @Test + void retry_ReturnsRetryDecision() { + final RetryDecision decision = RetryDecision.retry(); + + assertThat(decision, notNullValue()); + assertThat(decision.isShouldStop(), equalTo(false)); + assertThat(decision.getException(), equalTo(Optional.empty())); + } + + @Test + void stop_ReturnsStopDecision() { + final RetryDecision decision = RetryDecision.stop(); + + assertThat(decision, notNullValue()); + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException(), equalTo(Optional.empty())); + } + + @Test + void stopWithException_ReturnsStopDecisionWithException() { + final RuntimeException exception = new RuntimeException("Test exception"); + + final RetryDecision decision = RetryDecision.stopWithException(exception); + + assertThat(decision, notNullValue()); + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException().get(), equalTo(exception)); + assertThat(decision.getException().get().getMessage(), equalTo("Test exception")); + } + + @Test + void stopWithException_WithSecurityException_PreservesExceptionType() { + final SecurityException exception = new SecurityException("Access denied"); + + final RetryDecision decision = RetryDecision.stopWithException(exception); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException().get(), equalTo(exception)); + assertThat(decision.getException().get().getMessage(), equalTo("Access denied")); + } + + @Test + void stopWithException_WithNullException_AcceptsNull() { + final RetryDecision decision = RetryDecision.stopWithException(null); + + assertThat(decision, notNullValue()); + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException(), equalTo(Optional.ofNullable(null))); + } + + @Test + void retry_CreatesIndependentInstances() { + final RetryDecision decision1 = RetryDecision.retry(); + final RetryDecision decision2 = RetryDecision.retry(); + + assertThat(decision1, notNullValue()); + assertThat(decision2, notNullValue()); + assertThat(decision1 == decision2, equalTo(false)); + } + + @Test + void stop_CreatesIndependentInstances() { + final RetryDecision decision1 = RetryDecision.stop(); + final RetryDecision decision2 = RetryDecision.stop(); + + assertThat(decision1, notNullValue()); + assertThat(decision2, notNullValue()); + assertThat(decision1 == decision2, equalTo(false)); + } + + @Test + void stopWithException_WithDifferentExceptions_MaintainsDistinctState() { + final RuntimeException exception1 = new RuntimeException("Exception 1"); + final RuntimeException exception2 = new RuntimeException("Exception 2"); + + final RetryDecision decision1 = RetryDecision.stopWithException(exception1); + final RetryDecision decision2 = RetryDecision.stopWithException(exception2); + + assertThat(decision1.getException().get(), equalTo(exception1)); + assertThat(decision2.getException().get(), equalTo(exception2)); + assertThat(decision1.getException().get() == decision2.getException().get(), equalTo(false)); + } + + @Test + void getShouldStop_ReturnsCorrectValue() { + final RetryDecision retryDecision = RetryDecision.retry(); + final RetryDecision stopDecision = RetryDecision.stop(); + + assertThat(retryDecision.isShouldStop(), equalTo(false)); + assertThat(stopDecision.isShouldStop(), equalTo(true)); + } + + @Test + void getException_ReturnsCorrectValue() { + final RuntimeException exception = new RuntimeException("Test"); + final RetryDecision decisionWithException = RetryDecision.stopWithException(exception); + final RetryDecision decisionWithoutException = RetryDecision.stop(); + + assertThat(decisionWithException.getException().get(), equalTo(exception)); + assertThat(decisionWithoutException.getException(), equalTo(Optional.empty())); + } + + @Test + void stopWithException_WithIllegalArgumentException_PreservesExceptionType() { + final IllegalArgumentException exception = new IllegalArgumentException( + "Invalid argument"); + + final RetryDecision decision = RetryDecision.stopWithException(exception); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException().get(), equalTo(exception)); + assertThat(decision.getException().get().getMessage(), equalTo("Invalid argument")); + } + + @Test + void stopWithException_WithNestedCause_PreservesFullException() { + final Exception cause = new Exception("Root cause"); + final RuntimeException exception = new RuntimeException("Wrapper exception", cause); + + final RetryDecision decision = RetryDecision.stopWithException(exception); + + assertThat(decision.isShouldStop(), equalTo(true)); + assertThat(decision.getException().get(), equalTo(exception)); + assertThat(decision.getException().get().getCause(), equalTo(cause)); + assertThat(decision.getException().get().getCause().getMessage(), equalTo("Root cause")); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java new file mode 100644 index 0000000000..e2a9b20934 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java @@ -0,0 +1,344 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RetryHandlerTest { + + private static final int MAX_RETRIES = 3; + private static final long SLEEP_TIME_MS = 100L; + @Mock + private RetryStrategy retryStrategy; + @Mock + private StatusCodeHandler statusCodeHandler; + @Mock + private Runnable credentialRenewal; + private RetryHandler retryHandler; + + @BeforeEach + void setUp() { + lenient().when(retryStrategy.getMaxRetries()).thenReturn(MAX_RETRIES); + lenient().when(retryStrategy.calculateSleepTime(any(Exception.class), anyInt())) + .thenReturn(SLEEP_TIME_MS); + retryHandler = new RetryHandler(retryStrategy, statusCodeHandler); + } + + @Test + void constructor_WithValidParams_InitializesSuccessfully() { + final RetryHandler handler = new RetryHandler(retryStrategy, statusCodeHandler); + assertThat(handler, notNullValue()); + } + + @Test + void executeWithRetry_WithNullOperation_ThrowsIllegalArgumentException() { + final SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, + () -> retryHandler.executeWithRetry(null, credentialRenewal)); + + assertThat(exception.getMessage(), equalTo("Operation cannot be null")); + } + + @Test + void executeWithRetry_WithNullCredentialRenewal_ThrowsIllegalArgumentException() { + final Supplier operation = () -> "success"; + + final SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, + () -> retryHandler.executeWithRetry(operation, null)); + + assertThat(exception.getMessage(), equalTo("Credential renewal cannot be null")); + } + + @Test + void executeWithRetry_WithSuccessfulOperation_ReturnsResultImmediately() { + final String expectedResult = "success"; + final Supplier operation = () -> expectedResult; + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo(expectedResult)); + verify(statusCodeHandler, never()).handleStatusCode(any(), anyInt(), any()); + } + + @Test + void executeWithRetry_WithRetryableError_RetriesAndSucceeds() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() < 2) { + throw new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE); + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo("success")); + assertThat(attemptCount.get(), equalTo(3)); + verify(statusCodeHandler, times(2)).handleStatusCode(any(HttpServerErrorException.class), + anyInt(), eq(credentialRenewal)); + } + + @Test + void executeWithRetry_WithNonRetryableError_ThrowsExceptionImmediately() { + final HttpClientErrorException clientException = new HttpClientErrorException( + HttpStatus.BAD_REQUEST, "Bad Request"); + final Supplier operation = () -> { + throw clientException; + }; + + when(statusCodeHandler.handleStatusCode(eq(clientException), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.stop()); + + final HttpClientErrorException exception = assertThrows(HttpClientErrorException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal)); + + assertThat(exception, equalTo(clientException)); + verify(statusCodeHandler, times(1)).handleStatusCode(eq(clientException), eq(0), + eq(credentialRenewal)); + } + + @Test + void executeWithRetry_WithStopDecisionAndException_ThrowsCustomException() { + final HttpClientErrorException clientException = new HttpClientErrorException( + HttpStatus.FORBIDDEN, "Forbidden"); + final SecurityException customException = new SecurityException("Access denied"); + final Supplier operation = () -> { + throw clientException; + }; + + when(statusCodeHandler.handleStatusCode(eq(clientException), eq(0), + eq(credentialRenewal))) + .thenReturn(RetryDecision.stopWithException(customException)); + + final SecurityException exception = assertThrows(SecurityException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal)); + + assertThat(exception.getMessage(), equalTo("Access forbidden: Access denied")); + } + + @Test + void executeWithRetry_ExceedingMaxRetries_ThrowsHttpServerErrorException() { + final Supplier operation = () -> { + throw new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE); + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final HttpServerErrorException exception = assertThrows(HttpServerErrorException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal)); + + assertThat(exception.getStatusCode(), equalTo(HttpStatus.SERVICE_UNAVAILABLE)); + verify(statusCodeHandler, times(MAX_RETRIES)).handleStatusCode( + any(HttpServerErrorException.class), anyInt(), eq(credentialRenewal)); + } + + @Test + void executeWithRetry_WithHttpClientErrorException_HandlesCorrectly() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final HttpClientErrorException clientException = new HttpClientErrorException( + HttpStatus.UNAUTHORIZED); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() < 1) { + throw clientException; + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(eq(clientException), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo("success")); + verify(statusCodeHandler, times(1)).handleStatusCode(eq(clientException), eq(0), + eq(credentialRenewal)); + verify(credentialRenewal, never()).run(); + } + + @Test + void executeWithRetry_WithMultipleRetries_UsesCorrectRetryCount() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() < 2) { + throw new HttpServerErrorException(HttpStatus.INTERNAL_SERVER_ERROR); + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), eq(1), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo("success")); + verify(statusCodeHandler, times(1)).handleStatusCode(any(HttpServerErrorException.class), + eq(0), eq(credentialRenewal)); + verify(statusCodeHandler, times(1)).handleStatusCode(any(HttpServerErrorException.class), + eq(1), eq(credentialRenewal)); + } + + @Test + void executeWithRetry_WithRateLimitError_RetriesWithBackoff() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final HttpClientErrorException rateLimitException = new HttpClientErrorException( + HttpStatus.TOO_MANY_REQUESTS); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() < 1) { + throw rateLimitException; + } + return "success"; + }; + + when(retryStrategy.calculateSleepTime(eq(rateLimitException), eq(0))) + .thenReturn(5000L); + when(statusCodeHandler.handleStatusCode(eq(rateLimitException), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo("success")); + verify(retryStrategy, times(1)).calculateSleepTime(eq(rateLimitException), eq(0)); + } + + @Test + void executeWithRetry_WithInterruptedException_ThrowsRuntimeException() { + final Supplier operation = () -> { + throw new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE); + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + when(retryStrategy.calculateSleepTime(any(Exception.class), anyInt())) + .thenAnswer(invocation -> { + Thread.currentThread().interrupt(); + return SLEEP_TIME_MS; + }); + + final RuntimeException exception = assertThrows(RuntimeException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal)); + + assertThat(exception.getMessage(), equalTo("Retry interrupted")); + assertThat(exception.getCause(), instanceOf(InterruptedException.class)); + } + + @Test + void executeWithRetry_WithNotFoundError_StopsRetrying() { + final HttpClientErrorException notFoundException = new HttpClientErrorException( + HttpStatus.NOT_FOUND); + final Supplier operation = () -> { + throw notFoundException; + }; + + when(statusCodeHandler.handleStatusCode(eq(notFoundException), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.stop()); + + final HttpClientErrorException exception = assertThrows(HttpClientErrorException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal)); + + assertThat(exception, equalTo(notFoundException)); + verify(statusCodeHandler, times(1)).handleStatusCode(eq(notFoundException), eq(0), + eq(credentialRenewal)); + } + + @Test + void executeWithRetry_WithSuccessAfterOneRetry_CallsOperationTwice() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() == 0) { + throw new HttpServerErrorException(HttpStatus.BAD_GATEWAY); + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo("success")); + assertThat(attemptCount.get(), equalTo(2)); + } + + @Test + void executeWithRetry_WithDifferentExceptionTypes_HandlesCorrectly() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final Supplier operation = () -> { + final int attempt = attemptCount.getAndIncrement(); + if (attempt == 0) { + throw new HttpClientErrorException(HttpStatus.UNAUTHORIZED); + } else if (attempt == 1) { + throw new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE); + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(any(HttpClientErrorException.class), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), eq(1), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo("success")); + assertThat(attemptCount.get(), equalTo(3)); + } + + @Test + void executeWithRetry_WithMaxRetriesReached_VerifiesRetryCount() { + when(retryStrategy.getMaxRetries()).thenReturn(2); + final RetryHandler handlerWithLimitedRetries = new RetryHandler(retryStrategy, + statusCodeHandler); + final Supplier operation = () -> { + throw new HttpServerErrorException(HttpStatus.INTERNAL_SERVER_ERROR); + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final HttpServerErrorException exception = assertThrows(HttpServerErrorException.class, + () -> handlerWithLimitedRetries.executeWithRetry(operation, credentialRenewal)); + + assertThat(exception.getStatusCode(), equalTo(HttpStatus.INTERNAL_SERVER_ERROR)); + verify(statusCodeHandler, times(2)).handleStatusCode(any(HttpServerErrorException.class), + anyInt(), eq(credentialRenewal)); + } +} From c5983ed4f04255dc454a543bcf6635c2de3b2164 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Tue, 16 Dec 2025 08:49:29 -0600 Subject: [PATCH 21/51] Remove usage of buffer accumulator from Kafka custom consumer (#6357) Signed-off-by: Taylor Gray Signed-off-by: Nathan Wand --- .../kafka/consumer/KafkaCustomConsumer.java | 35 ++++++++----------- .../consumer/KafkaCustomConsumerTest.java | 20 ++++++++--- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java b/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java index f51c5cec2a..ac94189820 100644 --- a/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java +++ b/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java @@ -17,13 +17,12 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.errors.RebalanceInProgressException; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.AuthenticationException; +import org.apache.kafka.common.errors.RebalanceInProgressException; import org.apache.kafka.common.errors.RecordDeserializationException; -import org.opensearch.dataprepper.buffer.common.BufferAccumulator; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.buffer.Buffer; @@ -70,8 +69,8 @@ public class KafkaCustomConsumer implements Runnable, ConsumerRebalanceListener private static final Logger LOG = LoggerFactory.getLogger(KafkaCustomConsumer.class); private static final Long COMMIT_OFFSET_INTERVAL_MS = 300000L; - private static final int DEFAULT_NUMBER_OF_RECORDS_TO_ACCUMULATE = 1; private static final int RETRY_ON_EXCEPTION_SLEEP_MS = 1000; + private static final int BUFFER_WRITE_TIMEOUT = 2000; static final String DEFAULT_KEY = "message"; private volatile long lastCommitTime; @@ -81,7 +80,6 @@ public class KafkaCustomConsumer implements Runnable, ConsumerRebalanceListener private final TopicConsumerConfig topicConfig; private MessageFormat schema; private boolean paused; - private final BufferAccumulator> bufferAccumulator; private final Buffer> buffer; private static final ObjectMapper objectMapper = new ObjectMapper(); private final JsonFactory jsonFactory = new JsonFactory(); @@ -123,7 +121,7 @@ public KafkaCustomConsumer(final KafkaConsumer consumer, this.paused = false; this.byteDecoder = byteDecoder; this.topicMetrics = topicMetrics; - this.maxRetriesOnException = topicConfig.getMaxPollInterval().toMillis() / (2 * RETRY_ON_EXCEPTION_SLEEP_MS); + this.maxRetriesOnException = topicConfig.getMaxPollInterval().toMillis() / (2 * (RETRY_ON_EXCEPTION_SLEEP_MS + BUFFER_WRITE_TIMEOUT)); this.pauseConsumePredicate = pauseConsumePredicate; this.topicMetrics.register(consumer); this.offsetsToCommit = new HashMap<>(); @@ -137,8 +135,6 @@ public KafkaCustomConsumer(final KafkaConsumer consumer, this.partitionCommitTrackerMap = new HashMap<>(); this.partitionsToReset = Collections.synchronizedSet(new HashSet<>()); this.schema = MessageFormat.getByMessageFormatByName(schemaType); - Duration bufferTimeout = Duration.ofSeconds(1); - this.bufferAccumulator = BufferAccumulator.create(buffer, DEFAULT_NUMBER_OF_RECORDS_TO_ACCUMULATE, bufferTimeout); this.lastCommitTime = System.currentTimeMillis(); this.numberOfAcksPending = new AtomicInteger(0); this.errLogRateLimiter = new LogRateLimiter(2, System.currentTimeMillis()); @@ -492,23 +488,19 @@ private Record getRecord(ConsumerRecord consumerRecord, in return new Record(event); } - private void processRecord(final AcknowledgementSet acknowledgementSet, final Record record) { + private void processRecords(final AcknowledgementSet acknowledgementSet, final List> eventRecords) { // Always add record to acknowledgementSet before adding to // buffer because another thread may take and process // buffer contents before the event record is added // to acknowledgement set if (acknowledgementSet != null) { - acknowledgementSet.add(record.getData()); + eventRecords.forEach(record -> acknowledgementSet.add(record.getData())); } long numRetries = 0; while (true) { LOG.debug("In while loop for processing records, paused = {}", paused); try { - if (numRetries == 0) { - bufferAccumulator.add(record); - } else { - bufferAccumulator.flush(); - } + buffer.writeAll(eventRecords, BUFFER_WRITE_TIMEOUT); break; } catch (Exception e) { if (!paused && numRetries++ > maxRetriesOnException) { @@ -559,6 +551,7 @@ private void iterateRecordPartitions(ConsumerRecords records, fin } List> partitionRecords = records.records(topicPartition); + final List> eventRecords = new ArrayList<>(); for (ConsumerRecord consumerRecord : partitionRecords) { if (schema == MessageFormat.BYTES) { InputStream byteInputStream = new ByteArrayInputStream((byte[])consumerRecord.value()); @@ -567,24 +560,24 @@ private void iterateRecordPartitions(ConsumerRecords records, fin if(byteDecoder != null) { final long receivedTimeStamp = getRecordTimeStamp(consumerRecord, Instant.now().toEpochMilli()); - byteDecoder.parse(decompressedInputStream, Instant.ofEpochMilli(receivedTimeStamp), (record) -> { - processRecord(acknowledgementSet, record); - }); + byteDecoder.parse(decompressedInputStream, Instant.ofEpochMilli(receivedTimeStamp), eventRecords::add); } else { JsonNode jsonNode = objectMapper.readValue(decompressedInputStream, JsonNode.class); Event event = JacksonLog.builder().withData(jsonNode).build(); Record record = new Record<>(event); - processRecord(acknowledgementSet, record); + eventRecords.add(record); } } else { Record record = getRecord(consumerRecord, topicPartition.partition()); if (record != null) { - processRecord(acknowledgementSet, record); + eventRecords.add(record); } } } + processRecords(acknowledgementSet, eventRecords); + long lastOffset = partitionRecords.get(partitionRecords.size() - 1).offset(); long firstOffset = partitionRecords.get(0).offset(); Range offsetRange = Range.between(firstOffset, lastOffset); diff --git a/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java b/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java index 702fd26849..5606f6ee15 100644 --- a/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java +++ b/data-prepper-plugins/kafka-plugins/src/test/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumerTest.java @@ -22,6 +22,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -52,7 +55,9 @@ import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.equalTo; @@ -62,8 +67,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -230,14 +235,15 @@ public void testGetRecordTimeStamp() { assertThat(consumer.getRecordTimeStamp(consumerRecord3, nowMs), equalTo(nowMs)); } - @Test - public void testBufferOverflowPauseResume() throws InterruptedException, Exception { + @ParameterizedTest + @MethodSource("provideExceptionsFromBufferWrite") + public void testBufferOverflowPauseResume(final Exception bufferException) throws InterruptedException, Exception { when(topicConfig.getMaxPollInterval()).thenReturn(Duration.ofMillis(4000)); String topic = topicConfig.getName(); consumerRecords = createPlainTextRecords(topic, 0L); doAnswer((i)-> { if (!paused && !resumed) - throw new SizeOverflowException("size overflow"); + throw bufferException; buffer.writeAll(i.getArgument(0), i.getArgument(1)); return null; }).when(mockBuffer).writeAll(any(), anyInt()); @@ -690,6 +696,12 @@ private ConsumerRecords createJsonRecords(String topic) throws Exception { records.put(new TopicPartition(topic, testJsonPartition), Arrays.asList(record1, record2)); return new ConsumerRecords(records); } + + private static Stream provideExceptionsFromBufferWrite() { + return Stream.of( + Arguments.of(new SizeOverflowException("size overflow")), + Arguments.of(new TimeoutException())); + } } From ff8e7650835dd955ab40220124e18f34c2619487 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Tue, 16 Dec 2025 14:42:17 -0800 Subject: [PATCH 22/51] Add forward_to support to opensearch sink (#6349) * Add forward_to support to opensearch sink Signed-off-by: Krishna Kondaka * Added integration test Signed-off-by: Krishna Kondaka * Addressed review comments Signed-off-by: Krishna Kondaka * Addressed review comments Signed-off-by: Krishna Kondaka --------- Signed-off-by: Krishna Kondaka Signed-off-by: Nathan Wand --- .../configuration/SinkForwardConfig.java | 5 ++ .../dataprepper/model/sink/SinkContext.java | 1 + .../model/sink/SinkForwardRecordsContext.java | 16 ++--- .../configuration/SinkForwardConfigTest.java | 19 +++++- .../model/sink/SinkContextTest.java | 17 +++-- .../sink/SinkForwardRecordsContextTest.java | 32 ++++----- .../sink/opensearch/OpenSearchSinkIT.java | 42 +++++++++++- .../sink/opensearch/BulkRetryStrategy.java | 22 ++++-- .../sink/opensearch/OpenSearchSink.java | 32 ++++++++- .../opensearch/BulkRetryStrategyTests.java | 30 +++++++++ .../sink/opensearch/OpenSearchSinkTest.java | 67 +++++++++++++++++++ 11 files changed, 237 insertions(+), 46 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfig.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfig.java index 68dbed2280..97830c3c59 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfig.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfig.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.model.configuration; +import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -30,6 +31,9 @@ public SinkForwardConfig( @JsonProperty("pipelines") final List pipelineNames, @JsonProperty("with_data") final Map withData, @JsonProperty("with_metadata") final Map withMetadata) { + if (pipelineNames.size() != 1) { + throw new InvalidPluginConfigurationException("Supports only one forwarding pipeline"); + } this.pipelineNames = pipelineNames; this.withData = withData; this.withMetadata = withMetadata; @@ -46,5 +50,6 @@ public Map getWithMetadata() { public Map getWithData() { return withData; } + } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkContext.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkContext.java index 0d3b9485e1..010b027e8f 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkContext.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkContext.java @@ -97,6 +97,7 @@ public boolean forwardRecords(final SinkForwardRecordsContext sinkForwardRecords for (Map.Entry entry: forwardToPipelines.entrySet()) { entry.getValue().sendEvents(records); } + sinkForwardRecordsContext.clearRecords(); return true; } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContext.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContext.java index f972aad3cb..0528a67a76 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContext.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContext.java @@ -7,7 +7,6 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.InternalEventHandle; import java.util.ArrayList; import java.util.Collection; @@ -25,28 +24,21 @@ public SinkForwardRecordsContext(SinkContext sinkContext) { public void addRecord(Record record) { if (!forwardPipelinesPresent) return; - InternalEventHandle eventHandle = (InternalEventHandle)record.getData().getEventHandle(); - if (eventHandle != null) { - eventHandle.acquireReference(); - } records.add(record); } public void addRecords(Collection> newRecords) { if (!forwardPipelinesPresent) return; - newRecords.forEach((record) -> { - Event event = record.getData(); - InternalEventHandle eventHandle = (InternalEventHandle)event.getEventHandle(); - if (eventHandle != null) { - eventHandle.acquireReference(); - } - }); records.addAll(newRecords); } public List> getRecords() { return records; } + + public void clearRecords() { + records.clear(); + } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfigTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfigTest.java index a9b64daed9..f962842533 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfigTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkForwardConfigTest.java @@ -5,12 +5,14 @@ package org.opensearch.dataprepper.model.configuration; +import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.junit.jupiter.api.Test; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; import static org.mockito.Mockito.mock; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.List; import java.util.Map; @@ -26,8 +28,8 @@ void testDefaults() { } @Test - void testCustomValues() { - List pipelines = mock(List.class); + void pipelines_lsit_with_one_pipeline_succeeds() { + List pipelines = List.of("pipeline1"); Map withData = mock(Map.class); Map withMetadata = mock(Map.class); SinkForwardConfig sinkForwardConfig = new SinkForwardConfig(pipelines, withData, withMetadata); @@ -35,5 +37,18 @@ void testCustomValues() { assertThat(sinkForwardConfig.getWithData(), equalTo(withData)); assertThat(sinkForwardConfig.getWithMetadata(), equalTo(withMetadata)); } + + @Test + void pipelines_list_with_two_or_more_pipelines_throws_exception() { + List pipelines = List.of("pipeline1", "pipeline2"); + Map withData = mock(Map.class); + Map withMetadata = mock(Map.class); + assertThrows(InvalidPluginConfigurationException.class, ()->new SinkForwardConfig(pipelines, withData, withMetadata)); + } + + @Test + void empty_pipelines_list_throws_exception() { + assertThrows(InvalidPluginConfigurationException.class, ()->new SinkForwardConfig(List.of(), Map.of(), Map.of())); + } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkContextTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkContextTest.java index 72a5f64c6b..6f409aec16 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkContextTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkContextTest.java @@ -93,17 +93,21 @@ public void testForwardToPipelinesWithPipelineMap() { verify(forwardPipeline2, times(0)).sendEvents(eq(records)); sinkForwardRecordsContext.addRecords(records); assertThat(sinkContext.forwardRecords(sinkForwardRecordsContext, Map.of("datakey1", "datavalue1"), Map.of("metadataKey1", "metadataValue1")), equalTo(true)); - verify(forwardPipeline1, times(1)).sendEvents(eq(records)); - verify(forwardPipeline2, times(1)).sendEvents(eq(records)); + verify(forwardPipeline1, times(1)).sendEvents(any()); + verify(forwardPipeline2, times(1)).sendEvents(any()); verify(event, times(1)).put(any(String.class), any(Object.class)); verify(event, times(1)).getMetadata(); verify(eventMetadata, times(1)).setAttribute(any(String.class), any(Object.class)); + records = Collections.singletonList(record); + sinkForwardRecordsContext.addRecords(records); assertThat(sinkContext.forwardRecords(sinkForwardRecordsContext, null, null), equalTo(true)); - verify(forwardPipeline1, times(2)).sendEvents(eq(records)); - verify(forwardPipeline2, times(2)).sendEvents(eq(records)); + verify(forwardPipeline1, times(2)).sendEvents(any()); + verify(forwardPipeline2, times(2)).sendEvents(any()); + records = Collections.singletonList(record); + sinkForwardRecordsContext.addRecords(records); assertThat(sinkContext.forwardRecords(sinkForwardRecordsContext, Map.of(), Map.of()), equalTo(true)); - verify(forwardPipeline1, times(3)).sendEvents(eq(records)); - verify(forwardPipeline2, times(3)).sendEvents(eq(records)); + verify(forwardPipeline1, times(3)).sendEvents(any()); + verify(forwardPipeline2, times(3)).sendEvents(any()); } @Test @@ -148,6 +152,7 @@ public void testWithNoForwardToPipelines() { SinkForwardRecordsContext sinkForwardRecordsContext = new SinkForwardRecordsContext(sinkContext); sinkForwardRecordsContext.addRecords(List.of(record)); assertThat(sinkContext.forwardRecords(sinkForwardRecordsContext, Map.of(), Map.of()), equalTo(false)); + assertThat(sinkForwardRecordsContext.getRecords().size(), equalTo(0)); } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContextTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContextTest.java index 84450db5dc..abf7bf4a66 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContextTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/sink/SinkForwardRecordsContextTest.java @@ -9,13 +9,11 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.DefaultEventHandle; import org.opensearch.dataprepper.model.pipeline.HeadlessPipeline; import static org.mockito.Mockito.mock; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.when; import java.util.List; @@ -30,16 +28,9 @@ public void testSinkForwardRecordContextBasic() { SinkContext sinkContext = mock(SinkContext.class); when(sinkContext.getForwardToPipelines()).thenReturn(Map.of()); sinkForwardRecordsContext = new SinkForwardRecordsContext(sinkContext); - Event event = mock(Event.class); - DefaultEventHandle eventHandle = mock(DefaultEventHandle.class); - doNothing().when(eventHandle).acquireReference(); Record record1 = mock(Record.class); Record record2 = mock(Record.class); Record record3 = mock(Record.class); - when(record1.getData()).thenReturn(event); - when(record2.getData()).thenReturn(event); - when(record3.getData()).thenReturn(event); - when(event.getEventHandle()).thenReturn(eventHandle); sinkForwardRecordsContext.addRecord(record1); sinkForwardRecordsContext.addRecords(List.of(record2, record3)); List> records = sinkForwardRecordsContext.getRecords(); @@ -52,19 +43,28 @@ public void testSinkForwardRecordContextWithForwardingPipelines() { HeadlessPipeline headlessPipeline = mock(HeadlessPipeline.class); when(sinkContext.getForwardToPipelines()).thenReturn(Map.of("pipeline1", headlessPipeline)); sinkForwardRecordsContext = new SinkForwardRecordsContext(sinkContext); - Event event = mock(Event.class); - DefaultEventHandle eventHandle = mock(DefaultEventHandle.class); - doNothing().when(eventHandle).acquireReference(); Record record1 = mock(Record.class); Record record2 = mock(Record.class); Record record3 = mock(Record.class); - when(record1.getData()).thenReturn(event); - when(record2.getData()).thenReturn(event); - when(record3.getData()).thenReturn(event); - when(event.getEventHandle()).thenReturn(eventHandle); sinkForwardRecordsContext.addRecord(record1); sinkForwardRecordsContext.addRecords(List.of(record2, record3)); List> records = sinkForwardRecordsContext.getRecords(); assertThat(records.size(), equalTo(3)); + sinkForwardRecordsContext.clearRecords(); + assertThat(sinkForwardRecordsContext.getRecords().size(), equalTo(0)); + } + + @Test + public void testSinkForwardRecordContextClearRecords() { + SinkContext sinkContext = mock(SinkContext.class); + HeadlessPipeline headlessPipeline = mock(HeadlessPipeline.class); + when(sinkContext.getForwardToPipelines()).thenReturn(Map.of("pipeline1", headlessPipeline)); + sinkForwardRecordsContext = new SinkForwardRecordsContext(sinkContext); + Record record1 = mock(Record.class); + Record record2 = mock(Record.class); + sinkForwardRecordsContext.addRecords(List.of(record1, record2)); + assertThat(sinkForwardRecordsContext.getRecords().size(), equalTo(2)); + sinkForwardRecordsContext.clearRecords(); + assertThat(sinkForwardRecordsContext.getRecords().size(), equalTo(0)); } } diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 67ad1caef4..2a5fae4dc8 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -38,6 +38,7 @@ import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.model.pipeline.HeadlessPipeline; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; @@ -94,6 +95,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -151,21 +153,25 @@ public class OpenSearchSinkIT { private PluginConfigObservable pluginConfigObservable; public OpenSearchSink createObjectUnderTest(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { + sinkContext = mock(SinkContext.class); + when(sinkContext.getTagsTargetKey()).thenReturn(null); + when(sinkContext.getForwardToPipelines()).thenReturn(Map.of()); when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); when(pluginSetting.getPipelineName()).thenReturn(PIPELINE_NAME); when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); OpenSearchSink sink = new OpenSearchSink( - pluginSetting, null, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); + pluginSetting, sinkContext, expressionEvaluator, awsCredentialsSupplier, pipelineDescription, pluginConfigObservable, openSearchSinkConfig); if (doInitialize) { sink.doInitialize(); } return sink; } - public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig openSearchSinkConfig, boolean doInitialize) { + public OpenSearchSink createObjectUnderTestWithSinkContext(OpenSearchSinkConfig openSearchSinkConfig, final Map forwardPipelineMap, boolean doInitialize) { sinkContext = mock(SinkContext.class); testTagsTargetKey = RandomStringUtils.randomAlphabetic(5); when(sinkContext.getTagsTargetKey()).thenReturn(testTagsTargetKey); + when(sinkContext.getForwardToPipelines()).thenReturn(forwardPipelineMap); when(pipelineDescription.getPipelineName()).thenReturn(PIPELINE_NAME); when(pluginSetting.getPipelineName()).thenReturn(PIPELINE_NAME); when(pluginSetting.getName()).thenReturn(PLUGIN_NAME); @@ -846,6 +852,36 @@ public Stream provideArguments(ExtensionContext context) { } } + @Test + public void testOutputForwardsCreatedDocumentsToAPipeline() throws IOException, InterruptedException { + HeadlessPipeline forwardPipeline1 = mock(HeadlessPipeline.class); + Map forwardPipelineMap = Map.of("fwd_pipeline1", forwardPipeline1); + final String testIndexAlias = "test-alias"; + final String testTemplateFile = Objects.requireNonNull( + getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); + final String testIdField = "someId"; + final String testId = "foo"; + final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); + Map metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile); + metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); + final OpenSearchSink sink = createObjectUnderTestWithSinkContext(openSearchSinkConfig, forwardPipelineMap, true); + sink.output(testRecords); + final List> retSources = getSearchResponseDocSources(testIndexAlias); + assertThat(retSources.size(), equalTo(1)); + assertThat(getDocumentCount(testIndexAlias, "_id", testId), equalTo(Integer.valueOf(1))); + sink.shutdown(); + + // verify metrics + final List bulkRequestLatencies = MetricsTestUtil.getMeasurementList( + new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME) + .add(OpenSearchSink.BULKREQUEST_LATENCY).toString()); + assertThat(bulkRequestLatencies.size(), equalTo(3)); + // COUNT + Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0); + verify(sinkContext).forwardRecords(any(), eq(null), eq(null)); + } + @Test public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIndexAlias = "test-alias"; @@ -1255,7 +1291,7 @@ public void testEventOutputWithTags() throws IOException, InterruptedException { final List> testRecords = Collections.singletonList(new Record<>(testEvent)); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); - final OpenSearchSink sink = createObjectUnderTestWithSinkContext(openSearchSinkConfig, true); + final OpenSearchSink sink = createObjectUnderTestWithSinkContext(openSearchSinkConfig, Map.of(), true); sink.output(testRecords); final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java index 06d7cbe785..90e056c528 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategy.java @@ -28,11 +28,13 @@ import java.net.SocketTimeoutException; import java.time.Duration; import java.util.Arrays; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.function.Supplier; public final class BulkRetryStrategy { @@ -114,6 +116,7 @@ public final class BulkRetryStrategy { private final RequestFunction, BulkResponse> requestFunction; private final BiConsumer, Throwable> logFailure; + private final Consumer> successfulOperationsHandler; private final PluginMetrics pluginMetrics; private final Supplier bulkRequestSupplier; private final int maxRetries; @@ -158,6 +161,7 @@ String getExceptionMessage() { public BulkRetryStrategy(final RequestFunction, BulkResponse> requestFunction, final BiConsumer, Throwable> logFailure, + final Consumer> successfulOperationsHandler, final PluginMetrics pluginMetrics, final int maxRetries, final Supplier bulkRequestSupplier, @@ -167,6 +171,7 @@ public BulkRetryStrategy(final RequestFunction successfulOperations = new ArrayList<>(bulkRequestForRetry.getOperations()); + successfulOperationsHandler.accept(successfulOperations); final int totalDuplicateDocuments = bulkResponse.items().stream().filter(this::isDuplicateDocument).mapToInt(i -> 1).sum(); documentsDuplicates.increment(totalDuplicateDocuments); } @@ -384,6 +388,7 @@ private AccumulatingBulkRequest createBulkReq final AccumulatingBulkRequest requestToReissue = bulkRequestSupplier.get(); final ImmutableList.Builder nonRetryableFailures = ImmutableList.builder(); int index = 0; + List successfulOperations = new ArrayList<>(response.items().size()); for (final BulkResponseItem bulkItemResponse : response.items()) { BulkOperationWrapper bulkOperation = (BulkOperationWrapper)request.getOperationAt(index); @@ -399,6 +404,8 @@ private AccumulatingBulkRequest createBulkReq } else if (bulkItemResponse.error() != null && VERSION_CONFLICT_EXCEPTION_TYPE.equals(bulkItemResponse.error().type())) { documentsVersionConflictErrors.increment(); LOG.debug("Index: {}, Received version conflict from OpenSearch: {}", bulkItemResponse.index(), bulkItemResponse.error().reason()); + // This is not a successfully sent document, so do not add to "successfulOperations" + // and just release the eventHandle bulkOperation.releaseEventHandle(true); } else { nonRetryableFailures.add(FailedBulkOperation.builder() @@ -413,10 +420,11 @@ private AccumulatingBulkRequest createBulkReq if(isDuplicateDocument(bulkItemResponse)) { documentsDuplicates.increment(); } - bulkOperation.releaseEventHandle(true); + successfulOperations.add(bulkOperation); } index++; } + successfulOperationsHandler.accept(successfulOperations); final ImmutableList failedBulkOperations = nonRetryableFailures.build(); if(!failedBulkOperations.isEmpty()) { logFailure.accept(failedBulkOperations, null); @@ -428,6 +436,7 @@ private AccumulatingBulkRequest createBulkReq private void handleFailures(final AccumulatingBulkRequest accumulatingBulkRequest, final List itemResponses) { assert accumulatingBulkRequest.getOperationsCount() == itemResponses.size(); final ImmutableList.Builder failures = ImmutableList.builder(); + final List successfulOperations = new ArrayList<>(itemResponses.size()); for (int i = 0; i < itemResponses.size(); i++) { final BulkResponseItem bulkItemResponse = itemResponses.get(i); final BulkOperationWrapper bulkOperation = accumulatingBulkRequest.getOperationAt(i); @@ -435,6 +444,8 @@ private void handleFailures(final AccumulatingBulkRequest> { private volatile boolean initialized; private final SinkContext sinkContext; private final ExpressionEvaluator expressionEvaluator; + private final boolean useEventInBulkOperation; private FailedBulkOperationConverter failedBulkOperationConverter; private DataStreamDetector dataStreamDetector; @@ -167,6 +170,7 @@ public class OpenSearchSink extends AbstractSink> { private final ExecutorService queryExecutorService; private final int processWorkerThreads; + private final SinkForwardRecordsContext sinkForwardRecordsContext; @DataPrepperPluginConstructor public OpenSearchSink(final PluginSetting pluginSetting, @@ -180,8 +184,10 @@ public OpenSearchSink(final PluginSetting pluginSetting, this.processWorkerThreads = pipelineDescription.getNumberOfProcessWorkers(); this.awsCredentialsSupplier = awsCredentialsSupplier; this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + sinkForwardRecordsContext = new SinkForwardRecordsContext(sinkContext); this.expressionEvaluator = expressionEvaluator; this.pipeline = pipelineDescription.getPipelineName(); + this.useEventInBulkOperation = (getFailurePipeline() != null || sinkContext.getForwardToPipelines().size() > 0); bulkRequestTimer = pluginMetrics.timer(BULKREQUEST_LATENCY); bulkRequestErrorsCounter = pluginMetrics.counter(BULKREQUEST_ERRORS); invalidActionErrorsCounter = pluginMetrics.counter(INVALID_ACTION_ERRORS); @@ -300,6 +306,7 @@ private void doInitializeInternal() throws IOException { () -> openSearchClientRefresher.get()); bulkRetryStrategy = new BulkRetryStrategy(bulkRequest -> bulkApiWrapper.bulk(bulkRequest.getRequest()), this::logFailureForBulkRequests, + this::successfulOperationsHandler, pluginMetrics, maxRetries, bulkRequestSupplier, @@ -524,7 +531,7 @@ public void doOutput(final Collection> records) { final String queryTermKey = openSearchSinkConfig.getIndexConfiguration().getQueryTerm(); final String termValue = queryTermKey != null ? event.get(queryTermKey, String.class) : null; - BulkOperationWrapper bulkOperationWrapper = getFailurePipeline() != null ? + BulkOperationWrapper bulkOperationWrapper = (useEventInBulkOperation) ? new BulkOperationWrapper(bulkOperation, event, serializedJsonNode, termValue) : new BulkOperationWrapper(bulkOperation, event.getEventHandle(), serializedJsonNode, termValue); @@ -592,6 +599,27 @@ private void flushBatch(AccumulatingBulkRequest accumulatingBulkRequest) { }); } + @VisibleForTesting + void successfulOperationsHandler(final List successfulOperations) { + if (successfulOperations.size() == 0) { + return; + } + if (sinkContext.getForwardToPipelines().size() == 0) { + for (final BulkOperationWrapper bulkOperation: successfulOperations) { + if (bulkOperation.getEvent() != null) { + bulkOperation.getEvent().getEventHandle().release(true); + } else { + bulkOperation.getEventHandle().release(true); + } + } + return; + } + for (final BulkOperationWrapper bulkOperation: successfulOperations) { + sinkForwardRecordsContext.addRecord(new Record<>(bulkOperation.getEvent())); + } + sinkContext.forwardRecords(sinkForwardRecordsContext, null, null); + } + private void logFailureForBulkRequests(final List failedBulkOperations, final Throwable failure) { final List dlqObjects = failedBulkOperations.stream() @@ -724,7 +752,7 @@ private DlqObject createDlqObjectFromEvent(final Event event, .withPipelineName(pipeline) .withPluginId(PLUGIN_NAME); - if (getFailurePipeline() != null) { + if (useEventInBulkOperation) { builder.withEvent(event); } else { builder.withEventHandle(event.getEventHandle()); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java index 3ffdf43e11..002486b738 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/BulkRetryStrategyTests.java @@ -129,6 +129,16 @@ public BulkRetryStrategy createObjectUnderTest( return new BulkRetryStrategy( requestFunction, logFailure, + (operations) -> { + for (BulkOperationWrapper operation: operations) { + if (operation.getEvent() != null) { + operation.getEvent().getEventHandle().release(true); + } + if (operation.getEventHandle() != null) { + operation.getEventHandle().release(true); + } + } + }, pluginMetrics, Integer.MAX_VALUE, bulkRequestSupplier, @@ -146,6 +156,16 @@ public BulkRetryStrategy createObjectUnderTest( return new BulkRetryStrategy( requestFunction, logFailure, + (operations) -> { + for (BulkOperationWrapper operation: operations) { + if (operation.getEvent() != null) { + operation.getEvent().getEventHandle().release(true); + } + if (operation.getEventHandle() != null) { + operation.getEventHandle().release(true); + } + } + }, pluginMetrics, maxRetries, bulkRequestSupplier, @@ -164,6 +184,16 @@ public BulkRetryStrategy createObjectUnderTest( return new BulkRetryStrategy( requestFunction, logFailure, + (operations) -> { + for (BulkOperationWrapper operation: operations) { + if (operation.getEvent() != null) { + operation.getEvent().getEventHandle().release(true); + } + if (operation.getEventHandle() != null) { + operation.getEventHandle().release(true); + } + } + }, pluginMetrics, maxRetries, bulkRequestSupplier, diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java index bb37642659..74573c0d2c 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkTest.java @@ -24,6 +24,8 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.pipeline.HeadlessPipeline; +import org.opensearch.dataprepper.model.sink.SinkForwardRecordsContext; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventHandle; import org.opensearch.dataprepper.model.event.JacksonEvent; @@ -47,7 +49,9 @@ import java.io.IOException; import java.util.Collections; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -230,6 +234,69 @@ void test_initialization_with_failure_and_retry_with_query_manager() throws IOEx verify(pluginConfigObservable, times(2)).addPluginConfigObserver(any()); } + @Test + void test_sink_successful_records_handling_without_forwarding_pipelines_bulk_operations_with_event_handles() throws Exception { + when(sinkContext.getForwardToPipelines()).thenReturn(Map.of()); + final EventHandle eventHandle = mock(EventHandle.class); + final OpenSearchSink objectUnderTest = createObjectUnderTest(); + BulkOperationWrapper op1 = mock(BulkOperationWrapper.class); + BulkOperationWrapper op2 = mock(BulkOperationWrapper.class); + when(op1.getEvent()).thenReturn(null); + when(op2.getEvent()).thenReturn(null); + when(op1.getEventHandle()).thenReturn(eventHandle); + when(op2.getEventHandle()).thenReturn(eventHandle); + List operationsList = new ArrayList<>(); + operationsList.add(op1); + operationsList.add(op2); + objectUnderTest.successfulOperationsHandler(operationsList); + verify(eventHandle, times(2)).release(eq(true)); + + } + + @Test + void test_sink_successful_records_handling_with_forwarding_pipelines() throws Exception { + HeadlessPipeline forwardPipeline = mock(HeadlessPipeline.class); + when(sinkContext.getForwardToPipelines()).thenReturn(Map.of("fwd_pipeline", forwardPipeline)); + final EventHandle eventHandle = mock(EventHandle.class); + final OpenSearchSink objectUnderTest = createObjectUnderTest(); + BulkOperationWrapper op1 = mock(BulkOperationWrapper.class); + BulkOperationWrapper op2 = mock(BulkOperationWrapper.class); + Event event = mock(Event.class); + when(op1.getEvent()).thenReturn(event); + when(op2.getEvent()).thenReturn(event); + List operationsList = new ArrayList<>(); + operationsList.add(op1); + operationsList.add(op2); + objectUnderTest.successfulOperationsHandler(operationsList); + verify(eventHandle, times(0)).release(eq(true)); + verify(op1, times(1)).getEvent(); + verify(op2, times(1)).getEvent(); + verify(sinkContext, times(1)).forwardRecords(any(SinkForwardRecordsContext.class), eq(null), eq(null)); + + } + + + @Test + void test_sink_successful_records_handling_without_forwarding_pipelines_bulk_operations_with_events() throws Exception { + when(sinkContext.getForwardToPipelines()).thenReturn(Map.of()); + final EventHandle eventHandle = mock(EventHandle.class); + final OpenSearchSink objectUnderTest = createObjectUnderTest(); + BulkOperationWrapper op1 = mock(BulkOperationWrapper.class); + BulkOperationWrapper op2 = mock(BulkOperationWrapper.class); + Event event = mock(Event.class); + when(event.getEventHandle()).thenReturn(eventHandle); + when(op1.getEvent()).thenReturn(event); + when(op2.getEvent()).thenReturn(event); + List operationsList = new ArrayList<>(); + operationsList.add(op1); + operationsList.add(op2); + objectUnderTest.successfulOperationsHandler(operationsList); + verify(eventHandle, times(2)).release(eq(true)); + verify(op1, times(0)).getEventHandle(); + verify(op2, times(0)).getEventHandle(); + + } + @Test void doOutput_with_invalid_version_expression_catches_NumberFormatException_and_creates_DLQObject() throws IOException { when(pluginSetting.getName()).thenReturn("opensearch"); From 2f45cffc4f737783668b2a9e138448ce363e77a4 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Wed, 17 Dec 2025 14:31:54 -0800 Subject: [PATCH 23/51] Fixed PrometheusSinkBufferWriter getBuffer() to return non-duplicate and sorted time series (#6358) * Fixed PrometheusSinkBufferWriter getBuffer() to return non-duplicate and sorted timeseries Signed-off-by: Krishna Kondaka * Fixed CheckStyle error Signed-off-by: Krishna Kondaka --------- Signed-off-by: Krishna Kondaka Signed-off-by: Nathan Wand --- .../service/PrometheusSinkBufferWriter.java | 36 ++++++--- .../service/PrometheusTimeSeries.java | 8 ++ .../PrometheusSinkBufferWriterTest.java | 73 ++++++++++++++++--- 3 files changed, 95 insertions(+), 22 deletions(-) diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java index f0f48b15f2..a308d734ca 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java @@ -15,29 +15,45 @@ import org.opensearch.dataprepper.common.sink.SinkFlushContext; import org.opensearch.dataprepper.common.sink.SinkMetrics; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; public class PrometheusSinkBufferWriter implements SinkBufferWriter { - - private final List buffer; + + // Each buffer entry is a metric. + // Duplicate entries for the same metric at the same time is not allowed + // If there are multiple entries in the buffer for the same metric (with different time stamps), + // They must be sorted by time before sending to Prometheus + private final Map> buffer; private final SinkMetrics sinkMetrics; public PrometheusSinkBufferWriter(SinkMetrics sinkMetrics) { - buffer = new ArrayList<>(); + this.buffer = new HashMap<>(); this.sinkMetrics = sinkMetrics; } public boolean writeToBuffer(SinkBufferEntry bufferEntry) { - buffer.add(bufferEntry); + PrometheusTimeSeries timeSeries = ((PrometheusSinkBufferEntry)bufferEntry).getTimeSeries(); + if (timeSeries == null) { + return false; + } + + buffer.computeIfAbsent(timeSeries.getMetricName(), k -> new HashMap<>()) + .put(timeSeries.getTimeStamp(), bufferEntry); return true; } + @Override public SinkFlushableBuffer getBuffer(final SinkFlushContext sinkFlushContext) { - return new PrometheusSinkFlushableBuffer(buffer, sinkMetrics, sinkFlushContext); + List bufferList = buffer.values().stream() + .flatMap(timeSeriesMap -> timeSeriesMap.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue)) + .collect(Collectors.toList()); + + buffer.clear(); + return new PrometheusSinkFlushableBuffer(bufferList, sinkMetrics, sinkFlushContext); } - } - - - diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java index a8ba7cc23e..c8fba17abe 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java @@ -102,6 +102,14 @@ public PrometheusTimeSeries(Metric metric, final boolean sanitizeNames) throws E processResourceAndScopeAttributes(metric); } + public long getTimeStamp() { + return timestamp; + } + + public String getMetricName() { + return metricName; + } + private void processAttributes(Map attributesMap, String prefix) { if (attributesMap == null) return; diff --git a/data-prepper-plugins/prometheus-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterTest.java b/data-prepper-plugins/prometheus-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterTest.java index aaffb62fcd..942fc126e4 100644 --- a/data-prepper-plugins/prometheus-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterTest.java +++ b/data-prepper-plugins/prometheus-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterTest.java @@ -12,11 +12,15 @@ import org.opensearch.dataprepper.model.metric.JacksonGauge; import org.opensearch.dataprepper.common.sink.SinkMetrics; +import org.opensearch.dataprepper.model.metric.Gauge; +import org.opensearch.dataprepper.model.event.Event; import org.mockito.Mock; import static org.mockito.Mockito.mock; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; @@ -28,17 +32,17 @@ public class PrometheusSinkBufferWriterTest { @Mock private PrometheusSinkFlushContext sinkFlushContext; - private JacksonGauge gauge; + private JacksonGauge gauge1; private PrometheusSinkBufferEntry prometheusSinkBufferEntry; private PrometheusSinkBufferWriter prometheusSinkBufferWriter; - - + + @BeforeEach void setUp() throws Exception { sinkMetrics = mock(SinkMetrics.class); sinkFlushContext = mock(PrometheusSinkFlushContext.class); - gauge = createGaugeMetric(); - prometheusSinkBufferEntry = new PrometheusSinkBufferEntry(gauge, true); + gauge1 = createGaugeMetric("gauge1", Instant.now(), 1.0d); + prometheusSinkBufferEntry = new PrometheusSinkBufferEntry(gauge1, true); } PrometheusSinkBufferWriter createObjectUnderTest() { @@ -50,18 +54,63 @@ public void testPrometheusSinkBufferWriter() throws Exception { prometheusSinkBufferWriter = createObjectUnderTest(); prometheusSinkBufferWriter.writeToBuffer(prometheusSinkBufferEntry); PrometheusSinkFlushableBuffer prometheusSinkFlushableBuffer = (PrometheusSinkFlushableBuffer)prometheusSinkBufferWriter.getBuffer(sinkFlushContext); - assertThat(prometheusSinkFlushableBuffer.getEvents().get(0), sameInstance(gauge)); + assertThat(prometheusSinkFlushableBuffer.getEvents().get(0), sameInstance(gauge1)); + } + + @Test + public void testPrometheusSinkBufferWriterWithDuplicateTimeEntries() throws Exception { + prometheusSinkBufferWriter = createObjectUnderTest(); + Instant t2 = Instant.now().plusSeconds(5); + // Same metric with same name but different value, only most recent one is kept + Gauge gauge2 = createGaugeMetric("gauge2", t2, 10.0d); + Gauge gauge3 = createGaugeMetric("gauge2", t2, 20.0d); + PrometheusSinkBufferEntry entry2 = new PrometheusSinkBufferEntry(gauge2, true); + PrometheusSinkBufferEntry entry3 = new PrometheusSinkBufferEntry(gauge3, true); + prometheusSinkBufferWriter.writeToBuffer(prometheusSinkBufferEntry); + prometheusSinkBufferWriter.writeToBuffer(entry2); + prometheusSinkBufferWriter.writeToBuffer(entry3); + PrometheusSinkFlushContext sinkFlushContext = mock(PrometheusSinkFlushContext.class); + PrometheusSinkFlushableBuffer prometheusSinkFlushableBuffer = (PrometheusSinkFlushableBuffer)prometheusSinkBufferWriter.getBuffer(sinkFlushContext); + assertThat(prometheusSinkFlushableBuffer.getEvents().size(), equalTo(2)); + Event ev1 = prometheusSinkFlushableBuffer.getEvents().get(0); + Event ev2 = prometheusSinkFlushableBuffer.getEvents().get(1); + assertTrue(ev1 == gauge1 || ev1 == gauge3); + assertTrue(ev2 == gauge1 || ev2 == gauge3); } - private JacksonGauge createGaugeMetric() { + @Test + public void testPrometheusSinkBufferWriterWithOutOfOrderEntries() throws Exception { + prometheusSinkBufferWriter = createObjectUnderTest(); + // Same metric with same name but different value and different times but out of order times + // Expected result is sorted by time + Instant t2 = Instant.now().plusSeconds(50); + Gauge gauge2 = createGaugeMetric("gauge1", t2, 10.0d); + Instant t3 = t2.minusSeconds(150); + Gauge gauge3 = createGaugeMetric("gauge1", t3, 20.0d); + PrometheusSinkBufferEntry entry2 = new PrometheusSinkBufferEntry(gauge2, true); + PrometheusSinkBufferEntry entry3 = new PrometheusSinkBufferEntry(gauge3, true); + prometheusSinkBufferWriter.writeToBuffer(prometheusSinkBufferEntry); + prometheusSinkBufferWriter.writeToBuffer(entry2); + prometheusSinkBufferWriter.writeToBuffer(entry3); + PrometheusSinkFlushContext sinkFlushContext = mock(PrometheusSinkFlushContext.class); + PrometheusSinkFlushableBuffer prometheusSinkFlushableBuffer = (PrometheusSinkFlushableBuffer)prometheusSinkBufferWriter.getBuffer(sinkFlushContext); + assertThat(prometheusSinkFlushableBuffer.getEvents().size(), equalTo(3)); + assertThat(prometheusSinkFlushableBuffer.getEvents().get(0), sameInstance(gauge3)); + assertThat(prometheusSinkFlushableBuffer.getEvents().get(1), sameInstance(gauge1)); + assertThat(prometheusSinkFlushableBuffer.getEvents().get(2), sameInstance(gauge2)); + + } + + + private JacksonGauge createGaugeMetric(final String name, final Instant time, final double value) { return JacksonGauge.builder() - .withName("gauge") + .withName(name) .withDescription("Test Gauge Metric") - .withTimeReceived(Instant.now()) - .withTime(Instant.now().plusSeconds(10).toString()) - .withStartTime(Instant.now().plusSeconds(5).toString()) + .withTimeReceived(time) + .withTime(time.plusSeconds(10).toString()) + .withStartTime(time.plusSeconds(5).toString()) .withUnit("1") - .withValue(1.0d) + .withValue(value) .build(false); } From f408d22066f42c5f2c19955cea3707cb8e894054 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Wed, 17 Dec 2025 15:14:21 -0800 Subject: [PATCH 24/51] Make CWL retry indefinitely for retryable errors when no DLQ configured (#6355) * Make CWL retry indefinitely for retryable errors when no DLQ configured Signed-off-by: Krishna Kondaka * Added tests Signed-off-by: Krishna Kondaka --------- Signed-off-by: Krishna Kondaka Signed-off-by: Nathan Wand --- .../cloudwatch_logs/CloudWatchLogsSink.java | 2 +- .../client/CloudWatchLogsDispatcher.java | 5 +- .../client/CloudWatchLogsMetrics.java | 7 ++ .../CloudWatchLogsSinkTest.java | 85 ++++++++++++++++--- .../client/CloudWatchLogsDispatcherTest.java | 36 ++++++-- 5 files changed, 113 insertions(+), 22 deletions(-) diff --git a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSink.java b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSink.java index a6eafe04c4..e9111c6231 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSink.java +++ b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSink.java @@ -94,7 +94,7 @@ public CloudWatchLogsSink(final PluginSetting pluginSetting, .dropIfDlqNotConfigured(true) .logGroup(cloudWatchLogsSinkConfig.getLogGroup()) .logStream(cloudWatchLogsSinkConfig.getLogStream()) - .retryCount(cloudWatchLogsSinkConfig.getMaxRetries()) + .retryCount(dlqPushHandler == null ? Integer.MAX_VALUE : cloudWatchLogsSinkConfig.getMaxRetries()) .executor(executor) .build(); diff --git a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcher.java b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcher.java index b1a61781e1..99483dad4d 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcher.java +++ b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcher.java @@ -107,6 +107,7 @@ public void dispatchLogs(List inputLogEvents, List e @Builder protected static class Uploader implements Runnable { static final long INITIAL_DELAY_MS = 50; + static final int MULTIPLE_FAILURES_METRIC_COUNT = 5; static final long MAXIMUM_DELAY_MS = Duration.ofMinutes(10).toMillis(); private final CloudWatchLogsClient cloudWatchLogsClient; private final CloudWatchLogsMetrics cloudWatchLogsMetrics; @@ -141,7 +142,9 @@ public void upload() { failureMessage = e.getMessage(); LOG.error(NOISY, "Failed to push logs with error: {}", e.getMessage()); cloudWatchLogsMetrics.increaseRequestFailCounter(1); - failCount++; + if (++failCount % MULTIPLE_FAILURES_METRIC_COUNT == 0) { + cloudWatchLogsMetrics.increaseRequestMultiFailCounter(1); + } final long delayMillis = backoff.nextDelayMillis(failCount); if (delayMillis > 0) { Thread.sleep(delayMillis); diff --git a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsMetrics.java b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsMetrics.java index 3bf920c525..63fc55cf4e 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsMetrics.java +++ b/data-prepper-plugins/cloudwatch-logs/src/main/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsMetrics.java @@ -19,6 +19,7 @@ public class CloudWatchLogsMetrics { public static final String CLOUDWATCH_LOGS_EVENTS_SUCCEEDED = "cloudWatchLogsEventsSucceeded"; public static final String CLOUDWATCH_LOGS_EVENTS_FAILED = "cloudWatchLogsEventsFailed"; public static final String CLOUDWATCH_LOGS_REQUESTS_FAILED = "cloudWatchLogsRequestsFailed"; + public static final String CLOUDWATCH_LOGS_REQUEST_MULTI_FAILED = "cloudWatchLogsRequestMultipleFailures"; public static final String CLOUDWATCH_LOGS_LARGE_EVENTS_DROPPED = "cloudWatchLogsLargeEventsDropped"; public static final String CLOUDWATCH_LOGS_LOG_SIZE = "cloudWatchLogsLogSize"; public static final String CLOUDWATCH_LOGS_REQUEST_SIZE = "cloudWatchLogsRequestSize"; @@ -26,6 +27,7 @@ public class CloudWatchLogsMetrics { private final Counter logEventFailCounter; private final Counter requestSuccessCount; private final Counter requestFailCount; + private final Counter requestMultiFailCount; private final Counter logLargeEventsDroppedCounter; private final DistributionSummary logSizeMetric; private final DistributionSummary requestSizeMetric; @@ -33,6 +35,7 @@ public class CloudWatchLogsMetrics { public CloudWatchLogsMetrics(final PluginMetrics pluginMetrics) { this.logEventSuccessCounter = pluginMetrics.counter(CloudWatchLogsMetrics.CLOUDWATCH_LOGS_EVENTS_SUCCEEDED); this.requestFailCount = pluginMetrics.counter(CloudWatchLogsMetrics.CLOUDWATCH_LOGS_REQUESTS_FAILED); + this.requestMultiFailCount = pluginMetrics.counter(CloudWatchLogsMetrics.CLOUDWATCH_LOGS_REQUEST_MULTI_FAILED); this.logEventFailCounter = pluginMetrics.counter(CloudWatchLogsMetrics.CLOUDWATCH_LOGS_EVENTS_FAILED); this.requestSuccessCount = pluginMetrics.counter(CloudWatchLogsMetrics.CLOUDWATCH_LOGS_REQUESTS_SUCCEEDED); this.logLargeEventsDroppedCounter = pluginMetrics.counter(CloudWatchLogsMetrics.CLOUDWATCH_LOGS_LARGE_EVENTS_DROPPED); @@ -56,6 +59,10 @@ public void increaseRequestFailCounter(int value) { requestFailCount.increment(value); } + public void increaseRequestMultiFailCounter(int value) { + requestMultiFailCount.increment(value); + } + public void increaseLogLargeEventsDroppedCounter(int value) { logLargeEventsDroppedCounter.increment(value); } diff --git a/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSinkTest.java b/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSinkTest.java index 750e9f7537..3f19fe5ba8 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSinkTest.java +++ b/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/CloudWatchLogsSinkTest.java @@ -8,26 +8,34 @@ import io.micrometer.core.instrument.DistributionSummary; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.dlq.DlqPushHandler; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.client.CloudWatchLogsClientFactory; +import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.client.CloudWatchLogsDispatcher; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.client.CloudWatchLogsMetrics; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.config.AwsConfig; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.config.CloudWatchLogsSinkConfig; import org.opensearch.dataprepper.plugins.sink.cloudwatch_logs.config.ThresholdConfig; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; +import software.amazon.awssdk.regions.Region; + import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -36,12 +44,14 @@ import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class CloudWatchLogsSinkTest { + private static int TEST_MAX_RETRIES = 3; private PluginSetting mockPluginSetting; private PluginMetrics mockPluginMetrics; private PluginFactory mockPluginFactory; @@ -57,6 +67,7 @@ class CloudWatchLogsSinkTest { private static final String TEST_PLUGIN_NAME = "testPluginName"; private static final String TEST_PIPELINE_NAME = "testPipelineName"; private static final String TEST_BUFFER_TYPE = "in_memory"; + private int numRetries; @BeforeEach void setUp() { mockPluginSetting = mock(PluginSetting.class); @@ -73,12 +84,13 @@ void setUp() { DistributionSummary summary = mock(DistributionSummary.class); when(mockPluginMetrics.summary(anyString())).thenReturn(summary); + when(mockCloudWatchLogsSinkConfig.getDlq()).thenReturn(null); when(mockCloudWatchLogsSinkConfig.getAwsConfig()).thenReturn(mockAwsConfig); when(mockCloudWatchLogsSinkConfig.getThresholdConfig()).thenReturn(thresholdConfig); when(mockCloudWatchLogsSinkConfig.getHeaderOverrides()).thenReturn(new HashMap<>()); when(mockCloudWatchLogsSinkConfig.getLogGroup()).thenReturn(TEST_LOG_GROUP); when(mockCloudWatchLogsSinkConfig.getLogStream()).thenReturn(TEST_LOG_STREAM); - when(mockCloudWatchLogsSinkConfig.getMaxRetries()).thenReturn(3); + when(mockCloudWatchLogsSinkConfig.getMaxRetries()).thenReturn(TEST_MAX_RETRIES); when(mockCloudWatchLogsSinkConfig.getWorkers()).thenReturn(10); when(mockPluginSetting.getName()).thenReturn(TEST_PLUGIN_NAME); @@ -167,17 +179,17 @@ void WHEN_given_sample_empty_records_THEN_records_are_not_processed() { void WHEN_header_overrides_is_empty_THEN_empty_map_is_passed_to_client_factory() { Map emptyHeaders = new HashMap<>(); when(mockCloudWatchLogsSinkConfig.getHeaderOverrides()).thenReturn(emptyHeaders); - + try(MockedStatic mockedStatic = mockStatic(CloudWatchLogsClientFactory.class)) { mockedStatic.when(() -> CloudWatchLogsClientFactory.createCwlClient(any(AwsConfig.class), any(AwsCredentialsSupplier.class), any(), any())) .thenReturn(mockClient); CloudWatchLogsSink testCloudWatchSink = getTestCloudWatchSink(); - + mockedStatic.verify(() -> CloudWatchLogsClientFactory.createCwlClient( - eq(mockAwsConfig), - eq(mockCredentialSupplier), + eq(mockAwsConfig), + eq(mockCredentialSupplier), eq(emptyHeaders), any())); } @@ -186,17 +198,17 @@ void WHEN_header_overrides_is_empty_THEN_empty_map_is_passed_to_client_factory() @Test void WHEN_header_overrides_is_provided_THEN_headers_are_passed_to_client_factory() { when(mockCloudWatchLogsSinkConfig.getHeaderOverrides()).thenReturn(mockHeaderOverrides); - + try(MockedStatic mockedStatic = mockStatic(CloudWatchLogsClientFactory.class)) { mockedStatic.when(() -> CloudWatchLogsClientFactory.createCwlClient(any(AwsConfig.class), any(AwsCredentialsSupplier.class), any(), any())) .thenReturn(mockClient); CloudWatchLogsSink testCloudWatchSink = getTestCloudWatchSink(); - + mockedStatic.verify(() -> CloudWatchLogsClientFactory.createCwlClient( - eq(mockAwsConfig), - eq(mockCredentialSupplier), + eq(mockAwsConfig), + eq(mockCredentialSupplier), eq(mockHeaderOverrides), any())); } @@ -205,7 +217,7 @@ void WHEN_header_overrides_is_provided_THEN_headers_are_passed_to_client_factory @Test void WHEN_sink_initialization_with_header_overrides_THEN_sink_is_ready() { when(mockCloudWatchLogsSinkConfig.getHeaderOverrides()).thenReturn(mockHeaderOverrides); - + try(MockedStatic mockedStatic = mockStatic(CloudWatchLogsClientFactory.class)) { mockedStatic.when(() -> CloudWatchLogsClientFactory.createCwlClient(any(AwsConfig.class), any(AwsCredentialsSupplier.class), any(), any())) @@ -213,8 +225,59 @@ void WHEN_sink_initialization_with_header_overrides_THEN_sink_is_ready() { CloudWatchLogsSink testCloudWatchSink = getTestCloudWatchSink(); testCloudWatchSink.doInitialize(); - + assertTrue(testCloudWatchSink.isReady()); } } + + @Test + void WHEN_sink_has_no_dlq_config_THEN_retries_set_to_maxint() { + when(mockCloudWatchLogsSinkConfig.getHeaderOverrides()).thenReturn(mockHeaderOverrides); + + try(MockedStatic mockedStatic = mockStatic(CloudWatchLogsClientFactory.class)) { + final MockedConstruction dispatcherMock = + mockConstruction(CloudWatchLogsDispatcher.class, (mock, context) -> { + numRetries = (int)context.arguments().get(7); + }); + + mockedStatic.when(() -> CloudWatchLogsClientFactory.createCwlClient(any(AwsConfig.class), + any(AwsCredentialsSupplier.class), any(), any())) + .thenReturn(mockClient); + + CloudWatchLogsSink testCloudWatchSink = getTestCloudWatchSink(); + testCloudWatchSink.doInitialize(); + dispatcherMock.close(); + + } + assertThat(numRetries, equalTo(Integer.MAX_VALUE)); + } + + @Test + void WHEN_sink_has_dlq_config_THEN_retries_set_to_user_configured_value() { + PluginModel dlqConfig = mock(PluginModel.class); + when(mockCloudWatchLogsSinkConfig.getDlq()).thenReturn(dlqConfig); + when(mockCloudWatchLogsSinkConfig.getHeaderOverrides()).thenReturn(mockHeaderOverrides); + when(mockAwsConfig.getAwsRegion()).thenReturn(Region.of("us-west-2")); + when(mockAwsConfig.getAwsStsRoleArn()).thenReturn("role"); + + try(MockedStatic mockedStatic = mockStatic(CloudWatchLogsClientFactory.class)) { + final MockedConstruction dispatcherMock = + mockConstruction(CloudWatchLogsDispatcher.class, (mock, context) -> { + numRetries = (int)context.arguments().get(7); + }); + final MockedConstruction dlqMock = + mockConstruction(DlqPushHandler.class, (mock, context) -> { + }); + + mockedStatic.when(() -> CloudWatchLogsClientFactory.createCwlClient(any(AwsConfig.class), + any(AwsCredentialsSupplier.class), any(), any())) + .thenReturn(mockClient); + + CloudWatchLogsSink testCloudWatchSink = getTestCloudWatchSink(); + testCloudWatchSink.doInitialize(); + dispatcherMock.close(); + } + assertThat(numRetries, equalTo(TEST_MAX_RETRIES)); + } + } diff --git a/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcherTest.java b/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcherTest.java index 31941b42fc..04c73fc259 100644 --- a/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcherTest.java +++ b/data-prepper-plugins/cloudwatch-logs/src/test/java/org/opensearch/dataprepper/plugins/sink/cloudwatch_logs/client/CloudWatchLogsDispatcherTest.java @@ -68,14 +68,14 @@ List getSampleEventHandles() { return eventHandles; } - CloudWatchLogsDispatcher getCloudWatchLogsDispatcher() { + CloudWatchLogsDispatcher getCloudWatchLogsDispatcher(int retryCount) { return CloudWatchLogsDispatcher.builder() .cloudWatchLogsClient(mockCloudWatchLogsClient) .cloudWatchLogsMetrics(mockCloudWatchLogsMetrics) .executor(mockExecutor) .logGroup(LOG_GROUP) .logStream(LOG_STREAM) - .retryCount(RETRY_COUNT) + .retryCount(retryCount) .dropIfDlqNotConfigured(true) .build(); } @@ -88,7 +88,7 @@ private void executeDispatcherRunnable() { @Test void GIVEN_valid_input_log_events_SHOULD_call_executor() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); final PutLogEventsResponse response = mock(PutLogEventsResponse.class); @@ -105,7 +105,7 @@ void GIVEN_valid_input_log_events_SHOULD_call_executor() { @Test void GIVEN_too_old_events_SHOULD_not_release_old_events() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); final PutLogEventsResponse response = mock(PutLogEventsResponse.class); @@ -134,7 +134,7 @@ void GIVEN_too_old_events_SHOULD_not_release_old_events() { @Test void GIVEN_too_new_events_SHOULD_not_release_new_events() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); final PutLogEventsResponse response = mock(PutLogEventsResponse.class); @@ -164,7 +164,7 @@ void GIVEN_too_new_events_SHOULD_not_release_new_events() { @Test void GIVEN_both_old_and_new_rejected_events_SHOULD_only_release_valid_events() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); final PutLogEventsResponse response = mock(PutLogEventsResponse.class); @@ -197,7 +197,7 @@ void GIVEN_both_old_and_new_rejected_events_SHOULD_only_release_valid_events() { @Test void GIVEN_client_exception_SHOULD_retry() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); when(mockCloudWatchLogsClient.putLogEvents(any(PutLogEventsRequest.class))) @@ -213,9 +213,27 @@ void GIVEN_client_exception_SHOULD_retry() { verify(mockCloudWatchLogsMetrics, times(1)).increaseRequestSuccessCounter(1); } + @Test + void GIVEN_cloudwatch_exception_SHOULD_retry_forever() { + final int TEST_RETRY_COUNT = CloudWatchLogsDispatcher.Uploader.MULTIPLE_FAILURES_METRIC_COUNT+1; + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(TEST_RETRY_COUNT); + + final List eventHandles = getSampleEventHandles(); + when(mockCloudWatchLogsClient.putLogEvents(any(PutLogEventsRequest.class))) + .thenThrow(CloudWatchLogsException.class); + List inputLogEventList = cloudWatchLogsDispatcher.prepareInputLogEvents(getSampleBufferedData()); + cloudWatchLogsDispatcher.dispatchLogs(inputLogEventList, eventHandles); + + executeDispatcherRunnable(); + + verify(mockCloudWatchLogsMetrics, times(TEST_RETRY_COUNT)).increaseRequestFailCounter(1); + verify(mockCloudWatchLogsMetrics, times(0)).increaseRequestSuccessCounter(1); + verify(mockCloudWatchLogsMetrics, times(1)).increaseRequestMultiFailCounter(1); + } + @Test void GIVEN_cloudwatch_exception_SHOULD_retry() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); when(mockCloudWatchLogsClient.putLogEvents(any(PutLogEventsRequest.class))) @@ -233,7 +251,7 @@ void GIVEN_cloudwatch_exception_SHOULD_retry() { @Test void GIVEN_max_retries_exceeded_SHOULD_not_release_events() { - cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(); + cloudWatchLogsDispatcher = getCloudWatchLogsDispatcher(RETRY_COUNT); final List eventHandles = getSampleEventHandles(); when(mockCloudWatchLogsClient.putLogEvents(any(PutLogEventsRequest.class))) From 96f11af2c7929c83b0cd6e92326ef63b38ec539b Mon Sep 17 00:00:00 2001 From: chrisale000 Date: Thu, 18 Dec 2025 14:55:11 -0800 Subject: [PATCH 25/51] Metric Centralization through Dependency Injection (#6354) This change centralizes metrics creation and management by implementing dependency injection for metrics in the Office365 source plugin and other remaining components. This ensures consistent metrics handling across the codebase. Signed-off-by: Alexander Christensen Co-authored-by: Alexander Christensen Signed-off-by: Nathan Wand --- .../Office365RestClient.java | 67 +-- .../Office365RestClientConfiguration.java | 58 ++ .../Office365RestClientTest.java | 403 +++++--------- .../metrics/VendorAPIMetricsRecorder.java | 242 +++++++++ .../source_crawler/utils/MetricsHelper.java | 1 - .../utils/retry/RetryHandler.java | 16 +- .../metrics/VendorAPIMetricsRecorderTest.java | 506 ++++++++++++++++++ .../utils/MetricsHelperTest.java | 48 +- .../utils/retry/RetryHandlerTest.java | 95 ++++ 9 files changed, 1092 insertions(+), 344 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorder.java create mode 100644 data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorderTest.java diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java index 7fc4644b79..a4601787fa 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java @@ -10,12 +10,12 @@ package org.opensearch.dataprepper.plugins.source.microsoft_office365; import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface; import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException; import org.opensearch.dataprepper.plugins.source.microsoft_office365.models.AuditLogsResponse; +import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder; import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.RetryHandler; import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultRetryStrategy; import org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry.DefaultStatusCodeHandler; @@ -33,18 +33,9 @@ import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.CONTENT_TYPES; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.getErrorTypeMetricCounterMap; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishErrorTypeMetricCounter; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishGetResponseSizeMetricInBytes; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishGetRequestsSuccessMetric; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.provideGetRequestsFailureCounter; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishSearchResponseSizeMetricInBytes; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.publishSearchRequestsSuccessMetric; -import static org.opensearch.dataprepper.plugins.source.source_crawler.utils.MetricsHelper.provideSearchRequestFailureCounter; /** * REST client for interacting with Office 365 Management API. @@ -53,34 +44,21 @@ @Slf4j @Named public class Office365RestClient { - private static final String AUDIT_LOG_FETCH_LATENCY = "auditLogFetchLatency"; - private static final String API_CALLS = "apiCalls"; - private static final String AUDIT_LOGS_REQUESTED = "auditLogsRequested"; - private static final String SEARCH_CALL_LATENCY = "searchCallLatency"; - private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/"; + private static final String API_CALLS = "apiCalls"; private final RestTemplate restTemplate = new RestTemplate(); private final RetryHandler retryHandler; private final Office365AuthenticationInterface authConfig; - private final Timer auditLogFetchLatencyTimer; - private final Timer searchCallLatencyTimer; - private final Counter auditLogsRequestedCounter; + private final VendorAPIMetricsRecorder metricsRecorder; private final Counter apiCallsCounter; - private final PluginMetrics pluginMetrics; - - private Map errorTypeMetricCounterMap; public Office365RestClient(final Office365AuthenticationInterface authConfig, - final PluginMetrics pluginMetrics) { - // TODO: Abstract into a Office365PluginMetrics + final PluginMetrics pluginMetrics, + final VendorAPIMetricsRecorder metricsRecorder) { this.authConfig = authConfig; - this.pluginMetrics = pluginMetrics; - this.auditLogFetchLatencyTimer = pluginMetrics.timer(AUDIT_LOG_FETCH_LATENCY); - this.searchCallLatencyTimer = pluginMetrics.timer(SEARCH_CALL_LATENCY); - this.auditLogsRequestedCounter = pluginMetrics.counter(AUDIT_LOGS_REQUESTED); + this.metricsRecorder = metricsRecorder; this.apiCallsCounter = pluginMetrics.counter(API_CALLS); - this.errorTypeMetricCounterMap = getErrorTypeMetricCounterMap(pluginMetrics); this.retryHandler = new RetryHandler( new DefaultRetryStrategy(), new DefaultStatusCodeHandler()); @@ -93,7 +71,6 @@ public void startSubscriptions() { log.info("Starting Office 365 subscriptions for audit logs"); try { HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); // TODO: Only start the subscriptions only if the call commented @@ -141,7 +118,7 @@ public void startSubscriptions() { }, authConfig::renewCredentials); } } catch (Exception e) { - publishErrorTypeMetricCounter(e, this.errorTypeMetricCounterMap); + metricsRecorder.recordError(e); log.error(NOISY, "Failed to initialize subscriptions", e); throw new SaaSCrawlerException("Failed to initialize subscriptions: " + e.getMessage(), e, true); } @@ -174,12 +151,12 @@ public AuditLogsResponse searchAuditLogs(final String contentType, log.debug("Searching audit logs with URL: {}", url); final HttpHeaders headers = new HttpHeaders(); - return searchCallLatencyTimer.record(() -> { + return metricsRecorder.recordSearchLatency(() -> { try { return retryHandler.executeWithRetry( () -> { headers.setBearerAuth(authConfig.getAccessToken()); - apiCallsCounter.increment(); + metricsRecorder.recordDataApiRequest(); ResponseEntity>> response = restTemplate.exchange( url, @@ -205,9 +182,8 @@ public AuditLogsResponse searchAuditLogs(final String contentType, } } - // Publish centralized search metrics - publishSearchResponseSizeMetricInBytes(pluginMetrics, response); - publishSearchRequestsSuccessMetric(pluginMetrics); + metricsRecorder.recordSearchResponseSize(response); + metricsRecorder.recordSearchSuccess(); // Extract NextPageUri from response headers List nextPageHeaders = response.getHeaders().get("NextPageUri"); @@ -221,10 +197,11 @@ public AuditLogsResponse searchAuditLogs(final String contentType, return new AuditLogsResponse(response.getBody(), nextPageUri); }, authConfig::renewCredentials, - Optional.of(provideSearchRequestFailureCounter(pluginMetrics)) + metricsRecorder::recordSearchFailure ); } catch (Exception e) { - publishErrorTypeMetricCounter(e, this.errorTypeMetricCounterMap); + metricsRecorder.recordError(e); + metricsRecorder.recordSearchFailure(); log.error(NOISY, "Error while fetching audit logs for content type {} from URL: {}", contentType, url, e); throw new SaaSCrawlerException("Failed to fetch audit logs", e, true); @@ -245,14 +222,14 @@ public String getAuditLog(String contentUri) { } log.debug("Getting audit log from content URI: {}", contentUri); - auditLogsRequestedCounter.increment(); + metricsRecorder.recordLogsRequested(); final HttpHeaders headers = new HttpHeaders(); - return auditLogFetchLatencyTimer.record(() -> { + return metricsRecorder.recordGetLatency(() -> { try { String response = retryHandler.executeWithRetry(() -> { headers.setBearerAuth(authConfig.getAccessToken()); - apiCallsCounter.increment(); + metricsRecorder.recordDataApiRequest(); ResponseEntity responseEntity = restTemplate.exchange( contentUri, HttpMethod.GET, @@ -261,7 +238,7 @@ public String getAuditLog(String contentUri) { ); return responseEntity.getBody(); - }, authConfig::renewCredentials, Optional.of(provideGetRequestsFailureCounter(pluginMetrics))); + }, authConfig::renewCredentials, metricsRecorder::recordGetFailure); // Log response details if (response == null) { @@ -278,13 +255,13 @@ public String getAuditLog(String contentUri) { } } - // Publish centralized GET request metrics - publishGetResponseSizeMetricInBytes(pluginMetrics, response); - publishGetRequestsSuccessMetric(pluginMetrics); + metricsRecorder.recordGetResponseSize(response); + metricsRecorder.recordGetSuccess(); return response; } catch (Exception e) { - publishErrorTypeMetricCounter(e, this.errorTypeMetricCounterMap); + metricsRecorder.recordError(e); + metricsRecorder.recordGetFailure(); log.error(NOISY, "Error while fetching audit log content from URI: {}", contentUri, e); throw new SaaSCrawlerException("Failed to fetch audit log", e, true); } diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java new file mode 100644 index 0000000000..60ab4644bb --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/configuration/Office365RestClientConfiguration.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugins.source.microsoft_office365.configuration; + +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.microsoft_office365.Office365RestClient; +import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface; +import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Spring configuration for Microsoft Office 365 RestClient. + * + * This configuration class creates the Office365RestClient with the required dependencies: + * 1. Office365AuthenticationInterface for authentication + * 2. PluginMetrics for unified metrics recording + * + * The Office365RestClient internally creates VendorAPIMetricsRecorder instances + * for different operation types (GET, SEARCH, AUTH) from the provided PluginMetrics. + */ +@Configuration +public class Office365RestClientConfiguration { + + /** + * Creates VendorAPIMetricsRecorder with unified metrics for all operations. + * + * @param pluginMetrics The system plugin metrics instance + * @return Configured VendorAPIMetricsRecorder + */ + @Bean + public VendorAPIMetricsRecorder vendorAPIMetricsRecorder(PluginMetrics pluginMetrics) { + return new VendorAPIMetricsRecorder(pluginMetrics); + } + + /** + * Creates Office365RestClient with unified metrics recorder. + * + * @param authConfig The Office 365 authentication provider + * @param pluginMetrics The system plugin metrics instance + * @param vendorAPIMetricsRecorder The unified metrics recorder + * @return Configured Office365RestClient + */ + @Bean + public Office365RestClient office365RestClient( + Office365AuthenticationInterface authConfig, + PluginMetrics pluginMetrics, + VendorAPIMetricsRecorder vendorAPIMetricsRecorder) { + return new Office365RestClient(authConfig, pluginMetrics, vendorAPIMetricsRecorder); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java index 8741fc7cb7..2edf221303 100644 --- a/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClientTest.java @@ -9,12 +9,11 @@ package org.opensearch.dataprepper.plugins.source.microsoft_office365; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Timer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import io.micrometer.core.instrument.Counter; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -22,6 +21,7 @@ import org.opensearch.dataprepper.plugins.source.microsoft_office365.auth.Office365AuthenticationInterface; import org.opensearch.dataprepper.plugins.source.microsoft_office365.models.AuditLogsResponse; import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException; +import org.opensearch.dataprepper.plugins.source.source_crawler.metrics.VendorAPIMetricsRecorder; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.HttpEntity; @@ -48,35 +48,87 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.CONTENT_TYPES; +/** + * Unit tests for Office365RestClient. + * + * Tests REST API interactions with Office 365 Management API including: + * - Subscription management (create, handle existing subscriptions) + * - Audit log search and retrieval operations + * - Error handling and retry logic + * - Authentication and token renewal flows + * - Pagination support for large result sets + */ @ExtendWith(MockitoExtension.class) class Office365RestClientTest { @Mock private RestTemplate restTemplate; @Mock private Office365AuthenticationInterface authConfig; - - private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("Office365RestClientTest", "microsoft-office365"); + @Mock + private PluginMetrics pluginMetrics; + @Mock + private VendorAPIMetricsRecorder metricsRecorder; + @Mock + private Counter apiCallsCounter; private Office365RestClient office365RestClient; @BeforeEach - void setUp() throws NoSuchFieldException, IllegalAccessException{ - office365RestClient = new Office365RestClient(authConfig, pluginMetrics); + void setUp() throws NoSuchFieldException, IllegalAccessException { + // Setup VendorAPIMetricsRecorder method mocks - use lenient to avoid unnecessary stubbing errors + lenient().when(metricsRecorder.recordAuthLatency(any(java.util.function.Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, java.util.function.Supplier.class).get()); + lenient().when(metricsRecorder.recordSearchLatency(any(java.util.function.Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, java.util.function.Supplier.class).get()); + lenient().when(metricsRecorder.recordGetLatency(any(java.util.function.Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, java.util.function.Supplier.class).get()); + + // Setup Runnable overload mocks - execute the runnable when called + lenient().doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0, Runnable.class); + runnable.run(); + return null; + }).when(metricsRecorder).recordAuthLatency(any(Runnable.class)); + + lenient().doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0, Runnable.class); + runnable.run(); + return null; + }).when(metricsRecorder).recordSearchLatency(any(Runnable.class)); + + lenient().doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0, Runnable.class); + runnable.run(); + return null; + }).when(metricsRecorder).recordGetLatency(any(Runnable.class)); + + // Void methods don't need stubbing - Mockito handles them automatically + + // Mock the counter creation + when(pluginMetrics.counter("apiCalls")).thenReturn(apiCallsCounter); + + office365RestClient = new Office365RestClient(authConfig, pluginMetrics, metricsRecorder); ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "restTemplate", restTemplate); } + /** + * Tests successful subscription creation for all Office 365 content types. + * Verifies that POST requests are made for each content type and no exceptions are thrown. + */ @Test void testStartSubscriptionsSuccess() { + when(authConfig.getTenantId()).thenReturn("test-tenant"); + when(authConfig.getAccessToken()).thenReturn("test-token"); + ResponseEntity mockResponse = new ResponseEntity<>("{\"status\":\"enabled\"}", HttpStatus.OK); when(restTemplate.exchange(anyString(), eq(HttpMethod.POST), any(), eq(String.class))) .thenReturn(mockResponse); assertDoesNotThrow(() -> office365RestClient.startSubscriptions()); + verify(restTemplate, times(CONTENT_TYPES.length)).exchange( anyString(), eq(HttpMethod.POST), @@ -85,8 +137,15 @@ void testStartSubscriptionsSuccess() { ); } + /** + * Tests handling of AF20024 error code when subscriptions are already enabled. + * Verifies that this specific error is treated as success and doesn't throw an exception. + */ @Test void testStartSubscriptionsAlreadyEnabled() { + when(authConfig.getTenantId()).thenReturn("test-tenant"); + when(authConfig.getAccessToken()).thenReturn("test-token"); + HttpClientErrorException af20024Exception = new HttpClientErrorException( HttpStatus.BAD_REQUEST, "Bad Request", @@ -100,8 +159,15 @@ void testStartSubscriptionsAlreadyEnabled() { assertDoesNotThrow(() -> office365RestClient.startSubscriptions()); } + /** + * Tests error handling for subscription failures other than AF20024. + * Verifies that non-AF20024 errors are propagated as retryable SaaSCrawlerException. + */ @Test void testStartSubscriptionsOtherError() { + when(authConfig.getTenantId()).thenReturn("test-tenant"); + when(authConfig.getAccessToken()).thenReturn("test-token"); + HttpClientErrorException otherException = new HttpClientErrorException( HttpStatus.BAD_REQUEST, "Bad Request", @@ -109,15 +175,20 @@ void testStartSubscriptionsOtherError() { null ); when(restTemplate.exchange(anyString(), eq(HttpMethod.POST), any(), eq(String.class))) - .thenThrow(otherException); + .thenThrow(otherException) + .thenThrow(otherException) // Retry will call this again + .thenThrow(otherException); // Final retry SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, () -> office365RestClient.startSubscriptions()); - assertEquals("Failed to initialize subscriptions: 400 Bad Request", - exception.getMessage()); + assertTrue(exception.getMessage().contains("Failed to initialize subscriptions")); assertTrue(exception.isRetryable()); } + /** + * Tests audit log search functionality with URL and header validation. + * Verifies URL construction, authentication headers, and response mapping. + */ @Test void testSearchAuditLogs() { Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); @@ -142,7 +213,7 @@ void testSearchAuditLogs() { any(ParameterizedTypeReference.class) )).thenReturn(mockResponse); - office365RestClient.searchAuditLogs( + AuditLogsResponse result = office365RestClient.searchAuditLogs( "Audit.AzureActiveDirectory", startTime, endTime, @@ -160,13 +231,22 @@ void testSearchAuditLogs() { // Verify headers contain correct access token HttpHeaders headers = entityCaptor.getValue().getHeaders(); assertEquals("Bearer test-access-token", headers.getFirst("Authorization")); + + // Verify result + assertEquals(mockResults, result.getItems()); } + /** + * Tests pagination support in audit log search. + * Verifies that pageUri is used correctly and NextPageUri header is extracted from response. + */ @Test void testSearchAuditLogsWithPagination() { // Test with pageUri provided String pageUri = "https://next-page-url"; + when(authConfig.getAccessToken()).thenReturn("test-token"); + // Setup mock response with NextPageUri header HttpHeaders headers = new HttpHeaders(); headers.add("NextPageUri", "https://another-page"); @@ -193,35 +273,51 @@ void testSearchAuditLogsWithPagination() { assertEquals("https://another-page", response.getNextPageUri()); } + /** + * Tests error handling for audit log search failures. + * Verifies that persistent errors are propagated as retryable SaaSCrawlerException. + */ @Test void testSearchAuditLogsFailure() { Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); Instant endTime = Instant.now(); - // Mock REST template to throw an error + when(authConfig.getTenantId()).thenReturn("test-tenant"); + when(authConfig.getAccessToken()).thenReturn("test-token"); + + // Mock REST template to throw an error that persists through retries + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.INTERNAL_SERVER_ERROR); when(restTemplate.exchange( anyString(), eq(HttpMethod.GET), any(), any(ParameterizedTypeReference.class) - )).thenThrow(new HttpClientErrorException(HttpStatus.INTERNAL_SERVER_ERROR)); + )).thenThrow(exception) + .thenThrow(exception) // Retry 1 + .thenThrow(exception); // Retry 2 // Verify that the exception is propagated - SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, + SaaSCrawlerException crawlerException = assertThrows(SaaSCrawlerException.class, () -> office365RestClient.searchAuditLogs( "Audit.AzureActiveDirectory", startTime, endTime, null )); - assertEquals("Failed to fetch audit logs", exception.getMessage()); - assertTrue(exception.isRetryable()); + assertEquals("Failed to fetch audit logs", crawlerException.getMessage()); + assertTrue(crawlerException.isRetryable()); } + /** + * Tests basic audit log retrieval functionality. + * Verifies that audit log content can be fetched from a specific URI. + */ @Test void testGetAuditLog() { String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; String mockAuditLog = "{\"id\":\"123\",\"contentType\":\"Audit.AzureActiveDirectory\"}"; + + when(authConfig.getAccessToken()).thenReturn("test-token"); ResponseEntity mockResponse = new ResponseEntity<>(mockAuditLog, HttpStatus.OK); when(restTemplate.exchange( @@ -236,24 +332,38 @@ void testGetAuditLog() { assertEquals(mockAuditLog, result); } + /** + * Tests error handling for audit log retrieval failures. + * Verifies that persistent errors are propagated as retryable SaaSCrawlerException. + */ @Test void testGetAuditLogFailure() { String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - // Mock REST template to throw an exception + + when(authConfig.getAccessToken()).thenReturn("test-token"); + + // Mock REST template to throw an exception that persists through retries + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.INTERNAL_SERVER_ERROR); when(restTemplate.exchange( eq(contentUri), eq(HttpMethod.GET), any(), eq(String.class) - )).thenThrow(new HttpClientErrorException(HttpStatus.INTERNAL_SERVER_ERROR)); + )).thenThrow(exception) + .thenThrow(exception) // Retry 1 + .thenThrow(exception); // Retry 2 // Verify that the exception is propagated - SaaSCrawlerException exception = assertThrows(SaaSCrawlerException.class, + SaaSCrawlerException crawlerException = assertThrows(SaaSCrawlerException.class, () -> office365RestClient.getAuditLog(contentUri)); - assertEquals("Failed to fetch audit log", exception.getMessage()); - assertTrue(exception.isRetryable()); + assertEquals("Failed to fetch audit log", crawlerException.getMessage()); + assertTrue(crawlerException.isRetryable()); } + /** + * Tests automatic token renewal when receiving 401 Unauthorized response. + * Verifies that failed authentication triggers credential renewal and retry succeeds. + */ @Test void testTokenRenewal() { // Setup @@ -299,257 +409,4 @@ void testTokenRenewal() { assertEquals("Bearer token-1", requestTokens.get(1), "Second request should use token-1"); } - @Test - void testSearchAuditLogsFailureCounterIncrementsOnEachRetry() throws Exception { - Instant startTime = Instant.now().minus(1, ChronoUnit.HOURS); - Instant endTime = Instant.now(); - - when(authConfig.getTenantId()).thenReturn("test-tenant-id"); - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - when(restTemplate.exchange( - anyString(), - eq(HttpMethod.GET), - any(), - any(ParameterizedTypeReference.class) - )).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); - - assertThrows(SaaSCrawlerException.class, () -> - office365RestClient.searchAuditLogs( - "Audit.AzureActiveDirectory", - startTime, - endTime, - null - ) - ); - } - - @Test - void testGetAuditLogFailureCounterIncrementsOnEachRetry() throws Exception { - String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - when(restTemplate.exchange( - eq(contentUri), - eq(HttpMethod.GET), - any(), - eq(String.class) - )).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); - - assertThrows(SaaSCrawlerException.class, () -> - office365RestClient.getAuditLog(contentUri) - ); - } - - @Test - void testMetricsInitialization() { - // Test metrics initialization during construction. This approach is used for metrics that are called - // inside RetryHandler.executeWithRetry() static method calls, which would require complex static mocking - // to test for invocation. Testing initialization ensures the metrics infrastructure is properly set up. - - // Mock only the local timers and counters for Office365RestClient constructor - PluginMetrics mockPluginMetrics = org.mockito.Mockito.mock(PluginMetrics.class); - Timer mockAuditLogFetchLatencyTimer = org.mockito.Mockito.mock(Timer.class); - Timer mockSearchCallLatencyTimer = org.mockito.Mockito.mock(Timer.class); - Counter mockAuditLogsRequestedCounter = org.mockito.Mockito.mock(Counter.class); - Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - - when(mockPluginMetrics.timer("auditLogFetchLatency")).thenReturn(mockAuditLogFetchLatencyTimer); - when(mockPluginMetrics.timer("searchCallLatency")).thenReturn(mockSearchCallLatencyTimer); - when(mockPluginMetrics.counter("auditLogsRequested")).thenReturn(mockAuditLogsRequestedCounter); - when(mockPluginMetrics.counter("apiCalls")).thenReturn(mockApiCallsCounter); - - // Create Office365RestClient with mocked metrics - Office365RestClient testClient = new Office365RestClient(authConfig, mockPluginMetrics); - - // Verify only local metrics were requested during construction - verify(mockPluginMetrics).timer("auditLogFetchLatency"); - verify(mockPluginMetrics).timer("searchCallLatency"); - verify(mockPluginMetrics).counter("auditLogsRequested"); - verify(mockPluginMetrics).counter("apiCalls"); - } - - @Test - void testGetAuditLogMetricsInvocation() throws NoSuchFieldException, IllegalAccessException { - // Create mock metrics for only local fields that still exist - Counter mockAuditLogsRequestedCounter = org.mockito.Mockito.mock(Counter.class); - Timer mockAuditLogFetchLatencyTimer = org.mockito.Mockito.mock(Timer.class); - - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogsRequestedCounter", mockAuditLogsRequestedCounter); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "auditLogFetchLatencyTimer", mockAuditLogFetchLatencyTimer); - - // Mock timer.record() to execute the lambda - when(mockAuditLogFetchLatencyTimer.record(any(java.util.function.Supplier.class))).thenAnswer(invocation -> { - java.util.function.Supplier supplier = invocation.getArgument(0); - return supplier.get(); - }); - - String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - - // Test success scenario - String mockAuditLog = "{\"id\":\"123\",\"contentType\":\"Audit.AzureActiveDirectory\"}"; - ResponseEntity mockResponse = new ResponseEntity<>(mockAuditLog, HttpStatus.OK); - when(restTemplate.exchange(eq(contentUri), eq(HttpMethod.GET), any(), eq(String.class))) - .thenReturn(mockResponse); - - office365RestClient.getAuditLog(contentUri); - - // Verify only local metrics that still exist - verify(mockAuditLogsRequestedCounter).increment(); // Called directly before RetryHandler - verify(mockAuditLogFetchLatencyTimer).record(any(java.util.function.Supplier.class)); // Timer wrapper - } - - @Test - void testSearchAuditLogsMetricsInvocation() throws NoSuchFieldException, IllegalAccessException { - // Create mock metrics for only local fields that still exist - Timer mockSearchCallLatencyTimer = org.mockito.Mockito.mock(Timer.class); - - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "searchCallLatencyTimer", mockSearchCallLatencyTimer); - - // Mock timer.record() to execute the lambda - when(mockSearchCallLatencyTimer.record(any(java.util.function.Supplier.class))).thenAnswer(invocation -> { - java.util.function.Supplier supplier = invocation.getArgument(0); - return supplier.get(); - }); - - // Test success scenario - List> mockResults = Collections.singletonList(new HashMap<>()); - HttpHeaders responseHeaders = new HttpHeaders(); - responseHeaders.setContentLength(1024L); // Mock content length - ResponseEntity>> mockResponse = new ResponseEntity<>(mockResults, responseHeaders, HttpStatus.OK); - when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), any(), any(ParameterizedTypeReference.class))) - .thenReturn(mockResponse); - - office365RestClient.searchAuditLogs( - "Audit.AzureActiveDirectory", - Instant.now().minus(1, ChronoUnit.HOURS), - Instant.now(), - null - ); - - // Verify only local metrics that still exist - verify(mockSearchCallLatencyTimer).record(any(java.util.function.Supplier.class)); // Timer wrapper - } - - - @Test - void testApiCallsCounterIncrementForStartSubscriptions() throws NoSuchFieldException, IllegalAccessException { - // Test that apiCallsCounter is incremented for each subscription start call - Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "apiCallsCounter", mockApiCallsCounter); - - // Mock auth config - when(authConfig.getTenantId()).thenReturn("test-tenant-id"); - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock successful response - ResponseEntity mockResponse = new ResponseEntity<>("{\"status\":\"enabled\"}", HttpStatus.OK); - when(restTemplate.exchange(anyString(), eq(HttpMethod.POST), any(), eq(String.class))) - .thenReturn(mockResponse); - - // Execute - office365RestClient.startSubscriptions(); - - // Verify apiCallsCounter was incremented once for each content type - verify(mockApiCallsCounter, times(CONTENT_TYPES.length)).increment(); - } - - @Test - void testApiCallsCounterIncrementForSearchAuditLogs() throws NoSuchFieldException, IllegalAccessException { - // Test that apiCallsCounter is incremented for search audit logs call - Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "apiCallsCounter", mockApiCallsCounter); - - // Mock auth config - when(authConfig.getTenantId()).thenReturn("test-tenant-id"); - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock successful response - List> mockResults = Collections.singletonList(new HashMap<>()); - ResponseEntity>> mockResponse = new ResponseEntity<>(mockResults, HttpStatus.OK); - when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), any(), any(ParameterizedTypeReference.class))) - .thenReturn(mockResponse); - - // Execute - office365RestClient.searchAuditLogs( - "Audit.AzureActiveDirectory", - Instant.now().minus(1, ChronoUnit.HOURS), - Instant.now(), - null - ); - - // Verify apiCallsCounter was incremented once - verify(mockApiCallsCounter, times(1)).increment(); - } - - @Test - void testApiCallsCounterIncrementForGetAuditLog() throws NoSuchFieldException, IllegalAccessException { - // Test that apiCallsCounter is incremented for get audit log call - Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "apiCallsCounter", mockApiCallsCounter); - - // Mock auth config - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock successful response - String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - String mockAuditLog = "{\"id\":\"123\",\"contentType\":\"Audit.AzureActiveDirectory\"}"; - ResponseEntity mockResponse = new ResponseEntity<>(mockAuditLog, HttpStatus.OK); - when(restTemplate.exchange(eq(contentUri), eq(HttpMethod.GET), any(), eq(String.class))) - .thenReturn(mockResponse); - - // Execute - office365RestClient.getAuditLog(contentUri); - - // Verify apiCallsCounter was incremented once - verify(mockApiCallsCounter, times(1)).increment(); - } - - @Test - void testApiCallsCounterIncrementOnRetries() throws NoSuchFieldException, IllegalAccessException { - // Test that apiCallsCounter is incremented for each retry attempt - Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "apiCallsCounter", mockApiCallsCounter); - - // Mock auth config - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock failure response that will trigger retries - String contentUri = "https://manage.office.com/api/v1.0/test-tenant/activity/feed/audit/123"; - when(restTemplate.exchange(eq(contentUri), eq(HttpMethod.GET), any(), eq(String.class))) - .thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); - - // Execute and expect exception - assertThrows(RuntimeException.class, () -> office365RestClient.getAuditLog(contentUri)); - - // Verify apiCallsCounter was incremented 6 times (once for each retry attempt) - verify(mockApiCallsCounter, times(6)).increment(); - } - - @Test - void testApiCallsCounterIncrementForSearchAuditLogsWithRetries() throws NoSuchFieldException, IllegalAccessException { - // Test that apiCallsCounter is incremented for each retry attempt in searchAuditLogs - Counter mockApiCallsCounter = org.mockito.Mockito.mock(Counter.class); - ReflectivelySetField.setField(Office365RestClient.class, office365RestClient, "apiCallsCounter", mockApiCallsCounter); - - // Mock auth config - when(authConfig.getTenantId()).thenReturn("test-tenant-id"); - when(authConfig.getAccessToken()).thenReturn("test-access-token"); - - // Mock failure response that will trigger retries - when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), any(), any(ParameterizedTypeReference.class))) - .thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); - - // Execute and expect exception - assertThrows(RuntimeException.class, () -> office365RestClient.searchAuditLogs( - "Audit.AzureActiveDirectory", - Instant.now().minus(1, ChronoUnit.HOURS), - Instant.now(), - null - )); - - // Verify apiCallsCounter was incremented 6 times (once for each retry attempt) - verify(mockApiCallsCounter, times(6)).increment(); - } } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorder.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorder.java new file mode 100644 index 0000000000..64ae70ba73 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorder.java @@ -0,0 +1,242 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Timer; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; + +import java.time.Duration; +import java.util.function.Supplier; + +/** + * Comprehensive metrics recorder for vendor API operations in SaaS source plugins. + * + * This class provides a unified interface for recording metrics across different types of vendor API operations: + * - Search operations: latency, success/failure rates, and response sizes + * - Get/retrieval operations: latency, success/failure rates, and response sizes + * - Authentication operations: latency, success/failure rates + * - General API operations: request counts, logs requested, error categorization + * + * Most methods return void for efficient standalone usage. The error() method supports chaining for error handling scenarios. + */ +public class VendorAPIMetricsRecorder { + + // Search operation metrics + private final Counter searchSuccessCounter; + private final Counter searchFailureCounter; + private final Timer searchLatencyTimer; + private final DistributionSummary searchResponseSizeSummary; + + // Get operation metrics + private final Counter getSuccessCounter; + private final Counter getFailureCounter; + private final Timer getLatencyTimer; + private final DistributionSummary getResponseSizeSummary; + + // Authentication operation metrics + private final Counter authSuccessCounter; + private final Counter authFailureCounter; + private final Timer authLatencyTimer; + + // Shared metrics + private final Counter totalDataApiRequestsCounter; + private final Counter logsRequestedCounter; + + // Error metrics + private final Counter requestAccessDeniedCounter; + private final Counter requestThrottledCounter; + private final Counter resourceNotFoundCounter; + + private final PluginMetrics pluginMetrics; + + /** + * Creates a unified VendorAPIMetricsRecorder with all operation types. + * + * @param pluginMetrics The plugin metrics instance + */ + public VendorAPIMetricsRecorder(PluginMetrics pluginMetrics) { + this.pluginMetrics = pluginMetrics; + + // Search metrics + this.searchSuccessCounter = pluginMetrics.counter("searchRequestsSuccess"); + this.searchFailureCounter = pluginMetrics.counter("searchRequestsFailed"); + this.searchLatencyTimer = pluginMetrics.timer("searchRequestLatency"); + this.searchResponseSizeSummary = pluginMetrics.summary("searchResponseSizeBytes"); + + // Get metrics + this.getSuccessCounter = pluginMetrics.counter("getRequestsSuccess"); + this.getFailureCounter = pluginMetrics.counter("getRequestsFailed"); + this.getLatencyTimer = pluginMetrics.timer("getRequestLatency"); + this.getResponseSizeSummary = pluginMetrics.summary("getResponseSizeBytes"); + + // Auth metrics + this.authSuccessCounter = pluginMetrics.counter("authenticationRequestsSuccess"); + this.authFailureCounter = pluginMetrics.counter("authenticationRequestsFailed"); + this.authLatencyTimer = pluginMetrics.timer("authenticationRequestLatency"); + + // Shared metrics + this.totalDataApiRequestsCounter = pluginMetrics.counter("totalDataApiRequests"); + this.logsRequestedCounter = pluginMetrics.counter("logsRequested"); + + // Error metrics + this.requestAccessDeniedCounter = pluginMetrics.counter("requestAccessDenied"); + this.requestThrottledCounter = pluginMetrics.counter("requestThrottled"); + this.resourceNotFoundCounter = pluginMetrics.counter("resourceNotFound"); + } + + // Search operation methods + public void recordSearchSuccess() { + searchSuccessCounter.increment(); + } + + public void recordSearchFailure() { + searchFailureCounter.increment(); + } + + public T recordSearchLatency(Supplier operation) { + return searchLatencyTimer.record(operation); + } + + public void recordSearchLatency(Runnable operation) { + searchLatencyTimer.record(operation); + } + + public void recordSearchLatency(Duration duration) { + searchLatencyTimer.record(duration); + } + + public void recordSearchResponseSize(ResponseEntity response) { + if (response != null) { + String contentLength = response.getHeaders().getFirst("Content-Length"); + if (contentLength != null) { + try { + searchResponseSizeSummary.record(Long.parseLong(contentLength)); + } catch (NumberFormatException e) { + searchResponseSizeSummary.record(0L); + } + } else { + searchResponseSizeSummary.record(0L); + } + } else { + searchResponseSizeSummary.record(0L); + } + } + + public void recordSearchResponseSize(long bytes) { + searchResponseSizeSummary.record(bytes); + } + + public void recordSearchResponseSize(String response) { + if (response != null) { + searchResponseSizeSummary.record(response.getBytes().length); + } else { + searchResponseSizeSummary.record(0L); + } + } + + // Get operation methods + public void recordGetSuccess() { + getSuccessCounter.increment(); + } + + public void recordGetFailure() { + getFailureCounter.increment(); + } + + public T recordGetLatency(Supplier operation) { + return getLatencyTimer.record(operation); + } + + public void recordGetLatency(Runnable operation) { + getLatencyTimer.record(operation); + } + + public void recordGetLatency(Duration duration) { + getLatencyTimer.record(duration); + } + + public void recordGetResponseSize(String response) { + if (response != null) { + getResponseSizeSummary.record(response.getBytes().length); + } else { + getResponseSizeSummary.record(0L); + } + } + + public void recordGetResponseSize(long bytes) { + getResponseSizeSummary.record(bytes); + } + + public void recordGetResponseSize(ResponseEntity response) { + if (response != null && response.getBody() != null) { + getResponseSizeSummary.record(response.getHeaders().getContentLength()); + } else { + getResponseSizeSummary.record(0L); + } + } + + // Authentication operation methods + public void recordAuthSuccess() { + authSuccessCounter.increment(); + } + + public void recordAuthFailure() { + authFailureCounter.increment(); + } + + public T recordAuthLatency(Supplier operation) { + return authLatencyTimer.record(operation); + } + + public void recordAuthLatency(Runnable operation) { + authLatencyTimer.record(operation); + } + + public void recordAuthLatency(Duration duration) { + authLatencyTimer.record(duration); + } + + // Shared operation methods + public void recordDataApiRequest() { + totalDataApiRequestsCounter.increment(); + } + + public void recordLogsRequested() { + logsRequestedCounter.increment(); + } + + /** + * Records error metrics based on exception type and HTTP status code. + * Maps specific HTTP errors to business-meaningful metrics: + * - 401/403 -> requestAccessDenied + * - 429 -> requestThrottled + * - 404 -> resourceNotFound + * - SecurityException -> requestAccessDenied (treated as FORBIDDEN) + * + * @param exception The exception that occurred + */ + public void recordError(Exception exception) { + if (exception instanceof HttpClientErrorException) { + HttpClientErrorException httpE = (HttpClientErrorException) exception; + HttpStatus status = httpE.getStatusCode(); + + if (HttpStatus.FORBIDDEN == status || HttpStatus.UNAUTHORIZED == status) { + requestAccessDeniedCounter.increment(); + } else if (HttpStatus.TOO_MANY_REQUESTS == status) { + requestThrottledCounter.increment(); + } else if (HttpStatus.NOT_FOUND == status) { + resourceNotFoundCounter.increment(); + } + } else if (exception instanceof SecurityException) { + requestAccessDeniedCounter.increment(); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java index a348e6f658..5edfdcc065 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelper.java @@ -43,7 +43,6 @@ public class MetricsHelper { private static final String SEARCH_REQUESTS_SUCCESS = "searchRequestsSuccess"; private static final String SEARCH_RESPONSE_SIZE = "searchResponseSizeBytes"; - // other errors in crawlerClient public static final String REQUEST_ERRORS = "requestErrors"; diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java index 48bd0cdc3e..396f4a1b5a 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandler.java @@ -10,13 +10,11 @@ package org.opensearch.dataprepper.plugins.source.source_crawler.utils.retry; -import io.micrometer.core.instrument.Counter; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.plugins.source.source_crawler.exception.SaaSCrawlerException; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.HttpServerErrorException; -import java.util.Optional; import java.util.function.Supplier; @Slf4j @@ -37,7 +35,7 @@ public RetryHandler(RetryStrategy retryStrategy, StatusCodeHandler statusCodeHan /** * Executes the given operation with retry logic, optional credential renewal, - * and failure counter. + * and failure handler. * * @param operation The operation to execute. * @param credentialRenewal The action to renew credentials if needed. @@ -46,21 +44,21 @@ public RetryHandler(RetryStrategy retryStrategy, StatusCodeHandler statusCodeHan * @return The result of the operation. */ public T executeWithRetry(Supplier operation, Runnable credentialRenewal) { - return executeWithRetry(operation, credentialRenewal, Optional.empty()); + return executeWithRetry(operation, credentialRenewal, null); } /** * Executes the given operation with retry logic, optional credential renewal, - * and failure counter. + * and failure handler. * * @param operation The operation to execute. * @param credentialRenewal The action to renew credentials if needed. - * @param failureCounter The counter to increment on each failed attempt + * @param failureHandler The handler to run on each failed attempt * (optional). * @param The return type of the operation. * @return The result of the operation. */ - public T executeWithRetry(Supplier operation, Runnable credentialRenewal, Optional failureCounter) { + public T executeWithRetry(Supplier operation, Runnable credentialRenewal, Runnable failureHandler) { if (operation == null) { throw new SaaSCrawlerException("Operation cannot be null", false); } @@ -97,8 +95,8 @@ public T executeWithRetry(Supplier operation, Runnable credentialRenewal, long sleepTimeMs = retryStrategy.calculateSleepTime(ex, retryCount); sleep(sleepTimeMs); } finally { - if (!operationSucceeded) { - failureCounter.ifPresent(Counter::increment); + if (!operationSucceeded && failureHandler != null) { + failureHandler.run(); } } retryCount++; diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorderTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorderTest.java new file mode 100644 index 0000000000..01842867bd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/metrics/VendorAPIMetricsRecorderTest.java @@ -0,0 +1,506 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Timer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; + +import java.time.Duration; +import java.util.function.Supplier; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class VendorAPIMetricsRecorderTest { + + @Mock + private PluginMetrics pluginMetrics; + + // Search metrics + @Mock + private Counter searchSuccessCounter; + @Mock + private Counter searchFailureCounter; + @Mock + private Timer searchLatencyTimer; + @Mock + private DistributionSummary searchResponseSizeSummary; + + // Get metrics + @Mock + private Counter getSuccessCounter; + @Mock + private Counter getFailureCounter; + @Mock + private Timer getLatencyTimer; + @Mock + private DistributionSummary getResponseSizeSummary; + + // Auth metrics + @Mock + private Counter authSuccessCounter; + @Mock + private Counter authFailureCounter; + @Mock + private Timer authLatencyTimer; + + // Shared metrics + @Mock + private Counter totalDataApiRequestsCounter; + @Mock + private Counter logsRequestedCounter; + + // Error metrics + @Mock + private Counter requestAccessDeniedCounter; + @Mock + private Counter requestThrottledCounter; + @Mock + private Counter resourceNotFoundCounter; + + private VendorAPIMetricsRecorder recorder; + + @BeforeEach + void setUp() { + // Setup search metrics mocks + when(pluginMetrics.counter("searchRequestsSuccess")).thenReturn(searchSuccessCounter); + when(pluginMetrics.counter("searchRequestsFailed")).thenReturn(searchFailureCounter); + when(pluginMetrics.timer("searchRequestLatency")).thenReturn(searchLatencyTimer); + when(pluginMetrics.summary("searchResponseSizeBytes")).thenReturn(searchResponseSizeSummary); + + // Setup get metrics mocks + when(pluginMetrics.counter("getRequestsSuccess")).thenReturn(getSuccessCounter); + when(pluginMetrics.counter("getRequestsFailed")).thenReturn(getFailureCounter); + when(pluginMetrics.timer("getRequestLatency")).thenReturn(getLatencyTimer); + when(pluginMetrics.summary("getResponseSizeBytes")).thenReturn(getResponseSizeSummary); + + // Setup auth metrics mocks + when(pluginMetrics.counter("authenticationRequestsSuccess")).thenReturn(authSuccessCounter); + when(pluginMetrics.counter("authenticationRequestsFailed")).thenReturn(authFailureCounter); + when(pluginMetrics.timer("authenticationRequestLatency")).thenReturn(authLatencyTimer); + + // Setup shared metrics mocks + when(pluginMetrics.counter("totalDataApiRequests")).thenReturn(totalDataApiRequestsCounter); + when(pluginMetrics.counter("logsRequested")).thenReturn(logsRequestedCounter); + + // Setup error metrics mocks + when(pluginMetrics.counter("requestAccessDenied")).thenReturn(requestAccessDeniedCounter); + when(pluginMetrics.counter("requestThrottled")).thenReturn(requestThrottledCounter); + when(pluginMetrics.counter("resourceNotFound")).thenReturn(resourceNotFoundCounter); + + recorder = new VendorAPIMetricsRecorder(pluginMetrics); + } + + @Test + void constructor_CreatesAllMetricsCorrectly() { + assertThat(recorder, notNullValue()); + + // Verify search metrics creation + verify(pluginMetrics).counter("searchRequestsSuccess"); + verify(pluginMetrics).counter("searchRequestsFailed"); + verify(pluginMetrics).timer("searchRequestLatency"); + verify(pluginMetrics).summary("searchResponseSizeBytes"); + + // Verify get metrics creation + verify(pluginMetrics).counter("getRequestsSuccess"); + verify(pluginMetrics).counter("getRequestsFailed"); + verify(pluginMetrics).timer("getRequestLatency"); + verify(pluginMetrics).summary("getResponseSizeBytes"); + + // Verify auth metrics creation + verify(pluginMetrics).counter("authenticationRequestsSuccess"); + verify(pluginMetrics).counter("authenticationRequestsFailed"); + verify(pluginMetrics).timer("authenticationRequestLatency"); + + // Verify shared metrics creation + verify(pluginMetrics).counter("totalDataApiRequests"); + verify(pluginMetrics).counter("logsRequested"); + + // Verify error metrics creation + verify(pluginMetrics).counter("requestAccessDenied"); + verify(pluginMetrics).counter("requestThrottled"); + verify(pluginMetrics).counter("resourceNotFound"); + } + + @Test + void recordSearchSuccess_IncrementsSearchSuccessCounter() { + recorder.recordSearchSuccess(); + + verify(searchSuccessCounter).increment(); + } + + @Test + void recordSearchFailure_IncrementsSearchFailureCounter() { + recorder.recordSearchFailure(); + + verify(searchFailureCounter).increment(); + } + + @Test + void recordGetSuccess_IncrementsGetSuccessCounter() { + recorder.recordGetSuccess(); + + verify(getSuccessCounter).increment(); + } + + @Test + void recordGetFailure_IncrementsGetFailureCounter() { + recorder.recordGetFailure(); + + verify(getFailureCounter).increment(); + } + + @Test + void recordAuthSuccess_IncrementsAuthSuccessCounter() { + recorder.recordAuthSuccess(); + + verify(authSuccessCounter).increment(); + } + + @Test + void recordAuthFailure_IncrementsAuthFailureCounter() { + recorder.recordAuthFailure(); + + verify(authFailureCounter).increment(); + } + + @Test + void recordSearchLatency_WithSupplier_RecordsLatencyAndReturnsResult() { + String expectedResult = "search result"; + Supplier operation = () -> expectedResult; + when(searchLatencyTimer.record(any(Supplier.class))).thenReturn(expectedResult); + + String result = recorder.recordSearchLatency(operation); + + verify(searchLatencyTimer).record(eq(operation)); + assertThat(result, equalTo(expectedResult)); + } + + @Test + void recordGetLatency_WithSupplier_RecordsLatencyAndReturnsResult() { + String expectedResult = "get result"; + Supplier operation = () -> expectedResult; + when(getLatencyTimer.record(any(Supplier.class))).thenReturn(expectedResult); + + String result = recorder.recordGetLatency(operation); + + verify(getLatencyTimer).record(eq(operation)); + assertThat(result, equalTo(expectedResult)); + } + + @Test + void recordAuthLatency_WithSupplier_RecordsLatencyAndReturnsResult() { + String expectedResult = "auth result"; + Supplier operation = () -> expectedResult; + when(authLatencyTimer.record(any(Supplier.class))).thenReturn(expectedResult); + + String result = recorder.recordAuthLatency(operation); + + verify(authLatencyTimer).record(eq(operation)); + assertThat(result, equalTo(expectedResult)); + } + + @Test + void recordSearchLatency_WithRunnable_RecordsLatency() { + Runnable operation = () -> { /* void operation */ }; + + recorder.recordSearchLatency(operation); + + verify(searchLatencyTimer).record(eq(operation)); + } + + @Test + void recordGetLatency_WithRunnable_RecordsLatency() { + Runnable operation = () -> { /* void operation */ }; + + recorder.recordGetLatency(operation); + + verify(getLatencyTimer).record(eq(operation)); + } + + @Test + void recordAuthLatency_WithRunnable_RecordsLatency() { + Runnable operation = () -> { /* void operation */ }; + + recorder.recordAuthLatency(operation); + + verify(authLatencyTimer).record(eq(operation)); + } + + @Test + void recordSearchLatency_WithDuration_RecordsLatency() { + Duration duration = Duration.ofMillis(500); + + recorder.recordSearchLatency(duration); + + verify(searchLatencyTimer).record(duration); + } + + @Test + void recordGetLatency_WithDuration_RecordsLatency() { + Duration duration = Duration.ofMillis(300); + + recorder.recordGetLatency(duration); + + verify(getLatencyTimer).record(duration); + } + + @Test + void recordAuthLatency_WithDuration_RecordsLatency() { + Duration duration = Duration.ofMillis(200); + + recorder.recordAuthLatency(duration); + + verify(authLatencyTimer).record(duration); + } + + @Test + void recordSearchResponseSize_WithBytes_RecordsSize() { + long responseSize = 1024L; + + recorder.recordSearchResponseSize(responseSize); + + verify(searchResponseSizeSummary).record(responseSize); + } + + @Test + void recordSearchResponseSize_WithResponseEntity_RecordsContentLength() { + ResponseEntity response = ResponseEntity.ok() + .header("Content-Length", "512") + .body("test"); + + recorder.recordSearchResponseSize(response); + + verify(searchResponseSizeSummary).record(512L); + } + + @Test + void recordSearchResponseSize_WithString_RecordsByteLength() { + String response = "test response"; + + recorder.recordSearchResponseSize(response); + + verify(searchResponseSizeSummary).record(response.getBytes().length); + } + + @Test + void recordGetResponseSize_WithString_RecordsByteLength() { + String response = "get response"; + + recorder.recordGetResponseSize(response); + + verify(getResponseSizeSummary).record(response.getBytes().length); + } + + @Test + void recordGetResponseSize_WithResponseEntity_RecordsContentLength() { + // Create a proper ResponseEntity with Content-Length header + ResponseEntity response = ResponseEntity.ok() + .header("Content-Length", "256") + .body("test"); + + recorder.recordGetResponseSize(response); + + verify(getResponseSizeSummary).record(256L); + } + + @Test + void recordGetResponseSize_WithBytes_RecordsSize() { + long responseSize = 2048L; + + recorder.recordGetResponseSize(responseSize); + + verify(getResponseSizeSummary).record(responseSize); + } + + @Test + void recordDataApiRequest_IncrementsRequestCounter() { + recorder.recordDataApiRequest(); + + verify(totalDataApiRequestsCounter).increment(); + } + + @Test + void recordLogsRequested_IncrementsLogsRequestedCounter() { + recorder.recordLogsRequested(); + + verify(logsRequestedCounter).increment(); + } + + + @Test + void standaloneOperations_WorkCorrectly() { + // Test standalone usage pattern (the primary use case) + String response = "test response"; + + recorder.recordSearchSuccess(); + recorder.recordSearchResponseSize(response.getBytes().length); + recorder.recordDataApiRequest(); + + verify(searchSuccessCounter).increment(); + verify(searchResponseSizeSummary).record(response.getBytes().length); + verify(totalDataApiRequestsCounter).increment(); + } + + @Test + void mixedOperations_WorkCorrectly() { + // Test that we can use different operation types on the same recorder instance + recorder.recordSearchSuccess(); + recorder.recordGetSuccess(); + recorder.recordAuthSuccess(); + + verify(searchSuccessCounter).increment(); + verify(getSuccessCounter).increment(); + verify(authSuccessCounter).increment(); + } + + // Edge case tests for comprehensive coverage + + @Test + void recordSearchResponseSize_WithNullResponseEntity_RecordsZero() { + recorder.recordSearchResponseSize((ResponseEntity) null); + + verify(searchResponseSizeSummary).record(0L); + } + + @Test + void recordSearchResponseSize_WithNullString_RecordsZero() { + recorder.recordSearchResponseSize((String) null); + + verify(searchResponseSizeSummary).record(0L); + } + + @Test + void recordSearchResponseSize_WithResponseEntityMissingContentLength_RecordsZero() { + ResponseEntity response = ResponseEntity.ok().body("test"); + + recorder.recordSearchResponseSize(response); + + verify(searchResponseSizeSummary).record(0L); + } + + @Test + void recordSearchResponseSize_WithInvalidContentLength_RecordsZero() { + ResponseEntity response = ResponseEntity.ok() + .header("Content-Length", "invalid") + .body("test"); + + recorder.recordSearchResponseSize(response); + + verify(searchResponseSizeSummary).record(0L); + } + + @Test + void recordGetResponseSize_WithNullString_RecordsZero() { + recorder.recordGetResponseSize((String) null); + + verify(getResponseSizeSummary).record(0L); + } + + @Test + void recordGetResponseSize_WithNullResponseEntity_RecordsZero() { + recorder.recordGetResponseSize((ResponseEntity) null); + + verify(getResponseSizeSummary).record(0L); + } + + // recordError method tests + + @Test + void recordError_WithUnauthorized_IncrementsRequestAccessDeniedCounter() { + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.UNAUTHORIZED); + + recorder.recordError(exception); + + verify(requestAccessDeniedCounter).increment(); + } + + @Test + void recordError_WithForbidden_IncrementsRequestAccessDeniedCounter() { + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.FORBIDDEN); + + recorder.recordError(exception); + + verify(requestAccessDeniedCounter).increment(); + } + + @Test + void recordError_WithTooManyRequests_IncrementsRequestThrottledCounter() { + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS); + + recorder.recordError(exception); + + verify(requestThrottledCounter).increment(); + } + + @Test + void recordError_WithNotFound_IncrementsResourceNotFoundCounter() { + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.NOT_FOUND); + + recorder.recordError(exception); + + verify(resourceNotFoundCounter).increment(); + } + + @Test + void recordError_WithSecurityException_IncrementsRequestAccessDeniedCounter() { + SecurityException exception = new SecurityException("Access denied"); + + recorder.recordError(exception); + + verify(requestAccessDeniedCounter).increment(); + } + + @Test + void recordError_WithHttpServerErrorException_DoesNotIncrementAnyCounters() { + HttpServerErrorException exception = new HttpServerErrorException(HttpStatus.INTERNAL_SERVER_ERROR); + + recorder.recordError(exception); + + verify(requestAccessDeniedCounter, org.mockito.Mockito.never()).increment(); + verify(requestThrottledCounter, org.mockito.Mockito.never()).increment(); + verify(resourceNotFoundCounter, org.mockito.Mockito.never()).increment(); + } + + @Test + void recordError_WithOtherHttpClientError_DoesNotIncrementAnyCounters() { + HttpClientErrorException exception = new HttpClientErrorException(HttpStatus.BAD_REQUEST); + + recorder.recordError(exception); + + verify(requestAccessDeniedCounter, org.mockito.Mockito.never()).increment(); + verify(requestThrottledCounter, org.mockito.Mockito.never()).increment(); + verify(resourceNotFoundCounter, org.mockito.Mockito.never()).increment(); + } + + @Test + void recordError_WithGenericException_DoesNotIncrementAnyCounters() { + RuntimeException exception = new RuntimeException("Generic error"); + + recorder.recordError(exception); + + verify(requestAccessDeniedCounter, org.mockito.Mockito.never()).increment(); + verify(requestThrottledCounter, org.mockito.Mockito.never()).increment(); + verify(resourceNotFoundCounter, org.mockito.Mockito.never()).increment(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java index 96546397c3..42e7f88e25 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/MetricsHelperTest.java @@ -12,6 +12,7 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Timer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -44,7 +45,18 @@ import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class MetricsHelperTest { +class MetricsHelperTest { + + // Private test constants - duplicated from MetricsHelper for test isolation + private static final String REQUEST_ACCESS_DENIED = "requestAccessDenied"; + private static final String REQUEST_THROTTLED = "requestThrottled"; + private static final String RESOURCE_NOT_FOUND = "resourceNotFound"; + private static final String GET_REQUESTS_FAILED = "getRequestsFailed"; + private static final String GET_REQUESTS_SUCCESS = "getRequestsSuccess"; + private static final String GET_RESPONSE_SIZE = "getResponseSizeBytes"; + private static final String SEARCH_REQUESTS_FAILED = "searchRequestsFailed"; + private static final String SEARCH_REQUESTS_SUCCESS = "searchRequestsSuccess"; + private static final String SEARCH_RESPONSE_SIZE = "searchResponseSizeBytes"; @Mock private PluginMetrics pluginMetrics; @@ -55,10 +67,14 @@ public class MetricsHelperTest { @Mock private DistributionSummary mockDistributionSummary; + @Mock + private Timer mockTimer; + @BeforeEach void setUp() { lenient().when(pluginMetrics.counter(anyString())).thenReturn(mockCounter); lenient().when(pluginMetrics.summary(anyString())).thenReturn(mockDistributionSummary); + lenient().when(pluginMetrics.timer(anyString())).thenReturn(mockTimer); } @Test @@ -77,9 +93,9 @@ void testGetErrorTypeMetricCounterMap() { result.values().forEach(counter -> assertEquals(mockCounter, counter)); // requestAccessDenied is called twice for FORBIDDEN and UNAUTHORIZED - verify(pluginMetrics, times(2)).counter("requestAccessDenied"); - verify(pluginMetrics).counter("requestThrottled"); - verify(pluginMetrics).counter("resourceNotFound"); + verify(pluginMetrics, times(2)).counter(REQUEST_ACCESS_DENIED); + verify(pluginMetrics).counter(REQUEST_THROTTLED); + verify(pluginMetrics).counter(RESOURCE_NOT_FOUND); } @Test @@ -198,10 +214,10 @@ static Stream stringTestCases() { static Stream metricMethods() { return Stream.of( - Arguments.of("search", "searchResponseSizeBytes", + Arguments.of("search", SEARCH_RESPONSE_SIZE, (BiConsumer>) MetricsHelper::publishSearchResponseSizeMetricInBytes, (BiConsumer) MetricsHelper::publishSearchResponseSizeMetricInBytes), - Arguments.of("get", "getResponseSizeBytes", + Arguments.of("get", GET_RESPONSE_SIZE, (BiConsumer>) MetricsHelper::publishGetResponseSizeMetricInBytes, (BiConsumer) MetricsHelper::publishGetResponseSizeMetricInBytes) ); @@ -273,8 +289,8 @@ void testSuccessMetrics(String methodType, String expectedMetricName) { static Stream successMetricMethods() { return Stream.of( - Arguments.of("search", "searchRequestsSuccess"), - Arguments.of("get", "getRequestsSuccess") + Arguments.of("search", SEARCH_REQUESTS_SUCCESS), + Arguments.of("get", GET_REQUESTS_SUCCESS) ); } @@ -295,8 +311,8 @@ void testFailureCounterProviders(String methodType, String expectedMetricName) { static Stream failureCounterMethods() { return Stream.of( - Arguments.of("search", "searchRequestsFailed"), - Arguments.of("get", "getRequestsFailed") + Arguments.of("search", SEARCH_REQUESTS_FAILED), + Arguments.of("get", GET_REQUESTS_FAILED) ); } @@ -321,8 +337,8 @@ void testResponseEntityCompatibilityWithVariousResponseTypes() { MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, getResponseEntity); // Verify both method types use their respective metric names - verify(pluginMetrics).summary("searchResponseSizeBytes"); - verify(pluginMetrics).summary("getResponseSizeBytes"); + verify(pluginMetrics).summary(SEARCH_RESPONSE_SIZE); + verify(pluginMetrics).summary(GET_RESPONSE_SIZE); // Test with different ResponseEntity generic types HttpHeaders genericHeaders = new HttpHeaders(); @@ -356,7 +372,7 @@ void testSearchResponseSizeMetricWithGenericResponseEntity() { // This should work with the new ResponseEntity overload MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, genericResponse); - verify(pluginMetrics).summary("searchResponseSizeBytes"); + verify(pluginMetrics).summary(SEARCH_RESPONSE_SIZE); verify(mockDistributionSummary).record(3000L); } @@ -380,7 +396,7 @@ void testGetResponseSizeMetricWithGenericResponseEntity() { // This should work with the new ResponseEntity overload MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, genericResponse); - verify(pluginMetrics).summary("getResponseSizeBytes"); + verify(pluginMetrics).summary(GET_RESPONSE_SIZE); verify(mockDistributionSummary).record(1500L); } @@ -393,7 +409,7 @@ void testGenericResponseEntityOverloadsWithNullValues() { // Test null response MetricsHelper.publishSearchResponseSizeMetricInBytes(pluginMetrics, (ResponseEntity) null); - verify(pluginMetrics).summary("searchResponseSizeBytes"); + verify(pluginMetrics).summary(SEARCH_RESPONSE_SIZE); verify(mockDistributionSummary).record(-1L); reset(mockDistributionSummary); @@ -411,7 +427,7 @@ void testGenericResponseEntityOverloadsWithNullValues() { // Test GET request with null MetricsHelper.publishGetResponseSizeMetricInBytes(pluginMetrics, (ResponseEntity) null); - verify(pluginMetrics).summary("getResponseSizeBytes"); + verify(pluginMetrics).summary(GET_RESPONSE_SIZE); verify(mockDistributionSummary).record(-1L); } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java index e2a9b20934..114cf2f15c 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/retry/RetryHandlerTest.java @@ -48,6 +48,8 @@ class RetryHandlerTest { private StatusCodeHandler statusCodeHandler; @Mock private Runnable credentialRenewal; + @Mock + private Runnable failureHandler; private RetryHandler retryHandler; @BeforeEach @@ -341,4 +343,97 @@ void executeWithRetry_WithMaxRetriesReached_VerifiesRetryCount() { verify(statusCodeHandler, times(2)).handleStatusCode(any(HttpServerErrorException.class), anyInt(), eq(credentialRenewal)); } + + // Tests for failureHandler functionality + @Test + void executeWithRetry_WithFailureHandler_CallsFailureHandlerOnEachFailure() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() < 2) { + throw new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE); + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal, failureHandler); + + assertThat(result, equalTo("success")); + verify(failureHandler, times(2)).run(); // Called on first 2 failed attempts + } + + @Test + void executeWithRetry_WithFailureHandler_DoesNotCallOnSuccess() { + final String expectedResult = "success"; + final Supplier operation = () -> expectedResult; + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal, failureHandler); + + assertThat(result, equalTo(expectedResult)); + verify(failureHandler, never()).run(); // Should not be called on success + } + + @Test + void executeWithRetry_WithNullFailureHandler_DoesNotThrowException() { + final AtomicInteger attemptCount = new AtomicInteger(0); + final Supplier operation = () -> { + if (attemptCount.getAndIncrement() < 1) { + throw new HttpServerErrorException(HttpStatus.SERVICE_UNAVAILABLE); + } + return "success"; + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal, null); + + assertThat(result, equalTo("success")); // Should succeed without NPE + } + + @Test + void executeWithRetry_WithFailureHandlerAndMaxRetries_CallsFailureHandlerCorrectTimes() { + final Supplier operation = () -> { + throw new HttpServerErrorException(HttpStatus.INTERNAL_SERVER_ERROR); + }; + + when(statusCodeHandler.handleStatusCode(any(HttpServerErrorException.class), anyInt(), + eq(credentialRenewal))).thenReturn(RetryDecision.retry()); + + assertThrows(HttpServerErrorException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal, failureHandler)); + + verify(failureHandler, times(MAX_RETRIES)).run(); // Called exactly MAX_RETRIES times + } + + @Test + void executeWithRetry_WithFailureHandlerAndNonRetryableError_CallsFailureHandlerOnce() { + final HttpClientErrorException clientException = new HttpClientErrorException( + HttpStatus.BAD_REQUEST, "Bad Request"); + final Supplier operation = () -> { + throw clientException; + }; + + when(statusCodeHandler.handleStatusCode(eq(clientException), eq(0), + eq(credentialRenewal))).thenReturn(RetryDecision.stop()); + + assertThrows(HttpClientErrorException.class, + () -> retryHandler.executeWithRetry(operation, credentialRenewal, failureHandler)); + + verify(failureHandler, times(1)).run(); // Called once before stopping + } + + @Test + void executeWithRetry_TwoParameterOverload_CallsThreeParameterWithNullFailureHandler() { + final String expectedResult = "success"; + final Supplier operation = () -> expectedResult; + + final String result = retryHandler.executeWithRetry(operation, credentialRenewal); + + assertThat(result, equalTo(expectedResult)); + // This test verifies the two-parameter overload works correctly by calling + // the three-parameter version with null failureHandler (no NPE should occur) + } } From b094611d6b91cbc245b45dcfa38f4f50b36f4550 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Mon, 22 Dec 2025 17:03:41 -0800 Subject: [PATCH 26/51] Rebased to latest to resolve conflicts (#6365) Signed-off-by: Krishna Kondaka Signed-off-by: Nathan Wand --- .../common/sink/DefaultSinkBuffer.java | 6 +- .../sink/DefaultSinkOutputStrategy.java | 3 + .../common/sink/SinkBufferWriter.java | 7 + .../sink/prometheus/PrometheusSinkAMPIT.java | 94 +++++++++--- .../PrometheusSinkConfiguration.java | 8 + .../service/PrometheusSinkBuffer.java | 34 +++++ .../service/PrometheusSinkBufferEntry.java | 30 +--- .../service/PrometheusSinkBufferWriter.java | 83 +++++++++-- .../PrometheusSinkBufferWriterEntry.java | 119 +++++++++++++++ .../PrometheusSinkFlushableBuffer.java | 15 +- .../service/PrometheusSinkService.java | 8 +- .../service/PrometheusTimeSeries.java | 123 ++++++++++------ .../PrometheusSinkBufferWriterEntryTest.java | 132 +++++++++++++++++ .../PrometheusSinkBufferWriterTest.java | 70 ++++++++- .../service/PrometheusSinkServiceTest.java | 46 ++++-- .../service/PrometheusTimeSeriesTest.java | 139 ++++++++++++++++++ 16 files changed, 775 insertions(+), 142 deletions(-) create mode 100644 data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBuffer.java create mode 100644 data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterEntry.java create mode 100644 data-prepper-plugins/prometheus-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterEntryTest.java diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkBuffer.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkBuffer.java index 85d1109e30..ad11e26d43 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkBuffer.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkBuffer.java @@ -12,9 +12,9 @@ import java.time.Instant; public class DefaultSinkBuffer implements SinkBuffer { - private final SinkBufferWriter sinkBufferWriter; - private final long maxEvents; - private final long maxRequestSize; + protected final SinkBufferWriter sinkBufferWriter; + protected final long maxEvents; + protected final long maxRequestSize; private final long flushIntervalMs; private long lastFlushedTimeMs; private long numEvents; diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkOutputStrategy.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkOutputStrategy.java index cb8242f737..4d79925b7d 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkOutputStrategy.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/DefaultSinkOutputStrategy.java @@ -33,6 +33,9 @@ public void flushBuffer() { long startTime = System.nanoTime(); // getFlushableBuffer() should return the buffer contents SinkFlushableBuffer flushableBuffer = sinkBuffer.getFlushableBuffer(sinkFlushContext); + if (flushableBuffer == null) { + return; + } List events = flushableBuffer.getEvents(); try { SinkFlushResult flushResult = flushableBuffer.flush(); diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkBufferWriter.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkBufferWriter.java index 8e3c6af9a2..b71fd94941 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkBufferWriter.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/sink/SinkBufferWriter.java @@ -8,4 +8,11 @@ public interface SinkBufferWriter { public boolean writeToBuffer(SinkBufferEntry sinkBufferEntry); public SinkFlushableBuffer getBuffer(final SinkFlushContext sinkFlushContext); + default boolean isMaxEventsLimitReached(final long maxEvents) { + return false; + } + + default boolean willExceedMaxRequestSizeBytes(final SinkBufferEntry sinkBufferEntry, final long maxRequestSize) { + return false; + } } diff --git a/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java b/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java index 9fd75fa5e8..a3e8a71a88 100644 --- a/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java +++ b/data-prepper-plugins/prometheus-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/prometheus/PrometheusSinkAMPIT.java @@ -62,6 +62,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -216,6 +218,7 @@ void setUp() { lenient().when(thresholdConfig.getFlushInterval()).thenReturn(60L); prometheusSinkConfig = mock(PrometheusSinkConfiguration.class); when(prometheusSinkConfig.getMaxRetries()).thenReturn(5); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(0)); when(prometheusSinkConfig.getSanitizeNames()).thenReturn(false); when(prometheusSinkConfig.getUrl()).thenReturn(remoteWriteUrl); when(prometheusSinkConfig.getContentType()).thenReturn("application/x-protobuf"); @@ -292,16 +295,23 @@ private void getMetricsFromAMP(final String metricName, final String qs) throws } - @Test - void TestSumMetrics() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestSumMetrics(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); PrometheusSink sink = createObjectUnderTest(); long startTimeSeconds = testStartTime.getEpochSecond(); Instant time = Instant.now(); Collection> records = getSumRecordList(NUM_RECORDS, sumMetricName, 0); sink.doOutput(records); + Thread.sleep(window*1000); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } metricsInAMP = 0; Set expectedMetrics = new HashSet<>(); for (Record record: records) { @@ -393,8 +403,11 @@ void TestSumMetricsFailuresWithDLQ() throws Exception { verify(eventHandle, times(NUM_RECORDS)).release(eq(true)); } - @Test - void TestSumMetricsFailuresWithoutDLQ() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestSumMetricsFailuresWithoutDLQ(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); when(thresholdConfig.getMaxEvents()).thenReturn(1); PrometheusSink sink = createObjectUnderTest(); @@ -402,9 +415,13 @@ void TestSumMetricsFailuresWithoutDLQ() throws Exception { Instant time = Instant.now(); Collection> records = getSumRecordList(NUM_RECORDS-1, sumMetricName, 1); sink.doOutput(records); + Thread.sleep(window*1000); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } metricsInAMP = 0; Set expectedMetrics = new HashSet<>(); for (Record record: records) { @@ -467,16 +484,23 @@ private Collection> getSumRecordList(int numberOfRecords, final St return records; } - @Test - void TestGaugeMetrics() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestGaugeMetrics(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); PrometheusSink sink = createObjectUnderTest(); Collection> records = getGaugeRecordList(NUM_RECORDS); sink.doOutput(records); + Thread.sleep(window*1000); long startTimeSeconds = testStartTime.getEpochSecond(); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } metricsInAMP = 0; long endTimeSeconds = Instant.now().getEpochSecond(); getMetricsFromAMP(gaugeMetricName, ""); @@ -504,18 +528,25 @@ void TestGaugeMetrics() throws Exception { verify(eventHandle, times(NUM_RECORDS)).release(eq(true)); } - @Test - void TestGaugeMetricsWithMaxRequestSizeLimitAndFlushTimeout() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestGaugeMetricsWithMaxRequestSizeLimitAndFlushTimeout(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); when(thresholdConfig.getMaxRequestSizeBytes()).thenReturn(220L); lenient().when(thresholdConfig.getFlushInterval()).thenReturn(20L); PrometheusSink sink = createObjectUnderTest(); Collection> records = getGaugeRecordList(NUM_RECORDS); sink.doOutput(records); + Thread.sleep(window*1000); long startTimeSeconds = testStartTime.getEpochSecond(); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } metricsInAMP = 0; sink.doOutput(Collections.emptyList()); long endTimeSeconds = Instant.now().getEpochSecond(); @@ -569,16 +600,23 @@ private Collection> getGaugeRecordList(int numberOfRecords) { return records; } - @Test - void TestSummaryMetrics() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestSummaryMetrics(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); PrometheusSink sink = createObjectUnderTest(); Collection> records = getSummaryRecordList(NUM_RECORDS); sink.doOutput(records); + Thread.sleep(window*1000); long startTimeSeconds = testStartTime.getEpochSecond(); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } long endTimeSeconds = Instant.now().getEpochSecond()+10; metricsInAMP = 0; getMetricsFromAMP(summaryMetricName, "summary"); @@ -674,16 +712,23 @@ private Collection> getSummaryRecordList(int numberOfRecords) { return records; } - @Test - void TestHistogramMetrics() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestHistogramMetrics(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); PrometheusSink sink = createObjectUnderTest(); Collection> records = getHistogramRecordList(NUM_RECORDS); sink.doOutput(records); + Thread.sleep(window*1000); long startTimeSeconds = testStartTime.getEpochSecond(); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } metricsInAMP = 0; long endTimeSeconds = Instant.now().getEpochSecond()+10; getMetricsFromAMP(histogramMetricName, "histogram"); @@ -734,16 +779,23 @@ void TestHistogramMetrics() throws Exception { } - @Test - void TestExponentialHistogramMetrics() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + void TestExponentialHistogramMetrics(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); PrometheusSink sink = createObjectUnderTest(); Collection> records = getExponentialHistogramRecordList(NUM_RECORDS); sink.doOutput(records); + Thread.sleep(window*1000); long startTimeSeconds = testStartTime.getEpochSecond(); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } metricsInAMP = 0; long endTimeSeconds = Instant.now().getEpochSecond()+10; getMetricsFromAMP(exponentialHistogramMetricName, "exphistogram"); @@ -797,8 +849,11 @@ void TestExponentialHistogramMetrics() throws Exception { verify(eventHandle, times(NUM_RECORDS)).release(eq(true)); } - @Test - public void TestMultipleMetrics() throws Exception { + @ParameterizedTest + @ValueSource(ints = {0, 2, 5}) + public void TestMultipleMetrics(final int window) throws Exception { + lenient().when(thresholdConfig.getFlushInterval()).thenReturn(6L); + when(prometheusSinkConfig.getOutOfOrderWindow()).thenReturn(Duration.ofSeconds(window)); when(thresholdConfig.getMaxEvents()).thenReturn(1); long startTimeSeconds = testStartTime.getEpochSecond(); PrometheusSink sink = createObjectUnderTest(); @@ -808,9 +863,13 @@ public void TestMultipleMetrics() throws Exception { records.addAll(getGaugeRecordList(NUM_RECORDS/5)); records.addAll(getSumRecordList(NUM_RECORDS/5, sumMetricName, 0)); sink.doOutput(records); + Thread.sleep(window*1000); await().atMost(Duration.ofSeconds(60)) .untilAsserted(() -> { + if (window > 0) { + sink.doOutput(Collections.emptyList()); + } int totalMetrics = 0; metricsInAMP = 0; @@ -836,9 +895,8 @@ public void TestMultipleMetrics() throws Exception { assertThat(metricsInAMP, greaterThanOrEqualTo(1)); totalMetrics += metricsInAMP; - assertThat(totalMetrics, greaterThanOrEqualTo(NUM_RECORDS)); + verify(metricsSuccessCounter, times(10)).increment(1); }); - verify(metricsSuccessCounter, times(10)).increment(1); } private Collection> getHistogramRecordList(int numberOfRecords) { diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/configuration/PrometheusSinkConfiguration.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/configuration/PrometheusSinkConfiguration.java index 88e28f4e15..7be4ec9fea 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/configuration/PrometheusSinkConfiguration.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/configuration/PrometheusSinkConfiguration.java @@ -34,6 +34,7 @@ public class PrometheusSinkConfiguration { private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(60); private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(60); private static final Duration DEFAULT_IDLE_TIMEOUT = Duration.ofSeconds(60); + private static final Duration DEFAULT_OUT_OF_ORDER_WINDOW = Duration.ofSeconds(5); @JsonProperty("aws") @NotNull @@ -44,6 +45,9 @@ public class PrometheusSinkConfiguration { @JsonProperty("url") private String url; + @JsonProperty("out_of_order_window") + private Duration outOfOrderWindow = DEFAULT_OUT_OF_ORDER_WINDOW; + @JsonProperty("max_retries") private int maxRetries = DEFAULT_MAX_RETRIES; @@ -108,6 +112,10 @@ public String getContentType() { return contentType; } + public Duration getOutOfOrderWindow() { + return outOfOrderWindow; + } + public String getRemoteWriteVersion() { return remoteWriteVersion; } diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBuffer.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBuffer.java new file mode 100644 index 0000000000..21ddaa280c --- /dev/null +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBuffer.java @@ -0,0 +1,34 @@ + /* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.dataprepper.plugins.sink.prometheus.service; + +import org.opensearch.dataprepper.common.sink.DefaultSinkBuffer; +import org.opensearch.dataprepper.common.sink.SinkBufferWriter; +import org.opensearch.dataprepper.common.sink.SinkBufferEntry; + +public class PrometheusSinkBuffer extends DefaultSinkBuffer { + + public PrometheusSinkBuffer(final long maxEvents, final long maxRequestSize, + final long flushIntervalMs, final SinkBufferWriter sinkBufferWriter) { + + super(maxEvents, maxRequestSize, flushIntervalMs, sinkBufferWriter); + } + + @Override + public boolean isMaxEventsLimitReached() { + return sinkBufferWriter.isMaxEventsLimitReached(maxEvents); + } + + @Override + public boolean willExceedMaxRequestSizeBytes(final SinkBufferEntry sinkBufferEntry) { + return sinkBufferWriter.willExceedMaxRequestSizeBytes(sinkBufferEntry, maxRequestSize); + } + +} diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferEntry.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferEntry.java index 0708f595a0..c9f610ca67 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferEntry.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferEntry.java @@ -13,12 +13,7 @@ import org.opensearch.dataprepper.common.sink.SinkBufferEntry; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventType; -import org.opensearch.dataprepper.model.metric.ExponentialHistogram; -import org.opensearch.dataprepper.model.metric.Gauge; import org.opensearch.dataprepper.model.metric.Metric; -import org.opensearch.dataprepper.model.metric.Histogram; -import org.opensearch.dataprepper.model.metric.Sum; -import org.opensearch.dataprepper.model.metric.Summary; public class PrometheusSinkBufferEntry implements SinkBufferEntry { @@ -51,30 +46,7 @@ public Event getEvent() { private PrometheusTimeSeries getTimeSeriesForEvent(final boolean sanitizeNames) throws Exception { if (event.getMetadata().getEventType().equals(EventType.METRIC.toString())) { - try { - PrometheusTimeSeries timeSeries = new PrometheusTimeSeries((Metric)event, sanitizeNames); - if (event instanceof Gauge) { - final Gauge gauge = (Gauge) event; - timeSeries.addGaugeMetric(gauge); - } else if (event instanceof Sum) { - final Sum sum = (Sum) event; - timeSeries.addSumMetric(sum); - } else if (event instanceof Summary) { - final Summary summary = (Summary) event; - timeSeries.addSummaryMetric(summary); - } else if (event instanceof Histogram) { - final Histogram histogram = (Histogram) event; - timeSeries.addHistogramMetric(histogram); - } else if (event instanceof ExponentialHistogram) { - final ExponentialHistogram exponentialHistogram = (ExponentialHistogram) event; - timeSeries.addExponentialHistogramMetric(exponentialHistogram); - } else { - throw new RuntimeException("Unknown metric type"); - } - return timeSeries; - } catch (Exception e) { - throw e; - } + return new PrometheusTimeSeries((Metric)event, sanitizeNames); } throw new RuntimeException("Not metric type"); } diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java index a308d734ca..b415646f64 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriter.java @@ -9,51 +9,104 @@ */ package org.opensearch.dataprepper.plugins.sink.prometheus.service; +import com.google.common.annotations.VisibleForTesting; + import org.opensearch.dataprepper.common.sink.SinkBufferEntry; import org.opensearch.dataprepper.common.sink.SinkBufferWriter; import org.opensearch.dataprepper.common.sink.SinkFlushableBuffer; import org.opensearch.dataprepper.common.sink.SinkFlushContext; import org.opensearch.dataprepper.common.sink.SinkMetrics; +import org.opensearch.dataprepper.plugins.sink.prometheus.configuration.PrometheusSinkConfiguration; +import software.amazon.awssdk.utils.Pair; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; public class PrometheusSinkBufferWriter implements SinkBufferWriter { + private static final int MAX_EVENTS_PER_SERIES = 1000; // Each buffer entry is a metric. // Duplicate entries for the same metric at the same time is not allowed // If there are multiple entries in the buffer for the same metric (with different time stamps), // They must be sorted by time before sending to Prometheus - private final Map> buffer; + private final Map buffer; private final SinkMetrics sinkMetrics; + private final long outOfOrderWindowMillis; + private final long maxEvents; + private final long maxRequestSize; - public PrometheusSinkBufferWriter(SinkMetrics sinkMetrics) { + public PrometheusSinkBufferWriter(final PrometheusSinkConfiguration sinkConfig, final SinkMetrics sinkMetrics) { this.buffer = new HashMap<>(); this.sinkMetrics = sinkMetrics; + this.outOfOrderWindowMillis = sinkConfig.getOutOfOrderWindow().toMillis(); + this.maxEvents = sinkConfig.getThresholdConfig().getMaxEvents(); + this.maxRequestSize = sinkConfig.getThresholdConfig().getMaxRequestSizeBytes(); + } + + private String getSeriesKey(PrometheusSinkBufferEntry bufferEntry) { + return bufferEntry.getTimeSeries().getMetricKey(); + } + + @Override + public boolean isMaxEventsLimitReached(final long maxEvents) { + return buffer.values().stream() + .mapToLong(PrometheusSinkBufferWriterEntry::getNumberOfEntriesReadyToFlush) + .sum() >= maxEvents; + } + + @Override + public boolean willExceedMaxRequestSizeBytes(final SinkBufferEntry sinkBufferEntry, final long maxRequestSize) { + return buffer.values().stream() + .mapToLong(PrometheusSinkBufferWriterEntry::getSizeOfEntriesReadyToFlush) + .sum() >= maxRequestSize; } - public boolean writeToBuffer(SinkBufferEntry bufferEntry) { - PrometheusTimeSeries timeSeries = ((PrometheusSinkBufferEntry)bufferEntry).getTimeSeries(); - if (timeSeries == null) { + public boolean writeToBuffer(final SinkBufferEntry bufferEntry) { + PrometheusSinkBufferEntry entry = (PrometheusSinkBufferEntry)bufferEntry; + if (entry.getTimeSeries() == null || entry.getTimeSeries().getSize() >= maxRequestSize) { return false; } + final String seriesKey = getSeriesKey(entry); - buffer.computeIfAbsent(timeSeries.getMetricName(), k -> new HashMap<>()) - .put(timeSeries.getTimeStamp(), bufferEntry); - return true; + boolean result = buffer.computeIfAbsent(seriesKey, + k -> new PrometheusSinkBufferWriterEntry(this.outOfOrderWindowMillis, MAX_EVENTS_PER_SERIES)).add(entry); + if (!result) { + sinkMetrics.incrementEventsDroppedCounter(1); + } + return result; + } + + @VisibleForTesting + long getBufferSize() { + return buffer.size(); } @Override public SinkFlushableBuffer getBuffer(final SinkFlushContext sinkFlushContext) { - List bufferList = buffer.values().stream() - .flatMap(timeSeriesMap -> timeSeriesMap.entrySet().stream() - .sorted(Map.Entry.comparingByKey()) - .map(Map.Entry::getValue)) - .collect(Collectors.toList()); + List bufferList = new ArrayList<>(); + long curMaxEvents = maxEvents; + long curMaxSize = maxRequestSize; + long reqSize = 0L; + for (Map.Entry entry : buffer.entrySet()) { + if (curMaxEvents == 0 || curMaxSize == 0) { + break; + } + Pair, Long> result = entry.getValue().getEntriesReadyToFlush(curMaxEvents, curMaxSize); + if (!result.left().isEmpty()) { + curMaxEvents -= result.left().size(); + curMaxSize -= result.right(); + reqSize += result.right(); + bufferList.addAll(result.left()); + } + } + if (bufferList.isEmpty()) { + return null; + } + sinkMetrics.recordRequestSize(reqSize); + buffer.entrySet().removeIf(entry -> entry.getValue().getSize() == 0); - buffer.clear(); return new PrometheusSinkFlushableBuffer(bufferList, sinkMetrics, sinkFlushContext); } } diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterEntry.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterEntry.java new file mode 100644 index 0000000000..5a9eaf8104 --- /dev/null +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkBufferWriterEntry.java @@ -0,0 +1,119 @@ + /* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.dataprepper.plugins.sink.prometheus.service; + +import com.google.common.annotations.VisibleForTesting; +import software.amazon.awssdk.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +public class PrometheusSinkBufferWriterEntry { + private static final Logger LOG = LoggerFactory.getLogger(PrometheusSinkBufferWriterEntry.class); + private long lastSentTimestamp = -1L; + private final int maxEntries; + private final long windowMillis; + private long seriesMaxTimestamp = -1L; + private long realtimeSeriesMax; + private final TreeMap entries; + + public PrometheusSinkBufferWriterEntry(final long windowMillis, final int maxEntries) { + this.maxEntries = maxEntries; + this.windowMillis = windowMillis; + this.entries = new TreeMap<>(); + this.realtimeSeriesMax = Instant.now().toEpochMilli(); + } + + public boolean add(final PrometheusSinkBufferEntry bufferEntry) { + long time = bufferEntry.getTimeSeries().getTimestamp(); + if (time <= lastSentTimestamp) { + return false; + } + + if (time > seriesMaxTimestamp) { + seriesMaxTimestamp = time; + realtimeSeriesMax = Instant.now().toEpochMilli(); + } + + entries.put(time, bufferEntry); + if (entries.size() > maxEntries) { + LOG.warn("Number of entries exceeded maxEntries"); + return false; + } + return true; + } + + public long getSize() { + return entries.size(); + } + + @VisibleForTesting + long windowSize() { + long timeOffset = Instant.now().toEpochMilli() - realtimeSeriesMax; + return (seriesMaxTimestamp - windowMillis + timeOffset + 1); + } + + public long getNumberOfEntriesReadyToFlush() { + SortedMap readyToFlush = entries.headMap(windowSize()); + return readyToFlush.size(); + } + + public long getSizeOfEntriesReadyToFlush() { + return entries.headMap(windowSize()) + .values() + .stream() + .mapToLong(entry -> entry.getTimeSeries().getSize()) + .sum(); + } + + public Pair, Long> getEntriesReadyToFlush(final long maxEvents, final long maxSize) { + if (entries.isEmpty()) { + return Pair.of(Collections.emptyList(), 0L); + } + + final long cutoff = windowSize(); + List toFlush = new ArrayList<>(); + + SortedMap readyToFlush = entries.headMap(cutoff); + long numEntries = 0L; + long curSize = 0L; + if (!readyToFlush.isEmpty()) { + Iterator> iterator = readyToFlush.entrySet().iterator(); + + while (iterator.hasNext() && numEntries < maxEvents ) { + Map.Entry entry = iterator.next(); + long entrySize = entry.getValue().getTimeSeries().getSize(); + + if (curSize + entrySize > maxSize) { + break; + } + curSize += entrySize; + toFlush.add(entry.getValue()); + iterator.remove(); + numEntries++; + } + } + + if (!toFlush.isEmpty()) { + toFlush.sort(Comparator.comparing(bufferEntry -> bufferEntry.getTimeSeries().getTimestamp())); + this.lastSentTimestamp = toFlush.get(toFlush.size() - 1).getTimeSeries().getTimestamp(); + } + return Pair.of(toFlush, curSize); + } +} diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkFlushableBuffer.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkFlushableBuffer.java index d95882c607..5126db965f 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkFlushableBuffer.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkFlushableBuffer.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; public class PrometheusSinkFlushableBuffer implements SinkFlushableBuffer { List buffer; @@ -39,15 +40,15 @@ public PrometheusSinkFlushableBuffer(List buffer, final SinkMet @Override public SinkFlushResult flush() { - if (buffer.size() == 0) { + if (buffer.isEmpty()) { return null; } PrometheusHttpSender httpSender = sinkFlushContext.getHttpSender(); final Remote.WriteRequest.Builder writeRequestBuilder = Remote.WriteRequest.newBuilder(); - List allTimeSeries = new ArrayList<>(); + List allTimeSeries = new ArrayList<>(buffer.size() * 2); - List events = new ArrayList<>(); + List events = new ArrayList<>(buffer.size()); for (final SinkBufferEntry sinkBufferEntry : buffer) { PrometheusSinkBufferEntry bufferEntry = (PrometheusSinkBufferEntry)sinkBufferEntry; allTimeSeries.addAll(bufferEntry.getTimeSeries().getTimeSeriesList()); @@ -70,11 +71,9 @@ public SinkFlushResult flush() { @Override public List getEvents() { - List result = new ArrayList<>(); - for (final SinkBufferEntry bufferEntry: buffer) { - result.add(bufferEntry.getEvent()); - } - return result; + return buffer.stream() + .map(SinkBufferEntry::getEvent) + .collect(Collectors.toList()); } } diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkService.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkService.java index 8ad503c7cd..c75f217a3f 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkService.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusSinkService.java @@ -13,7 +13,6 @@ import com.google.common.annotations.VisibleForTesting; import org.opensearch.dataprepper.common.sink.DefaultSinkOutputStrategy; -import org.opensearch.dataprepper.common.sink.DefaultSinkBuffer; import org.opensearch.dataprepper.common.sink.SinkMetrics; import org.opensearch.dataprepper.common.sink.SinkBufferEntry; import org.opensearch.dataprepper.common.sink.ReentrantLockStrategy; @@ -30,7 +29,6 @@ import java.util.Collection; import java.util.List; - public class PrometheusSinkService extends DefaultSinkOutputStrategy { static final String PLUGIN_NAME = "prometheus"; private static final Logger LOG = LoggerFactory.getLogger(PrometheusSinkService.class); @@ -51,10 +49,10 @@ public PrometheusSinkService(final PrometheusSinkConfiguration prometheusSinkCon final HeadlessPipeline dlqPipeline, final PipelineDescription pipelineDescription) { super(new ReentrantLockStrategy(), - new DefaultSinkBuffer(prometheusSinkConfiguration.getThresholdConfig().getMaxEvents(), + new PrometheusSinkBuffer(prometheusSinkConfiguration.getThresholdConfig().getMaxEvents(), prometheusSinkConfiguration.getThresholdConfig().getMaxRequestSizeBytes(), prometheusSinkConfiguration.getThresholdConfig().getFlushIntervalMs(), - new PrometheusSinkBufferWriter(sinkMetrics)), + new PrometheusSinkBufferWriter(prometheusSinkConfiguration, sinkMetrics)), new PrometheusSinkFlushContext(httpSender), sinkMetrics); sanitizeNames = prometheusSinkConfiguration.getSanitizeNames(); @@ -81,7 +79,7 @@ public void setDlqPipeline(HeadlessPipeline pipeline) { } public void flushDlqList() { - if (dlqRecords.size() == 0) { + if (dlqRecords.isEmpty()) { return; } if (dlqPipeline != null) { diff --git a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java index c8fba17abe..6d3ce470a8 100644 --- a/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java +++ b/data-prepper-plugins/prometheus-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/prometheus/service/PrometheusTimeSeries.java @@ -27,6 +27,8 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -83,8 +85,9 @@ public class PrometheusTimeSeries { private long timestamp; private boolean sanitizeNames; private List timeSeriesList; - private List