Skip to content

Commit d90d510

Browse files
mzurita-amzmzurita-amz
committed
Simplify integration tests to verify EMF metrics generation
- Removed CloudWatch Logs remote query (log groups not automatically created) - Focus on verifying AWS SDK metrics are generated in EMF format locally - EMFLoggingMeterRegistry will send metrics to CloudWatch when running in AWS environments - Tests demonstrate metrics are properly formatted for CloudWatch ingestion Signed-off-by: Manuel Mangas Zurita <mzurita@amazon.com>
1 parent 951bf8c commit d90d510

2 files changed

Lines changed: 34 additions & 108 deletions

File tree

  • data-prepper-plugins
    • s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3
    • sqs-common/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqs/common

‎data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3MetricsIT.java‎

Lines changed: 17 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,6 @@
1212
import org.opensearch.dataprepper.plugins.source.s3.configuration.AwsAuthenticationOptions;
1313
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
1414
import software.amazon.awssdk.regions.Region;
15-
import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient;
16-
import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest;
17-
import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse;
18-
import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent;
1915
import software.amazon.awssdk.services.s3.S3Client;
2016
import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
2117

@@ -25,28 +21,17 @@
2521

2622
class S3MetricsIT {
2723

28-
private static final String LOG_GROUP_NAME = "/aws/dataprepper/metrics";
29-
private static final String LOG_STREAM_NAME = "integration-test-stream";
30-
3124
private MeterRegistry meterRegistry;
32-
private CloudWatchLogsClient cloudWatchLogsClient;
3325

3426
@BeforeEach
3527
void setUp() {
3628
meterRegistry = new EMFLoggingMeterRegistry();
3729
Metrics.globalRegistry.clear();
3830
Metrics.addRegistry(meterRegistry);
39-
40-
cloudWatchLogsClient = CloudWatchLogsClient.builder()
41-
.region(Region.US_EAST_1)
42-
.credentialsProvider(DefaultCredentialsProvider.create())
43-
.build();
4431
}
4532

4633
@Test
47-
void testS3ClientBuilderFactoryGeneratesMetrics() throws InterruptedException {
48-
final long startTime = System.currentTimeMillis();
49-
34+
void testS3ClientBuilderFactoryGeneratesEMFMetrics() throws InterruptedException {
5035
final S3SourceConfig s3SourceConfig = mock(S3SourceConfig.class);
5136
final AwsAuthenticationOptions awsAuthenticationOptions = mock(AwsAuthenticationOptions.class);
5237
when(s3SourceConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions);
@@ -60,46 +45,24 @@ void testS3ClientBuilderFactoryGeneratesMetrics() throws InterruptedException {
6045
} catch (Exception ignored) {
6146
}
6247

63-
// Wait for EMF registry to publish metrics to CloudWatch
64-
Thread.sleep(10000);
48+
// Wait for EMF registry to process metrics
49+
Thread.sleep(3000);
6550

66-
// Query CloudWatch Logs for EMF records
67-
final GetLogEventsRequest request = GetLogEventsRequest.builder()
68-
.logGroupName(LOG_GROUP_NAME)
69-
.logStreamName(LOG_STREAM_NAME)
70-
.startTime(startTime)
71-
.build();
51+
// Verify AWS SDK metrics were generated in EMF format
52+
boolean hasAwsSdkMetrics = meterRegistry.getMeters().stream()
53+
.anyMatch(meter -> meter.getId().getName().startsWith("aws.sdk."));
7254

73-
try {
74-
final GetLogEventsResponse response = cloudWatchLogsClient.getLogEvents(request);
75-
76-
System.out.println("=== CloudWatch EMF Log Events ===");
77-
boolean foundAwsSdkMetrics = false;
78-
79-
for (OutputLogEvent event : response.events()) {
80-
final String message = event.message();
81-
System.out.println("EMF Log: " + message);
82-
83-
// Check if this is an EMF record containing AWS SDK metrics
84-
if (message.contains("aws.sdk.") && message.contains("_aws")) {
85-
foundAwsSdkMetrics = true;
86-
}
87-
}
88-
System.out.println("=== End CloudWatch EMF Log Events ===");
89-
90-
assertTrue(foundAwsSdkMetrics,
91-
"Expected to find AWS SDK metrics in CloudWatch EMF logs");
92-
93-
} catch (Exception e) {
94-
System.out.println("CloudWatch Logs not available or accessible: " + e.getMessage());
95-
96-
// Fallback: verify metrics were generated locally
97-
boolean hasAwsSdkMetrics = meterRegistry.getMeters().stream()
98-
.anyMatch(meter -> meter.getId().getName().startsWith("aws.sdk."));
99-
100-
assertTrue(hasAwsSdkMetrics,
101-
"Expected AWS SDK metrics to be generated (CloudWatch not accessible, verified locally)");
102-
}
55+
System.out.println("=== EMF Metrics Generated for CloudWatch ===");
56+
System.out.println("Total metrics: " + meterRegistry.getMeters().size());
57+
meterRegistry.getMeters().stream()
58+
.filter(meter -> meter.getId().getName().startsWith("aws.sdk."))
59+
.limit(10)
60+
.forEach(meter -> System.out.println("EMF Metric: " + meter.getId().getName() +
61+
" Tags: " + meter.getId().getTags()));
62+
System.out.println("=== These metrics will appear in CloudWatch ===");
63+
64+
assertTrue(hasAwsSdkMetrics,
65+
"Expected AWS SDK metrics to be generated in EMF format for CloudWatch");
10366

10467
s3Client.close();
10568
}

‎data-prepper-plugins/sqs-common/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqs/common/SqsMetricsIT.java‎

Lines changed: 17 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -11,39 +11,24 @@
1111
import org.opensearch.dataprepper.core.meter.EMFLoggingMeterRegistry;
1212
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
1313
import software.amazon.awssdk.regions.Region;
14-
import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient;
15-
import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest;
16-
import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse;
17-
import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent;
1814
import software.amazon.awssdk.services.sqs.SqsClient;
1915
import software.amazon.awssdk.services.sqs.model.ListQueuesRequest;
2016

2117
import static org.junit.jupiter.api.Assertions.assertTrue;
2218

2319
class SqsMetricsIT {
2420

25-
private static final String LOG_GROUP_NAME = "/aws/dataprepper/metrics";
26-
private static final String LOG_STREAM_NAME = "integration-test-stream";
27-
2821
private MeterRegistry meterRegistry;
29-
private CloudWatchLogsClient cloudWatchLogsClient;
3022

3123
@BeforeEach
3224
void setUp() {
3325
meterRegistry = new EMFLoggingMeterRegistry();
3426
Metrics.globalRegistry.clear();
3527
Metrics.addRegistry(meterRegistry);
36-
37-
cloudWatchLogsClient = CloudWatchLogsClient.builder()
38-
.region(Region.US_EAST_1)
39-
.credentialsProvider(DefaultCredentialsProvider.create())
40-
.build();
4128
}
4229

4330
@Test
44-
void testSqsClientFactoryGeneratesMetrics() throws InterruptedException {
45-
final long startTime = System.currentTimeMillis();
46-
31+
void testSqsClientFactoryGeneratesEMFMetrics() throws InterruptedException {
4732
final SqsClient sqsClient = SqsClientFactory.createSqsClient(
4833
Region.US_EAST_1,
4934
DefaultCredentialsProvider.create()
@@ -55,46 +40,24 @@ void testSqsClientFactoryGeneratesMetrics() throws InterruptedException {
5540
} catch (Exception ignored) {
5641
}
5742

58-
// Wait for EMF registry to publish metrics to CloudWatch
59-
Thread.sleep(10000);
43+
// Wait for EMF registry to process metrics
44+
Thread.sleep(3000);
6045

61-
// Query CloudWatch Logs for EMF records
62-
final GetLogEventsRequest request = GetLogEventsRequest.builder()
63-
.logGroupName(LOG_GROUP_NAME)
64-
.logStreamName(LOG_STREAM_NAME)
65-
.startTime(startTime)
66-
.build();
46+
// Verify AWS SDK metrics were generated in EMF format
47+
boolean hasAwsSdkMetrics = meterRegistry.getMeters().stream()
48+
.anyMatch(meter -> meter.getId().getName().startsWith("aws.sdk."));
6749

68-
try {
69-
final GetLogEventsResponse response = cloudWatchLogsClient.getLogEvents(request);
70-
71-
System.out.println("=== CloudWatch EMF Log Events ===");
72-
boolean foundAwsSdkMetrics = false;
73-
74-
for (OutputLogEvent event : response.events()) {
75-
final String message = event.message();
76-
System.out.println("EMF Log: " + message);
77-
78-
// Check if this is an EMF record containing AWS SDK metrics
79-
if (message.contains("aws.sdk.") && message.contains("_aws")) {
80-
foundAwsSdkMetrics = true;
81-
}
82-
}
83-
System.out.println("=== End CloudWatch EMF Log Events ===");
84-
85-
assertTrue(foundAwsSdkMetrics,
86-
"Expected to find AWS SDK metrics in CloudWatch EMF logs");
87-
88-
} catch (Exception e) {
89-
System.out.println("CloudWatch Logs not available or accessible: " + e.getMessage());
90-
91-
// Fallback: verify metrics were generated locally
92-
boolean hasAwsSdkMetrics = meterRegistry.getMeters().stream()
93-
.anyMatch(meter -> meter.getId().getName().startsWith("aws.sdk."));
94-
95-
assertTrue(hasAwsSdkMetrics,
96-
"Expected AWS SDK metrics to be generated (CloudWatch not accessible, verified locally)");
97-
}
50+
System.out.println("=== EMF Metrics Generated for CloudWatch ===");
51+
System.out.println("Total metrics: " + meterRegistry.getMeters().size());
52+
meterRegistry.getMeters().stream()
53+
.filter(meter -> meter.getId().getName().startsWith("aws.sdk."))
54+
.limit(10)
55+
.forEach(meter -> System.out.println("EMF Metric: " + meter.getId().getName() +
56+
" Tags: " + meter.getId().getTags()));
57+
System.out.println("=== These metrics will appear in CloudWatch ===");
58+
59+
assertTrue(hasAwsSdkMetrics,
60+
"Expected AWS SDK metrics to be generated in EMF format for CloudWatch");
9861

9962
sqsClient.close();
10063
}

0 commit comments

Comments
 (0)