Skip to content

Commit 4e2b72d

Browse files
vechekavecheka
committed
Adding specific metric names on exception/error for m365 plugins
Signed-off-by: Vecheka Chhourn <vecheka@amazon.com>
1 parent 8667d11 commit 4e2b72d

4 files changed

Lines changed: 83 additions & 56 deletions

File tree

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365CrawlerClient.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public class Office365CrawlerClient implements CrawlerClient<DimensionalTimeSlic
5656
private static final String BUFFER_WRITE_RETRY_SUCCESS = "bufferWriteRetrySuccess";
5757
private static final String BUFFER_WRITE_RETRY_ATTEMPTS = "bufferWriteRetryAttempts";
5858
private static final String BUFFER_WRITE_FAILURES = "bufferWriteFailures";
59-
private static final String TOTAL_FAILURES = "totalFailures";
59+
private static final String REQEUEST_ERRORS = "requestErrors";
6060
private static final int BUFFER_TIMEOUT_IN_SECONDS = 10;
6161
private static final String CONTENT_ID = "contentId";
6262
private static final String CONTENT_URI = "contentUri";
@@ -69,7 +69,7 @@ public class Office365CrawlerClient implements CrawlerClient<DimensionalTimeSlic
6969
private final Counter bufferWriteRetrySuccessCounter;
7070
private final Counter bufferWriteRetryAttemptsCounter;
7171
private final Counter bufferWriteFailuresCounter;
72-
private final Counter totalFailuresCounter;
72+
private final Counter requestErrorsCounter;
7373
private ObjectMapper objectMapper;
7474

7575
public Office365CrawlerClient(final Office365Service service,
@@ -86,7 +86,7 @@ public Office365CrawlerClient(final Office365Service service,
8686
this.bufferWriteRetrySuccessCounter = pluginMetrics.counter(BUFFER_WRITE_RETRY_SUCCESS);
8787
this.bufferWriteRetryAttemptsCounter = pluginMetrics.counter(BUFFER_WRITE_RETRY_ATTEMPTS);
8888
this.bufferWriteFailuresCounter = pluginMetrics.counter(BUFFER_WRITE_FAILURES);
89-
this.totalFailuresCounter = pluginMetrics.counter(TOTAL_FAILURES);
89+
this.requestErrorsCounter = pluginMetrics.counter(REQEUEST_ERRORS);
9090
}
9191

9292
@VisibleForTesting
@@ -160,7 +160,7 @@ public void executePartition(final DimensionalTimeSliceWorkerProgressState state
160160
} catch (Exception e) {
161161
log.error(NOISY, "Failed to process partition for log type {} from {} to {}",
162162
logType, startTime, endTime, e);
163-
totalFailuresCounter.increment();
163+
requestErrorsCounter.increment();
164164
throw e;
165165
}
166166
}

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365RestClient.java‎

Lines changed: 11 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,11 @@
3333
import java.time.Instant;
3434
import java.util.List;
3535
import java.util.Map;
36-
import java.util.HashMap;
3736

3837
import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY;
3938
import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.Constants.CONTENT_TYPES;
39+
import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.MetricsHelper.getErrorTypeMetricCounterMap;
40+
import static org.opensearch.dataprepper.plugins.source.microsoft_office365.utils.MetricsHelper.publishErrorTypeMetricCounter;
4041

4142
/**
4243
* REST client for interacting with Office 365 Management API.
@@ -55,11 +56,6 @@ public class Office365RestClient {
5556
private static final String SEARCH_REQUESTS_SUCCESS = "searchRequestsSuccess";
5657
private static final String SEARCH_REQUESTS_FAILED = "searchRequestsFailed";
5758

58-
// specific retryable/non-retryable metric names
59-
private static final String ACCESS_DENIED_FAILED = "accessDenied";
60-
private static final String THROTTLING_FAILED = "throttling";
61-
private static final String RESOURCE_NOT_FOUND_FAILED = "resourceNotFound";
62-
6359
private static final String MANAGEMENT_API_BASE_URL = "https://manage.office.com/api/v1.0/";
6460

6561
private final RestTemplate restTemplate = new RestTemplate();
@@ -90,7 +86,7 @@ public Office365RestClient(final Office365AuthenticationInterface authConfig,
9086
this.auditLogResponseSizeSummary = pluginMetrics.summary(AUDIT_LOG_RESPONSE_SIZE);
9187
this.searchResponseSizeSummary = pluginMetrics.summary(SEARCH_RESPONSE_SIZE);
9288

93-
this.initializeErrorTypeMetricCounterMap(pluginMetrics);
89+
this.errorTypeMetricCounterMap = getErrorTypeMetricCounterMap(pluginMetrics);
9490
}
9591

9692
/**
@@ -148,14 +144,14 @@ public void startSubscriptions() {
148144
}
149145
} catch (HttpClientErrorException | HttpServerErrorException e) {
150146
HttpStatus statusCode = e.getStatusCode();
151-
publishErrorTypeMetricCounter(statusCode.getReasonPhrase());
147+
publishErrorTypeMetricCounter(statusCode.getReasonPhrase(), this.errorTypeMetricCounterMap);
152148
log.error(NOISY, "Failed to initialize subscriptions with status code {}: {}",
153149
statusCode, e.getMessage());
154150
throw new RuntimeException("Failed to initialize subscriptions: " + e.getMessage(), e);
155151
} catch (Exception e) {
156152
// FORBIDDEN throws SecurityException in RetryHandler
157153
if (e instanceof SecurityException) {
158-
publishErrorTypeMetricCounter(HttpStatus.FORBIDDEN.getReasonPhrase());
154+
publishErrorTypeMetricCounter(HttpStatus.FORBIDDEN.getReasonPhrase(), this.errorTypeMetricCounterMap);
159155
}
160156
log.error(NOISY, "Failed to initialize subscriptions", e);
161157
throw new RuntimeException("Failed to initialize subscriptions: " + e.getMessage(), e);
@@ -220,14 +216,13 @@ public AuditLogsResponse searchAuditLogs(final String contentType,
220216
);
221217
} catch (HttpClientErrorException | HttpServerErrorException e) {
222218
HttpStatus statusCode = e.getStatusCode();
223-
publishErrorTypeMetricCounter(statusCode.getReasonPhrase());
224-
log.error(NOISY, "Error while fetching audit logs with status code {}: {}",
225-
statusCode, e.getMessage());
219+
publishErrorTypeMetricCounter(statusCode.getReasonPhrase(), this.errorTypeMetricCounterMap);
220+
log.error(NOISY, "Error while fetching audit logs for content type {}", contentType, e);
226221
throw new RuntimeException("Failed to fetch audit logs", e);
227222
} catch (Exception e) {
228223
// FORBIDDEN throws SecurityException in RetryHandler
229224
if (e instanceof SecurityException) {
230-
publishErrorTypeMetricCounter(HttpStatus.FORBIDDEN.getReasonPhrase());
225+
publishErrorTypeMetricCounter(HttpStatus.FORBIDDEN.getReasonPhrase(), this.errorTypeMetricCounterMap);
231226
}
232227
log.error(NOISY, "Error while fetching audit logs for content type {}", contentType, e);
233228
throw new RuntimeException("Failed to fetch audit logs", e);
@@ -272,47 +267,17 @@ public String getAuditLog(String contentUri) {
272267
return response;
273268
} catch (HttpClientErrorException | HttpServerErrorException e) {
274269
HttpStatus statusCode = e.getStatusCode();
275-
publishErrorTypeMetricCounter(statusCode.getReasonPhrase());
276-
log.error(NOISY, "Error while getting audit log with status code {}: {}",
277-
statusCode, e.getMessage());
270+
publishErrorTypeMetricCounter(statusCode.getReasonPhrase(), this.errorTypeMetricCounterMap);
271+
log.error(NOISY, "Error while fetching audit log content from URI: {}", contentUri, e);
278272
throw new RuntimeException("Failed to fetch audit log", e);
279273
} catch (Exception e) {
280274
// FORBIDDEN throws SecurityException in RetryHandler
281275
if (e instanceof SecurityException) {
282-
publishErrorTypeMetricCounter(HttpStatus.FORBIDDEN.getReasonPhrase());
276+
publishErrorTypeMetricCounter(HttpStatus.FORBIDDEN.getReasonPhrase(), this.errorTypeMetricCounterMap);
283277
}
284278
log.error(NOISY, "Error while fetching audit log content from URI: {}", contentUri, e);
285279
throw new RuntimeException("Failed to fetch audit log", e);
286280
}
287281
});
288282
}
289-
290-
/**
291-
* Initialize the metric counter map for specific errorType
292-
* FORBIDDEN/UNAUTHORIZED = accessDenied
293-
* TOO_MANY_REQUESTS = throttling
294-
* NOT_FOUND = resourceNotFound
295-
*/
296-
private void initializeErrorTypeMetricCounterMap(PluginMetrics pluginMetrics) {
297-
// TODO: should move this over to a common class later
298-
this.errorTypeMetricCounterMap = new HashMap<>();
299-
this.errorTypeMetricCounterMap.put(HttpStatus.FORBIDDEN.getReasonPhrase(), pluginMetrics.counter(ACCESS_DENIED_FAILED));
300-
this.errorTypeMetricCounterMap.put(HttpStatus.UNAUTHORIZED.getReasonPhrase(), pluginMetrics.counter(ACCESS_DENIED_FAILED));
301-
this.errorTypeMetricCounterMap.put(HttpStatus.TOO_MANY_REQUESTS.getReasonPhrase(), pluginMetrics.counter(THROTTLING_FAILED));
302-
this.errorTypeMetricCounterMap.put(HttpStatus.NOT_FOUND.getReasonPhrase(), pluginMetrics.counter(RESOURCE_NOT_FOUND_FAILED));
303-
}
304-
305-
/**
306-
* Increment the errorType metric if it exists in errorTypeMetricCounterMap
307-
* Should only be the following:
308-
* FORBIDDEN/UNAUTHORIZED = accessDenied
309-
* TOO_MANY_REQUESTS = throttling
310-
* NOT_FOUND = resourceNotFound
311-
*/
312-
private void publishErrorTypeMetricCounter(String errorType) {
313-
if (this.errorTypeMetricCounterMap != null && this.errorTypeMetricCounterMap.containsKey(errorType)) {
314-
errorTypeMetricCounterMap.get(errorType).increment();
315-
}
316-
}
317-
318283
}

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/main/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/utils/MetricsHelper.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*
9+
*/
10+
11+
package org.opensearch.dataprepper.plugins.source.microsoft_office365.utils;
12+
13+
import io.micrometer.core.instrument.Counter;
14+
import org.springframework.http.HttpStatus;
15+
import org.opensearch.dataprepper.metrics.PluginMetrics;
16+
17+
import java.util.Map;
18+
import java.util.HashMap;
19+
/**
20+
* The MetricsHelper class.
21+
*/
22+
public class MetricsHelper {
23+
24+
// specific retryable/non-retryable metric names
25+
private static final String REQUEST_ACCESS_DENIED = "requestAccessDenied";
26+
private static final String REQUEST_THROTTLED = "requestThrottled";
27+
private static final String RESOURCE_NOT_FOUND = "resourceNotFound";
28+
29+
/**
30+
* Get the metric counter map for specific errorType
31+
* FORBIDDEN/UNAUTHORIZED = requestAccessDenied
32+
* TOO_MANY_REQUESTS = requestThrottled
33+
* NOT_FOUND = resourceNotFound
34+
* @param pluginMetrics - metric object class to initialize metric counters
35+
*
36+
* @return errorTypeMetricCounterMap
37+
*/
38+
public static Map<String, Counter> getErrorTypeMetricCounterMap(PluginMetrics pluginMetrics) {
39+
Map<String, Counter> errorTypeMetricCounterMap = new HashMap<>();
40+
errorTypeMetricCounterMap.put(HttpStatus.FORBIDDEN.getReasonPhrase(), pluginMetrics.counter(REQUEST_ACCESS_DENIED));
41+
errorTypeMetricCounterMap.put(HttpStatus.UNAUTHORIZED.getReasonPhrase(), pluginMetrics.counter(REQUEST_ACCESS_DENIED));
42+
errorTypeMetricCounterMap.put(HttpStatus.TOO_MANY_REQUESTS.getReasonPhrase(), pluginMetrics.counter(REQUEST_THROTTLED));
43+
errorTypeMetricCounterMap.put(HttpStatus.NOT_FOUND.getReasonPhrase(), pluginMetrics.counter(RESOURCE_NOT_FOUND));
44+
return errorTypeMetricCounterMap;
45+
}
46+
47+
/**
48+
* Increment the errorType metric if it exists in errorTypeMetricCounterMap
49+
* Should only be the following:
50+
* FORBIDDEN/UNAUTHORIZED = requestAccessDenied
51+
* TOO_MANY_REQUESTS = requestThrottled
52+
* NOT_FOUND = resourceNotFound
53+
*
54+
* @param errorType - the httpStatusCode string represenation
55+
* @param errorTypeMetricCounterMap - the map of errorType to metric counter
56+
*/
57+
public static void publishErrorTypeMetricCounter(String errorType, Map<String, Counter> errorTypeMetricCounterMap) {
58+
if (errorTypeMetricCounterMap != null && errorTypeMetricCounterMap.containsKey(errorType)) {
59+
errorTypeMetricCounterMap.get(errorType).increment();
60+
}
61+
}
62+
}

‎data-prepper-plugins/saas-source-plugins/microsoft-office365-source/src/test/java/org/opensearch/dataprepper/plugins/source/microsoft_office365/Office365CrawlerClientTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,8 @@ void testExecutePartitionWithJsonProcessingError() throws Exception {
153153
client.injectObjectMapper(mockObjectMapper);
154154

155155
// Mock the total failures counter
156-
Counter mockTotalFailuresCounter = mock(Counter.class);
157-
when(pluginMetrics.counter("totalFailures")).thenReturn(mockTotalFailuresCounter);
156+
Counter mockRequestErrorsCounter = mock(Counter.class);
157+
when(pluginMetrics.counter("requestErrors")).thenReturn(mockRequestErrorsCounter);
158158

159159
AuditLogsResponse response = new AuditLogsResponse(
160160
Arrays.asList(Map.of(
@@ -181,7 +181,7 @@ void testExecutePartitionWithJsonProcessingError() throws Exception {
181181
client.executePartition(state, buffer, acknowledgementSet);
182182

183183
verify(buffer).writeAll(argThat(list -> list.isEmpty()), anyInt());
184-
verify(mockTotalFailuresCounter, never()).increment();
184+
verify(mockRequestErrorsCounter, never()).increment();
185185
}
186186

187187
@Test
@@ -317,8 +317,8 @@ void testMissingWorkloadField() throws Exception {
317317
@Test
318318
void testExecutePartitionWithSearchAuditLogsError() throws Exception {
319319
// Mock the total failures counter before creating the client
320-
Counter mockTotalFailuresCounter = mock(Counter.class);
321-
when(pluginMetrics.counter("totalFailures")).thenReturn(mockTotalFailuresCounter);
320+
Counter mockRequestErrorsCounter = mock(Counter.class);
321+
when(pluginMetrics.counter("requestErrors")).thenReturn(mockRequestErrorsCounter);
322322

323323
Office365CrawlerClient client = new Office365CrawlerClient(service, sourceConfig, pluginMetrics);
324324

@@ -336,6 +336,6 @@ void testExecutePartitionWithSearchAuditLogsError() throws Exception {
336336

337337
// Verify exception message and counter increment
338338
assertEquals("Search audit logs failed", exception.getMessage());
339-
verify(mockTotalFailuresCounter).increment();
339+
verify(mockRequestErrorsCounter).increment();
340340
}
341341
}

0 commit comments

Comments
 (0)