Skip to content

Commit ac1e33e

Browse files
ravisharmravisharm
committed
Add monitor to find length of queue at Routers
1 parent e03d29f commit ac1e33e

4 files changed

Lines changed: 188 additions & 0 deletions

File tree

‎docs/operations/metrics.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,14 @@ Most metric values reset each emission period, as specified in `druid.monitoring
162162
|`jetty/threadPool/max`|Number of maximum threads allocatable.|`druid.server.http.numThreads` plus a small fixed number of threads allocated for Jetty acceptors and selectors.|
163163
|`jetty/threadPool/queueSize`|Size of the worker queue.|Not much higher than `druid.server.http.queueSize`.|
164164

165+
### Router HTTP Client
166+
167+
These metrics are only available on the Router.
168+
169+
|Metric|Description|Normal value|
170+
|------|-----------|------------|
171+
|`router/http/numRequestsQueued`|Total number of requests queued across all destinations in the Router's outbound HTTP client.|Should remain well below `druid.router.http.numRequestsQueued` (default 1024). Sustained high values indicate downstream brokers may be slow or overloaded.|
172+
165173
### Cache
166174

167175
|Metric|Description|Dimensions|Normal value|

‎server/src/main/java/org/apache/druid/server/metrics/RouterHttpClientMonitor.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.server.metrics;
21+
22+
import com.google.inject.Inject;
23+
import com.google.inject.Provider;
24+
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
25+
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
26+
import org.apache.druid.java.util.metrics.AbstractMonitor;
27+
import org.apache.druid.server.router.Router;
28+
import org.eclipse.jetty.client.Destination;
29+
import org.eclipse.jetty.client.HttpClient;
30+
import org.eclipse.jetty.client.transport.HttpDestination;
31+
32+
/**
33+
* Monitor that emits metrics for the Router's outbound Jetty HttpClient,
34+
* including the total number of requests currently queued across all destinations.
35+
*/
36+
public class RouterHttpClientMonitor extends AbstractMonitor
37+
{
38+
private final Provider<HttpClient> httpClientProvider;
39+
40+
@Inject
41+
public RouterHttpClientMonitor(@Router Provider<HttpClient> httpClientProvider)
42+
{
43+
this.httpClientProvider = httpClientProvider;
44+
}
45+
46+
@Override
47+
public boolean doMonitor(ServiceEmitter emitter)
48+
{
49+
final HttpClient httpClient = httpClientProvider.get();
50+
final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder();
51+
52+
int totalQueuedRequests = 0;
53+
for (Destination destination : httpClient.getDestinations()) {
54+
if (destination instanceof HttpDestination) {
55+
totalQueuedRequests += ((HttpDestination) destination).getQueuedRequestCount();
56+
}
57+
}
58+
59+
emitter.emit(builder.setMetric("router/http/numRequestsQueued", totalQueuedRequests));
60+
61+
return true;
62+
}
63+
}

‎server/src/test/java/org/apache/druid/server/metrics/RouterHttpClientMonitorTest.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.druid.server.metrics;
21+
22+
import com.google.inject.Provider;
23+
import org.apache.druid.java.util.metrics.StubServiceEmitter;
24+
import org.eclipse.jetty.client.HttpClient;
25+
import org.eclipse.jetty.client.transport.HttpDestination;
26+
import org.junit.Assert;
27+
import org.junit.Before;
28+
import org.junit.Test;
29+
import org.mockito.Mockito;
30+
31+
import java.util.Collections;
32+
import java.util.List;
33+
34+
public class RouterHttpClientMonitorTest
35+
{
36+
private HttpClient httpClient;
37+
private Provider<HttpClient> httpClientProvider;
38+
private RouterHttpClientMonitor monitor;
39+
40+
@Before
41+
public void setUp()
42+
{
43+
httpClient = Mockito.mock(HttpClient.class);
44+
httpClientProvider = () -> httpClient;
45+
monitor = new RouterHttpClientMonitor(httpClientProvider);
46+
}
47+
48+
@Test
49+
public void testDoMonitorReturnsTrue()
50+
{
51+
Mockito.when(httpClient.getDestinations()).thenReturn(Collections.emptyList());
52+
final StubServiceEmitter emitter = new StubServiceEmitter("router", "localhost");
53+
Assert.assertTrue(monitor.doMonitor(emitter));
54+
}
55+
56+
@Test
57+
public void testNoDestinationsEmitsZero()
58+
{
59+
Mockito.when(httpClient.getDestinations()).thenReturn(Collections.emptyList());
60+
final StubServiceEmitter emitter = new StubServiceEmitter("router", "localhost");
61+
monitor.doMonitor(emitter);
62+
63+
Assert.assertEquals(1, emitter.getEvents().size());
64+
Assert.assertEquals("router/http/numRequestsQueued", emitter.getEvents().get(0).toMap().get("metric"));
65+
Assert.assertEquals(0, emitter.getEvents().get(0).toMap().get("value"));
66+
}
67+
68+
@Test
69+
public void testSingleDestinationQueueCount()
70+
{
71+
HttpDestination dest = Mockito.mock(HttpDestination.class);
72+
Mockito.when(dest.getQueuedRequestCount()).thenReturn(7);
73+
Mockito.when(httpClient.getDestinations()).thenReturn(Collections.singletonList(dest));
74+
75+
final StubServiceEmitter emitter = new StubServiceEmitter("router", "localhost");
76+
monitor.doMonitor(emitter);
77+
78+
Assert.assertEquals(1, emitter.getEvents().size());
79+
Assert.assertEquals(7, emitter.getEvents().get(0).toMap().get("value"));
80+
}
81+
82+
@Test
83+
public void testMultipleDestinationsQueueCountSummed()
84+
{
85+
HttpDestination dest1 = Mockito.mock(HttpDestination.class);
86+
HttpDestination dest2 = Mockito.mock(HttpDestination.class);
87+
Mockito.when(dest1.getQueuedRequestCount()).thenReturn(3);
88+
Mockito.when(dest2.getQueuedRequestCount()).thenReturn(5);
89+
Mockito.when(httpClient.getDestinations()).thenReturn(List.of(dest1, dest2));
90+
91+
final StubServiceEmitter emitter = new StubServiceEmitter("router", "localhost");
92+
monitor.doMonitor(emitter);
93+
94+
Assert.assertEquals(1, emitter.getEvents().size());
95+
Assert.assertEquals(8, emitter.getEvents().get(0).toMap().get("value"));
96+
}
97+
98+
@Test
99+
public void testNonHttpDestinationIsSkipped()
100+
{
101+
// A non-HttpDestination should not contribute to the count.
102+
org.eclipse.jetty.client.Destination plainDest =
103+
Mockito.mock(org.eclipse.jetty.client.Destination.class);
104+
HttpDestination httpDest = Mockito.mock(HttpDestination.class);
105+
Mockito.when(httpDest.getQueuedRequestCount()).thenReturn(4);
106+
Mockito.when(httpClient.getDestinations()).thenReturn(List.of(plainDest, httpDest));
107+
108+
final StubServiceEmitter emitter = new StubServiceEmitter("router", "localhost");
109+
monitor.doMonitor(emitter);
110+
111+
Assert.assertEquals(1, emitter.getEvents().size());
112+
Assert.assertEquals(4, emitter.getEvents().get(0).toMap().get("value"));
113+
}
114+
}

‎services/src/main/java/org/apache/druid/cli/CliRouter.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@
4646
import org.apache.druid.server.http.RouterResource;
4747
import org.apache.druid.server.http.SelfDiscoveryResource;
4848
import org.apache.druid.server.initialization.jetty.JettyServerInitializer;
49+
import org.apache.druid.server.metrics.MetricsModule;
4950
import org.apache.druid.server.metrics.QueryCountStatsProvider;
51+
import org.apache.druid.server.metrics.RouterHttpClientMonitor;
5052
import org.apache.druid.server.router.AvaticaConnectionBalancer;
5153
import org.apache.druid.server.router.CoordinatorRuleManager;
5254
import org.apache.druid.server.router.ManagementProxyConfig;
@@ -115,6 +117,7 @@ protected List<? extends Module> getModules()
115117
.in(LazySingleton.class);
116118

117119
binder.bind(QueryCountStatsProvider.class).to(AsyncQueryForwardingServlet.class).in(LazySingleton.class);
120+
MetricsModule.register(binder, RouterHttpClientMonitor.class);
118121
binder.bind(JettyServerInitializer.class).to(RouterJettyServerInitializer.class).in(LazySingleton.class);
119122

120123
Jerseys.addResource(binder, RouterResource.class);

0 commit comments

Comments
 (0)