1616package com .google .cloud .bigtable .gaxx .grpc ;
1717
1818import com .google .api .core .SettableApiFuture ;
19+ import com .google .auto .value .AutoValue ;
1920import com .google .bigtable .v2 .PingAndWarmResponse ;
20- import com .google .cloud .bigtable .data .v2 .stub .BigtableChannelPrimer ;
2121import com .google .cloud .bigtable .gaxx .grpc .BigtableChannelPool .Entry ;
2222import com .google .common .annotations .VisibleForTesting ;
2323import com .google .common .collect .ImmutableList ;
3737public class ChannelPoolHealthChecker {
3838
3939 // Configuration constants
40+ // Window_Duration is the duration over which we keep probe results
4041 private static final Duration WINDOW_DURATION = Duration .ofMinutes (5 );
41- static final Duration PROBE_RATE = Duration .ofSeconds (30 );
42+ // Interval at which we probe channel health
43+ static final Duration PROBE_INTERVAL = Duration .ofSeconds (30 );
44+ // Timeout deadline for a probe
4245 @ VisibleForTesting static final Duration PROBE_DEADLINE = Duration .ofMillis (500 );
46+ // Minimum interval between new idle channel evictions
4347 private static final Duration MIN_EVICTION_INTERVAL = Duration .ofMinutes (10 );
48+ // Minimum number of probes that must be sent to a channel before it will be considered for
49+ // eviction
4450 private static final int MIN_PROBES_FOR_EVALUATION = 4 ;
51+ // Percentage of probes that must fail for a channel to be considered unhealthy
4552 private static final int SINGLE_CHANNEL_FAILURE_PERCENT_THRESHOLD = 60 ;
53+ // "Circuitbreaker" - If this or a higher percentage of channels in a pool are bad, we will not
54+ // evict any channels
4655 private static final int POOLWIDE_BAD_CHANNEL_CIRCUITBREAKER_PERCENT = 70 ;
4756
4857 /** Inner class to represent the result of a single probe. */
49- static class ProbeResult {
50- final Instant startTime ;
51- final boolean success ;
58+ @ AutoValue
59+ abstract static class ProbeResult {
60+ abstract Instant startTime () ;
5261
53- ProbeResult (Instant startTime , boolean success ) {
54- this .startTime = startTime ;
55- this .success = success ;
56- }
62+ abstract boolean isSuccessful ();
5763
58- public boolean isSuccessful ( ) {
59- return success ;
64+ static ProbeResult create ( Instant startTime , boolean success ) {
65+ return new AutoValue_ChannelPoolHealthChecker_ProbeResult ( startTime , success ) ;
6066 }
6167 }
6268
63- // Class fields
6469 private final Supplier <ImmutableList <Entry >> entrySupplier ;
6570 private Instant lastEviction ;
6671 private ScheduledExecutorService executor ;
6772
68- private BigtableChannelPrimer channelPrimer ;
73+ private ChannelPrimer channelPrimer ;
6974
7075 private final Clock clock ;
7176
7277 /** Constructor for the pool health checker. */
7378 public ChannelPoolHealthChecker (
7479 Supplier <ImmutableList <Entry >> entrySupplier ,
75- BigtableChannelPrimer channelPrimer ,
80+ ChannelPrimer channelPrimer ,
7681 ScheduledExecutorService executor ,
7782 Clock clock ) {
7883 this .entrySupplier = entrySupplier ;
@@ -84,18 +89,18 @@ public ChannelPoolHealthChecker(
8489
8590 void start () {
8691 Duration initialDelayProbe =
87- Duration .ofMillis (ThreadLocalRandom .current ().nextLong (PROBE_RATE .toMillis ()));
92+ Duration .ofMillis (ThreadLocalRandom .current ().nextLong (PROBE_INTERVAL .toMillis ()));
8893 executor .scheduleAtFixedRate (
8994 this ::runProbes ,
9095 initialDelayProbe .toMillis (),
91- PROBE_RATE .toMillis (),
96+ PROBE_INTERVAL .toMillis (),
9297 TimeUnit .MILLISECONDS );
9398 Duration initialDelayDetect =
94- Duration .ofMillis (ThreadLocalRandom .current ().nextLong (PROBE_RATE .toMillis ()));
99+ Duration .ofMillis (ThreadLocalRandom .current ().nextLong (PROBE_INTERVAL .toMillis ()));
95100 executor .scheduleAtFixedRate (
96101 this ::detectAndRemoveOutlierEntries ,
97102 initialDelayDetect .toMillis (),
98- PROBE_RATE .toMillis (),
103+ PROBE_INTERVAL .toMillis (),
99104 TimeUnit .MILLISECONDS );
100105 }
101106
@@ -127,7 +132,7 @@ void onComplete(
127132 } catch (Exception e ) {
128133 success = false ;
129134 }
130- addProbeResult (entry , new ProbeResult (startTime , success ));
135+ addProbeResult (entry , ProbeResult . create (startTime , success ));
131136 }
132137
133138 @ VisibleForTesting
@@ -144,7 +149,7 @@ void addProbeResult(Entry entry, ProbeResult result) {
144149 void pruneHistoryFor (Entry entry ) {
145150 Instant windowStart = clock .instant ().minus (WINDOW_DURATION );
146151 while (!entry .probeHistory .isEmpty ()
147- && entry .probeHistory .peek ().startTime .isBefore (windowStart )) {
152+ && entry .probeHistory .peek ().startTime () .isBefore (windowStart )) {
148153 ProbeResult removedResult = entry .probeHistory .poll ();
149154 if (removedResult .isSuccessful ()) {
150155 entry .successfulProbesInWindow .decrementAndGet ();
@@ -157,8 +162,6 @@ void pruneHistoryFor(Entry entry) {
157162 /** Checks if a single entry is currently healthy based on its probe history. */
158163 @ VisibleForTesting
159164 boolean isEntryHealthy (Entry entry ) {
160- pruneHistoryFor (entry ); // Ensure window is current before calculation
161-
162165 int failedProbes = entry .failedProbesInWindow .get ();
163166 int totalProbes = failedProbes + entry .successfulProbesInWindow .get ();
164167
@@ -178,10 +181,6 @@ boolean isEntryHealthy(Entry entry) {
178181 @ Nullable
179182 @ VisibleForTesting
180183 Entry findOutlierEntry () {
181- if (lastEviction .plus (WINDOW_DURATION ).isAfter (clock .instant ())) {
182- return null ;
183- }
184-
185184 List <Entry > unhealthyEntries =
186185 this .entrySupplier .get ().stream ()
187186 .peek (this ::pruneHistoryFor )
0 commit comments