Skip to content
This repository was archived by the owner on May 8, 2026. It is now read-only.

Commit 45e1137

Browse files
mutianfmutianf
committed
fix broken outstanding rpc metrics
1 parent c0f6ebf commit 45e1137

3 files changed

Lines changed: 89 additions & 11 deletions

File tree

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableExporterUtils.java‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static com.google.api.MetricDescriptor.ValueType.DISTRIBUTION;
2626
import static com.google.api.MetricDescriptor.ValueType.DOUBLE;
2727
import static com.google.api.MetricDescriptor.ValueType.INT64;
28+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
2829
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BIGTABLE_CLIENT_METRICS;
2930
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BIGTABLE_PROJECT_ID_KEY;
3031
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_UID_KEY;
@@ -99,6 +100,14 @@ class BigtableExporterUtils {
99100
ImmutableSet.of(
100101
BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, TABLE_ID_KEY, CLUSTER_ID_KEY, ZONE_ID_KEY);
101102

103+
// These labels are defined on the Bigtable client resource. It's hard coded when we create the
104+
// internal exporter for connection level metrics. For request level metrics, these attributes
105+
// are updated with the actual value from the request and we use them to update the monitored
106+
// schema. This is needed for BigtableDataClientFactory, when the otel instance is created with
107+
// one instance / app profile but the actual call is on a different instance / app profile.
108+
private static final Set<AttributeKey<String>> BIGTABLE_CLIENT_RESOURCE_LABEL =
109+
ImmutableSet.of(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY);
110+
102111
private static final Map<GCPPlatformDetector.SupportedPlatform, String> SUPPORTED_PLATFORM_MAP =
103112
ImmutableMap.of(
104113
GCPPlatformDetector.SupportedPlatform.GOOGLE_COMPUTE_ENGINE, "gcp_compute_engine",
@@ -318,7 +327,9 @@ private static Optional<TimeSeries> createInternalMetricsTimeSeries(
318327
// - the useless views should be removed
319328
// - internal metrics should use relative metric names w/o the prefix
320329
if (BIGTABLE_CLIENT_METRICS.contains(metricData.getName())) {
321-
metricBuilder = newApplicationMetricBuilder(metricData.getName(), pointData.getAttributes());
330+
metricBuilder =
331+
newApplicationMetricBuilder(
332+
metricData.getName(), pointData.getAttributes(), applicationResource);
322333
} else if (GRPC_METRICS.containsKey(metricData.getName())) {
323334
metricBuilder = newGrpcMetricBuilder(metricData.getName(), pointData.getAttributes());
324335
} else {
@@ -343,11 +354,17 @@ private static Optional<TimeSeries> createInternalMetricsTimeSeries(
343354
}
344355

345356
private static Metric.Builder newApplicationMetricBuilder(
346-
String metricName, Attributes attributes) {
357+
String metricName, Attributes attributes, MonitoredResource applicationResource) {
358+
MonitoredResource.Builder updatedResource = applicationResource.toBuilder();
347359
// TODO: unify handling of metric prefixes
348360
Metric.Builder metricBuilder = Metric.newBuilder().setType(metricName);
349361
for (Map.Entry<AttributeKey<?>, Object> e : attributes.asMap().entrySet()) {
350-
metricBuilder.putLabels(e.getKey().getKey(), String.valueOf(e.getValue()));
362+
AttributeKey<?> key = e.getKey();
363+
if (BIGTABLE_CLIENT_RESOURCE_LABEL.contains(key)) {
364+
updatedResource.putLabels(key.getKey(), String.valueOf(e.getValue()));
365+
} else {
366+
metricBuilder.putLabels(e.getKey().getKey(), String.valueOf(e.getValue()));
367+
}
351368
}
352369
return metricBuilder;
353370
}

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ public class BuiltinMetricsConstants {
5757
static final AttributeKey<String> TRANSPORT_REGION = AttributeKey.stringKey("transport_region");
5858
static final AttributeKey<String> TRANSPORT_ZONE = AttributeKey.stringKey("transport_zone");
5959
static final AttributeKey<String> TRANSPORT_SUBZONE = AttributeKey.stringKey("transport_subzone");
60+
static final AttributeKey<String> LB_POLICY_KEY = AttributeKey.stringKey("lb_policy");
6061

6162
// gRPC attribute keys
6263
// Note that these attributes keys from transformed from
@@ -280,22 +281,38 @@ public static Map<InstrumentSelector, View> getBigtableClientViews() {
280281
InstrumentType.HISTOGRAM,
281282
"1",
282283
ImmutableSet.<AttributeKey>builder()
283-
.add(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY, CLIENT_NAME_KEY)
284+
.add(
285+
BIGTABLE_PROJECT_ID_KEY,
286+
INSTANCE_ID_KEY,
287+
APP_PROFILE_KEY,
288+
TRANSPORT_TYPE,
289+
STREAMING_KEY,
290+
LB_POLICY_KEY)
284291
.build());
285292
defineView(
286293
views,
287294
BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME,
288295
null,
289296
InstrumentType.GAUGE,
290297
"1",
291-
ImmutableSet.<AttributeKey>builder().add(METHOD_KEY).build());
298+
ImmutableSet.<AttributeKey>builder()
299+
.add(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY, METHOD_KEY)
300+
.build());
292301
defineView(
293302
views,
294303
BATCH_WRITE_FLOW_CONTROL_FACTOR_NAME,
295304
AGGREGATION_BATCH_WRITE_FLOW_CONTROL_FACTOR_HISTOGRAM,
296305
InstrumentType.HISTOGRAM,
297306
"1",
298-
ImmutableSet.<AttributeKey>builder().add(STATUS_KEY, APPLIED_KEY, METHOD_KEY).build());
307+
ImmutableSet.<AttributeKey>builder()
308+
.add(
309+
BIGTABLE_PROJECT_ID_KEY,
310+
INSTANCE_ID_KEY,
311+
APP_PROFILE_KEY,
312+
STATUS_KEY,
313+
APPLIED_KEY,
314+
METHOD_KEY)
315+
.build());
299316
return views.build();
300317
}
301318

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,17 @@
1717

1818
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APPLICATION_BLOCKING_LATENCIES_NAME;
1919
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APPLIED_KEY;
20+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.APP_PROFILE_KEY;
2021
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.ATTEMPT_LATENCIES_NAME;
2122
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BATCH_WRITE_FLOW_CONTROL_FACTOR_NAME;
2223
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME;
24+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.BIGTABLE_PROJECT_ID_KEY;
2325
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_BLOCKING_LATENCIES_NAME;
2426
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLIENT_NAME_KEY;
2527
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CLUSTER_ID_KEY;
2628
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CONNECTIVITY_ERROR_COUNT_NAME;
2729
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.FIRST_RESPONSE_LATENCIES_NAME;
30+
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.INSTANCE_ID_KEY;
2831
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METHOD_KEY;
2932
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME;
3033
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME;
@@ -832,7 +835,13 @@ public void testBatchWriteFlowControlTargetQpsIncreased() throws InterruptedExce
832835

833836
MetricData targetQpsMetric =
834837
getMetricData(metricReader, BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME);
835-
Attributes targetQpsAttributes = Attributes.of(METHOD_KEY, "Bigtable.MutateRows");
838+
Attributes targetQpsAttributes =
839+
Attributes.builder()
840+
.put(METHOD_KEY, "Bigtable.MutateRows")
841+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
842+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
843+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
844+
.build();
836845
double actual_qps = getAggregatedDoubleValue(targetQpsMetric, targetQpsAttributes);
837846
double expected_qps = 12;
838847
assertThat(expected_qps).isEqualTo(actual_qps);
@@ -843,6 +852,9 @@ public void testBatchWriteFlowControlTargetQpsIncreased() throws InterruptedExce
843852
.put(METHOD_KEY, "Bigtable.MutateRows")
844853
.put(APPLIED_KEY, true)
845854
.put(STATUS_KEY, "OK")
855+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
856+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
857+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
846858
.build();
847859
double actual_factor_mean = getAggregatedDoubleValue(factorMetric, factorAttributes);
848860
double expected_factor_mean = 1.2;
@@ -862,7 +874,12 @@ public void testBatchWriteFlowControlTargetQpsDecreased() throws InterruptedExce
862874
MetricData targetQpsMetric =
863875
getMetricData(metricReader, BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME);
864876
Attributes targetQpsAttributes =
865-
Attributes.builder().put(METHOD_KEY, "Bigtable.MutateRows").build();
877+
Attributes.builder()
878+
.put(METHOD_KEY, "Bigtable.MutateRows")
879+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
880+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
881+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
882+
.build();
866883
double actual_qps = getAggregatedDoubleValue(targetQpsMetric, targetQpsAttributes);
867884
double expected_qps = 8.0;
868885
assertThat(expected_qps).isEqualTo(actual_qps);
@@ -873,6 +890,9 @@ public void testBatchWriteFlowControlTargetQpsDecreased() throws InterruptedExce
873890
.put(METHOD_KEY, "Bigtable.MutateRows")
874891
.put(APPLIED_KEY, true)
875892
.put(STATUS_KEY, "OK")
893+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
894+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
895+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
876896
.build();
877897
double actual_factor_mean = getAggregatedDoubleValue(factorMetric, factorAttributes);
878898
double expected_factor_mean = 0.8;
@@ -892,7 +912,12 @@ public void testBatchWriteFlowControlTargetQpsCappedOnMaxFactor() throws Interru
892912
MetricData targetQpsMetric =
893913
getMetricData(metricReader, BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME);
894914
Attributes targetQpsAttributes =
895-
Attributes.builder().put(METHOD_KEY, "Bigtable.MutateRows").build();
915+
Attributes.builder()
916+
.put(METHOD_KEY, "Bigtable.MutateRows")
917+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
918+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
919+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
920+
.build();
896921
double actual_qps = getAggregatedDoubleValue(targetQpsMetric, targetQpsAttributes);
897922
// Factor is 1.8 but capped at 1.3 so updated QPS is 13.
898923
double expected_qps = 13;
@@ -904,6 +929,9 @@ public void testBatchWriteFlowControlTargetQpsCappedOnMaxFactor() throws Interru
904929
.put(METHOD_KEY, "Bigtable.MutateRows")
905930
.put(APPLIED_KEY, true)
906931
.put(STATUS_KEY, "OK")
932+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
933+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
934+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
907935
.build();
908936
double actual_factor_mean = getAggregatedDoubleValue(factorMetric, factorAttributes);
909937
// Factor is 1.8 but capped at 1.3
@@ -924,7 +952,12 @@ public void testBatchWriteFlowControlTargetQpsCappedOnMinFactor() throws Interru
924952
MetricData targetQpsMetric =
925953
getMetricData(metricReader, BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME);
926954
Attributes targetQpsAttributes =
927-
Attributes.builder().put(METHOD_KEY, "Bigtable.MutateRows").build();
955+
Attributes.builder()
956+
.put(METHOD_KEY, "Bigtable.MutateRows")
957+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
958+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
959+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
960+
.build();
928961
double actual_qps = getAggregatedDoubleValue(targetQpsMetric, targetQpsAttributes);
929962
// Factor is 0.5 but capped at 0.7 so updated QPS is 7.
930963
double expected_qps = 7;
@@ -936,6 +969,9 @@ public void testBatchWriteFlowControlTargetQpsCappedOnMinFactor() throws Interru
936969
.put(METHOD_KEY, "Bigtable.MutateRows")
937970
.put(APPLIED_KEY, true)
938971
.put(STATUS_KEY, "OK")
972+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
973+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
974+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
939975
.build();
940976
double actual_factor_mean = getAggregatedDoubleValue(factorMetric, factorAttributes);
941977
// Factor is 0.5 but capped at 0.7
@@ -957,7 +993,12 @@ public void testBatchWriteFlowControlTargetQpsDecreasedForError() throws Interru
957993
MetricData targetQpsMetric =
958994
getMetricData(metricReader, BATCH_WRITE_FLOW_CONTROL_TARGET_QPS_NAME);
959995
Attributes targetQpsAttributes =
960-
Attributes.builder().put(METHOD_KEY, "Bigtable.MutateRows").build();
996+
Attributes.builder()
997+
.put(METHOD_KEY, "Bigtable.MutateRows")
998+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
999+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
1000+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
1001+
.build();
9611002
double actual_qps = getAggregatedDoubleValue(targetQpsMetric, targetQpsAttributes);
9621003
// On error, min factor is applied.
9631004
double expected_qps = 7;
@@ -969,6 +1010,9 @@ public void testBatchWriteFlowControlTargetQpsDecreasedForError() throws Interru
9691010
.put(METHOD_KEY, "Bigtable.MutateRows")
9701011
.put(APPLIED_KEY, true)
9711012
.put(STATUS_KEY, "UNAVAILABLE")
1013+
.put(BIGTABLE_PROJECT_ID_KEY, PROJECT_ID)
1014+
.put(INSTANCE_ID_KEY, INSTANCE_ID)
1015+
.put(APP_PROFILE_KEY, APP_PROFILE_ID)
9721016
.build();
9731017
double actual_factor_mean = getAggregatedDoubleValue(factorMetric, factorAttributes);
9741018
// On error, min factor is applied.

0 commit comments

Comments
 (0)