Skip to content

Commit 40ac6e4

Browse files
diegomarquezpdiegomarquezp
authored
impl(o11y): introduce url.full (#12209)
- Adds HTTP_URL_FULL_ATTRIBUTE to ObservabilityAttributes - Introduces requestUrlResolved into ApiTracer - Implements redaction logic in ObservabilityUtils - Passes tracer into HTTP/JSON transport to record the URL Note: Following Footnote 3, the spec indicates logging implementations MAY expose a configuration to override redacted fields. This implementation does not expose this configurability to provide a consistent and optimally safe baseline out of the box.
1 parent 965761a commit 40ac6e4

File tree

10 files changed

+257
-4
lines changed

10 files changed

+257
-4
lines changed

‎sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpRequestRunnable.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import com.google.api.client.json.JsonObjectParser;
4545
import com.google.api.client.json.gson.GsonFactory;
4646
import com.google.api.client.util.GenericData;
47+
import com.google.api.gax.tracing.ApiTracer;
4748
import com.google.auth.Credentials;
4849
import com.google.auth.http.HttpCredentialsAdapter;
4950
import com.google.auto.value.AutoValue;
@@ -188,6 +189,11 @@ HttpRequest createHttpRequest() throws IOException {
188189
}
189190
}
190191

192+
ApiTracer tracer = httpJsonCallOptions.getTracer();
193+
if (tracer != null) {
194+
tracer.requestUrlResolved(url.build());
195+
}
196+
191197
HttpRequest httpRequest = buildRequest(requestFactory, url, jsonHttpContent);
192198

193199
for (Map.Entry<String, Object> entry : headers.getHeaders().entrySet()) {

‎sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpRequestRunnableTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.google.api.client.http.EmptyContent;
3333
import com.google.api.client.http.HttpRequest;
3434
import com.google.api.client.testing.http.MockHttpTransport;
35+
import com.google.api.gax.tracing.ApiTracer;
3536
import com.google.common.truth.Truth;
3637
import com.google.longrunning.ListOperationsRequest;
3738
import com.google.protobuf.Empty;
@@ -123,6 +124,32 @@ void testRequestUrl() throws IOException {
123124
Truth.assertThat(httpRequest.getUrl().toString()).isEqualTo(expectedUrl);
124125
}
125126

127+
@Test
128+
void testApiTracerRequestUrlResolved() throws IOException {
129+
ApiTracer tracer = Mockito.mock(ApiTracer.class);
130+
ApiMethodDescriptor<Field, Empty> methodDescriptor =
131+
ApiMethodDescriptor.<Field, Empty>newBuilder()
132+
.setFullMethodName("house.cat.get")
133+
.setHttpMethod(null)
134+
.setRequestFormatter(requestFormatter)
135+
.setResponseParser(responseParser)
136+
.build();
137+
138+
HttpRequestRunnable<Field, Empty> httpRequestRunnable =
139+
new HttpRequestRunnable<>(
140+
requestMessage,
141+
methodDescriptor,
142+
ENDPOINT,
143+
HttpJsonCallOptions.newBuilder().setTracer(tracer).build(),
144+
new MockHttpTransport(),
145+
HttpJsonMetadata.newBuilder().build(),
146+
(result) -> {});
147+
148+
httpRequestRunnable.createHttpRequest();
149+
String expectedUrl = ENDPOINT + "/name/feline" + "?food=bird&food=mouse&size=small";
150+
Mockito.verify(tracer).requestUrlResolved(expectedUrl);
151+
}
152+
126153
@Test
127154
void testRequestUrlUnnormalized() throws IOException {
128155
ApiMethodDescriptor<Field, Empty> methodDescriptor =

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,14 @@ default void requestSent() {}
196196
default void batchRequestSent(long elementCount, long requestSize) {}
197197
;
198198

199+
/**
200+
* Annotates the attempt with the full resolved HTTP URL. Only relevant for HTTP transport.
201+
*
202+
* @param requestUrl the full URL of the request
203+
*/
204+
default void requestUrlResolved(String requestUrl) {}
205+
;
206+
199207
/**
200208
* A context class to be used with {@link #inScope()} and a try-with-resources block. Closing a
201209
* {@link Scope} removes any context that the underlying implementation might've set in {@link

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ public class ObservabilityAttributes {
9797
/** The destination resource id of the request (e.g. projects/p/locations/l/topics/t). */
9898
public static final String DESTINATION_RESOURCE_ID_ATTRIBUTE = "gcp.resource.destination.id";
9999

100+
/** The full URL of the HTTP request, with sensitive query parameters redacted. */
101+
public static final String HTTP_URL_FULL_ATTRIBUTE = "url.full";
102+
100103
/** The type of error that occurred (e.g., from google.rpc.ErrorInfo.reason). */
101104
public static final String ERROR_TYPE_ATTRIBUTE = "error.type";
102105

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java‎

Lines changed: 118 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,132 @@
3131

3232
import com.google.api.gax.rpc.ApiException;
3333
import com.google.api.gax.rpc.StatusCode;
34+
import com.google.common.base.Joiner;
35+
import com.google.common.base.Splitter;
36+
import com.google.common.collect.ImmutableSet;
3437
import com.google.rpc.ErrorInfo;
3538
import io.opentelemetry.api.common.Attributes;
3639
import io.opentelemetry.api.common.AttributesBuilder;
3740
import java.util.Map;
3841
import java.util.concurrent.CancellationException;
3942
import javax.annotation.Nullable;
4043

41-
class ObservabilityUtils {
44+
final class ObservabilityUtils {
4245

43-
/** Function to extract the status of the error as a string */
44-
static String extractStatus(@Nullable Throwable error) {
46+
private ObservabilityUtils() {}
47+
48+
/** Constant for redacted values. */
49+
private static final String REDACTED_VALUE = "REDACTED";
50+
51+
/**
52+
* A set of lowercase query parameter keys whose values should be redacted in URLs for
53+
* observability. These include direct credentials (access keys), cryptographic signatures (to
54+
* prevent replay attacks or leak of authorization), and session identifiers (like upload_id).
55+
*/
56+
private static final ImmutableSet<String> SENSITIVE_QUERY_KEYS =
57+
ImmutableSet.of(
58+
"awsaccesskeyid", // AWS S3-compatible access keys
59+
"signature", // General cryptographic signature
60+
"sig", // General cryptographic signature (abbreviated)
61+
"x-goog-signature", // Google Cloud specific signature
62+
"upload_id", // Resumable upload session identifiers
63+
"access_token", // OAuth2 explicit tokens
64+
"key", // API Keys
65+
"api_key"); // API Keys
66+
67+
/**
68+
* Sanitizes an HTTP URL by redacting sensitive query parameters and credentials in the user-info
69+
* component. If the provided URL cannot be parsed (e.g. invalid syntax), it returns the original
70+
* string.
71+
*
72+
* <p>This sanitization process conforms to the recommendations in footnote 3 of the OpenTelemetry
73+
* semantic conventions for HTTP URL attributes:
74+
* https://opentelemetry.io/docs/specs/semconv/registry/attributes/url/
75+
*
76+
* <ul>
77+
* <li><i>"url.full MUST NOT contain credentials passed via URL in form of
78+
* https://user:pass@example.com/. In such case username and password SHOULD be redacted and
79+
* attribute's value SHOULD be https://REDACTED:REDACTED@example.com/."</i> - Handled by
80+
* stripping the raw user info component.
81+
* <li><i>"url.full SHOULD capture the absolute URL when it is available (or can be
82+
* reconstructed)."</i> - Handled by parsing and rebuilding the generic URI.
83+
* <li><i>"When a query string value is redacted, the query string key SHOULD still be
84+
* preserved, e.g. https://www.example.com/path?color=blue&sig=REDACTED."</i> - Handled by
85+
* the redactSensitiveQueryValues method.
86+
* </ul>
87+
*
88+
* @param url the raw URL string
89+
* @return the sanitized URL string, or the original if unparsable
90+
*/
91+
static String sanitizeUrlFull(final String url) {
92+
try {
93+
java.net.URI uri = new java.net.URI(url);
94+
String sanitizedUserInfo =
95+
uri.getRawUserInfo() != null ? REDACTED_VALUE + ":" + REDACTED_VALUE : null;
96+
String sanitizedQuery = redactSensitiveQueryValues(uri.getRawQuery());
97+
java.net.URI sanitizedUri =
98+
new java.net.URI(
99+
uri.getScheme(),
100+
sanitizedUserInfo,
101+
uri.getHost(),
102+
uri.getPort(),
103+
uri.getRawPath(),
104+
sanitizedQuery,
105+
uri.getRawFragment());
106+
return sanitizedUri.toString();
107+
} catch (java.net.URISyntaxException | IllegalArgumentException ex) {
108+
return "";
109+
}
110+
}
111+
112+
/**
113+
* Redacts the values of sensitive keys within a raw URI query string.
114+
*
115+
* <p>This logic splits the query string by the {@code &} delimiter without full URL decoding,
116+
* ensures only values belonging to predefined sensitive keys are replaced with {@code
117+
* REDACTED_VALUE}. The check is strictly case-insensitive.
118+
*
119+
* <p>Note regarding Footnote 3: The OpenTelemetry spec recommends case-sensitive matching for
120+
* query parameters. However, we intentionally utilize case-insensitive matching (by lowercasing
121+
* all query keys) to prevent credentials bypassing validation when sent with mixed casings (e.g.,
122+
* Sig=..., API_KEY=...).
123+
*
124+
* @param rawQuery the raw query string from a java.net.URI
125+
* @return a reconstructed query sequence with sensitive values redacted
126+
*/
127+
private static String redactSensitiveQueryValues(final String rawQuery) {
128+
if (rawQuery == null || rawQuery.isEmpty()) {
129+
return rawQuery;
130+
}
131+
132+
java.util.List<String> redactedParams =
133+
Splitter.on('&').splitToList(rawQuery).stream()
134+
.map(
135+
param -> {
136+
int equalsIndex = param.indexOf('=');
137+
if (equalsIndex < 0) {
138+
return param;
139+
}
140+
String key = param.substring(0, equalsIndex);
141+
// Case-insensitive match utilizing the fact that all
142+
// predefined keys are in lowercase
143+
if (SENSITIVE_QUERY_KEYS.contains(key.toLowerCase())) {
144+
return key + "=" + REDACTED_VALUE;
145+
}
146+
return param;
147+
})
148+
.collect(java.util.stream.Collectors.toList());
149+
150+
return Joiner.on('&').join(redactedParams);
151+
}
152+
153+
/**
154+
* Function to extract the status of the error as a string.
155+
*
156+
* @param error the thrown throwable error
157+
* @return the extracted status string
158+
*/
159+
static String extractStatus(@Nullable final Throwable error) {
45160
final String statusString;
46161

47162
if (error == null) {

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,4 +204,16 @@ private void endAttempt() {
204204
attemptSpan = null;
205205
}
206206
}
207+
208+
@Override
209+
public void requestUrlResolved(String url) {
210+
if (attemptSpan == null) {
211+
return;
212+
}
213+
String sanitizedUrlString = ObservabilityUtils.sanitizeUrlFull(url);
214+
if (sanitizedUrlString.isEmpty()) {
215+
return;
216+
}
217+
attemptSpan.setAttribute(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, sanitizedUrlString);
218+
}
207219
}

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/retrying/RetryAlgorithmTest.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,9 @@ void testCreateFirstAttemptWithUnusedContext() {
6464
void testCreateFirstAttemptWithContext() {
6565
TimedRetryAlgorithmWithContext timedAlgorithm = mock(TimedRetryAlgorithmWithContext.class);
6666
RetryAlgorithm<Void> algorithm =
67-
new RetryAlgorithm<>(mock(ResultRetryAlgorithmWithContext.class), timedAlgorithm);
67+
new RetryAlgorithm<>(
68+
(ResultRetryAlgorithmWithContext<Void>) mock(ResultRetryAlgorithmWithContext.class),
69+
(TimedRetryAlgorithmWithContext) timedAlgorithm);
6870

6971
RetryingContext context = mock(RetryingContext.class);
7072
algorithm.createFirstAttempt(context);

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/ObservabilityUtilsTest.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,43 @@ void testToOtelAttributes_shouldMapIntAttributes() {
118118
void testToOtelAttributes_shouldReturnEmptyAttributes_nullInput() {
119119
assertThat(ObservabilityUtils.toOtelAttributes(null)).isEqualTo(Attributes.empty());
120120
}
121+
122+
@Test
123+
void testSanitizeUrlFull_redactsUserInfo() {
124+
String url = "https://user:password@example.com/some/path?foo=bar";
125+
String sanitized = ObservabilityUtils.sanitizeUrlFull(url);
126+
assertThat(sanitized).isEqualTo("https://REDACTED:REDACTED@example.com/some/path?foo=bar");
127+
}
128+
129+
@Test
130+
void testSanitizeUrlFull_redactsSensitiveQueryParameters_caseInsensitive() {
131+
String url =
132+
"https://example.com/some/path?upload_Id=secret&AWSAccessKeyId=123&foo=bar&API_KEY=my_key";
133+
String sanitized = ObservabilityUtils.sanitizeUrlFull(url);
134+
assertThat(sanitized)
135+
.isEqualTo(
136+
"https://example.com/some/path?upload_Id=REDACTED&AWSAccessKeyId=REDACTED&foo=bar&API_KEY=REDACTED");
137+
}
138+
139+
@Test
140+
void testSanitizeUrlFull_handlesKeyOnlyParameters() {
141+
String url = "https://example.com/some/path?api_key&foo=bar";
142+
String sanitized = ObservabilityUtils.sanitizeUrlFull(url);
143+
assertThat(sanitized).isEqualTo("https://example.com/some/path?api_key&foo=bar");
144+
}
145+
146+
@Test
147+
void testSanitizeUrlFull_handlesMalformedUrl() {
148+
String url = "invalid::url:";
149+
String sanitized = ObservabilityUtils.sanitizeUrlFull(url);
150+
// Unparsable URLs should be returned as empty string
151+
assertThat(sanitized).isEmpty();
152+
}
153+
154+
@Test
155+
void testSanitizeUrlFull_noQueryOrUserInfo() {
156+
String url = "https://example.com/some/path";
157+
String sanitized = ObservabilityUtils.sanitizeUrlFull(url);
158+
assertThat(sanitized).isEqualTo(url);
159+
}
121160
}

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
import static com.google.common.truth.Truth.assertThat;
3333
import static org.mockito.ArgumentMatchers.any;
3434
import static org.mockito.ArgumentMatchers.anyString;
35+
import static org.mockito.ArgumentMatchers.eq;
36+
import static org.mockito.Mockito.never;
3537
import static org.mockito.Mockito.verify;
3638
import static org.mockito.Mockito.when;
3739

@@ -246,4 +248,28 @@ void testAttemptStarted_retryAttributes_http() {
246248
ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE),
247249
5L);
248250
}
251+
252+
@Test
253+
void testRequestUrlResolved_setsAttribute() {
254+
spanTracer.attemptStarted(new Object(), 1);
255+
256+
String rawUrl = "https://example.com?api_key=secret";
257+
spanTracer.requestUrlResolved(rawUrl);
258+
259+
verify(span)
260+
.setAttribute(
261+
ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE,
262+
"https://example.com?api_key=REDACTED");
263+
}
264+
265+
@Test
266+
void testRequestUrlResolved_badUrl_notSet() {
267+
spanTracer.attemptStarted(new Object(), 1);
268+
269+
String rawUrl = "htps:::://the-example";
270+
spanTracer.requestUrlResolved(rawUrl);
271+
272+
verify(span, never())
273+
.setAttribute(eq(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE), anyString());
274+
}
249275
}

‎sdk-platform-java/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import com.google.rpc.Status;
4747
import com.google.showcase.v1beta1.EchoClient;
4848
import com.google.showcase.v1beta1.EchoRequest;
49+
import com.google.showcase.v1beta1.EchoResponse;
4950
import com.google.showcase.v1beta1.EchoSettings;
5051
import com.google.showcase.v1beta1.it.util.TestClientInitializer;
5152
import com.google.showcase.v1beta1.stub.EchoStub;
@@ -69,6 +70,7 @@ class ITOtelTracing {
6970
private static final long SHOWCASE_SERVER_PORT = 7469;
7071
private static final String SHOWCASE_REPO = "googleapis/sdk-platform-java";
7172
private static final String SHOWCASE_ARTIFACT = "com.google.cloud:gapic-showcase";
73+
private static final String SHOWCASE_USER_URL = "http://localhost:7469/v1beta1/echo:echo";
7274

7375
private InMemorySpanExporter spanExporter;
7476
private OpenTelemetrySdk openTelemetrySdk;
@@ -206,6 +208,19 @@ void testTracing_successfulEcho_httpjson() throws Exception {
206208
.getAttributes()
207209
.get(AttributeKey.stringKey(ObservabilityAttributes.HTTP_URL_TEMPLATE_ATTRIBUTE)))
208210
.isEqualTo("v1beta1/echo:echo");
211+
assertThat(
212+
attemptSpan
213+
.getAttributes()
214+
.get(AttributeKey.stringKey(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE)))
215+
.isEqualTo(SHOWCASE_USER_URL);
216+
EchoResponse fetchedEcho = EchoResponse.newBuilder().setContent("tracing-test").build();
217+
long expectedMagnitude = computeExpectedHttpJsonResponseSize(fetchedEcho);
218+
Long observedMagnitude =
219+
attemptSpan
220+
.getAttributes()
221+
.get(AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_BODY_SIZE));
222+
assertThat(observedMagnitude).isNotNull();
223+
assertThat(observedMagnitude).isAtLeast((long) (expectedMagnitude * (1 - 0.15)));
209224
assertThat(attemptSpan.getInstrumentationScopeInfo().getName()).isEqualTo(SHOWCASE_ARTIFACT);
210225
}
211226
}

0 commit comments

Comments
 (0)