Skip to content

Commit e3033dd

Browse files
committed
fix(bigquery-analytics): make GCS offload projection-aware (google#321)
Content parsing/offload ran before row projection, so denying content_parts (which holds the offload object reference) could still upload the payload to GCS with no retained reference -- a payload leak + cost. And denying both content and content_parts still did the full parse/offload for a row that keeps neither payload column. - When content_parts is denied, do not construct the GCS offloader (large / binary content is kept inline + truncated instead of uploaded); log a warning so the disabled offload is visible. - When both content and content_parts are denied, skip content parsing entirely (no inline summary, no parts, no offload). Adds 2 tests asserting the storage upload mock is not called for payload_column_denylist=["content_parts"] and ["content","content_parts"] with gcs_bucket_name set. Full plugin suite: 294 passed, 6 skipped.
1 parent 9e7cbe9 commit e3033dd

2 files changed

Lines changed: 111 additions & 14 deletions

File tree

src/google/adk/plugins/bigquery_agent_analytics_plugin.py

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2545,14 +2545,28 @@ async def _lazy_setup(self, **kwargs) -> None:
25452545

25462546
self.offloader = None
25472547
if self.config.gcs_bucket_name:
2548-
self.offloader = GCSOffloader(
2549-
self.project_id,
2550-
self.config.gcs_bucket_name,
2551-
self._executor,
2552-
storage_client=storage.Client(
2553-
project=self.project_id, credentials=self._credentials
2554-
),
2555-
)
2548+
if "content_parts" in self._denied_columns:
2549+
# #321: GCS offload stores its object reference in the
2550+
# ``content_parts`` column. With ``content_parts`` projected out,
2551+
# an upload would be orphaned -- payload leaks to GCS and incurs
2552+
# cost with no retained reference. Disable offload and keep
2553+
# content inline (truncated) instead.
2554+
logger.warning(
2555+
"GCS offload disabled: payload_column_denylist drops"
2556+
" 'content_parts', which holds the offloaded object reference;"
2557+
" large/binary content is kept inline (truncated) instead of"
2558+
" being uploaded to %s.",
2559+
self.config.gcs_bucket_name,
2560+
)
2561+
else:
2562+
self.offloader = GCSOffloader(
2563+
self.project_id,
2564+
self.config.gcs_bucket_name,
2565+
self._executor,
2566+
storage_client=storage.Client(
2567+
project=self.project_id, credentials=self._credentials
2568+
),
2569+
)
25562570

25572571
self.parser = HybridContentParser(
25582572
self.offloader,
@@ -3419,12 +3433,18 @@ async def _log_event(
34193433
logger.warning("Parser not initialized; skipping event %s.", event_type)
34203434
return
34213435

3422-
# Update parser's trace/span IDs for GCS pathing (reuse instance)
3423-
self.parser.trace_id = trace_id or "no_trace"
3424-
self.parser.span_id = span_id or "no_span"
3425-
content_json, content_parts, parser_truncated = await self.parser.parse(
3426-
raw_content
3427-
)
3436+
# #321: when both payload columns are projected out, skip content parsing
3437+
# entirely -- no inline summary, no parts, and (critically) no GCS offload
3438+
# work for a row that retains neither payload column.
3439+
if {"content", "content_parts"} <= self._denied_columns:
3440+
content_json, content_parts, parser_truncated = None, [], False
3441+
else:
3442+
# Update parser's trace/span IDs for GCS pathing (reuse instance)
3443+
self.parser.trace_id = trace_id or "no_trace"
3444+
self.parser.span_id = span_id or "no_span"
3445+
content_json, content_parts, parser_truncated = await self.parser.parse(
3446+
raw_content
3447+
)
34283448
is_truncated = is_truncated or parser_truncated
34293449

34303450
latency_json = self._extract_latency(event_data)

tests/unittests/plugins/test_bigquery_agent_analytics_plugin.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9210,3 +9210,80 @@ def test_enrich_attributes_skips_otel_when_attributes_denied(callback_context):
92109210
bigquery_agent_analytics_plugin.EventData(), callback_context
92119211
)
92129212
assert "otel" not in attrs
9213+
9214+
9215+
@pytest.mark.asyncio
9216+
async def test_content_parts_denied_disables_gcs_offload(
9217+
mock_write_client,
9218+
callback_context,
9219+
mock_auth_default,
9220+
mock_bq_client,
9221+
mock_to_arrow_schema,
9222+
dummy_arrow_schema,
9223+
mock_storage_client,
9224+
):
9225+
# #321: denying content_parts (which holds the offload object reference)
9226+
# must disable GCS offload, otherwise the payload is uploaded with no
9227+
# retained reference (leak + cost).
9228+
config = bigquery_agent_analytics_plugin.BigQueryLoggerConfig(
9229+
gcs_bucket_name="test-bucket",
9230+
payload_column_denylist=["content_parts"],
9231+
)
9232+
async with managed_plugin(
9233+
PROJECT_ID, DATASET_ID, table_id=TABLE_ID, config=config
9234+
) as plugin:
9235+
await plugin._ensure_started(
9236+
storage_client=mock_storage_client.return_value
9237+
)
9238+
assert plugin.offloader is None
9239+
mock_blob = (
9240+
mock_storage_client.return_value.bucket.return_value.blob.return_value
9241+
)
9242+
large_text = "A" * (32 * 1024 + 1)
9243+
llm_request = llm_request_lib.LlmRequest(
9244+
model="gemini-pro",
9245+
contents=[types.Content(parts=[types.Part(text=large_text)])],
9246+
)
9247+
await plugin.before_model_callback(
9248+
callback_context=callback_context, llm_request=llm_request
9249+
)
9250+
await plugin.flush()
9251+
mock_blob.upload_from_string.assert_not_called()
9252+
9253+
9254+
@pytest.mark.asyncio
9255+
async def test_both_payload_columns_denied_skips_parse_and_offload(
9256+
mock_write_client,
9257+
callback_context,
9258+
mock_auth_default,
9259+
mock_bq_client,
9260+
mock_to_arrow_schema,
9261+
dummy_arrow_schema,
9262+
mock_storage_client,
9263+
):
9264+
# #321: with both content and content_parts denied, parsing is skipped
9265+
# entirely -- no inline summary, no parts, and no GCS upload work.
9266+
config = bigquery_agent_analytics_plugin.BigQueryLoggerConfig(
9267+
gcs_bucket_name="test-bucket",
9268+
payload_column_denylist=["content", "content_parts"],
9269+
)
9270+
async with managed_plugin(
9271+
PROJECT_ID, DATASET_ID, table_id=TABLE_ID, config=config
9272+
) as plugin:
9273+
await plugin._ensure_started(
9274+
storage_client=mock_storage_client.return_value
9275+
)
9276+
assert plugin.offloader is None
9277+
mock_blob = (
9278+
mock_storage_client.return_value.bucket.return_value.blob.return_value
9279+
)
9280+
large_text = "A" * (32 * 1024 + 1)
9281+
llm_request = llm_request_lib.LlmRequest(
9282+
model="gemini-pro",
9283+
contents=[types.Content(parts=[types.Part(text=large_text)])],
9284+
)
9285+
await plugin.before_model_callback(
9286+
callback_context=callback_context, llm_request=llm_request
9287+
)
9288+
await plugin.flush()
9289+
mock_blob.upload_from_string.assert_not_called()

0 commit comments

Comments
 (0)