This repository was archived by the owner on Mar 13, 2026. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 129
feat: support biglake tables in pandas_gbq.sample #1014
Merged
Merged
Changes from 3 commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
8784664
feat: support biglake tables in pandas_gbq.sample
tswast 1164fcc
Merge remote-tracking branch 'origin/main' into tswast-biglake-sample
tswast dc62471
chore: fix biglake implementation
tswast cbbff09
test: add tests
tswast c57a30e
fix: regex
tswast c0cb0e5
add tests
tswast 7723f0d
chore: lint
tswast 686193e
chore: more lint
tswast a848d69
chore: more lint
tswast 53e714d
chore: fix docs build
tswast 8172803
test: improve coverage
tswast 482a644
lint
tswast 2dff7cb
chore: adjust coverage target
tswast e3cf528
chore: adjust coverage target
tswast 4fc4a28
chore: adjust coverage target
tswast File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,6 +51,7 @@ docs.metadata | |
| # Virtual environment | ||
| env/ | ||
| venv/ | ||
| .venv/ | ||
|
|
||
| # Test logs | ||
| coverage.xml | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| # Copyright (c) 2026 pandas-gbq Authors All rights reserved. | ||
| # Use of this source code is governed by a BSD-style | ||
| # license that can be found in the LICENSE file. | ||
|
|
||
| """ | ||
| Utilities for working with BigLake tables. | ||
| """ | ||
|
|
||
| # TODO(tswast): Synchronize with bigframes/session/iceberg.py, which uses | ||
| # pyiceberg and the BigLake APIs, rather than relying on dry run. | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import dataclasses | ||
| from typing import Sequence | ||
|
|
||
| import google.auth.transport.requests | ||
| import google.cloud.bigquery | ||
| import google.oauth2.credentials | ||
|
|
||
| import pandas_gbq.core.resource_references | ||
|
|
||
|
|
||
| _DRY_RUN_TEMPLATE = """ | ||
| SELECT * | ||
| FROM `{project}.{catalog}.{namespace}.{table}` | ||
| """ | ||
|
|
||
|
|
||
| _COUNT_TEMPLATE = """ | ||
| SELECT COUNT(*) as total_rows | ||
| FROM `{project}.{catalog}.{namespace}.{table}` | ||
| """ | ||
|
|
||
| @dataclasses.dataclass(frozen=True) | ||
| class BigLakeTableMetadata: | ||
| schema: Sequence[google.cloud.bigquery.SchemaField] | ||
| num_rows: int | ||
|
|
||
|
|
||
| def get_table_metadata( | ||
| *, | ||
| reference: pandas_gbq.core.resource_references.BigLakeTableId, | ||
| bqclient: google.cloud.bigquery.Client, | ||
| ) -> BigLakeTableMetadata: | ||
| """ | ||
| Get the schema for a BigLake table. | ||
|
|
||
| Currently, this does some BigQuery queries. In the future, we'll want to get | ||
| other metadata like the number of rows and storage bytes so that we can do a | ||
| more accurate estimate of how many rows to sample. | ||
| """ | ||
| dry_run_config = google.cloud.bigquery.QueryJobConfig(dry_run=True) | ||
| query = _DRY_RUN_TEMPLATE.format( | ||
| project=reference.project, | ||
| catalog=reference.catalog, | ||
| namespace=".".join(reference.namespace), | ||
| table=reference.table, | ||
| ) | ||
| job = bqclient.query(query, job_config=dry_run_config) | ||
| job.result() | ||
| schema = job.schema | ||
|
|
||
| count_rows = list(bqclient.query_and_wait(_COUNT_TEMPLATE.format( | ||
| project=reference.project, | ||
| catalog=reference.catalog, | ||
| namespace=".".join(reference.namespace), | ||
| table=reference.table, | ||
| ))) | ||
| assert len(count_rows) == 1, "got unexpected query response when determining number of rows" | ||
| total_rows = count_rows[0].total_rows | ||
|
|
||
| return BigLakeTableMetadata( | ||
| schema=schema if schema is not None else [], | ||
| num_rows=total_rows, | ||
| ) | ||
|
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| # Copyright (c) 2026 pandas-gbq Authors All rights reserved. | ||
| # Use of this source code is governed by a BSD-style | ||
| # license that can be found in the LICENSE file. | ||
|
|
||
| import dataclasses | ||
| import re | ||
|
|
||
|
|
||
| _TABLE_REFEREENCE_PATTERN = re.compile( | ||
| # In the past, organizations could prefix their project IDs with a domain | ||
| # name. Such projects still exist, especially at Google. | ||
| r"^(?P<legacy_project_domain>[^:]+:)?" | ||
| r"(?P<project>[^.]+)\." | ||
| # Dataset for native BigQuery tables, catalog + namespace(s) for BigLake. | ||
| r"(?P<inner_parts>([^.\s]+\.?)+)\." | ||
|
Check failure on line 15 in pandas_gbq/core/resource_references.py
|
||
| # Table names can't contain ".", as that's used as the separator. | ||
| r"(?P<table>[^.]+)$" | ||
| ) | ||
|
|
||
|
|
||
| @dataclasses.dataclass(frozen=True) | ||
| class BigLakeTableId: | ||
| project: str | ||
| catalog: str | ||
| namespace: tuple[str, ...] | ||
| table: str | ||
|
|
||
|
|
||
| @dataclasses.dataclass(frozen=True) | ||
| class BigQueryTableId: | ||
| project_id: str | ||
| dataset_id: str | ||
| table_id: str | ||
|
|
||
|
|
||
| def parse_table_id(table_id: str) -> BigLakeTableId | BigQueryTableId: | ||
| """Turn a string into a BigLakeTableId or BigQueryTableId. | ||
|
|
||
| Raises: | ||
| ValueError: If the table ID is invalid. | ||
| """ | ||
| regex_match = _TABLE_REFEREENCE_PATTERN.match(table_id) | ||
| if not regex_match: | ||
| raise ValueError(f"Invalid table ID: {table_id}") | ||
|
|
||
| inner_parts = regex_match.group("inner_parts").split(".") | ||
| if len(inner_parts) == 1: | ||
| return BigQueryTableId( | ||
| project_id=regex_match.group("project"), | ||
| dataset_id=inner_parts[0], | ||
| table_id=regex_match.group("table"), | ||
| ) | ||
|
|
||
| return BigLakeTableId( | ||
| project=regex_match.group("project"), | ||
| catalog=inner_parts[0], | ||
| namespace=tuple(inner_parts[1:]), | ||
| table=regex_match.group("table"), | ||
| ) | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.