From 92109dc16865710ae1dbca2b98164b96032ff75c Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 11 Mar 2025 13:15:49 +0000 Subject: [PATCH 01/17] [Feature Request]: Request to use BASIC enum when calling tables.get() in BigQuery #34075 --- sdks/python/apache_beam/io/gcp/bigquery.py | 2 +- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 9f60b5af6726..3eef0b6157a2 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -925,7 +925,7 @@ def _export_files(self, bq): else: table_ref = self.table_reference table = bq.get_table( - table_ref.projectId, table_ref.datasetId, table_ref.tableId) + table_ref.projectId, table_ref.datasetId, table_ref.tableId, table_view='BASIC') return table.schema, metadata_list diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 28935aa5cdd9..6c0c9ffb00e3 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -787,7 +787,7 @@ def get_table(self, project_id, dataset_id, table_id): HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, datasetId=dataset_id, tableId=table_id) + projectId=project_id, datasetId=dataset_id, tableId=table_id, table_view='BASIC') response = self.client.tables.Get(request) return response From 294f23e99c910e2cf704a26f52df69d5c52aabc0 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 11 Mar 2025 15:42:30 +0000 Subject: [PATCH 02/17] pass table_view as parameter --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 6c0c9ffb00e3..a6cfe80ec350 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -772,7 +772,7 @@ def _insert_all_rows( @retry.with_exponential_backoff( num_retries=MAX_RETRIES, retry_filter=retry.retry_on_server_errors_timeout_or_quota_issues_filter) - def get_table(self, project_id, dataset_id, table_id): + def get_table(self, project_id, dataset_id, table_id, table_view='BASIC'): """Lookup a table's metadata object. Args: @@ -787,7 +787,7 @@ def get_table(self, project_id, dataset_id, table_id): HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, datasetId=dataset_id, tableId=table_id, table_view='BASIC') + projectId=project_id, datasetId=dataset_id, tableId=table_id, table_view=table_view) response = self.client.tables.Get(request) return response From 638ba4f18543e573db2c3a010b9bd37195b70623 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 11 Mar 2025 15:56:19 +0000 Subject: [PATCH 03/17] add docstring for table_view --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index a6cfe80ec350..80b108c3b556 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -780,6 +780,7 @@ def get_table(self, project_id, dataset_id, table_id, table_view='BASIC'): project_id: table lookup parameter dataset_id: table lookup parameter table_id: table lookup parameter + table_view: table information parameter Returns: bigquery.Table instance From 95bf0c4ae1cd6abfe6b00c8b308e6dd8c9c38c59 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 18 Mar 2025 08:13:27 +0000 Subject: [PATCH 04/17] view instead of table_view --- sdks/python/apache_beam/io/gcp/bigquery.py | 2 +- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 3eef0b6157a2..7f912949d0ce 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -925,7 +925,7 @@ def _export_files(self, bq): else: table_ref = self.table_reference table = bq.get_table( - table_ref.projectId, table_ref.datasetId, table_ref.tableId, table_view='BASIC') + table_ref.projectId, table_ref.datasetId, table_ref.tableId, view='BASIC') return table.schema, metadata_list diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 80b108c3b556..5d721f869f8c 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -772,7 +772,7 @@ def _insert_all_rows( @retry.with_exponential_backoff( num_retries=MAX_RETRIES, retry_filter=retry.retry_on_server_errors_timeout_or_quota_issues_filter) - def get_table(self, project_id, dataset_id, table_id, table_view='BASIC'): + def get_table(self, project_id, dataset_id, table_id, view='BASIC'): """Lookup a table's metadata object. Args: @@ -780,7 +780,7 @@ def get_table(self, project_id, dataset_id, table_id, table_view='BASIC'): project_id: table lookup parameter dataset_id: table lookup parameter table_id: table lookup parameter - table_view: table information parameter + view: table information parameter Returns: bigquery.Table instance @@ -788,7 +788,7 @@ def get_table(self, project_id, dataset_id, table_id, table_view='BASIC'): HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, datasetId=dataset_id, tableId=table_id, table_view=table_view) + projectId=project_id, datasetId=dataset_id, tableId=table_id, view=view) response = self.client.tables.Get(request) return response From 1c0da8da7b4d2cfe1f5b472483a5dbcfcd124e7b Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Wed, 19 Mar 2025 09:16:52 +0000 Subject: [PATCH 05/17] fix error --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 5d721f869f8c..0d3a65176462 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -788,7 +788,8 @@ def get_table(self, project_id, dataset_id, table_id, view='BASIC'): HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, datasetId=dataset_id, tableId=table_id, view=view) + projectId=project_id, datasetId=dataset_id, tableId=table_id) + request.view = view response = self.client.tables.Get(request) return response From dc57f7efb27b485e5abfc27b4836fa93e1f910a1 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 11 Apr 2025 13:31:51 +0000 Subject: [PATCH 06/17] re-define view --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 0d3a65176462..52c722b4e77e 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -788,8 +788,7 @@ def get_table(self, project_id, dataset_id, table_id, view='BASIC'): HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, datasetId=dataset_id, tableId=table_id) - request.view = view + projectId=project_id, datasetId=dataset_id, tableId=table_id, view='BASIC') response = self.client.tables.Get(request) return response From 93a4501679f23699230906af5f339b64eec866b2 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 11 Apr 2025 13:34:28 +0000 Subject: [PATCH 07/17] define view in BigqueryTablesGetRequest --- .../clients/bigquery/bigquery_v2_messages.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py index fd0bc9694071..268f108883e7 100644 --- a/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py +++ b/sdks/python/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_messages.py @@ -935,12 +935,29 @@ class BigqueryTablesGetRequest(_messages.Message): selectedFields: List of fields to return (comma-separated). If unspecified, all fields are returned tableId: Table ID of the requested table + view: table information parameter """ datasetId = _messages.StringField(1, required=True) projectId = _messages.StringField(2, required=True) selectedFields = _messages.StringField(3) tableId = _messages.StringField(4, required=True) + view = _messages.EnumField('ViewValueValuesEnum', 5, default='BASIC') + # @@protoc_insertion_point(class_scope:google.cloud.bigquery_v2.BigqueryTablesGetRequest) + # @@protoc_insertion_point(enum_scope:google.cloud.bigquery_v2.BigqueryTablesGetRequest.ViewValueValuesEnum) + class ViewValueValuesEnum(_messages.Enum): + r"""table information parameter + + Values: + TABLE_METADATA_VIEW_UNSPECIFIED: The default value. Default to the STORAGE_STATS view. + BASIC: Includes basic table information, such as schema and partitioning specification. This view does not include storage statistics like `numRows` or `numBytes`. It is significantly more efficient and suitable for high query rates. + STORAGE_STATS: Includes all information from the BASIC view, along with additional storage statistics such as numBytes, numLongTermBytes, numRows, and lastModifiedTime. + FULL: Includes all table information, including storage statistics. It returns same information as STORAGE_STATS view, but may contain additional information in the future. + """ + TABLE_METADATA_VIEW_UNSPECIFIED = 0 + BASIC = 1 + STORAGE_STATS = 2 + FULL = 3 class BigqueryTablesInsertRequest(_messages.Message): From 383c9016c41f99909c29ea68f311b28762a04e58 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 11 Apr 2025 13:49:50 +0000 Subject: [PATCH 08/17] yapf beam/sdks/python/apache_beam/io/gcp/bigquery_tools.py --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 52c722b4e77e..a0e8fbd338e1 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -788,7 +788,10 @@ def get_table(self, project_id, dataset_id, table_id, view='BASIC'): HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, datasetId=dataset_id, tableId=table_id, view='BASIC') + projectId=project_id, + datasetId=dataset_id, + tableId=table_id, + view='BASIC') response = self.client.tables.Get(request) return response @@ -1283,8 +1286,8 @@ def insert_rows( # can happen during retries on failures. # TODO(silviuc): Must add support to writing TableRow's instead of dicts. insert_ids = [ - str(self.unique_row_id) if not insert_ids else insert_ids[i] for i, - _ in enumerate(rows) + str(self.unique_row_id) if not insert_ids else insert_ids[i] + for i, _ in enumerate(rows) ] rows = [ fast_json_loads(fast_json_dumps(r, default=default_encoder)) @@ -1393,6 +1396,7 @@ class RowAsDictJsonCoder(coders.Coder): This is the default coder for sources and sinks if the coder argument is not specified. """ + def encode(self, table_row): # The normal error when dumping NAN/INF values is: # ValueError: Out of range float values are not JSON compliant @@ -1420,6 +1424,7 @@ class JsonRowWriter(io.IOBase): A writer which provides an IOBase-like interface for writing table rows (represented as dicts) as newline-delimited JSON strings. """ + def __init__(self, file_handle): """Initialize an JsonRowWriter. @@ -1460,6 +1465,7 @@ class AvroRowWriter(io.IOBase): A writer which provides an IOBase-like interface for writing table rows (represented as dicts) as Avro records. """ + def __init__(self, file_handle, schema): """Initialize an AvroRowWriter. @@ -1553,6 +1559,7 @@ class AppendDestinationsFn(DoFn): Experimental; no backwards compatibility guarantees. """ + def __init__(self, destination): self._display_destination = destination self.destination = AppendDestinationsFn._get_table_fn(destination) @@ -1659,6 +1666,7 @@ def get_table_schema_from_string(schema): def table_schema_to_dict(table_schema): """Create a dictionary representation of table schema for serialization """ + def get_table_field(field): """Create a dictionary representation of a table field """ From b1eb3341de69a8c2ce2b6d7272e49f7f0720f715 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 11 Apr 2025 15:44:29 +0000 Subject: [PATCH 09/17] fix ValidationError --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index a0e8fbd338e1..fda26e3f0506 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -791,7 +791,7 @@ def get_table(self, project_id, dataset_id, table_id, view='BASIC'): projectId=project_id, datasetId=dataset_id, tableId=table_id, - view='BASIC') + view=view) response = self.client.tables.Get(request) return response From dcea2ab84a9ed85794c49ba4669e3361511fe14c Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 30 May 2025 14:14:46 +0000 Subject: [PATCH 10/17] fix ValidationError --- sdks/python/apache_beam/io/gcp/bigquery.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 3ccf9b3c4945..000613a4665c 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -374,7 +374,6 @@ def chain_after(result): import fastavro from objsize import get_deep_size - import apache_beam as beam from apache_beam import coders from apache_beam import pvalue @@ -422,6 +421,10 @@ def chain_after(result): from apache_beam.typehints.schemas import schema_from_element_type from apache_beam.utils import retry from apache_beam.utils.annotations import deprecated +from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import BigqueryTablesGetRequest +from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import ViewValueValuesEnum + + try: from apache_beam.io.gcp.internal.clients.bigquery import DatasetReference @@ -925,7 +928,7 @@ def _export_files(self, bq): else: table_ref = self.table_reference table = bq.get_table( - table_ref.projectId, table_ref.datasetId, table_ref.tableId, view='BASIC') + table_ref.projectId, table_ref.datasetId, table_ref.tableId, view=BigqueryTablesGetRequest.ViewValueValuesEnum) return table.schema, metadata_list From b5059463151354d2cb0c415ae81c68536e8fa063 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 30 May 2025 15:36:40 +0000 Subject: [PATCH 11/17] fix ValidationError --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index fda26e3f0506..d25bc93d2075 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -43,6 +43,7 @@ from typing import TypeVar from typing import Union +from beam.sdks.python.apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import BigqueryTablesGetRequest import fastavro import numpy as np import regex @@ -772,7 +773,7 @@ def _insert_all_rows( @retry.with_exponential_backoff( num_retries=MAX_RETRIES, retry_filter=retry.retry_on_server_errors_timeout_or_quota_issues_filter) - def get_table(self, project_id, dataset_id, table_id, view='BASIC'): + def get_table(self, project_id, dataset_id, table_id, view=BigqueryTablesGetRequest.ViewValueValuesEnum.BASIC): """Lookup a table's metadata object. Args: From 47ee407100f88ba6318152d3716e6b9d1465f7a1 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Fri, 30 May 2025 15:58:09 +0000 Subject: [PATCH 12/17] Fix Python formatting issues detected by yapf --- .../python/apache_beam/io/gcp/bigquery_tools.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index d25bc93d2075..f3760c612cd1 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -773,7 +773,12 @@ def _insert_all_rows( @retry.with_exponential_backoff( num_retries=MAX_RETRIES, retry_filter=retry.retry_on_server_errors_timeout_or_quota_issues_filter) - def get_table(self, project_id, dataset_id, table_id, view=BigqueryTablesGetRequest.ViewValueValuesEnum.BASIC): + def get_table( + self, + project_id, + dataset_id, + table_id, + view=BigqueryTablesGetRequest.ViewValueValuesEnum.BASIC): """Lookup a table's metadata object. Args: @@ -789,10 +794,7 @@ def get_table(self, project_id, dataset_id, table_id, view=BigqueryTablesGetRequ HttpError: if lookup failed. """ request = bigquery.BigqueryTablesGetRequest( - projectId=project_id, - datasetId=dataset_id, - tableId=table_id, - view=view) + projectId=project_id, datasetId=dataset_id, tableId=table_id, view=view) response = self.client.tables.Get(request) return response @@ -1397,7 +1399,6 @@ class RowAsDictJsonCoder(coders.Coder): This is the default coder for sources and sinks if the coder argument is not specified. """ - def encode(self, table_row): # The normal error when dumping NAN/INF values is: # ValueError: Out of range float values are not JSON compliant @@ -1425,7 +1426,6 @@ class JsonRowWriter(io.IOBase): A writer which provides an IOBase-like interface for writing table rows (represented as dicts) as newline-delimited JSON strings. """ - def __init__(self, file_handle): """Initialize an JsonRowWriter. @@ -1466,7 +1466,6 @@ class AvroRowWriter(io.IOBase): A writer which provides an IOBase-like interface for writing table rows (represented as dicts) as Avro records. """ - def __init__(self, file_handle, schema): """Initialize an AvroRowWriter. @@ -1560,7 +1559,6 @@ class AppendDestinationsFn(DoFn): Experimental; no backwards compatibility guarantees. """ - def __init__(self, destination): self._display_destination = destination self.destination = AppendDestinationsFn._get_table_fn(destination) @@ -1667,7 +1665,6 @@ def get_table_schema_from_string(schema): def table_schema_to_dict(table_schema): """Create a dictionary representation of table schema for serialization """ - def get_table_field(field): """Create a dictionary representation of a table field """ From c57a248cbe5780dba6f6256cd820577036b2995d Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Mon, 2 Jun 2025 15:54:21 +0000 Subject: [PATCH 13/17] fix module error --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index f3760c612cd1..fa876d12b2b1 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -43,7 +43,6 @@ from typing import TypeVar from typing import Union -from beam.sdks.python.apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import BigqueryTablesGetRequest import fastavro import numpy as np import regex @@ -67,6 +66,8 @@ from apache_beam.typehints.typehints import Any from apache_beam.utils import retry from apache_beam.utils.histogram import LinearBucket +from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import BigqueryTablesGetRequest + # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position From 73723b657aba745e43857fad0a8a4ac5b8980ba5 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 3 Jun 2025 05:02:23 +0000 Subject: [PATCH 14/17] fix module error --- sdks/python/apache_beam/yaml/yaml_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py index 67a5e65bda93..ed2f38e4fff2 100644 --- a/sdks/python/apache_beam/yaml/yaml_io.py +++ b/sdks/python/apache_beam/yaml/yaml_io.py @@ -35,7 +35,7 @@ import apache_beam as beam import apache_beam.io as beam_io from apache_beam import coders -from apache_beam.io import ReadFromBigQuery +from apache_beam.io.gcp.bigquery import ReadFromBigQuery from apache_beam.io import ReadFromTFRecord from apache_beam.io import WriteToBigQuery from apache_beam.io import WriteToTFRecord From 654265ad31da5d1b548c965ab81c468efb96341f Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 3 Jun 2025 05:06:40 +0000 Subject: [PATCH 15/17] fix import error --- sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py | 2 +- sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py b/sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py index 0b4cfe2ecbae..897aecfee275 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py @@ -55,7 +55,7 @@ from apache_beam import Map from apache_beam import ParDo -from apache_beam.io import BigQueryDisposition +from apache_beam.io.gcp.bigquery import BigQueryDisposition from apache_beam.io import BigQuerySource from apache_beam.io import Read from apache_beam.io import WriteToBigQuery diff --git a/sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py b/sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py index 1aafb1b60a85..a2eabb49302f 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py @@ -55,7 +55,7 @@ from apache_beam import Map from apache_beam import ParDo -from apache_beam.io import BigQueryDisposition +from apache_beam.io.gcp.bigquery import BigQueryDisposition from apache_beam.io import Read from apache_beam.io import WriteToBigQuery from apache_beam.io.gcp.bigquery_tools import parse_table_schema_from_json From 6f25b9fc4d87e8f009ec3357e067126b5abe5f66 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Tue, 3 Jun 2025 06:16:13 +0000 Subject: [PATCH 16/17] fix module error --- sdks/python/apache_beam/io/gcp/bigquery.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index ea449e56be8a..f22cfaf46160 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -422,10 +422,6 @@ def chain_after(result): from apache_beam.utils import retry from apache_beam.utils.annotations import deprecated from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import BigqueryTablesGetRequest -from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import ViewValueValuesEnum - - - try: from apache_beam.io.gcp.internal.clients.bigquery import DatasetReference from apache_beam.io.gcp.internal.clients.bigquery import TableReference @@ -929,7 +925,7 @@ def _export_files(self, bq): else: table_ref = self.table_reference table = bq.get_table( - table_ref.projectId, table_ref.datasetId, table_ref.tableId, view=BigqueryTablesGetRequest.ViewValueValuesEnum) + table_ref.projectId, table_ref.datasetId, table_ref.tableId, view=BigqueryTablesGetRequest.ViewValueValuesEnum.BASIC) return table.schema, metadata_list From ab38977102dd62be64066cc2e10f3aa4d1d6e1e6 Mon Sep 17 00:00:00 2001 From: Tamanna Kakkar Date: Wed, 4 Jun 2025 10:00:31 +0000 Subject: [PATCH 17/17] fix yapf --- sdks/python/apache_beam/io/gcp/bigquery.py | 5 ++++- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index f22cfaf46160..4f88b406161e 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -925,7 +925,10 @@ def _export_files(self, bq): else: table_ref = self.table_reference table = bq.get_table( - table_ref.projectId, table_ref.datasetId, table_ref.tableId, view=BigqueryTablesGetRequest.ViewValueValuesEnum.BASIC) + table_ref.projectId, + table_ref.datasetId, + table_ref.tableId, + view=BigqueryTablesGetRequest.ViewValueValuesEnum.BASIC) return table.schema, metadata_list diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index fa876d12b2b1..60dd13ca78f6 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -68,7 +68,6 @@ from apache_beam.utils.histogram import LinearBucket from apache_beam.io.gcp.internal.clients.bigquery.bigquery_v2_messages import BigqueryTablesGetRequest - # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: