Skip to content

Commit c11df77

Browse files
shuoweilshuoweil
committed
feat: implement pandas-gbq telemetry and version checks
1 parent d4bd53d commit c11df77

2 files changed

Lines changed: 8 additions & 194 deletions

File tree

‎packages/google-cloud-bigquery/google/cloud/bigquery/table.py‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2711,14 +2711,6 @@ def to_dataframe(
27112711
is not supported dtype.
27122712
27132713
"""
2714-
if not _versions_helpers.PANDAS_GBQ_VERSIONS.is_delegation_supported:
2715-
warnings.warn(
2716-
"Retrieving dataframes via the core client is deprecated. "
2717-
"Please install 'pandas-gbq' for the new high-performance backend.",
2718-
PendingDeprecationWarning,
2719-
stacklevel=2,
2720-
)
2721-
27222714
_pandas_helpers.verify_pandas_imports()
27232715

27242716
if geography_as_object and shapely is None:
@@ -2828,26 +2820,6 @@ def to_dataframe(
28282820
f"{ua} pandas-gbq/{pandas_gbq_version}".strip()
28292821
)
28302822

2831-
return pandas_gbq.pandas.from_row_iterator(
2832-
self,
2833-
bqstorage_client=bqstorage_client,
2834-
dtypes=dtypes,
2835-
progress_bar_type=progress_bar_type,
2836-
create_bqstorage_client=create_bqstorage_client,
2837-
geography_as_object=geography_as_object,
2838-
bool_dtype=bool_dtype,
2839-
int_dtype=int_dtype,
2840-
float_dtype=float_dtype,
2841-
string_dtype=string_dtype,
2842-
date_dtype=date_dtype,
2843-
datetime_dtype=datetime_dtype,
2844-
time_dtype=time_dtype,
2845-
timestamp_dtype=timestamp_dtype,
2846-
range_date_dtype=range_date_dtype,
2847-
range_datetime_dtype=range_datetime_dtype,
2848-
range_timestamp_dtype=range_timestamp_dtype,
2849-
timeout=timeout,
2850-
)
28512823

28522824
record_batch = self.to_arrow(
28532825
progress_bar_type=progress_bar_type,

‎packages/google-cloud-bigquery/tests/unit/test_table.py‎

Lines changed: 8 additions & 166 deletions
Original file line numberDiff line numberDiff line change
@@ -5715,167 +5715,12 @@ def test_rowiterator_to_geodataframe_delegation(self, to_dataframe):
57155715

57165716
self.assertEqual([v.__class__.__name__ for v in df.g], ["Point"])
57175717

5718-
def test_to_dataframe_delegated_when_supported_no_range_types(self):
5719-
import sys
5720-
5721-
db_dtypes = pytest.importorskip("db_dtypes")
5722-
pandas = pytest.importorskip("pandas")
5723-
mock_pandas_gbq = mock.Mock()
5724-
mock_pandas_gbq.pandas.from_row_iterator.return_value = mock.sentinel.dataframe
5725-
mock_pandas_gbq.__version__ = "1.0.0"
5726-
5727-
with mock.patch(
5728-
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
5729-
new_callable=mock.PropertyMock,
5730-
return_value=True,
5731-
):
5732-
with mock.patch(
5733-
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
5734-
False,
5735-
):
5736-
with mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}):
5737-
row_iterator = self._make_one_from_data(
5738-
(("name", "STRING"),), (("foo",),)
5739-
)
5740-
df = row_iterator.to_dataframe(
5741-
progress_bar_type="tqdm", timeout=5.0
5742-
)
5743-
5744-
mock_pandas_gbq.pandas.from_row_iterator.assert_called_once_with(
5745-
row_iterator,
5746-
bqstorage_client=None,
5747-
dtypes={},
5748-
progress_bar_type="tqdm",
5749-
create_bqstorage_client=True,
5750-
geography_as_object=False,
5751-
bool_dtype=pandas.BooleanDtype(),
5752-
int_dtype=pandas.Int64Dtype(),
5753-
float_dtype=None,
5754-
string_dtype=None,
5755-
date_dtype=DefaultPandasDTypes.DATE_DTYPE,
5756-
datetime_dtype=None,
5757-
time_dtype=db_dtypes.TimeDtype(),
5758-
timestamp_dtype=None,
5759-
range_date_dtype=None,
5760-
range_datetime_dtype=None,
5761-
range_timestamp_dtype=None,
5762-
timeout=5.0,
5763-
)
5764-
self.assertEqual(df, mock.sentinel.dataframe)
5765-
5766-
def test_to_dataframe_delegated_when_supported_with_range_types(self):
5767-
import sys
5768-
5769-
db_dtypes = pytest.importorskip("db_dtypes")
5770-
pandas = pytest.importorskip("pandas")
5771-
if not hasattr(pandas, "ArrowDtype"):
5772-
pytest.skip("pandas.ArrowDtype is not available in this environment.")
5773-
pyarrow = pytest.importorskip("pyarrow")
5774-
mock_pandas_gbq = mock.Mock()
5775-
mock_pandas_gbq.pandas.from_row_iterator.return_value = mock.sentinel.dataframe
5776-
mock_pandas_gbq.__version__ = "1.0.0"
5777-
5778-
with mock.patch(
5779-
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
5780-
new_callable=mock.PropertyMock,
5781-
return_value=True,
5782-
):
5783-
with mock.patch(
5784-
"google.cloud.bigquery._versions_helpers.SUPPORTS_RANGE_PYARROW",
5785-
True,
5786-
):
5787-
with mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}):
5788-
row_iterator = self._make_one_from_data(
5789-
(("name", "STRING"),), (("foo",),)
5790-
)
5791-
df = row_iterator.to_dataframe(
5792-
progress_bar_type="tqdm", timeout=5.0
5793-
)
5794-
5795-
expected_range_date = pandas.ArrowDtype(
5796-
pyarrow.struct(
5797-
[("start", pyarrow.date32()), ("end", pyarrow.date32())]
5798-
)
5799-
)
5800-
expected_range_datetime = pandas.ArrowDtype(
5801-
pyarrow.struct(
5802-
[
5803-
("start", pyarrow.timestamp("us")),
5804-
("end", pyarrow.timestamp("us")),
5805-
]
5806-
)
5807-
)
5808-
expected_range_timestamp = pandas.ArrowDtype(
5809-
pyarrow.struct(
5810-
[
5811-
("start", pyarrow.timestamp("us", tz="UTC")),
5812-
("end", pyarrow.timestamp("us", tz="UTC")),
5813-
]
5814-
)
5815-
)
5816-
5817-
mock_pandas_gbq.pandas.from_row_iterator.assert_called_once_with(
5818-
row_iterator,
5819-
bqstorage_client=None,
5820-
dtypes={},
5821-
progress_bar_type="tqdm",
5822-
create_bqstorage_client=True,
5823-
geography_as_object=False,
5824-
bool_dtype=pandas.BooleanDtype(),
5825-
int_dtype=pandas.Int64Dtype(),
5826-
float_dtype=None,
5827-
string_dtype=None,
5828-
date_dtype=DefaultPandasDTypes.DATE_DTYPE,
5829-
datetime_dtype=None,
5830-
time_dtype=db_dtypes.TimeDtype(),
5831-
timestamp_dtype=None,
5832-
range_date_dtype=expected_range_date,
5833-
range_datetime_dtype=expected_range_datetime,
5834-
range_timestamp_dtype=expected_range_timestamp,
5835-
timeout=5.0,
5836-
)
5837-
self.assertEqual(df, mock.sentinel.dataframe)
5838-
5839-
def test_to_dataframe_not_delegated_when_unsupported(self):
5840-
import sys
5841-
5842-
pandas = pytest.importorskip("pandas")
5843-
pytest.importorskip("pyarrow")
5844-
mock_pandas_gbq = mock.Mock()
5845-
5846-
with mock.patch(
5847-
"google.cloud.bigquery._versions_helpers.PandasGBQVersions.is_delegation_supported",
5848-
new_callable=mock.PropertyMock,
5849-
return_value=False,
5850-
):
5851-
with mock.patch.dict(sys.modules, {"pandas_gbq": mock_pandas_gbq}):
5852-
row_iterator = self._make_one_from_data(
5853-
(("name", "STRING"),), (("foo",),)
5854-
)
5855-
5856-
with warnings.catch_warnings(record=True) as warned:
5857-
warnings.simplefilter("always")
5858-
df = row_iterator.to_dataframe(create_bqstorage_client=False)
5859-
5860-
mock_pandas_gbq.pandas.from_row_iterator.assert_not_called()
5861-
self.assertIsInstance(df, pandas.DataFrame)
5862-
self.assertEqual(df.name.tolist(), ["foo"])
5863-
5864-
deprecation_warnings = [
5865-
w
5866-
for w in warned
5867-
if issubclass(w.category, PendingDeprecationWarning)
5868-
and "pandas-gbq" in str(w.message)
5869-
]
5870-
self.assertEqual(len(deprecation_warnings), 1)
5871-
58725718
def test_to_dataframe_delegated_updates_user_agent(self):
58735719
import sys
58745720

58755721
pytest.importorskip("db_dtypes")
5876-
pytest.importorskip("pandas")
5722+
pandas = pytest.importorskip("pandas")
58775723
mock_pandas_gbq = mock.Mock()
5878-
mock_pandas_gbq.pandas.from_row_iterator.return_value = mock.sentinel.dataframe
58795724
mock_pandas_gbq.__version__ = "1.0.0"
58805725

58815726
mock_client_info = mock.Mock()
@@ -5901,7 +5746,7 @@ def test_to_dataframe_delegated_updates_user_agent(self):
59015746
df = row_iterator.to_dataframe(
59025747
progress_bar_type="tqdm", timeout=5.0
59035748
)
5904-
self.assertEqual(df, mock.sentinel.dataframe)
5749+
self.assertIsInstance(df, pandas.DataFrame)
59055750
self.assertEqual(
59065751
mock_client_info.user_agent,
59075752
"gl-python/3.10.0 pandas-gbq/1.0.0",
@@ -5911,9 +5756,8 @@ def test_to_dataframe_delegated_does_not_duplicate_user_agent(self):
59115756
import sys
59125757

59135758
pytest.importorskip("db_dtypes")
5914-
pytest.importorskip("pandas")
5759+
pandas = pytest.importorskip("pandas")
59155760
mock_pandas_gbq = mock.Mock()
5916-
mock_pandas_gbq.pandas.from_row_iterator.return_value = mock.sentinel.dataframe
59175761
mock_pandas_gbq.__version__ = "1.0.0"
59185762

59195763
mock_client_info = mock.Mock()
@@ -5939,7 +5783,7 @@ def test_to_dataframe_delegated_does_not_duplicate_user_agent(self):
59395783
df = row_iterator.to_dataframe(
59405784
progress_bar_type="tqdm", timeout=5.0
59415785
)
5942-
self.assertEqual(df, mock.sentinel.dataframe)
5786+
self.assertIsInstance(df, pandas.DataFrame)
59435787
self.assertEqual(
59445788
mock_client_info.user_agent,
59455789
"gl-python/3.10.0 pandas-gbq/1.0.0",
@@ -5949,9 +5793,8 @@ def test_to_dataframe_delegated_when_client_info_is_none(self):
59495793
import sys
59505794

59515795
pytest.importorskip("db_dtypes")
5952-
pytest.importorskip("pandas")
5796+
pandas = pytest.importorskip("pandas")
59535797
mock_pandas_gbq = mock.Mock()
5954-
mock_pandas_gbq.pandas.from_row_iterator.return_value = mock.sentinel.dataframe
59555798
mock_pandas_gbq.__version__ = "1.0.0"
59565799

59575800
mock_client = _mock_client()
@@ -5974,15 +5817,14 @@ def test_to_dataframe_delegated_when_client_info_is_none(self):
59745817
df = row_iterator.to_dataframe(
59755818
progress_bar_type="tqdm", timeout=5.0
59765819
)
5977-
self.assertEqual(df, mock.sentinel.dataframe)
5820+
self.assertIsInstance(df, pandas.DataFrame)
59785821

59795822
def test_to_dataframe_delegated_when_user_agent_is_none(self):
59805823
import sys
59815824

59825825
pytest.importorskip("db_dtypes")
5983-
pytest.importorskip("pandas")
5826+
pandas = pytest.importorskip("pandas")
59845827
mock_pandas_gbq = mock.Mock()
5985-
mock_pandas_gbq.pandas.from_row_iterator.return_value = mock.sentinel.dataframe
59865828
mock_pandas_gbq.__version__ = "1.0.0"
59875829

59885830
mock_client_info = mock.Mock()
@@ -6008,7 +5850,7 @@ def test_to_dataframe_delegated_when_user_agent_is_none(self):
60085850
df = row_iterator.to_dataframe(
60095851
progress_bar_type="tqdm", timeout=5.0
60105852
)
6011-
self.assertEqual(df, mock.sentinel.dataframe)
5853+
self.assertIsInstance(df, pandas.DataFrame)
60125854
self.assertEqual(
60135855
mock_client_info.user_agent,
60145856
"pandas-gbq/1.0.0",

0 commit comments

Comments
 (0)