Merge branch 'main' into shuowei-fix-describe-unsupported

Author: shuoweil

.github/workflows/docs.yml

@@ -36,6 +36,3 @@ jobs:
       run: |
         python -m pip install --upgrade setuptools pip wheel
         python -m pip install nox
-    - name: Run docfx
-      run: |
-        nox -s docfx

bigframes/bigquery/_operations/ai.py

@@ -745,7 +745,7 @@ def if_(
             or pandas Series.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
-            If not provided, the connection from the current session will be used.
+            If not provided, the query uses your end-user credential.
 
     Returns:
         bigframes.series.Series: A new series of bools.
@@ -756,7 +756,7 @@ def if_(
 
     operator = ai_ops.AIIf(
         prompt_context=tuple(prompt_context),
-        connection_id=_resolve_connection_id(series_list[0], connection_id),
+        connection_id=connection_id,
     )
 
     return series_list[0]._apply_nary_op(operator, series_list[1:])
@@ -800,7 +800,7 @@ def classify(
             Categories to classify the input into.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
-            If not provided, the connection from the current session will be used.
+            If not provided, the query uses your end-user credential.
 
     Returns:
         bigframes.series.Series: A new series of strings.
@@ -812,7 +812,7 @@ def classify(
     operator = ai_ops.AIClassify(
         prompt_context=tuple(prompt_context),
         categories=tuple(categories),
-        connection_id=_resolve_connection_id(series_list[0], connection_id),
+        connection_id=connection_id,
     )
 
     return series_list[0]._apply_nary_op(operator, series_list[1:])
@@ -853,7 +853,7 @@ def score(
             or pandas Series.
         connection_id (str, optional):
             Specifies the connection to use to communicate with the model. For example, `myproject.us.myconnection`.
-            If not provided, the connection from the current session will be used.
+            If not provided, the query uses your end-user credential.
 
     Returns:
         bigframes.series.Series: A new series of double (float) values.
@@ -864,7 +864,7 @@ def score(
 
     operator = ai_ops.AIScore(
         prompt_context=tuple(prompt_context),
-        connection_id=_resolve_connection_id(series_list[0], connection_id),
+        connection_id=connection_id,
     )
 
     return series_list[0]._apply_nary_op(operator, series_list[1:])

bigframes/core/bigframe_node.py

@@ -330,12 +330,32 @@ def top_down(
         """
         Perform a top-down transformation of the BigFrameNode tree.
         """
+        results: Dict[BigFrameNode, BigFrameNode] = {}
+        # Each stack entry is (node, t_node). t_node is None until transform(node) is called.
+        stack: list[tuple[BigFrameNode, typing.Optional[BigFrameNode]]] = [(self, None)]
 
-        @functools.cache
-        def recursive_transform(node: BigFrameNode) -> BigFrameNode:
-            return transform(node).transform_children(recursive_transform)
+        while stack:
+            node, t_node = stack[-1]
+
+            if t_node is None:
+                if node in results:
+                    stack.pop()
+                    continue
+                t_node = transform(node)
+                stack[-1] = (node, t_node)
+
+            all_done = True
+            for child in reversed(t_node.child_nodes):
+                if child not in results:
+                    stack.append((child, None))
+                    all_done = False
+                    break
+
+            if all_done:
+                results[node] = t_node.transform_children(lambda x: results[x])
+                stack.pop()
 
-        return recursive_transform(self)
+        return results[self]
 
     def bottom_up(
         self: BigFrameNode,

bigframes/core/compile/sqlglot/expressions/ai_ops.py

@@ -113,9 +113,9 @@ def _construct_named_args(op: ops.NaryOp) -> list[sge.Kwarg]:
             )
         )
 
-    endpoit = op_args.get("endpoint", None)
-    if endpoit is not None:
-        args.append(sge.Kwarg(this="endpoint", expression=sge.Literal.string(endpoit)))
+    endpoint = op_args.get("endpoint", None)
+    if endpoint is not None:
+        args.append(sge.Kwarg(this="endpoint", expression=sge.Literal.string(endpoint)))
 
     request_type = op_args.get("request_type", None)
     if request_type is not None:

bigframes/operations/ai_ops.py

@@ -123,7 +123,7 @@ class AIIf(base_ops.NaryOp):
     name: ClassVar[str] = "ai_if"
 
     prompt_context: Tuple[str | None, ...]
-    connection_id: str
+    connection_id: str | None
 
     def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType:
         return dtypes.BOOL_DTYPE
@@ -135,7 +135,7 @@ class AIClassify(base_ops.NaryOp):
 
     prompt_context: Tuple[str | None, ...]
     categories: tuple[str, ...]
-    connection_id: str
+    connection_id: str | None
 
     def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType:
         return dtypes.STRING_DTYPE
@@ -146,7 +146,7 @@ class AIScore(base_ops.NaryOp):
     name: ClassVar[str] = "ai_score"
 
     prompt_context: Tuple[str | None, ...]
-    connection_id: str
+    connection_id: str | None
 
     def output_type(self, *input_types: dtypes.ExpressionType) -> dtypes.ExpressionType:
         return dtypes.FLOAT_DTYPE

docs/conf.py

@@ -59,9 +59,12 @@
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",
     "sphinx_sitemap",
-    "myst_parser",
+    "myst_nb",
 ]
 
+# myst-nb configuration
+nb_execution_mode = "off"
+
 # autodoc/autosummary flags
 autoclass_content = "both"
 autodoc_default_options = {"members": True}
@@ -269,12 +272,14 @@
 
 
 suppress_warnings = [
+    # Allow unknown mimetype so we can use widgets in tutorial notebooks.
+    "mystnb.unknown_mime_type",
     # Temporarily suppress this to avoid "more than one target found for
     # cross-reference" warning, which are intractable for us to avoid while in
     # a mono-repo.
     # See https://github.com/sphinx-doc/sphinx/blob
     # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843
-    "ref.python"
+    "ref.python",
 ]
 
 # -- Options for LaTeX output ---------------------------------------------

docs/notebooks

@@ -0,0 +1 @@
+../notebooks

docs/user_guide/index.rst

@@ -9,3 +9,116 @@ User Guide
 
    Getting Started <https://docs.cloud.google.com/bigquery/docs/dataframes-quickstart>
    Cloud Docs User Guides <https://docs.cloud.google.com/bigquery/docs/bigquery-dataframes-introduction>
+
+.. toctree::
+   :caption: Getting Started
+   :maxdepth: 1
+
+   Quickstart Template <../notebooks/getting_started/bq_dataframes_template.ipynb>
+   Getting Started <../notebooks/getting_started/getting_started_bq_dataframes.ipynb>
+   Magics <../notebooks/getting_started/magics.ipynb>
+   ML Fundamentals <../notebooks/getting_started/ml_fundamentals_bq_dataframes.ipynb>
+
+.. toctree::
+   :caption: DataFrames
+   :maxdepth: 1
+
+   Anywidget Mode <../notebooks/dataframes/anywidget_mode.ipynb>
+   Dataframe <../notebooks/dataframes/dataframe.ipynb>
+   Index Col Null <../notebooks/dataframes/index_col_null.ipynb>
+   Integrations <../notebooks/dataframes/integrations.ipynb>
+   Pypi <../notebooks/dataframes/pypi.ipynb>
+
+.. toctree::
+   :caption: Data Types
+   :maxdepth: 1
+
+   Array <../notebooks/data_types/array.ipynb>
+   Json <../notebooks/data_types/json.ipynb>
+   Struct <../notebooks/data_types/struct.ipynb>
+   Timedelta <../notebooks/data_types/timedelta.ipynb>
+
+.. toctree::
+   :caption: Generative AI
+   :maxdepth: 1
+
+   AI Functions <../notebooks/generative_ai/ai_functions.ipynb>
+   AI Forecast <../notebooks/generative_ai/bq_dataframes_ai_forecast.ipynb>
+   LLM Code Generation <../notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb>
+   LLM KMeans <../notebooks/generative_ai/bq_dataframes_llm_kmeans.ipynb>
+   LLM Output Schema <../notebooks/generative_ai/bq_dataframes_llm_output_schema.ipynb>
+   LLM Vector Search <../notebooks/generative_ai/bq_dataframes_llm_vector_search.ipynb>
+   Drug Name Generation <../notebooks/generative_ai/bq_dataframes_ml_drug_name_generation.ipynb>
+   Large Language Models <../notebooks/generative_ai/large_language_models.ipynb>
+
+.. toctree::
+   :caption: Machine Learning
+   :maxdepth: 1
+
+   ML Cross Validation <../notebooks/ml/bq_dataframes_ml_cross_validation.ipynb>
+   Linear Regression <../notebooks/ml/bq_dataframes_ml_linear_regression.ipynb>
+   Linear Regression BBQ <../notebooks/ml/bq_dataframes_ml_linear_regression_bbq.ipynb>
+   Linear Regression Big <../notebooks/ml/bq_dataframes_ml_linear_regression_big.ipynb>
+   Easy Linear Regression <../notebooks/ml/easy_linear_regression.ipynb>
+   Sklearn Linear Regression <../notebooks/ml/sklearn_linear_regression.ipynb>
+   Timeseries Analysis <../notebooks/ml/timeseries_analysis.ipynb>
+
+.. toctree::
+   :caption: Visualization
+   :maxdepth: 1
+
+   COVID Line Graphs <../notebooks/visualization/bq_dataframes_covid_line_graphs.ipynb>
+   Tutorial <../notebooks/visualization/tutorial.ipynb>
+
+.. toctree::
+   :caption: Geospatial Data
+   :maxdepth: 1
+
+   Geoseries <../notebooks/geo/geoseries.ipynb>
+
+.. toctree::
+   :caption: Regionalized BigQuery
+   :maxdepth: 1
+
+   Regionalized <../notebooks/location/regionalized.ipynb>
+
+.. toctree::
+   :caption: Multimodal
+   :maxdepth: 1
+
+   Multimodal Dataframe <../notebooks/multimodal/multimodal_dataframe.ipynb>
+
+.. toctree::
+   :caption: Remote Functions
+   :maxdepth: 1
+
+   Remote Function <../notebooks/remote_functions/remote_function.ipynb>
+   Remote Function Usecases <../notebooks/remote_functions/remote_function_usecases.ipynb>
+   Remote Function Vertex Claude Model <../notebooks/remote_functions/remote_function_vertex_claude_model.ipynb>
+
+.. toctree::
+   :caption: Streaming
+   :maxdepth: 1
+
+   Streaming Dataframe <../notebooks/streaming/streaming_dataframe.ipynb>
+
+.. toctree::
+   :caption: Experimental
+   :maxdepth: 1
+
+   AI Operators <../notebooks/experimental/ai_operators.ipynb>
+   Semantic Operators <../notebooks/experimental/semantic_operators.ipynb>
+
+.. toctree::
+   :caption: Apps
+   :maxdepth: 1
+
+   Synthetic Data Generation <../notebooks/apps/synthetic_data_generation.ipynb>
+
+.. toctree::
+   :caption: Kaggle
+   :maxdepth: 1
+
+   AI Forecast <../notebooks/kaggle/bq_dataframes_ai_forecast.ipynb>
+   Describe Product Images <../notebooks/kaggle/describe-product-images-with-bigframes-multimodal.ipynb>
+   Vector Search Over National Jukebox <../notebooks/kaggle/vector-search-with-bigframes-over-national-jukebox.ipynb>

notebooks/dataframes/dataframe.ipynb

@@ -49,7 +49,7 @@
    "id": "13861abc-120c-4db6-ad0c-e414b85d3443",
    "metadata": {},
    "source": [
-    "### Select a subset of the DF"
+    "## Select a subset of the DF"
    ]
   },
   {

notebooks/dataframes/index_col_null.ipynb

@@ -358,7 +358,7 @@
    "id": "13861abc-120c-4db6-ad0c-e414b85d3443",
    "metadata": {},
    "source": [
-    "### Select a subset of the DataFrame\n",
+    "## Select a subset of the DataFrame",
     "\n",
     "Filter columns by selecting a list of columns from the DataFrame.\n",
     "\n",