diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 19009050a..ad1e20e88 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -59c4c0f3d5f0ef00cd5350b5674e941a7606d91a \ No newline at end of file +27cebd58ae24e19c95c675db3a93b6046abaca2a \ No newline at end of file diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index f60e0a370..ee4d53210 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -15,3 +15,10 @@ ### API Changes * Add `google_ads`, `tiktok_ads`, `salesforce_marketing_cloud`, `hubspot`, `workday_hcm`, `guidewire` and `zendesk` enum values for `databricks.sdk.service.pipelines.IngestionSourceType`. +* Add `batch_create_materialized_features()` method for [w.feature_engineering](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/ml/feature_engineering.html) workspace-level service. +* Add `lineage_context` field for `databricks.sdk.service.ml.Feature`. +* Add `autoscale_v2` enum value for `databricks.sdk.service.compute.EventDetailsCause`. +* Add `unsupported_conversation_type_exception` enum value for `databricks.sdk.service.dashboards.MessageErrorType`. +* Add `red_state` and `yellow_state` enum values for `databricks.sdk.service.vectorsearch.EndpointStatusState`. +* [Breaking] Change `table_names` field for `databricks.sdk.service.jobs.TableUpdateTriggerConfiguration` to be required. +* Change `table_names` field for `databricks.sdk.service.jobs.TableUpdateTriggerConfiguration` to be required. \ No newline at end of file diff --git a/databricks/sdk/__init__.py b/databricks/sdk/__init__.py index 0d285ccda..148bfdc43 100755 --- a/databricks/sdk/__init__.py +++ b/databricks/sdk/__init__.py @@ -711,7 +711,7 @@ def permissions(self) -> pkg_iam.PermissionsAPI: @property def pipelines(self) -> pkg_pipelines.PipelinesAPI: - """The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines.""" + """The Lakeflow Spark Declarative Pipelines API allows you to create, edit, delete, start, and view details about pipelines.""" return self._pipelines @property diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index 51ce52a40..a99c5405a 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -11210,6 +11210,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/catalogs", query=query, headers=headers) if "catalogs" in json: @@ -11404,6 +11406,8 @@ def list(self, *, max_results: Optional[int] = None, page_token: Optional[str] = "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/connections", query=query, headers=headers) if "connections" in json: @@ -12316,6 +12320,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/external-locations", query=query, headers=headers) if "external_locations" in json: @@ -12694,6 +12700,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/functions", query=query, headers=headers) if "functions" in json: @@ -13053,6 +13061,8 @@ def list(self, *, max_results: Optional[int] = None, page_token: Optional[str] = "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/metastores", query=query, headers=headers) if "metastores" in json: @@ -14894,6 +14904,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/schemas", query=query, headers=headers) if "schemas" in json: @@ -15122,6 +15134,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/storage-credentials", query=query, headers=headers) if "storage_credentials" in json: @@ -15381,6 +15395,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do( "GET", f"/api/2.1/unity-catalog/metastores/{metastore_id}/systemschemas", query=query, headers=headers @@ -15714,6 +15730,8 @@ def list( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do("GET", "/api/2.1/unity-catalog/tables", query=query, headers=headers) if "tables" in json: @@ -16247,6 +16265,8 @@ def get_bindings( "Accept": "application/json", } + if "max_results" not in query: + query["max_results"] = 0 while True: json = self._api.do( "GET", diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index 49f3c3b05..64e6f4b0a 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -3363,6 +3363,7 @@ class EventDetailsCause(Enum): AUTORECOVERY = "AUTORECOVERY" AUTOSCALE = "AUTOSCALE" + AUTOSCALE_V2 = "AUTOSCALE_V2" REPLACE_BAD_NODES = "REPLACE_BAD_NODES" USER_REQUEST = "USER_REQUEST" diff --git a/databricks/sdk/service/dashboards.py b/databricks/sdk/service/dashboards.py index b94b2a089..5bf772f27 100755 --- a/databricks/sdk/service/dashboards.py +++ b/databricks/sdk/service/dashboards.py @@ -1189,6 +1189,7 @@ class MessageErrorType(Enum): TOO_MANY_TABLES_EXCEPTION = "TOO_MANY_TABLES_EXCEPTION" UNEXPECTED_REPLY_PROCESS_EXCEPTION = "UNEXPECTED_REPLY_PROCESS_EXCEPTION" UNKNOWN_AI_MODEL = "UNKNOWN_AI_MODEL" + UNSUPPORTED_CONVERSATION_TYPE_EXCEPTION = "UNSUPPORTED_CONVERSATION_TYPE_EXCEPTION" WAREHOUSE_ACCESS_MISSING_EXCEPTION = "WAREHOUSE_ACCESS_MISSING_EXCEPTION" WAREHOUSE_NOT_FOUND_EXCEPTION = "WAREHOUSE_NOT_FOUND_EXCEPTION" diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 5842aef8c..1ca8e631c 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -7243,6 +7243,10 @@ def from_dict(cls, d: Dict[str, Any]) -> TableTriggerState: @dataclass class TableUpdateTriggerConfiguration: + table_names: List[str] + """A list of tables to monitor for changes. The table name must be in the format + `catalog_name.schema_name.table_name`.""" + condition: Optional[Condition] = None """The table(s) condition based on which to trigger a job run.""" @@ -7250,10 +7254,6 @@ class TableUpdateTriggerConfiguration: """If set, the trigger starts a run only after the specified amount of time has passed since the last time the trigger fired. The minimum allowed value is 60 seconds.""" - table_names: Optional[List[str]] = None - """A list of tables to monitor for changes. The table name must be in the format - `catalog_name.schema_name.table_name`.""" - wait_after_last_change_seconds: Optional[int] = None """If set, the trigger starts a run only after no table updates have occurred for the specified time and can be used to wait for a series of table updates before triggering a run. The minimum diff --git a/databricks/sdk/service/ml.py b/databricks/sdk/service/ml.py index b95021e3d..94fd823ca 100755 --- a/databricks/sdk/service/ml.py +++ b/databricks/sdk/service/ml.py @@ -201,6 +201,31 @@ def from_dict(cls, d: Dict[str, Any]) -> ApproveTransitionRequestResponse: return cls(activity=_from_dict(d, "activity", Activity)) +@dataclass +class BatchCreateMaterializedFeaturesResponse: + materialized_features: Optional[List[MaterializedFeature]] = None + """The created materialized features with assigned IDs.""" + + def as_dict(self) -> dict: + """Serializes the BatchCreateMaterializedFeaturesResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.materialized_features: + body["materialized_features"] = [v.as_dict() for v in self.materialized_features] + return body + + def as_shallow_dict(self) -> dict: + """Serializes the BatchCreateMaterializedFeaturesResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.materialized_features: + body["materialized_features"] = self.materialized_features + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> BatchCreateMaterializedFeaturesResponse: + """Deserializes the BatchCreateMaterializedFeaturesResponse from a dictionary.""" + return cls(materialized_features=_repeated_dict(d, "materialized_features", MaterializedFeature)) + + class CommentActivityAction(Enum): """An action that a user (with sufficient permissions) could take on an activity or comment. @@ -423,6 +448,31 @@ def from_dict(cls, d: Dict[str, Any]) -> CreateLoggedModelResponse: return cls(model=_from_dict(d, "model", LoggedModel)) +@dataclass +class CreateMaterializedFeatureRequest: + materialized_feature: MaterializedFeature + """The materialized feature to create.""" + + def as_dict(self) -> dict: + """Serializes the CreateMaterializedFeatureRequest into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.materialized_feature: + body["materialized_feature"] = self.materialized_feature.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the CreateMaterializedFeatureRequest into a shallow dictionary of its immediate attributes.""" + body = {} + if self.materialized_feature: + body["materialized_feature"] = self.materialized_feature + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> CreateMaterializedFeatureRequest: + """Deserializes the CreateMaterializedFeatureRequest from a dictionary.""" + return cls(materialized_feature=_from_dict(d, "materialized_feature", MaterializedFeature)) + + @dataclass class CreateModelResponse: registered_model: Optional[Model] = None @@ -1334,6 +1384,9 @@ class Feature: filter_condition: Optional[str] = None """The filter condition applied to the source data before aggregation.""" + lineage_context: Optional[LineageContext] = None + """Lineage context information for this feature.""" + def as_dict(self) -> dict: """Serializes the Feature into a dictionary suitable for use as a JSON request body.""" body = {} @@ -1347,6 +1400,8 @@ def as_dict(self) -> dict: body["function"] = self.function.as_dict() if self.inputs: body["inputs"] = [v for v in self.inputs] + if self.lineage_context: + body["lineage_context"] = self.lineage_context.as_dict() if self.source: body["source"] = self.source.as_dict() if self.time_window: @@ -1366,6 +1421,8 @@ def as_shallow_dict(self) -> dict: body["function"] = self.function if self.inputs: body["inputs"] = self.inputs + if self.lineage_context: + body["lineage_context"] = self.lineage_context if self.source: body["source"] = self.source if self.time_window: @@ -1381,6 +1438,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Feature: full_name=d.get("full_name", None), function=_from_dict(d, "function", Function), inputs=d.get("inputs", None), + lineage_context=_from_dict(d, "lineage_context", LineageContext), source=_from_dict(d, "source", DataSource), time_window=_from_dict(d, "time_window", TimeWindow), ) @@ -2192,6 +2250,38 @@ def from_dict(cls, d: Dict[str, Any]) -> InputTag: return cls(key=d.get("key", None), value=d.get("value", None)) +@dataclass +class JobContext: + job_id: Optional[int] = None + """The job ID where this API invoked.""" + + job_run_id: Optional[int] = None + """The job run ID where this API was invoked.""" + + def as_dict(self) -> dict: + """Serializes the JobContext into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.job_id is not None: + body["job_id"] = self.job_id + if self.job_run_id is not None: + body["job_run_id"] = self.job_run_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the JobContext into a shallow dictionary of its immediate attributes.""" + body = {} + if self.job_id is not None: + body["job_id"] = self.job_id + if self.job_run_id is not None: + body["job_run_id"] = self.job_run_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> JobContext: + """Deserializes the JobContext from a dictionary.""" + return cls(job_id=d.get("job_id", None), job_run_id=d.get("job_run_id", None)) + + @dataclass class JobSpec: job_id: str @@ -2269,6 +2359,42 @@ def from_dict(cls, d: Dict[str, Any]) -> JobSpecWithoutSecret: return cls(job_id=d.get("job_id", None), workspace_url=d.get("workspace_url", None)) +@dataclass +class LineageContext: + """Lineage context information for tracking where an API was invoked. This will allow us to track + lineage, which currently uses caller entity information for use across the Lineage Client and + Observability in Lumberjack.""" + + job_context: Optional[JobContext] = None + """Job context information including job ID and run ID.""" + + notebook_id: Optional[int] = None + """The notebook ID where this API was invoked.""" + + def as_dict(self) -> dict: + """Serializes the LineageContext into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.job_context: + body["job_context"] = self.job_context.as_dict() + if self.notebook_id is not None: + body["notebook_id"] = self.notebook_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the LineageContext into a shallow dictionary of its immediate attributes.""" + body = {} + if self.job_context: + body["job_context"] = self.job_context + if self.notebook_id is not None: + body["notebook_id"] = self.notebook_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> LineageContext: + """Deserializes the LineageContext from a dictionary.""" + return cls(job_context=_from_dict(d, "job_context", JobContext), notebook_id=d.get("notebook_id", None)) + + @dataclass class LinkedFeature: """Feature for model version. ([ML-57150] Renamed from Feature to LinkedFeature)""" @@ -6844,6 +6970,30 @@ class FeatureEngineeringAPI: def __init__(self, api_client): self._api = api_client + def batch_create_materialized_features( + self, requests: List[CreateMaterializedFeatureRequest] + ) -> BatchCreateMaterializedFeaturesResponse: + """Batch create materialized features. + + :param requests: List[:class:`CreateMaterializedFeatureRequest`] + The requests to create materialized features. + + :returns: :class:`BatchCreateMaterializedFeaturesResponse` + """ + + body = {} + if requests is not None: + body["requests"] = [v.as_dict() for v in requests] + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + + res = self._api.do( + "POST", "/api/2.0/feature-engineering/materialized-features:batchCreate", body=body, headers=headers + ) + return BatchCreateMaterializedFeaturesResponse.from_dict(res) + def create_feature(self, feature: Feature) -> Feature: """Create a Feature. diff --git a/databricks/sdk/service/pipelines.py b/databricks/sdk/service/pipelines.py index 458c4b388..9ab410419 100755 --- a/databricks/sdk/service/pipelines.py +++ b/databricks/sdk/service/pipelines.py @@ -556,8 +556,8 @@ class IngestionGatewayPipelineDefinition: gateway_storage_name: Optional[str] = None """Optional. The Unity Catalog-compatible name for the gateway storage location. This is the - destination to use for the data that is extracted by the gateway. Delta Live Tables system will - automatically create the storage location under the catalog and schema.""" + destination to use for the data that is extracted by the gateway. Spark Declarative Pipelines + system will automatically create the storage location under the catalog and schema.""" def as_dict(self) -> dict: """Serializes the IngestionGatewayPipelineDefinition into a dictionary suitable for use as a JSON request body.""" @@ -2979,8 +2979,8 @@ class TableSpecificConfig: """The SCD type to use to ingest the table.""" sequence_by: Optional[List[str]] = None - """The column names specifying the logical order of events in the source data. Delta Live Tables - uses this sequencing to handle change events that arrive out of order.""" + """The column names specifying the logical order of events in the source data. Spark Declarative + Pipelines uses this sequencing to handle change events that arrive out of order.""" workday_report_parameters: Optional[IngestionPipelineDefinitionWorkdayReportParameters] = None """(Optional) Additional custom parameters for Workday Report""" @@ -3254,16 +3254,17 @@ class UpdateStateInfoState(Enum): class PipelinesAPI: - """The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines. + """The Lakeflow Spark Declarative Pipelines API allows you to create, edit, delete, start, and view details + about pipelines. - Delta Live Tables is a framework for building reliable, maintainable, and testable data processing - pipelines. You define the transformations to perform on your data, and Delta Live Tables manages task - orchestration, cluster management, monitoring, data quality, and error handling. + Spark Declarative Pipelines is a framework for building reliable, maintainable, and testable data + processing pipelines. You define the transformations to perform on your data, and Spark Declarative + Pipelines manages task orchestration, cluster management, monitoring, data quality, and error handling. - Instead of defining your data pipelines using a series of separate Apache Spark tasks, Delta Live Tables - manages how your data is transformed based on a target schema you define for each processing step. You can - also enforce data quality with Delta Live Tables expectations. Expectations allow you to define expected - data quality and specify how to handle records that fail those expectations.""" + Instead of defining your data pipelines using a series of separate Apache Spark tasks, Spark Declarative + Pipelines manages how your data is transformed based on a target schema you define for each processing + step. You can also enforce data quality with Spark Declarative Pipelines expectations. Expectations allow + you to define expected data quality and specify how to handle records that fail those expectations.""" def __init__(self, api_client): self._api = api_client @@ -3626,7 +3627,7 @@ def list_pipelines( order_by: Optional[List[str]] = None, page_token: Optional[str] = None, ) -> Iterator[PipelineStateInfo]: - """Lists pipelines defined in the Delta Live Tables system. + """Lists pipelines defined in the Spark Declarative Pipelines system. :param filter: str (optional) Select a subset of results based on the specified criteria. The supported filters are: diff --git a/databricks/sdk/service/vectorsearch.py b/databricks/sdk/service/vectorsearch.py index 370ce78d4..a0b731ffa 100755 --- a/databricks/sdk/service/vectorsearch.py +++ b/databricks/sdk/service/vectorsearch.py @@ -598,6 +598,8 @@ class EndpointStatusState(Enum): OFFLINE = "OFFLINE" ONLINE = "ONLINE" PROVISIONING = "PROVISIONING" + RED_STATE = "RED_STATE" + YELLOW_STATE = "YELLOW_STATE" class EndpointType(Enum): @@ -1756,7 +1758,7 @@ def query_index( :param query_text: str (optional) Query text. Required for Delta Sync Index using model endpoint. :param query_type: str (optional) - The query type to use. Choices are `ANN` and `HYBRID`. Defaults to `ANN`. + The query type to use. Choices are `ANN` and `HYBRID` and `FULL_TEXT`. Defaults to `ANN`. :param query_vector: List[float] (optional) Query vector. Required for Direct Vector Access Index and Delta Sync Index using self-managed vectors. diff --git a/docs/account/iam/workspace_assignment.rst b/docs/account/iam/workspace_assignment.rst index 133b16f3d..2a8043172 100644 --- a/docs/account/iam/workspace_assignment.rst +++ b/docs/account/iam/workspace_assignment.rst @@ -43,9 +43,9 @@ a = AccountClient() - workspace_id = os.environ["TEST_WORKSPACE_ID"] + workspace_id = os.environ["DUMMY_WORKSPACE_ID"] - all = a.workspace_assignment.list(list=workspace_id) + all = a.workspace_assignment.list(workspace_id=workspace_id) Get the permission assignments for the specified Databricks account and Databricks workspace. @@ -74,9 +74,9 @@ spn_id = spn.id - workspace_id = os.environ["DUMMY_WORKSPACE_ID"] + workspace_id = os.environ["TEST_WORKSPACE_ID"] - _ = a.workspace_assignment.update( + a.workspace_assignment.update( workspace_id=workspace_id, principal_id=spn_id, permissions=[iam.WorkspacePermission.USER], diff --git a/docs/account/provisioning/credentials.rst b/docs/account/provisioning/credentials.rst index d63648d58..b71c1707e 100644 --- a/docs/account/provisioning/credentials.rst +++ b/docs/account/provisioning/credentials.rst @@ -24,15 +24,15 @@ a = AccountClient() - role = a.credentials.create( + creds = a.credentials.create( credentials_name=f"sdk-{time.time_ns()}", aws_credentials=provisioning.CreateCredentialAwsCredentials( - sts_role=provisioning.CreateCredentialStsRole(role_arn=os.environ["TEST_CROSSACCOUNT_ARN"]) + sts_role=provisioning.CreateCredentialStsRole(role_arn=os.environ["TEST_LOGDELIVERY_ARN"]) ), ) # cleanup - a.credentials.delete(credentials_id=role.credentials_id) + a.credentials.delete(credentials_id=creds.credentials_id) Creates a Databricks credential configuration that represents cloud cross-account credentials for a specified account. Databricks uses this to set up network infrastructure properly to host Databricks diff --git a/docs/account/provisioning/storage.rst b/docs/account/provisioning/storage.rst index b9f080e36..41a04deb3 100644 --- a/docs/account/provisioning/storage.rst +++ b/docs/account/provisioning/storage.rst @@ -16,7 +16,6 @@ .. code-block:: - import os import time from databricks.sdk import AccountClient @@ -24,13 +23,13 @@ a = AccountClient() - storage = a.storage.create( + bucket = a.storage.create( storage_configuration_name=f"sdk-{time.time_ns()}", - root_bucket_info=provisioning.RootBucketInfo(bucket_name=os.environ["TEST_ROOT_BUCKET"]), + root_bucket_info=provisioning.RootBucketInfo(bucket_name=f"sdk-{time.time_ns()}"), ) # cleanup - a.storage.delete(storage_configuration_id=storage.storage_configuration_id) + a.storage.delete(storage_configuration_id=bucket.storage_configuration_id) Creates a Databricks storage configuration for an account. diff --git a/docs/dbdataclasses/compute.rst b/docs/dbdataclasses/compute.rst index d5871432c..c6064252a 100644 --- a/docs/dbdataclasses/compute.rst +++ b/docs/dbdataclasses/compute.rst @@ -421,6 +421,9 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: AUTOSCALE :value: "AUTOSCALE" + .. py:attribute:: AUTOSCALE_V2 + :value: "AUTOSCALE_V2" + .. py:attribute:: REPLACE_BAD_NODES :value: "REPLACE_BAD_NODES" diff --git a/docs/dbdataclasses/dashboards.rst b/docs/dbdataclasses/dashboards.rst index 394036f3a..df004c847 100644 --- a/docs/dbdataclasses/dashboards.rst +++ b/docs/dbdataclasses/dashboards.rst @@ -298,6 +298,9 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: UNKNOWN_AI_MODEL :value: "UNKNOWN_AI_MODEL" + .. py:attribute:: UNSUPPORTED_CONVERSATION_TYPE_EXCEPTION + :value: "UNSUPPORTED_CONVERSATION_TYPE_EXCEPTION" + .. py:attribute:: WAREHOUSE_ACCESS_MISSING_EXCEPTION :value: "WAREHOUSE_ACCESS_MISSING_EXCEPTION" diff --git a/docs/dbdataclasses/ml.rst b/docs/dbdataclasses/ml.rst index 1d8fba9ef..844e66245 100644 --- a/docs/dbdataclasses/ml.rst +++ b/docs/dbdataclasses/ml.rst @@ -66,6 +66,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: BatchCreateMaterializedFeaturesResponse + :members: + :undoc-members: + .. py:class:: CommentActivityAction An action that a user (with sufficient permissions) could take on an activity or comment. @@ -114,6 +118,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: CreateMaterializedFeatureRequest + :members: + :undoc-members: + .. autoclass:: CreateModelResponse :members: :undoc-members: @@ -405,6 +413,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: JobContext + :members: + :undoc-members: + .. autoclass:: JobSpec :members: :undoc-members: @@ -413,6 +425,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: LineageContext + :members: + :undoc-members: + .. autoclass:: LinkedFeature :members: :undoc-members: diff --git a/docs/dbdataclasses/vectorsearch.rst b/docs/dbdataclasses/vectorsearch.rst index b6250cc64..b8bd46536 100644 --- a/docs/dbdataclasses/vectorsearch.rst +++ b/docs/dbdataclasses/vectorsearch.rst @@ -80,6 +80,12 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: PROVISIONING :value: "PROVISIONING" + .. py:attribute:: RED_STATE + :value: "RED_STATE" + + .. py:attribute:: YELLOW_STATE + :value: "YELLOW_STATE" + .. py:class:: EndpointType Type of endpoint. diff --git a/docs/workspace/catalog/catalogs.rst b/docs/workspace/catalog/catalogs.rst index 77de87dc4..17297d8dd 100644 --- a/docs/workspace/catalog/catalogs.rst +++ b/docs/workspace/catalog/catalogs.rst @@ -24,10 +24,10 @@ w = WorkspaceClient() - created = w.catalogs.create(name=f"sdk-{time.time_ns()}") + created_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") # cleanup - w.catalogs.delete(name=created.name, force=True) + w.catalogs.delete(name=created_catalog.name, force=True) Creates a new catalog instance in the parent metastore if the caller is a metastore admin or has the **CREATE_CATALOG** privilege. diff --git a/docs/workspace/catalog/external_locations.rst b/docs/workspace/catalog/external_locations.rst index df5bdb1b6..612800956 100644 --- a/docs/workspace/catalog/external_locations.rst +++ b/docs/workspace/catalog/external_locations.rst @@ -30,22 +30,20 @@ w = WorkspaceClient() - storage_credential = w.storage_credentials.create( + credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), - comment="created via SDK", ) - external_location = w.external_locations.create( + created = w.external_locations.create( name=f"sdk-{time.time_ns()}", - credential_name=storage_credential.name, - comment="created via SDK", - url="s3://" + os.environ["TEST_BUCKET"] + "/" + f"sdk-{time.time_ns()}", + credential_name=credential.name, + url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), ) # cleanup - w.storage_credentials.delete(name=storage_credential.name) - w.external_locations.delete(name=external_location.name) + w.storage_credentials.delete(name=credential.name) + w.external_locations.delete(name=created.name) Creates a new external location entry in the metastore. The caller must be a metastore admin or have the **CREATE_EXTERNAL_LOCATION** privilege on both the metastore and the associated storage @@ -107,20 +105,20 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', + url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), ) - _ = w.external_locations.get(get=created.name) + _ = w.external_locations.get(name=created.name) # cleanup - w.storage_credentials.delete(delete=credential.name) - w.external_locations.delete(delete=created.name) + w.storage_credentials.delete(name=credential.name) + w.external_locations.delete(name=created.name) Gets an external location from the metastore. The caller must be either a metastore admin, the owner of the external location, or a user that has some privilege on the external location. @@ -142,11 +140,10 @@ .. code-block:: from databricks.sdk import WorkspaceClient - from databricks.sdk.service import catalog w = WorkspaceClient() - all = w.external_locations.list(catalog.ListExternalLocationsRequest()) + all = w.external_locations.list() Gets an array of external locations (__ExternalLocationInfo__ objects) from the metastore. The caller must be a metastore admin, the owner of the external location, or a user that has some privilege on diff --git a/docs/workspace/catalog/storage_credentials.rst b/docs/workspace/catalog/storage_credentials.rst index ad6e4ebe5..d8111141e 100644 --- a/docs/workspace/catalog/storage_credentials.rst +++ b/docs/workspace/catalog/storage_credentials.rst @@ -30,13 +30,13 @@ w = WorkspaceClient() - created = w.storage_credentials.create( + credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) # cleanup - w.storage_credentials.delete(delete=created.name) + w.storage_credentials.delete(name=credential.name) Creates a new storage credential. @@ -98,13 +98,13 @@ created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) - by_name = w.storage_credentials.get(name=created.name) + by_name = w.storage_credentials.get(get=created.name) # cleanup - w.storage_credentials.delete(name=created.name) + w.storage_credentials.delete(delete=created.name) Gets a storage credential from the metastore. The caller must be a metastore admin, the owner of the storage credential, or have some permission on the storage credential. @@ -123,10 +123,11 @@ .. code-block:: from databricks.sdk import WorkspaceClient + from databricks.sdk.service import catalog w = WorkspaceClient() - all = w.storage_credentials.list() + all = w.storage_credentials.list(catalog.ListStorageCredentialsRequest()) Gets an array of storage credentials (as __StorageCredentialInfo__ objects). The array is limited to only those storage credentials the caller has permission to access. If the caller is a metastore diff --git a/docs/workspace/compute/clusters.rst b/docs/workspace/compute/clusters.rst index d46b8ecd0..db78626ff 100644 --- a/docs/workspace/compute/clusters.rst +++ b/docs/workspace/compute/clusters.rst @@ -647,11 +647,10 @@ .. code-block:: from databricks.sdk import WorkspaceClient - from databricks.sdk.service import compute w = WorkspaceClient() - all = w.clusters.list(compute.ListClustersRequest()) + nodes = w.clusters.list_node_types() Return information about all pinned and active clusters, and all clusters terminated within the last 30 days. Clusters terminated prior to this period are not included. diff --git a/docs/workspace/iam/current_user.rst b/docs/workspace/iam/current_user.rst index 2f95213e2..b2390ce63 100644 --- a/docs/workspace/iam/current_user.rst +++ b/docs/workspace/iam/current_user.rst @@ -17,7 +17,7 @@ w = WorkspaceClient() - me2 = w.current_user.me() + me = w.current_user.me() Get details about the current method caller's identity. diff --git a/docs/workspace/iam/permissions.rst b/docs/workspace/iam/permissions.rst index 15524c53e..ea24afd1a 100644 --- a/docs/workspace/iam/permissions.rst +++ b/docs/workspace/iam/permissions.rst @@ -44,7 +44,7 @@ obj = w.workspace.get_status(path=notebook_path) - levels = w.permissions.get_permission_levels(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) + _ = w.permissions.get(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) Gets the permissions of an object. Objects can inherit permissions from their parent objects or root object. diff --git a/docs/workspace/jobs/jobs.rst b/docs/workspace/jobs/jobs.rst index 39beecc1b..0b82986de 100644 --- a/docs/workspace/jobs/jobs.rst +++ b/docs/workspace/jobs/jobs.rst @@ -357,21 +357,23 @@ w.clusters.ensure_cluster_is_running(os.environ["DATABRICKS_CLUSTER_ID"]) and os.environ["DATABRICKS_CLUSTER_ID"] ) - run = w.jobs.submit( - run_name=f"sdk-{time.time_ns()}", + created_job = w.jobs.create( + name=f"sdk-{time.time_ns()}", tasks=[ - jobs.SubmitTask( + jobs.Task( + description="test", existing_cluster_id=cluster_id, notebook_task=jobs.NotebookTask(notebook_path=notebook_path), - task_key=f"sdk-{time.time_ns()}", + task_key="test", + timeout_seconds=0, ) ], - ).result() + ) - output = w.jobs.get_run_output(run_id=run.tasks[0].run_id) + by_id = w.jobs.get(job_id=created_job.job_id) # cleanup - w.jobs.delete_run(run_id=run.run_id) + w.jobs.delete(job_id=created_job.job_id) Get a single job. diff --git a/docs/workspace/ml/feature_engineering.rst b/docs/workspace/ml/feature_engineering.rst index 0ec7cc8bf..57c99a11c 100644 --- a/docs/workspace/ml/feature_engineering.rst +++ b/docs/workspace/ml/feature_engineering.rst @@ -6,6 +6,16 @@ [description] + .. py:method:: batch_create_materialized_features(requests: List[CreateMaterializedFeatureRequest]) -> BatchCreateMaterializedFeaturesResponse + + Batch create materialized features. + + :param requests: List[:class:`CreateMaterializedFeatureRequest`] + The requests to create materialized features. + + :returns: :class:`BatchCreateMaterializedFeaturesResponse` + + .. py:method:: create_feature(feature: Feature) -> Feature Create a Feature. diff --git a/docs/workspace/ml/model_registry.rst b/docs/workspace/ml/model_registry.rst index 2d34256e4..98d803a63 100644 --- a/docs/workspace/ml/model_registry.rst +++ b/docs/workspace/ml/model_registry.rst @@ -90,7 +90,7 @@ w = WorkspaceClient() - model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") + created = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") Creates a new registered model with the name specified in the request body. Throws `RESOURCE_ALREADY_EXISTS` if a registered model with the given name exists. @@ -120,7 +120,7 @@ model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - mv = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") + created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") Creates a model version. @@ -734,14 +734,13 @@ w = WorkspaceClient() - model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") + created = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") + model = w.model_registry.get_model(name=created.registered_model.name) - w.model_registry.update_model_version( + w.model_registry.update_model( + name=model.registered_model_databricks.name, description=f"sdk-{time.time_ns()}", - name=created.model_version.name, - version=created.model_version.version, ) Updates a registered model. diff --git a/docs/workspace/pipelines/pipelines.rst b/docs/workspace/pipelines/pipelines.rst index 502061df3..b0bada615 100644 --- a/docs/workspace/pipelines/pipelines.rst +++ b/docs/workspace/pipelines/pipelines.rst @@ -4,16 +4,17 @@ .. py:class:: PipelinesAPI - The Delta Live Tables API allows you to create, edit, delete, start, and view details about pipelines. + The Lakeflow Spark Declarative Pipelines API allows you to create, edit, delete, start, and view details + about pipelines. - Delta Live Tables is a framework for building reliable, maintainable, and testable data processing - pipelines. You define the transformations to perform on your data, and Delta Live Tables manages task - orchestration, cluster management, monitoring, data quality, and error handling. + Spark Declarative Pipelines is a framework for building reliable, maintainable, and testable data + processing pipelines. You define the transformations to perform on your data, and Spark Declarative + Pipelines manages task orchestration, cluster management, monitoring, data quality, and error handling. - Instead of defining your data pipelines using a series of separate Apache Spark tasks, Delta Live Tables - manages how your data is transformed based on a target schema you define for each processing step. You can - also enforce data quality with Delta Live Tables expectations. Expectations allow you to define expected - data quality and specify how to handle records that fail those expectations. + Instead of defining your data pipelines using a series of separate Apache Spark tasks, Spark Declarative + Pipelines manages how your data is transformed based on a target schema you define for each processing + step. You can also enforce data quality with Spark Declarative Pipelines expectations. Expectations allow + you to define expected data quality and specify how to handle records that fail those expectations. .. py:method:: create( [, allow_duplicate_names: Optional[bool], budget_policy_id: Optional[str], catalog: Optional[str], channel: Optional[str], clusters: Optional[List[PipelineCluster]], configuration: Optional[Dict[str, str]], continuous: Optional[bool], deployment: Optional[PipelineDeployment], development: Optional[bool], dry_run: Optional[bool], edition: Optional[str], environment: Optional[PipelinesEnvironment], event_log: Optional[EventLogSpec], filters: Optional[Filters], gateway_definition: Optional[IngestionGatewayPipelineDefinition], id: Optional[str], ingestion_definition: Optional[IngestionPipelineDefinition], libraries: Optional[List[PipelineLibrary]], name: Optional[str], notifications: Optional[List[Notifications]], photon: Optional[bool], restart_window: Optional[RestartWindow], root_path: Optional[str], run_as: Optional[RunAs], schema: Optional[str], serverless: Optional[bool], storage: Optional[str], tags: Optional[Dict[str, str]], target: Optional[str], trigger: Optional[PipelineTrigger], usage_policy_id: Optional[str]]) -> CreatePipelineResponse @@ -291,7 +292,7 @@ all = w.pipelines.list_pipelines(pipelines.ListPipelinesRequest()) - Lists pipelines defined in the Delta Live Tables system. + Lists pipelines defined in the Spark Declarative Pipelines system. :param filter: str (optional) Select a subset of results based on the specified criteria. The supported filters are: diff --git a/docs/workspace/sharing/providers.rst b/docs/workspace/sharing/providers.rst index fd81e1b24..1a7c88de9 100644 --- a/docs/workspace/sharing/providers.rst +++ b/docs/workspace/sharing/providers.rst @@ -101,12 +101,25 @@ .. code-block:: + import time + from databricks.sdk import WorkspaceClient - from databricks.sdk.service import sharing w = WorkspaceClient() - all = w.providers.list(sharing.ListProvidersRequest()) + public_share_recipient = """{ + "shareCredentialsVersion":1, + "bearerToken":"dapiabcdefghijklmonpqrstuvwxyz", + "endpoint":"https://sharing.delta.io/delta-sharing/" + } + """ + + created = w.providers.create(name=f"sdk-{time.time_ns()}", recipient_profile_str=public_share_recipient) + + shares = w.providers.list_shares(name=created.name) + + # cleanup + w.providers.delete(name=created.name) Gets an array of available authentication providers. The caller must either be a metastore admin or the owner of the providers. Providers not owned by the caller are not included in the response. There diff --git a/docs/workspace/sql/queries.rst b/docs/workspace/sql/queries.rst index f0081b3f2..0dfb63fbf 100644 --- a/docs/workspace/sql/queries.rst +++ b/docs/workspace/sql/queries.rst @@ -29,7 +29,7 @@ display_name=f"sdk-{time.time_ns()}", warehouse_id=srcs[0].warehouse_id, description="test query from Go SDK", - query_text="SHOW TABLES", + query_text="SELECT 1", ) ) diff --git a/docs/workspace/vectorsearch/vector_search_indexes.rst b/docs/workspace/vectorsearch/vector_search_indexes.rst index 11417c9da..c89f3579c 100644 --- a/docs/workspace/vectorsearch/vector_search_indexes.rst +++ b/docs/workspace/vectorsearch/vector_search_indexes.rst @@ -102,7 +102,7 @@ :param query_text: str (optional) Query text. Required for Delta Sync Index using model endpoint. :param query_type: str (optional) - The query type to use. Choices are `ANN` and `HYBRID`. Defaults to `ANN`. + The query type to use. Choices are `ANN` and `HYBRID` and `FULL_TEXT`. Defaults to `ANN`. :param query_vector: List[float] (optional) Query vector. Required for Direct Vector Access Index and Delta Sync Index using self-managed vectors. diff --git a/docs/workspace/workspace/workspace.rst b/docs/workspace/workspace/workspace.rst index 02f5e5931..e1b7d12b9 100644 --- a/docs/workspace/workspace/workspace.rst +++ b/docs/workspace/workspace/workspace.rst @@ -178,7 +178,7 @@ content=base64.b64encode(("CREATE LIVE TABLE dlt_sample AS SELECT 1").encode()).decode(), format=workspace.ImportFormat.SOURCE, language=workspace.Language.SQL, - overwrite=True, + overwrite=true_, path=notebook_path, )