From 47dfc6dc9ccbb0931dff9ed9a10a330239e75514 Mon Sep 17 00:00:00 2001 From: hectorcast-db Date: Mon, 19 Feb 2024 17:07:19 +0100 Subject: [PATCH] Release v0.20.0 (#550) Major Changes: * Updated behaviour for raw parameter in `ApiClient.do()` method. The raw data is not returned directly anymore, but as part of a dict with the `contents` key. This dict will also contain response headers if returned by the API. Internal Changes: * Add get_workspace_id to docgen blocklist ([#549](https://github.com/databricks/databricks-sdk-py/pull/549)). * Support HEAD operation and response Headers ([#547](https://github.com/databricks/databricks-sdk-py/pull/547)). API Changes: * Changed `delete()`, `get()` and `update()` methods for [w.connections](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/connections.html) workspace-level service with new required argument order. * Changed `update()` method for [w.lakehouse_monitors](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/lakehouse_monitors.html) workspace-level service with new required argument order. * Changed `delete()`, `get()` and `update()` methods for [w.volumes](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/volumes.html) workspace-level service with new required argument order. * Added [w.online_tables](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/online_tables.html) workspace-level service. * Renamed `name_arg` field to `name` for the following dataclasses: `databricks.sdk.service.catalog.DeleteConnectionRequest`, `databricks.sdk.service.catalog.GetConnectionRequest`, `databricks.sdk.service.catalog.UpdateConnection`, `databricks.sdk.service.sharing.DeleteCleanRoomRequest`, `databricks.sdk.service.sharing.GetCleanRoomRequest` and `databricks.sdk.service.sharing.UpdateCleanRoom`. * Removed `full_name_arg` field for `databricks.sdk.service.catalog.DeleteVolumeRequest`. * Added `name` field for `databricks.sdk.service.catalog.DeleteVolumeRequest`. * Added `max_results` field for `databricks.sdk.service.catalog.ListVolumesRequest`. * Added `page_token` field for `databricks.sdk.service.catalog.ListVolumesRequest`. * Added `next_page_token` field for `databricks.sdk.service.catalog.ListVolumesResponseContent`. * Removed `full_name_arg` field for `databricks.sdk.service.catalog.ReadVolumeRequest`. * Added `name` field for `databricks.sdk.service.catalog.ReadVolumeRequest`. * Removed `assets_dir` field for `databricks.sdk.service.catalog.UpdateMonitor`. * Removed `full_name_arg` field for `databricks.sdk.service.catalog.UpdateVolumeRequestContent`. * Added `name` field for `databricks.sdk.service.catalog.UpdateVolumeRequestContent`. * Added the following catalog dataclasses: `ContinuousUpdateStatus`, `DeleteOnlineTableRequest`, `FailedStatus`, `GetOnlineTableRequest`, `OnlineTable`, `OnlineTableSpec`, `OnlineTableState`, `OnlineTableStatus`, `PipelineProgress`, `ProvisioningStatus`, `TriggeredUpdateStatus` and `ViewData`. * Added `get_directory_metadata()` method for [w.files](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/files.html) workspace-level service. * Added `get_metadata()` method for [w.files](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/files.html) workspace-level service. * Added `content_length`, `content_type` and `last_modified` fields for `databricks.sdk.service.files.DownloadResponse`. * Added the following files dataclasses: `FileSize`, `GetDirectoryMetadataRequest`, `GetMetadataRequest`, `GetMetadataResponse` and `LastModifiedHttpDate`. * Removed `trigger_history` field for `databricks.sdk.service.jobs.Job`. * Removed `databricks.sdk.service.jobs.TriggerEvaluation` dataclass. * Removed `databricks.sdk.service.jobs.TriggerHistory` dataclass. * Added `table` field for `databricks.sdk.service.jobs.TriggerSettings`. * Added `databricks.sdk.service.jobs.Condition` dataclass. * Added `databricks.sdk.service.jobs.TableTriggerConfiguration` dataclass. * Removed `config` field for `databricks.sdk.service.serving.ExternalModel`. * Removed `databricks.sdk.service.serving.ExternalModelConfig` dataclass. Fields moved to `databricks.sdk.service.serving.ExternalModel`. * Added `max_provisioned_throughput` and `min_provisioned_throughput` fields for `databricks.sdk.service.serving.ServedEntityInput`. * Added `max_provisioned_throughput` and `min_provisioned_throughput` fields for `databricks.sdk.service.serving.ServedEntityOutput`. * Changed `delete()` method for [w.clean_rooms](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/clean_rooms.html) workspace-level service with new required argument order. * Changed `get()` method for [w.clean_rooms](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/clean_rooms.html) workspace-level service with new required argument order. * Changed `update()` method for [w.clean_rooms](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/clean_rooms.html) workspace-level service with new required argument order. * Added `enum_options` field for `databricks.sdk.service.sql.Parameter`. * Added `multi_values_options` field for `databricks.sdk.service.sql.Parameter`. * Added `query_id` field for `databricks.sdk.service.sql.Parameter`. * Added `databricks.sdk.service.sql.MultiValuesOptions` dataclass. OpenAPI SHA: cdd76a98a4fca7008572b3a94427566dd286c63b, Date: 2024-02-19 --- .codegen/_openapi_sha | 2 +- CHANGELOG.md | 61 +++ databricks/sdk/__init__.py | 9 +- databricks/sdk/service/catalog.py | 510 ++++++++++++++++-- databricks/sdk/service/files.py | 138 ++++- databricks/sdk/service/jobs.py | 264 +++++---- databricks/sdk/service/serving.py | 75 +-- databricks/sdk/service/sharing.py | 30 +- databricks/sdk/service/sql.py | 51 +- databricks/sdk/version.py | 2 +- docs/dbdataclasses/catalog.rst | 36 ++ docs/dbdataclasses/files.rst | 4 + docs/dbdataclasses/jobs.rst | 12 +- docs/dbdataclasses/serving.rst | 4 - docs/dbdataclasses/sql.rst | 4 + docs/workspace/catalog/connections.rst | 24 +- docs/workspace/catalog/index.rst | 1 + docs/workspace/catalog/lakehouse_monitors.rst | 4 +- docs/workspace/catalog/online_tables.rst | 47 ++ docs/workspace/catalog/volumes.rst | 43 +- docs/workspace/jobs/jobs.rst | 40 +- docs/workspace/sharing/clean_rooms.rst | 12 +- tests/integration/test_files.py | 5 +- 23 files changed, 1056 insertions(+), 322 deletions(-) create mode 100644 docs/workspace/catalog/online_tables.rst diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 2f884a290..013e5ffe8 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -6b897bc95b23abed8b9f5eff0e6b8ec034046180 \ No newline at end of file +cdd76a98a4fca7008572b3a94427566dd286c63b \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index aa6cdb36a..61dd444d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,66 @@ # Version changelog +## 0.20.0 + +Major Changes: + +* Updated behaviour for raw parameter in `ApiClient.do()` method. The raw data is not returned directly anymore, but as part of a dict with the `contents` key. This dict will also contain response headers if returned by the API. + +Internal Changes: + +* Add get_workspace_id to docgen blocklist ([#549](https://github.com/databricks/databricks-sdk-py/pull/549)). +* Support HEAD operation and response Headers ([#547](https://github.com/databricks/databricks-sdk-py/pull/547)). + +API Changes: + + * Changed `delete()`, `get()` and `update()` methods for [w.connections](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/connections.html) workspace-level service with new required argument order. + * Changed `update()` method for [w.lakehouse_monitors](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/lakehouse_monitors.html) workspace-level service with new required argument order. + * Changed `delete()`, `get()` and `update()` methods for [w.volumes](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/volumes.html) workspace-level service with new required argument order. + * Added [w.online_tables](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/online_tables.html) workspace-level service. + * Renamed `name_arg` field to `name` for the following dataclasses: `databricks.sdk.service.catalog.DeleteConnectionRequest`, + `databricks.sdk.service.catalog.GetConnectionRequest`, + `databricks.sdk.service.catalog.UpdateConnection`, + `databricks.sdk.service.sharing.DeleteCleanRoomRequest`, + `databricks.sdk.service.sharing.GetCleanRoomRequest` and + `databricks.sdk.service.sharing.UpdateCleanRoom`. + * Removed `full_name_arg` field for `databricks.sdk.service.catalog.DeleteVolumeRequest`. + * Added `name` field for `databricks.sdk.service.catalog.DeleteVolumeRequest`. + * Added `max_results` field for `databricks.sdk.service.catalog.ListVolumesRequest`. + * Added `page_token` field for `databricks.sdk.service.catalog.ListVolumesRequest`. + * Added `next_page_token` field for `databricks.sdk.service.catalog.ListVolumesResponseContent`. + * Removed `full_name_arg` field for `databricks.sdk.service.catalog.ReadVolumeRequest`. + * Added `name` field for `databricks.sdk.service.catalog.ReadVolumeRequest`. + * Removed `assets_dir` field for `databricks.sdk.service.catalog.UpdateMonitor`. + * Removed `full_name_arg` field for `databricks.sdk.service.catalog.UpdateVolumeRequestContent`. + * Added `name` field for `databricks.sdk.service.catalog.UpdateVolumeRequestContent`. + * Added the following catalog dataclasses: `ContinuousUpdateStatus`, `DeleteOnlineTableRequest`, `FailedStatus`, + `GetOnlineTableRequest`, `OnlineTable`, `OnlineTableSpec`, `OnlineTableState`, `OnlineTableStatus`, + `PipelineProgress`, `ProvisioningStatus`, `TriggeredUpdateStatus` and `ViewData`. + * Added `get_directory_metadata()` method for [w.files](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/files.html) workspace-level service. + * Added `get_metadata()` method for [w.files](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/files.html) workspace-level service. + * Added `content_length`, `content_type` and `last_modified` fields for `databricks.sdk.service.files.DownloadResponse`. + * Added the following files dataclasses: `FileSize`, `GetDirectoryMetadataRequest`, `GetMetadataRequest`, + `GetMetadataResponse` and `LastModifiedHttpDate`. + * Removed `trigger_history` field for `databricks.sdk.service.jobs.Job`. + * Removed `databricks.sdk.service.jobs.TriggerEvaluation` dataclass. + * Removed `databricks.sdk.service.jobs.TriggerHistory` dataclass. + * Added `table` field for `databricks.sdk.service.jobs.TriggerSettings`. + * Added `databricks.sdk.service.jobs.Condition` dataclass. + * Added `databricks.sdk.service.jobs.TableTriggerConfiguration` dataclass. + * Removed `config` field for `databricks.sdk.service.serving.ExternalModel`. + * Removed `databricks.sdk.service.serving.ExternalModelConfig` dataclass. Fields moved to `databricks.sdk.service.serving.ExternalModel`. + * Added `max_provisioned_throughput` and `min_provisioned_throughput` fields for `databricks.sdk.service.serving.ServedEntityInput`. + * Added `max_provisioned_throughput` and `min_provisioned_throughput` fields for `databricks.sdk.service.serving.ServedEntityOutput`. + * Changed `delete()` method for [w.clean_rooms](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/clean_rooms.html) workspace-level service with new required argument order. + * Changed `get()` method for [w.clean_rooms](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/clean_rooms.html) workspace-level service with new required argument order. + * Changed `update()` method for [w.clean_rooms](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/clean_rooms.html) workspace-level service with new required argument order. + * Added `enum_options` field for `databricks.sdk.service.sql.Parameter`. + * Added `multi_values_options` field for `databricks.sdk.service.sql.Parameter`. + * Added `query_id` field for `databricks.sdk.service.sql.Parameter`. + * Added `databricks.sdk.service.sql.MultiValuesOptions` dataclass. + +OpenAPI SHA: cdd76a98a4fca7008572b3a94427566dd286c63b, Date: 2024-02-19 + ## 0.19.1 New features: diff --git a/databricks/sdk/__init__.py b/databricks/sdk/__init__.py index ddf4d31f7..91575ddfd 100755 --- a/databricks/sdk/__init__.py +++ b/databricks/sdk/__init__.py @@ -15,6 +15,7 @@ ExternalLocationsAPI, FunctionsAPI, GrantsAPI, LakehouseMonitorsAPI, MetastoresAPI, ModelVersionsAPI, + OnlineTablesAPI, RegisteredModelsAPI, SchemasAPI, StorageCredentialsAPI, SystemSchemasAPI, @@ -180,6 +181,7 @@ def __init__(self, self._metastores = MetastoresAPI(self._api_client) self._model_registry = ModelRegistryAPI(self._api_client) self._model_versions = ModelVersionsAPI(self._api_client) + self._online_tables = OnlineTablesAPI(self._api_client) self._permissions = PermissionsAPI(self._api_client) self._pipelines = PipelinesAPI(self._api_client) self._policy_families = PolicyFamiliesAPI(self._api_client) @@ -322,7 +324,7 @@ def external_locations(self) -> ExternalLocationsAPI: @property def files(self) -> FilesAPI: - """The Files API allows you to read, write, and delete files and directories in Unity Catalog volumes.""" + """The Files API allows you to read, write, list, and delete files and directories.""" return self._files @property @@ -400,6 +402,11 @@ def model_versions(self) -> ModelVersionsAPI: """Databricks provides a hosted version of MLflow Model Registry in Unity Catalog.""" return self._model_versions + @property + def online_tables(self) -> OnlineTablesAPI: + """Online tables provide lower latency and higher QPS access to data from Delta tables.""" + return self._online_tables + @property def permissions(self) -> PermissionsAPI: """Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints.""" diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index 2f1be049c..d0175cb7b 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -794,6 +794,41 @@ class ConnectionType(Enum): SQLSERVER = 'SQLSERVER' +@dataclass +class ContinuousUpdateStatus: + """Detailed status of an online table. Shown if the online table is in the ONLINE_CONTINUOUS_UPDATE + or the ONLINE_UPDATING_PIPELINE_RESOURCES state.""" + + initial_pipeline_sync_progress: Optional[PipelineProgress] = None + """Progress of the initial data synchronization.""" + + last_processed_commit_version: Optional[int] = None + """The last source table Delta version that was synced to the online table. Note that this Delta + version may not be completely synced to the online table yet.""" + + timestamp: Optional[str] = None + """The timestamp of the last time any data was synchronized from the source table to the online + table.""" + + def as_dict(self) -> dict: + """Serializes the ContinuousUpdateStatus into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.initial_pipeline_sync_progress: + body['initial_pipeline_sync_progress'] = self.initial_pipeline_sync_progress.as_dict() + if self.last_processed_commit_version is not None: + body['last_processed_commit_version'] = self.last_processed_commit_version + if self.timestamp is not None: body['timestamp'] = self.timestamp + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> ContinuousUpdateStatus: + """Deserializes the ContinuousUpdateStatus from a dictionary.""" + return cls(initial_pipeline_sync_progress=_from_dict(d, 'initial_pipeline_sync_progress', + PipelineProgress), + last_processed_commit_version=d.get('last_processed_commit_version', None), + timestamp=d.get('timestamp', None)) + + @dataclass class CreateCatalog: name: str @@ -1808,6 +1843,35 @@ def from_dict(cls, d: Dict[str, any]) -> ExternalLocationInfo: url=d.get('url', None)) +@dataclass +class FailedStatus: + """Detailed status of an online table. Shown if the online table is in the OFFLINE_FAILED or the + ONLINE_PIPELINE_FAILED state.""" + + last_processed_commit_version: Optional[int] = None + """The last source table Delta version that was synced to the online table. Note that this Delta + version may only be partially synced to the online table. Only populated if the table is still + online and available for serving.""" + + timestamp: Optional[str] = None + """The timestamp of the last time any data was synchronized from the source table to the online + table. Only populated if the table is still online and available for serving.""" + + def as_dict(self) -> dict: + """Serializes the FailedStatus into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.last_processed_commit_version is not None: + body['last_processed_commit_version'] = self.last_processed_commit_version + if self.timestamp is not None: body['timestamp'] = self.timestamp + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> FailedStatus: + """Deserializes the FailedStatus from a dictionary.""" + return cls(last_processed_commit_version=d.get('last_processed_commit_version', None), + timestamp=d.get('timestamp', None)) + + @dataclass class ForeignKeyConstraint: name: str @@ -2548,18 +2612,25 @@ def from_dict(cls, d: Dict[str, any]) -> ListTablesResponse: @dataclass class ListVolumesResponseContent: + next_page_token: Optional[str] = None + """Opaque token to retrieve the next page of results. Absent if there are no more pages. + __page_token__ should be set to this value for the next request to retrieve the next page of + results.""" + volumes: Optional[List[VolumeInfo]] = None def as_dict(self) -> dict: """Serializes the ListVolumesResponseContent into a dictionary suitable for use as a JSON request body.""" body = {} + if self.next_page_token is not None: body['next_page_token'] = self.next_page_token if self.volumes: body['volumes'] = [v.as_dict() for v in self.volumes] return body @classmethod def from_dict(cls, d: Dict[str, any]) -> ListVolumesResponseContent: """Deserializes the ListVolumesResponseContent from a dictionary.""" - return cls(volumes=_repeated_dict(d, 'volumes', VolumeInfo)) + return cls(next_page_token=d.get('next_page_token', None), + volumes=_repeated_dict(d, 'volumes', VolumeInfo)) class MatchType(Enum): @@ -3223,6 +3294,158 @@ def from_dict(cls, d: Dict[str, any]) -> NamedTableConstraint: return cls(name=d.get('name', None)) +@dataclass +class OnlineTable: + """Online Table information.""" + + name: Optional[str] = None + """Full three-part (catalog, schema, table) name of the table.""" + + spec: Optional[OnlineTableSpec] = None + """Specification of the online table.""" + + status: Optional[OnlineTableStatus] = None + """Online Table status""" + + def as_dict(self) -> dict: + """Serializes the OnlineTable into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.name is not None: body['name'] = self.name + if self.spec: body['spec'] = self.spec.as_dict() + if self.status: body['status'] = self.status.as_dict() + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> OnlineTable: + """Deserializes the OnlineTable from a dictionary.""" + return cls(name=d.get('name', None), + spec=_from_dict(d, 'spec', OnlineTableSpec), + status=_from_dict(d, 'status', OnlineTableStatus)) + + +@dataclass +class OnlineTableSpec: + """Specification of an online table.""" + + perform_full_copy: Optional[bool] = None + """Whether to create a full-copy pipeline -- a pipeline that stops after creates a full copy of the + source table upon initialization and does not process any change data feeds (CDFs) afterwards. + The pipeline can still be manually triggered afterwards, but it always perform a full copy of + the source table and there are no incremental updates. This mode is useful for syncing views or + tables without CDFs to online tables. Note that the full-copy pipeline only supports "triggered" + scheduling policy.""" + + pipeline_id: Optional[str] = None + """ID of the associated pipeline. Generated by the server - cannot be set by the caller.""" + + primary_key_columns: Optional[List[str]] = None + """Primary Key columns to be used for data insert/update in the destination.""" + + run_continuously: Optional[Any] = None + """Pipeline runs continuously after generating the initial data.""" + + run_triggered: Optional[Any] = None + """Pipeline stops after generating the initial data and can be triggered later (manually, through a + cron job or through data triggers)""" + + source_table_full_name: Optional[str] = None + """Three-part (catalog, schema, table) name of the source Delta table.""" + + timeseries_key: Optional[str] = None + """Time series key to deduplicate (tie-break) rows with the same primary key.""" + + def as_dict(self) -> dict: + """Serializes the OnlineTableSpec into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.perform_full_copy is not None: body['perform_full_copy'] = self.perform_full_copy + if self.pipeline_id is not None: body['pipeline_id'] = self.pipeline_id + if self.primary_key_columns: body['primary_key_columns'] = [v for v in self.primary_key_columns] + if self.run_continuously: body['run_continuously'] = self.run_continuously + if self.run_triggered: body['run_triggered'] = self.run_triggered + if self.source_table_full_name is not None: + body['source_table_full_name'] = self.source_table_full_name + if self.timeseries_key is not None: body['timeseries_key'] = self.timeseries_key + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> OnlineTableSpec: + """Deserializes the OnlineTableSpec from a dictionary.""" + return cls(perform_full_copy=d.get('perform_full_copy', None), + pipeline_id=d.get('pipeline_id', None), + primary_key_columns=d.get('primary_key_columns', None), + run_continuously=d.get('run_continuously', None), + run_triggered=d.get('run_triggered', None), + source_table_full_name=d.get('source_table_full_name', None), + timeseries_key=d.get('timeseries_key', None)) + + +class OnlineTableState(Enum): + """The state of an online table.""" + + OFFLINE = 'OFFLINE' + OFFLINE_FAILED = 'OFFLINE_FAILED' + ONLINE = 'ONLINE' + ONLINE_CONTINUOUS_UPDATE = 'ONLINE_CONTINUOUS_UPDATE' + ONLINE_NO_PENDING_UPDATE = 'ONLINE_NO_PENDING_UPDATE' + ONLINE_PIPELINE_FAILED = 'ONLINE_PIPELINE_FAILED' + ONLINE_TABLE_STATE_UNSPECIFIED = 'ONLINE_TABLE_STATE_UNSPECIFIED' + ONLINE_TRIGGERED_UPDATE = 'ONLINE_TRIGGERED_UPDATE' + ONLINE_UPDATING_PIPELINE_RESOURCES = 'ONLINE_UPDATING_PIPELINE_RESOURCES' + PROVISIONING = 'PROVISIONING' + PROVISIONING_INITIAL_SNAPSHOT = 'PROVISIONING_INITIAL_SNAPSHOT' + PROVISIONING_PIPELINE_RESOURCES = 'PROVISIONING_PIPELINE_RESOURCES' + + +@dataclass +class OnlineTableStatus: + """Status of an online table.""" + + continuous_update_status: Optional[ContinuousUpdateStatus] = None + """Detailed status of an online table. Shown if the online table is in the ONLINE_CONTINUOUS_UPDATE + or the ONLINE_UPDATING_PIPELINE_RESOURCES state.""" + + detailed_state: Optional[OnlineTableState] = None + """The state of the online table.""" + + failed_status: Optional[FailedStatus] = None + """Detailed status of an online table. Shown if the online table is in the OFFLINE_FAILED or the + ONLINE_PIPELINE_FAILED state.""" + + message: Optional[str] = None + """A text description of the current state of the online table.""" + + provisioning_status: Optional[ProvisioningStatus] = None + """Detailed status of an online table. Shown if the online table is in the + PROVISIONING_PIPELINE_RESOURCES or the PROVISIONING_INITIAL_SNAPSHOT state.""" + + triggered_update_status: Optional[TriggeredUpdateStatus] = None + """Detailed status of an online table. Shown if the online table is in the ONLINE_TRIGGERED_UPDATE + or the ONLINE_NO_PENDING_UPDATE state.""" + + def as_dict(self) -> dict: + """Serializes the OnlineTableStatus into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.continuous_update_status: + body['continuous_update_status'] = self.continuous_update_status.as_dict() + if self.detailed_state is not None: body['detailed_state'] = self.detailed_state.value + if self.failed_status: body['failed_status'] = self.failed_status.as_dict() + if self.message is not None: body['message'] = self.message + if self.provisioning_status: body['provisioning_status'] = self.provisioning_status.as_dict() + if self.triggered_update_status: + body['triggered_update_status'] = self.triggered_update_status.as_dict() + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> OnlineTableStatus: + """Deserializes the OnlineTableStatus from a dictionary.""" + return cls(continuous_update_status=_from_dict(d, 'continuous_update_status', ContinuousUpdateStatus), + detailed_state=_enum(d, 'detailed_state', OnlineTableState), + failed_status=_from_dict(d, 'failed_status', FailedStatus), + message=d.get('message', None), + provisioning_status=_from_dict(d, 'provisioning_status', ProvisioningStatus), + triggered_update_status=_from_dict(d, 'triggered_update_status', TriggeredUpdateStatus)) + + @dataclass class PermissionsChange: add: Optional[List[Privilege]] = None @@ -3268,6 +3491,49 @@ def from_dict(cls, d: Dict[str, any]) -> PermissionsList: return cls(privilege_assignments=_repeated_dict(d, 'privilege_assignments', PrivilegeAssignment)) +@dataclass +class PipelineProgress: + """Progress information of the Online Table data synchronization pipeline.""" + + estimated_completion_time_seconds: Optional[float] = None + """The estimated time remaining to complete this update in seconds.""" + + latest_version_currently_processing: Optional[int] = None + """The source table Delta version that was last processed by the pipeline. The pipeline may not + have completely processed this version yet.""" + + sync_progress_completion: Optional[float] = None + """The completion ratio of this update. This is a number between 0 and 1.""" + + synced_row_count: Optional[int] = None + """The number of rows that have been synced in this update.""" + + total_row_count: Optional[int] = None + """The total number of rows that need to be synced in this update. This number may be an estimate.""" + + def as_dict(self) -> dict: + """Serializes the PipelineProgress into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.estimated_completion_time_seconds is not None: + body['estimated_completion_time_seconds'] = self.estimated_completion_time_seconds + if self.latest_version_currently_processing is not None: + body['latest_version_currently_processing'] = self.latest_version_currently_processing + if self.sync_progress_completion is not None: + body['sync_progress_completion'] = self.sync_progress_completion + if self.synced_row_count is not None: body['synced_row_count'] = self.synced_row_count + if self.total_row_count is not None: body['total_row_count'] = self.total_row_count + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> PipelineProgress: + """Deserializes the PipelineProgress from a dictionary.""" + return cls(estimated_completion_time_seconds=d.get('estimated_completion_time_seconds', None), + latest_version_currently_processing=d.get('latest_version_currently_processing', None), + sync_progress_completion=d.get('sync_progress_completion', None), + synced_row_count=d.get('synced_row_count', None), + total_row_count=d.get('total_row_count', None)) + + @dataclass class PrimaryKeyConstraint: name: str @@ -3385,6 +3651,29 @@ class ProvisioningInfoState(Enum): STATE_UNSPECIFIED = 'STATE_UNSPECIFIED' +@dataclass +class ProvisioningStatus: + """Detailed status of an online table. Shown if the online table is in the + PROVISIONING_PIPELINE_RESOURCES or the PROVISIONING_INITIAL_SNAPSHOT state.""" + + initial_pipeline_sync_progress: Optional[PipelineProgress] = None + """Details about initial data synchronization. Only populated when in the + PROVISIONING_INITIAL_SNAPSHOT state.""" + + def as_dict(self) -> dict: + """Serializes the ProvisioningStatus into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.initial_pipeline_sync_progress: + body['initial_pipeline_sync_progress'] = self.initial_pipeline_sync_progress.as_dict() + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> ProvisioningStatus: + """Deserializes the ProvisioningStatus from a dictionary.""" + return cls( + initial_pipeline_sync_progress=_from_dict(d, 'initial_pipeline_sync_progress', PipelineProgress)) + + @dataclass class RegisteredModelAlias: """Registered model alias.""" @@ -4109,6 +4398,40 @@ class TableType(Enum): VIEW = 'VIEW' +@dataclass +class TriggeredUpdateStatus: + """Detailed status of an online table. Shown if the online table is in the ONLINE_TRIGGERED_UPDATE + or the ONLINE_NO_PENDING_UPDATE state.""" + + last_processed_commit_version: Optional[int] = None + """The last source table Delta version that was synced to the online table. Note that this Delta + version may not be completely synced to the online table yet.""" + + timestamp: Optional[str] = None + """The timestamp of the last time any data was synchronized from the source table to the online + table.""" + + triggered_update_progress: Optional[PipelineProgress] = None + """Progress of the active data synchronization pipeline.""" + + def as_dict(self) -> dict: + """Serializes the TriggeredUpdateStatus into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.last_processed_commit_version is not None: + body['last_processed_commit_version'] = self.last_processed_commit_version + if self.timestamp is not None: body['timestamp'] = self.timestamp + if self.triggered_update_progress: + body['triggered_update_progress'] = self.triggered_update_progress.as_dict() + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> TriggeredUpdateStatus: + """Deserializes the TriggeredUpdateStatus from a dictionary.""" + return cls(last_processed_commit_version=d.get('last_processed_commit_version', None), + timestamp=d.get('timestamp', None), + triggered_update_progress=_from_dict(d, 'triggered_update_progress', PipelineProgress)) + + @dataclass class UpdateCatalog: comment: Optional[str] = None @@ -4163,7 +4486,7 @@ class UpdateConnection: options: Dict[str, str] """A map of key-value properties attached to the securable.""" - name_arg: Optional[str] = None + name: Optional[str] = None """Name of the connection.""" new_name: Optional[str] = None @@ -4175,7 +4498,7 @@ class UpdateConnection: def as_dict(self) -> dict: """Serializes the UpdateConnection into a dictionary suitable for use as a JSON request body.""" body = {} - if self.name_arg is not None: body['name_arg'] = self.name_arg + if self.name is not None: body['name'] = self.name if self.new_name is not None: body['new_name'] = self.new_name if self.options: body['options'] = self.options if self.owner is not None: body['owner'] = self.owner @@ -4184,7 +4507,7 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, any]) -> UpdateConnection: """Deserializes the UpdateConnection from a dictionary.""" - return cls(name_arg=d.get('name_arg', None), + return cls(name=d.get('name', None), new_name=d.get('new_name', None), options=d.get('options', None), owner=d.get('owner', None)) @@ -4401,9 +4724,6 @@ def from_dict(cls, d: Dict[str, any]) -> UpdateModelVersionRequest: @dataclass class UpdateMonitor: - assets_dir: str - """The directory to store monitoring assets (e.g. dashboard, metric tables).""" - output_schema_name: str """Schema where output metric tables are created.""" @@ -4445,7 +4765,6 @@ class UpdateMonitor: def as_dict(self) -> dict: """Serializes the UpdateMonitor into a dictionary suitable for use as a JSON request body.""" body = {} - if self.assets_dir is not None: body['assets_dir'] = self.assets_dir if self.baseline_table_name is not None: body['baseline_table_name'] = self.baseline_table_name if self.custom_metrics: body['custom_metrics'] = [v.as_dict() for v in self.custom_metrics] if self.data_classification_config: @@ -4463,8 +4782,7 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, any]) -> UpdateMonitor: """Deserializes the UpdateMonitor from a dictionary.""" - return cls(assets_dir=d.get('assets_dir', None), - baseline_table_name=d.get('baseline_table_name', None), + return cls(baseline_table_name=d.get('baseline_table_name', None), custom_metrics=_repeated_dict(d, 'custom_metrics', MonitorCustomMetric), data_classification_config=_from_dict(d, 'data_classification_config', MonitorDataClassificationConfig), @@ -4660,7 +4978,7 @@ class UpdateVolumeRequestContent: comment: Optional[str] = None """The comment attached to the volume""" - full_name_arg: Optional[str] = None + name: Optional[str] = None """The three-level (fully qualified) name of the volume""" new_name: Optional[str] = None @@ -4673,7 +4991,7 @@ def as_dict(self) -> dict: """Serializes the UpdateVolumeRequestContent into a dictionary suitable for use as a JSON request body.""" body = {} if self.comment is not None: body['comment'] = self.comment - if self.full_name_arg is not None: body['full_name_arg'] = self.full_name_arg + if self.name is not None: body['name'] = self.name if self.new_name is not None: body['new_name'] = self.new_name if self.owner is not None: body['owner'] = self.owner return body @@ -4682,7 +5000,7 @@ def as_dict(self) -> dict: def from_dict(cls, d: Dict[str, any]) -> UpdateVolumeRequestContent: """Deserializes the UpdateVolumeRequestContent from a dictionary.""" return cls(comment=d.get('comment', None), - full_name_arg=d.get('full_name_arg', None), + name=d.get('name', None), new_name=d.get('new_name', None), owner=d.get('owner', None)) @@ -4871,6 +5189,29 @@ class ValidationResultResult(Enum): SKIP = 'SKIP' +@dataclass +class ViewData: + """Online Table information.""" + + name: Optional[str] = None + """Full three-part (catalog, schema, table) name of the table.""" + + spec: Optional[OnlineTableSpec] = None + """Specification of the online table.""" + + def as_dict(self) -> dict: + """Serializes the ViewData into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.name is not None: body['name'] = self.name + if self.spec: body['spec'] = self.spec.as_dict() + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> ViewData: + """Deserializes the ViewData from a dictionary.""" + return cls(name=d.get('name', None), spec=_from_dict(d, 'spec', OnlineTableSpec)) + + @dataclass class VolumeInfo: access_point: Optional[str] = None @@ -5638,12 +5979,12 @@ def create(self, res = self._api.do('POST', '/api/2.1/unity-catalog/connections', body=body, headers=headers) return ConnectionInfo.from_dict(res) - def delete(self, name_arg: str): + def delete(self, name: str): """Delete a connection. Deletes the connection that matches the supplied name. - :param name_arg: str + :param name: str The name of the connection to be deleted. @@ -5651,14 +5992,14 @@ def delete(self, name_arg: str): headers = {'Accept': 'application/json', } - self._api.do('DELETE', f'/api/2.1/unity-catalog/connections/{name_arg}', headers=headers) + self._api.do('DELETE', f'/api/2.1/unity-catalog/connections/{name}', headers=headers) - def get(self, name_arg: str) -> ConnectionInfo: + def get(self, name: str) -> ConnectionInfo: """Get a connection. Gets a connection from it's name. - :param name_arg: str + :param name: str Name of the connection. :returns: :class:`ConnectionInfo` @@ -5666,7 +6007,7 @@ def get(self, name_arg: str) -> ConnectionInfo: headers = {'Accept': 'application/json', } - res = self._api.do('GET', f'/api/2.1/unity-catalog/connections/{name_arg}', headers=headers) + res = self._api.do('GET', f'/api/2.1/unity-catalog/connections/{name}', headers=headers) return ConnectionInfo.from_dict(res) def list(self) -> Iterator[ConnectionInfo]: @@ -5684,7 +6025,7 @@ def list(self) -> Iterator[ConnectionInfo]: return parsed if parsed is not None else [] def update(self, - name_arg: str, + name: str, options: Dict[str, str], *, new_name: Optional[str] = None, @@ -5693,7 +6034,7 @@ def update(self, Updates the connection that matches the supplied name. - :param name_arg: str + :param name: str Name of the connection. :param options: Dict[str,str] A map of key-value properties attached to the securable. @@ -5710,10 +6051,7 @@ def update(self, if owner is not None: body['owner'] = owner headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } - res = self._api.do('PATCH', - f'/api/2.1/unity-catalog/connections/{name_arg}', - body=body, - headers=headers) + res = self._api.do('PATCH', f'/api/2.1/unity-catalog/connections/{name}', body=body, headers=headers) return ConnectionInfo.from_dict(res) @@ -6429,7 +6767,6 @@ def run_refresh(self, full_name: str) -> MonitorRefreshInfo: def update(self, full_name: str, - assets_dir: str, output_schema_name: str, *, baseline_table_name: Optional[str] = None, @@ -6457,8 +6794,6 @@ def update(self, :param full_name: str Full name of the table. - :param assets_dir: str - The directory to store monitoring assets (e.g. dashboard, metric tables). :param output_schema_name: str Schema where output metric tables are created. :param baseline_table_name: str (optional) @@ -6487,7 +6822,6 @@ def update(self, :returns: :class:`MonitorInfo` """ body = {} - if assets_dir is not None: body['assets_dir'] = assets_dir if baseline_table_name is not None: body['baseline_table_name'] = baseline_table_name if custom_metrics is not None: body['custom_metrics'] = [v.as_dict() for v in custom_metrics] if data_classification_config is not None: @@ -6928,6 +7262,66 @@ def update(self, full_name: str, version: int, *, comment: Optional[str] = None) return ModelVersionInfo.from_dict(res) +class OnlineTablesAPI: + """Online tables provide lower latency and higher QPS access to data from Delta tables.""" + + def __init__(self, api_client): + self._api = api_client + + def create(self, *, name: Optional[str] = None, spec: Optional[OnlineTableSpec] = None) -> OnlineTable: + """Create an Online Table. + + Create a new Online Table. + + :param name: str (optional) + Full three-part (catalog, schema, table) name of the table. + :param spec: :class:`OnlineTableSpec` (optional) + Specification of the online table. + + :returns: :class:`OnlineTable` + """ + body = {} + if name is not None: body['name'] = name + if spec is not None: body['spec'] = spec.as_dict() + headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } + + res = self._api.do('POST', '/api/2.0/online-tables', body=body, headers=headers) + return OnlineTable.from_dict(res) + + def delete(self, name: str): + """Delete an Online Table. + + Delete an online table. Warning: This will delete all the data in the online table. If the source + Delta table was deleted or modified since this Online Table was created, this will lose the data + forever! + + :param name: str + Full three-part (catalog, schema, table) name of the table. + + + """ + + headers = {'Accept': 'application/json', } + + self._api.do('DELETE', f'/api/2.0/online-tables/{name}', headers=headers) + + def get(self, name: str) -> OnlineTable: + """Get an Online Table. + + Get information about an existing online table and its status. + + :param name: str + Full three-part (catalog, schema, table) name of the table. + + :returns: :class:`OnlineTable` + """ + + headers = {'Accept': 'application/json', } + + res = self._api.do('GET', f'/api/2.0/online-tables/{name}', headers=headers) + return OnlineTable.from_dict(res) + + class RegisteredModelsAPI: """Databricks provides a hosted version of MLflow Model Registry in Unity Catalog. Models in Unity Catalog provide centralized access control, auditing, lineage, and discovery of ML models across Databricks @@ -8039,7 +8433,7 @@ def create(self, res = self._api.do('POST', '/api/2.1/unity-catalog/volumes', body=body, headers=headers) return VolumeInfo.from_dict(res) - def delete(self, full_name_arg: str): + def delete(self, name: str): """Delete a Volume. Deletes a volume from the specified parent catalog and schema. @@ -8048,7 +8442,7 @@ def delete(self, full_name_arg: str): also be the owner or have the **USE_CATALOG** privilege on the parent catalog and the **USE_SCHEMA** privilege on the parent schema. - :param full_name_arg: str + :param name: str The three-level (fully qualified) name of the volume @@ -8056,12 +8450,17 @@ def delete(self, full_name_arg: str): headers = {} - self._api.do('DELETE', f'/api/2.1/unity-catalog/volumes/{full_name_arg}', headers=headers) + self._api.do('DELETE', f'/api/2.1/unity-catalog/volumes/{name}', headers=headers) - def list(self, catalog_name: str, schema_name: str) -> Iterator[VolumeInfo]: + def list(self, + catalog_name: str, + schema_name: str, + *, + max_results: Optional[int] = None, + page_token: Optional[str] = None) -> Iterator[VolumeInfo]: """List Volumes. - Gets an array of all volumes for the current metastore under the parent catalog and schema. + Gets an array of volumes for the current metastore under the parent catalog and schema. The returned volumes are filtered based on the privileges of the calling user. For example, the metastore admin is able to list all the volumes. A regular user needs to be the owner or have the @@ -8075,20 +8474,42 @@ def list(self, catalog_name: str, schema_name: str) -> Iterator[VolumeInfo]: The identifier of the catalog :param schema_name: str The identifier of the schema + :param max_results: int (optional) + Maximum number of volumes to return (page length). + + If not set, the page length is set to a server configured value (10000, as of 1/29/2024). - when set + to a value greater than 0, the page length is the minimum of this value and a server configured + value (10000, as of 1/29/2024); - when set to 0, the page length is set to a server configured value + (10000, as of 1/29/2024) (recommended); - when set to a value less than 0, an invalid parameter + error is returned; + + Note: this parameter controls only the maximum number of volumes to return. The actual number of + volumes returned in a page may be smaller than this value, including 0, even if there are more + pages. + :param page_token: str (optional) + Opaque token returned by a previous request. It must be included in the request to retrieve the next + page of results (pagination). :returns: Iterator over :class:`VolumeInfo` """ query = {} if catalog_name is not None: query['catalog_name'] = catalog_name + if max_results is not None: query['max_results'] = max_results + if page_token is not None: query['page_token'] = page_token if schema_name is not None: query['schema_name'] = schema_name headers = {'Accept': 'application/json', } - json = self._api.do('GET', '/api/2.1/unity-catalog/volumes', query=query, headers=headers) - parsed = ListVolumesResponseContent.from_dict(json).volumes - return parsed if parsed is not None else [] + while True: + json = self._api.do('GET', '/api/2.1/unity-catalog/volumes', query=query, headers=headers) + if 'volumes' in json: + for v in json['volumes']: + yield VolumeInfo.from_dict(v) + if 'next_page_token' not in json or not json['next_page_token']: + return + query['page_token'] = json['next_page_token'] - def read(self, full_name_arg: str) -> VolumeInfo: + def read(self, name: str) -> VolumeInfo: """Get a Volume. Gets a volume from the metastore for a specific catalog and schema. @@ -8097,7 +8518,7 @@ def read(self, full_name_arg: str) -> VolumeInfo: volume. For the latter case, the caller must also be the owner or have the **USE_CATALOG** privilege on the parent catalog and the **USE_SCHEMA** privilege on the parent schema. - :param full_name_arg: str + :param name: str The three-level (fully qualified) name of the volume :returns: :class:`VolumeInfo` @@ -8105,11 +8526,11 @@ def read(self, full_name_arg: str) -> VolumeInfo: headers = {'Accept': 'application/json', } - res = self._api.do('GET', f'/api/2.1/unity-catalog/volumes/{full_name_arg}', headers=headers) + res = self._api.do('GET', f'/api/2.1/unity-catalog/volumes/{name}', headers=headers) return VolumeInfo.from_dict(res) def update(self, - full_name_arg: str, + name: str, *, comment: Optional[str] = None, new_name: Optional[str] = None, @@ -8124,7 +8545,7 @@ def update(self, Currently only the name, the owner or the comment of the volume could be updated. - :param full_name_arg: str + :param name: str The three-level (fully qualified) name of the volume :param comment: str (optional) The comment attached to the volume @@ -8141,10 +8562,7 @@ def update(self, if owner is not None: body['owner'] = owner headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } - res = self._api.do('PATCH', - f'/api/2.1/unity-catalog/volumes/{full_name_arg}', - body=body, - headers=headers) + res = self._api.do('PATCH', f'/api/2.1/unity-catalog/volumes/{name}', body=body, headers=headers) return VolumeInfo.from_dict(res) diff --git a/databricks/sdk/service/files.py b/databricks/sdk/service/files.py index e08753daa..37e571db1 100755 --- a/databricks/sdk/service/files.py +++ b/databricks/sdk/service/files.py @@ -124,7 +124,7 @@ class DirectoryEntry: """Last modification time of given file in milliseconds since unix epoch.""" name: Optional[str] = None - """The name of the file or directory.""" + """The name of the file or directory. This is the last component of the path.""" path: Optional[str] = None """The absolute path of the file or directory.""" @@ -151,18 +151,30 @@ def from_dict(cls, d: Dict[str, any]) -> DirectoryEntry: @dataclass class DownloadResponse: + content_length: Optional[int] = None + + content_type: Optional[str] = None + contents: Optional[BinaryIO] = None + last_modified: Optional[str] = None + def as_dict(self) -> dict: """Serializes the DownloadResponse into a dictionary suitable for use as a JSON request body.""" body = {} + if self.content_length is not None: body['content-length'] = self.content_length + if self.content_type is not None: body['content-type'] = self.content_type if self.contents: body['contents'] = self.contents + if self.last_modified is not None: body['last-modified'] = self.last_modified return body @classmethod def from_dict(cls, d: Dict[str, any]) -> DownloadResponse: """Deserializes the DownloadResponse from a dictionary.""" - return cls(contents=d.get('contents', None)) + return cls(content_length=d.get('content-length', None), + content_type=d.get('content-type', None), + contents=d.get('contents', None), + last_modified=d.get('last-modified', None)) @dataclass @@ -197,6 +209,30 @@ def from_dict(cls, d: Dict[str, any]) -> FileInfo: path=d.get('path', None)) +@dataclass +class GetMetadataResponse: + content_length: Optional[int] = None + + content_type: Optional[str] = None + + last_modified: Optional[str] = None + + def as_dict(self) -> dict: + """Serializes the GetMetadataResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.content_length is not None: body['content-length'] = self.content_length + if self.content_type is not None: body['content-type'] = self.content_type + if self.last_modified is not None: body['last-modified'] = self.last_modified + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> GetMetadataResponse: + """Deserializes the GetMetadataResponse from a dictionary.""" + return cls(content_length=d.get('content-length', None), + content_type=d.get('content-type', None), + last_modified=d.get('last-modified', None)) + + @dataclass class ListDirectoryResponse: contents: Optional[List[DirectoryEntry]] = None @@ -580,7 +616,18 @@ def read(self, path: str, *, length: Optional[int] = None, offset: Optional[int] class FilesAPI: - """The Files API allows you to read, write, and delete files and directories in Unity Catalog volumes.""" + """The Files API allows you to read, write, list, and delete files and directories. We support Unity Catalog + volumes with paths starting with "/Volumes///". + + The Files API is designed like a standard HTTP API, rather than as a JSON RPC API. This is intended to + make it easier and more efficient to work with file contents as raw bytes. + + Because the Files API is a standard HTTP API, the URI path is used to specify the file or directory to + operate on. The path is always absolute. + + The Files API has separate endpoints for working with files, `/fs/files`, and working with directories, + `/fs/directories`. The standard HTTP methods `GET`, `HEAD`, `PUT`, and `DELETE` work as expected on these + endpoints.""" def __init__(self, api_client): self._api = api_client @@ -588,7 +635,9 @@ def __init__(self, api_client): def create_directory(self, directory_path: str): """Create a directory. - Creates an empty directory. If called on an existing directory, the API returns a success response. + Creates an empty directory. If necessary, also creates any parent directories of the new, empty + directory (like the shell command `mkdir -p`). If called on an existing directory, returns a success + response; this method is idempotent. :param directory_path: str The absolute path of a directory. @@ -603,7 +652,7 @@ def create_directory(self, directory_path: str): def delete(self, file_path: str): """Delete a file. - Deletes a file. + Deletes a file. If the request is successful, there is no response body. :param file_path: str The absolute path of the file. @@ -618,7 +667,10 @@ def delete(self, file_path: str): def delete_directory(self, directory_path: str): """Delete a directory. - Deletes an empty directory. If the directory is not empty, the API returns a HTTP 400 error. + Deletes an empty directory. + + To delete a non-empty directory, first delete all of its contents. This can be done by listing the + directory contents and deleting each file and subdirectory recursively. :param directory_path: str The absolute path of a directory. @@ -633,7 +685,8 @@ def delete_directory(self, directory_path: str): def download(self, file_path: str) -> DownloadResponse: """Download a file. - Downloads a file of up to 5 GiB. + Downloads a file of up to 5 GiB. The file contents are the response body. This is a standard HTTP file + download, not a JSON RPC. :param file_path: str The absolute path of the file. @@ -642,10 +695,54 @@ def download(self, file_path: str) -> DownloadResponse: """ headers = {'Accept': 'application/octet-stream', } - - res = self._api.do('GET', f'/api/2.0/fs/files{file_path}', headers=headers, raw=True) + response_headers = ['content-length', 'content-type', 'last-modified', ] + res = self._api.do('GET', + f'/api/2.0/fs/files{file_path}', + headers=headers, + response_headers=response_headers, + raw=True) return DownloadResponse.from_dict(res) + def get_directory_metadata(self, directory_path: str): + """Get directory metadata. + + Get the metadata of a directory. The response HTTP headers contain the metadata. There is no response + body. + + This method is useful to check if a directory exists and the caller has access to it. + + If you wish to ensure the directory exists, you can instead use `PUT`, which will create the directory + if it does not exist, and is idempotent (it will succeed if the directory already exists). + + :param directory_path: str + The absolute path of a directory. + + + """ + + headers = {} + + self._api.do('HEAD', f'/api/2.0/fs/directories{directory_path}', headers=headers) + + def get_metadata(self, file_path: str) -> GetMetadataResponse: + """Get file metadata. + + Get the metadata of a file. The response HTTP headers contain the metadata. There is no response body. + + :param file_path: str + The absolute path of the file. + + :returns: :class:`GetMetadataResponse` + """ + + headers = {} + response_headers = ['content-length', 'content-type', 'last-modified', ] + res = self._api.do('HEAD', + f'/api/2.0/fs/files{file_path}', + headers=headers, + response_headers=response_headers) + return GetMetadataResponse.from_dict(res) + def list_directory_contents(self, directory_path: str, *, @@ -659,16 +756,22 @@ def list_directory_contents(self, :param directory_path: str The absolute path of a directory. :param page_size: int (optional) - The maximum number of directory entries to return. The API may return fewer than this value. - Receiving fewer results does not imply there are no more results. As long as the response contains a - next_page_token, there may be more results. + The maximum number of directory entries to return. The response may contain fewer entries. If the + response contains a `next_page_token`, there may be more entries, even if fewer than `page_size` + entries are in the response. + + We recommend not to set this value unless you are intentionally listing less than the complete + directory contents. If unspecified, at most 1000 directory entries will be returned. The maximum value is 1000. Values above 1000 will be coerced to 1000. :param page_token: str (optional) - A page token, received from a previous `list` call. Provide this to retrieve the subsequent page. - When paginating, all other parameters provided to `list` must match the call that provided the page - token. + An opaque page token which was the `next_page_token` in the response of the previous request to list + the contents of this directory. Provide this token to retrieve the next page of directory entries. + When providing a `page_token`, all other parameters provided to the request must match the previous + request. To list all of the entries in a directory, it is necessary to continue requesting pages of + entries until the response contains no `next_page_token`. Note that the number of entries returned + must not be used to determine when the listing is complete. :returns: Iterator over :class:`DirectoryEntry` """ @@ -693,7 +796,10 @@ def list_directory_contents(self, def upload(self, file_path: str, contents: BinaryIO, *, overwrite: Optional[bool] = None): """Upload a file. - Uploads a file of up to 5 GiB. + Uploads a file of up to 5 GiB. The file contents should be sent as the request body as raw bytes (an + octet stream); do not encode or otherwise modify the bytes before sending. The contents of the + resulting file will be exactly the bytes sent in the request body. If the request is successful, there + is no response body. :param file_path: str The absolute path of the file. diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 3ef2d3457..1d7955b3e 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -170,9 +170,9 @@ class BaseRun: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * - `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. - - * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival.""" + `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: + Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is + triggered by a table update.""" trigger_info: Optional[TriggerInfo] = None @@ -340,6 +340,12 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterSpec: new_cluster=_from_dict(d, 'new_cluster', compute.ClusterSpec)) +class Condition(Enum): + + ALL_UPDATED = 'ALL_UPDATED' + ANY_UPDATED = 'ANY_UPDATED' + + @dataclass class ConditionTask: left: Optional[str] = None @@ -512,9 +518,9 @@ class CreateJob: """An optional timeout applied to each run of this job. A value of `0` means no timeout.""" trigger: Optional[TriggerSettings] = None - """Trigger settings for the job. Can be used to trigger a run when new files arrive in an external - location. The default behavior is that the job runs only when triggered by clicking “Run - Now” in the Jobs UI or sending an API request to `runNow`.""" + """A configuration to trigger a run when certain conditions are met. The default behavior is that + the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API + request to `runNow`.""" webhook_notifications: Optional[WebhookNotifications] = None """A collection of system notification IDs to notify when runs of this job begin or complete.""" @@ -838,7 +844,7 @@ class ForEachTask: task: Task concurrency: Optional[int] = None - """Controls the number of active iterations task runs. Default is 100 (maximal value).""" + """Controls the number of active iterations task runs. Default is 20, maximum allowed is 100.""" def as_dict(self) -> dict: """Serializes the ForEachTask into a dictionary suitable for use as a JSON request body.""" @@ -1061,9 +1067,6 @@ class Job: """Settings for this job and all of its runs. These settings can be updated using the `resetJob` method.""" - trigger_history: Optional[TriggerHistory] = None - """History of the file arrival trigger associated with the job.""" - def as_dict(self) -> dict: """Serializes the Job into a dictionary suitable for use as a JSON request body.""" body = {} @@ -1072,7 +1075,6 @@ def as_dict(self) -> dict: if self.job_id is not None: body['job_id'] = self.job_id if self.run_as_user_name is not None: body['run_as_user_name'] = self.run_as_user_name if self.settings: body['settings'] = self.settings.as_dict() - if self.trigger_history: body['trigger_history'] = self.trigger_history.as_dict() return body @classmethod @@ -1082,8 +1084,7 @@ def from_dict(cls, d: Dict[str, any]) -> Job: creator_user_name=d.get('creator_user_name', None), job_id=d.get('job_id', None), run_as_user_name=d.get('run_as_user_name', None), - settings=_from_dict(d, 'settings', JobSettings), - trigger_history=_from_dict(d, 'trigger_history', TriggerHistory)) + settings=_from_dict(d, 'settings', JobSettings)) @dataclass @@ -1591,9 +1592,9 @@ class JobSettings: """An optional timeout applied to each run of this job. A value of `0` means no timeout.""" trigger: Optional[TriggerSettings] = None - """Trigger settings for the job. Can be used to trigger a run when new files arrive in an external - location. The default behavior is that the job runs only when triggered by clicking “Run - Now” in the Jobs UI or sending an API request to `runNow`.""" + """A configuration to trigger a run when certain conditions are met. The default behavior is that + the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API + request to `runNow`.""" webhook_notifications: Optional[WebhookNotifications] = None """A collection of system notification IDs to notify when runs of this job begin or complete.""" @@ -1883,7 +1884,7 @@ class NotebookTask: :method:jobs/runNow with parameters specified, the two parameters maps are merged. If the same key is specified in `base_parameters` and in `run-now`, the value from `run-now` is used. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. If the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters, the default value from the notebook is used. @@ -1892,8 +1893,8 @@ class NotebookTask: The JSON representation of this field cannot exceed 1MB. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables - [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets""" + [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" source: Optional[Source] = None """Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved @@ -2084,8 +2085,9 @@ class RepairRun: be specified in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs.""" + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" job_parameters: Optional[Dict[str, str]] = None """Job-level parameters used in the run. for example `"param": "overriding_val"`""" @@ -2103,13 +2105,13 @@ class RepairRun: notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables - [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html""" + [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" pipeline_params: Optional[PipelineParams] = None @@ -2123,7 +2125,7 @@ class RepairRun: `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -2131,7 +2133,7 @@ class RepairRun: returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" rerun_all_failed_tasks: Optional[bool] = None """If true, repair all failed tasks. Only one of `rerun_tasks` or `rerun_all_failed_tasks` can be @@ -2151,7 +2153,7 @@ class RepairRun: parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -2159,7 +2161,7 @@ class RepairRun: returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" sql_params: Optional[Dict[str, str]] = None """A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john @@ -2538,9 +2540,9 @@ class Run: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * - `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. - - * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival.""" + `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: + Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is + triggered by a table update.""" trigger_info: Optional[TriggerInfo] = None @@ -2656,7 +2658,7 @@ class RunConditionTaskOp(Enum): @dataclass class RunForEachTask: concurrency: Optional[int] = None - """Controls the number of active iterations task runs. Default is 100 (maximal value).""" + """Controls the number of active iterations task runs. Default is 20, maximum allowed is 100.""" inputs: Optional[str] = None """Array for task to iterate on. This can be a JSON string or a reference to an array parameter.""" @@ -2793,8 +2795,9 @@ class RunNow: be specified in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs.""" + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" job_parameters: Optional[Dict[str, str]] = None """Job-level parameters used in the run. for example `"param": "overriding_val"`""" @@ -2808,13 +2811,13 @@ class RunNow: notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables - [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html""" + [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" pipeline_params: Optional[PipelineParams] = None @@ -2828,7 +2831,7 @@ class RunNow: `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -2836,7 +2839,7 @@ class RunNow: returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" queue: Optional[QueueSettings] = None """The queue settings of the run.""" @@ -2848,7 +2851,7 @@ class RunNow: parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -2856,7 +2859,7 @@ class RunNow: returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" sql_params: Optional[Dict[str, str]] = None """A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john @@ -2998,8 +3001,9 @@ class RunParameters: be specified in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs.""" + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" job_parameters: Optional[Dict[str, str]] = None """Job-level parameters used in the run. for example `"param": "overriding_val"`""" @@ -3013,13 +3017,13 @@ class RunParameters: notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables - [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html""" + [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" pipeline_params: Optional[PipelineParams] = None @@ -3033,7 +3037,7 @@ class RunParameters: `run-now`, it would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -3041,7 +3045,7 @@ class RunParameters: returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" spark_submit_params: Optional[List[str]] = None """A list of parameters for jobs with spark submit task, for example `"spark_submit_params": @@ -3050,7 +3054,7 @@ class RunParameters: parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -3058,7 +3062,7 @@ class RunParameters: returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" sql_params: Optional[Dict[str, str]] = None """A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john @@ -3405,9 +3409,9 @@ class SparkJarTask: parameters: Optional[List[str]] = None """Parameters passed to the main method. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" def as_dict(self) -> dict: """Serializes the SparkJarTask into a dictionary suitable for use as a JSON request body.""" @@ -3436,9 +3440,9 @@ class SparkPythonTask: parameters: Optional[List[str]] = None """Command line parameters passed to the Python file. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" source: Optional[Source] = None """Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file @@ -3470,9 +3474,9 @@ class SparkSubmitTask: parameters: Optional[List[str]] = None """Command-line parameters passed to spark submit. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables""" + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html""" def as_dict(self) -> dict: """Serializes the SparkSubmitTask into a dictionary suitable for use as a JSON request body.""" @@ -4137,6 +4141,44 @@ def from_dict(cls, d: Dict[str, any]) -> SubmitTask: webhook_notifications=_from_dict(d, 'webhook_notifications', WebhookNotifications)) +@dataclass +class TableTriggerConfiguration: + condition: Optional[Condition] = None + """The table(s) condition based on which to trigger a job run.""" + + min_time_between_triggers_seconds: Optional[int] = None + """If set, the trigger starts a run only after the specified amount of time has passed since the + last time the trigger fired. The minimum allowed value is 60 seconds.""" + + table_names: Optional[List[str]] = None + """A list of Delta tables to monitor for changes. The table name must be in the format + `catalog_name.schema_name.table_name`.""" + + wait_after_last_change_seconds: Optional[int] = None + """If set, the trigger starts a run only after no table updates have occurred for the specified + time and can be used to wait for a series of table updates before triggering a run. The minimum + allowed value is 60 seconds.""" + + def as_dict(self) -> dict: + """Serializes the TableTriggerConfiguration into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.condition is not None: body['condition'] = self.condition.value + if self.min_time_between_triggers_seconds is not None: + body['min_time_between_triggers_seconds'] = self.min_time_between_triggers_seconds + if self.table_names: body['table_names'] = [v for v in self.table_names] + if self.wait_after_last_change_seconds is not None: + body['wait_after_last_change_seconds'] = self.wait_after_last_change_seconds + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> TableTriggerConfiguration: + """Deserializes the TableTriggerConfiguration from a dictionary.""" + return cls(condition=_enum(d, 'condition', Condition), + min_time_between_triggers_seconds=d.get('min_time_between_triggers_seconds', None), + table_names=d.get('table_names', None), + wait_after_last_change_seconds=d.get('wait_after_last_change_seconds', None)) + + @dataclass class Task: task_key: str @@ -4434,62 +4476,6 @@ def from_dict(cls, d: Dict[str, any]) -> TaskNotificationSettings: no_alert_for_skipped_runs=d.get('no_alert_for_skipped_runs', None)) -@dataclass -class TriggerEvaluation: - description: Optional[str] = None - """Human-readable description of the the trigger evaluation result. Explains why the trigger - evaluation triggered or did not trigger a run, or failed.""" - - run_id: Optional[int] = None - """The ID of the run that was triggered by the trigger evaluation. Only returned if a run was - triggered.""" - - timestamp: Optional[int] = None - """Timestamp at which the trigger was evaluated.""" - - def as_dict(self) -> dict: - """Serializes the TriggerEvaluation into a dictionary suitable for use as a JSON request body.""" - body = {} - if self.description is not None: body['description'] = self.description - if self.run_id is not None: body['run_id'] = self.run_id - if self.timestamp is not None: body['timestamp'] = self.timestamp - return body - - @classmethod - def from_dict(cls, d: Dict[str, any]) -> TriggerEvaluation: - """Deserializes the TriggerEvaluation from a dictionary.""" - return cls(description=d.get('description', None), - run_id=d.get('run_id', None), - timestamp=d.get('timestamp', None)) - - -@dataclass -class TriggerHistory: - last_failed: Optional[TriggerEvaluation] = None - """The last time the trigger failed to evaluate.""" - - last_not_triggered: Optional[TriggerEvaluation] = None - """The last time the trigger was evaluated but did not trigger a run.""" - - last_triggered: Optional[TriggerEvaluation] = None - """The last time the run was triggered due to a file arrival.""" - - def as_dict(self) -> dict: - """Serializes the TriggerHistory into a dictionary suitable for use as a JSON request body.""" - body = {} - if self.last_failed: body['last_failed'] = self.last_failed.as_dict() - if self.last_not_triggered: body['last_not_triggered'] = self.last_not_triggered.as_dict() - if self.last_triggered: body['last_triggered'] = self.last_triggered.as_dict() - return body - - @classmethod - def from_dict(cls, d: Dict[str, any]) -> TriggerHistory: - """Deserializes the TriggerHistory from a dictionary.""" - return cls(last_failed=_from_dict(d, 'last_failed', TriggerEvaluation), - last_not_triggered=_from_dict(d, 'last_not_triggered', TriggerEvaluation), - last_triggered=_from_dict(d, 'last_triggered', TriggerEvaluation)) - - @dataclass class TriggerInfo: run_id: Optional[int] = None @@ -4515,18 +4501,23 @@ class TriggerSettings: pause_status: Optional[PauseStatus] = None """Whether this trigger is paused or not.""" + table: Optional[TableTriggerConfiguration] = None + """Table trigger settings.""" + def as_dict(self) -> dict: """Serializes the TriggerSettings into a dictionary suitable for use as a JSON request body.""" body = {} if self.file_arrival: body['file_arrival'] = self.file_arrival.as_dict() if self.pause_status is not None: body['pause_status'] = self.pause_status.value + if self.table: body['table'] = self.table.as_dict() return body @classmethod def from_dict(cls, d: Dict[str, any]) -> TriggerSettings: """Deserializes the TriggerSettings from a dictionary.""" return cls(file_arrival=_from_dict(d, 'file_arrival', FileArrivalTriggerConfiguration), - pause_status=_enum(d, 'pause_status', PauseStatus)) + pause_status=_enum(d, 'pause_status', PauseStatus), + table=_from_dict(d, 'table', TableTriggerConfiguration)) class TriggerType(Enum): @@ -4536,15 +4527,16 @@ class TriggerType(Enum): One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * - `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. - - * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival.""" + `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: + Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is + triggered by a table update.""" FILE_ARRIVAL = 'FILE_ARRIVAL' ONE_TIME = 'ONE_TIME' PERIODIC = 'PERIODIC' RETRY = 'RETRY' RUN_JOB_TASK = 'RUN_JOB_TASK' + TABLE = 'TABLE' @dataclass @@ -4903,9 +4895,9 @@ def create(self, :param timeout_seconds: int (optional) An optional timeout applied to each run of this job. A value of `0` means no timeout. :param trigger: :class:`TriggerSettings` (optional) - Trigger settings for the job. Can be used to trigger a run when new files arrive in an external - location. The default behavior is that the job runs only when triggered by clicking “Run Now” in - the Jobs UI or sending an API request to `runNow`. + A configuration to trigger a run when certain conditions are met. The default behavior is that the + job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to + `runNow`. :param webhook_notifications: :class:`WebhookNotifications` (optional) A collection of system notification IDs to notify when runs of this job begin or complete. @@ -5247,8 +5239,9 @@ def repair_run(self, in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param job_parameters: Dict[str,str] (optional) Job-level parameters used in the run. for example `"param": "overriding_val"` :param latest_repair_id: int (optional) @@ -5263,13 +5256,13 @@ def repair_run(self, notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param pipeline_params: :class:`PipelineParams` (optional) :param python_named_params: Dict[str,str] (optional) A map from keys to values for jobs with Python wheel task, for example `"python_named_params": @@ -5280,7 +5273,7 @@ def repair_run(self, would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -5288,7 +5281,7 @@ def repair_run(self, returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param rerun_all_failed_tasks: bool (optional) If true, repair all failed tasks. Only one of `rerun_tasks` or `rerun_all_failed_tasks` can be used. :param rerun_dependent_tasks: bool (optional) @@ -5303,7 +5296,7 @@ def repair_run(self, in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -5311,7 +5304,7 @@ def repair_run(self, returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param sql_params: Dict[str,str] (optional) A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. @@ -5441,8 +5434,9 @@ def run_now(self, in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param job_parameters: Dict[str,str] (optional) Job-level parameters used in the run. for example `"param": "overriding_val"` :param notebook_params: Dict[str,str] (optional) @@ -5454,13 +5448,13 @@ def run_now(self, notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param pipeline_params: :class:`PipelineParams` (optional) :param python_named_params: Dict[str,str] (optional) A map from keys to values for jobs with Python wheel task, for example `"python_named_params": @@ -5471,7 +5465,7 @@ def run_now(self, would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -5479,7 +5473,7 @@ def run_now(self, returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param queue: :class:`QueueSettings` (optional) The queue settings of the run. :param spark_submit_params: List[str] (optional) @@ -5489,7 +5483,7 @@ def run_now(self, in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -5497,7 +5491,7 @@ def run_now(self, returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param sql_params: Dict[str,str] (optional) A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. diff --git a/databricks/sdk/service/serving.py b/databricks/sdk/service/serving.py index 13e6286e7..ca2c0d836 100755 --- a/databricks/sdk/service/serving.py +++ b/databricks/sdk/service/serving.py @@ -764,52 +764,29 @@ class ExternalModel: task: str """The task type of the external model.""" - config: ExternalModelConfig - """The config for the external model, which must match the provider.""" - - def as_dict(self) -> dict: - """Serializes the ExternalModel into a dictionary suitable for use as a JSON request body.""" - body = {} - if self.config: body['config'] = self.config.as_dict() - if self.name is not None: body['name'] = self.name - if self.provider is not None: body['provider'] = self.provider.value - if self.task is not None: body['task'] = self.task - return body - - @classmethod - def from_dict(cls, d: Dict[str, any]) -> ExternalModel: - """Deserializes the ExternalModel from a dictionary.""" - return cls(config=_from_dict(d, 'config', ExternalModelConfig), - name=d.get('name', None), - provider=_enum(d, 'provider', ExternalModelProvider), - task=d.get('task', None)) - - -@dataclass -class ExternalModelConfig: ai21labs_config: Optional[Ai21LabsConfig] = None - """AI21Labs Config""" + """AI21Labs Config. Only required if the provider is 'ai21labs'.""" anthropic_config: Optional[AnthropicConfig] = None - """Anthropic Config""" + """Anthropic Config. Only required if the provider is 'anthropic'.""" aws_bedrock_config: Optional[AwsBedrockConfig] = None - """AWS Bedrock Config""" + """AWS Bedrock Config. Only required if the provider is 'aws-bedrock'.""" cohere_config: Optional[CohereConfig] = None - """Cohere Config""" + """Cohere Config. Only required if the provider is 'cohere'.""" databricks_model_serving_config: Optional[DatabricksModelServingConfig] = None - """Databricks Model Serving Config""" + """Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.""" openai_config: Optional[OpenAiConfig] = None - """OpenAI Config""" + """OpenAI Config. Only required if the provider is 'openai'.""" palm_config: Optional[PaLmConfig] = None - """PaLM Config""" + """PaLM Config. Only required if the provider is 'palm'.""" def as_dict(self) -> dict: - """Serializes the ExternalModelConfig into a dictionary suitable for use as a JSON request body.""" + """Serializes the ExternalModel into a dictionary suitable for use as a JSON request body.""" body = {} if self.ai21labs_config: body['ai21labs_config'] = self.ai21labs_config.as_dict() if self.anthropic_config: body['anthropic_config'] = self.anthropic_config.as_dict() @@ -817,21 +794,27 @@ def as_dict(self) -> dict: if self.cohere_config: body['cohere_config'] = self.cohere_config.as_dict() if self.databricks_model_serving_config: body['databricks_model_serving_config'] = self.databricks_model_serving_config.as_dict() + if self.name is not None: body['name'] = self.name if self.openai_config: body['openai_config'] = self.openai_config.as_dict() if self.palm_config: body['palm_config'] = self.palm_config.as_dict() + if self.provider is not None: body['provider'] = self.provider.value + if self.task is not None: body['task'] = self.task return body @classmethod - def from_dict(cls, d: Dict[str, any]) -> ExternalModelConfig: - """Deserializes the ExternalModelConfig from a dictionary.""" + def from_dict(cls, d: Dict[str, any]) -> ExternalModel: + """Deserializes the ExternalModel from a dictionary.""" return cls(ai21labs_config=_from_dict(d, 'ai21labs_config', Ai21LabsConfig), anthropic_config=_from_dict(d, 'anthropic_config', AnthropicConfig), aws_bedrock_config=_from_dict(d, 'aws_bedrock_config', AwsBedrockConfig), cohere_config=_from_dict(d, 'cohere_config', CohereConfig), databricks_model_serving_config=_from_dict(d, 'databricks_model_serving_config', DatabricksModelServingConfig), + name=d.get('name', None), openai_config=_from_dict(d, 'openai_config', OpenAiConfig), - palm_config=_from_dict(d, 'palm_config', PaLmConfig)) + palm_config=_from_dict(d, 'palm_config', PaLmConfig), + provider=_enum(d, 'provider', ExternalModelProvider), + task=d.get('task', None)) class ExternalModelProvider(Enum): @@ -1412,6 +1395,12 @@ class ServedEntityInput: instance_profile_arn: Optional[str] = None """ARN of the instance profile that the served entity uses to access AWS resources.""" + max_provisioned_throughput: Optional[int] = None + """The maximum tokens per second that the endpoint can scale up to.""" + + min_provisioned_throughput: Optional[int] = None + """The minimum tokens per second that the endpoint can scale down to.""" + name: Optional[str] = None """The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external @@ -1445,6 +1434,10 @@ def as_dict(self) -> dict: if self.environment_vars: body['environment_vars'] = self.environment_vars if self.external_model: body['external_model'] = self.external_model.as_dict() if self.instance_profile_arn is not None: body['instance_profile_arn'] = self.instance_profile_arn + if self.max_provisioned_throughput is not None: + body['max_provisioned_throughput'] = self.max_provisioned_throughput + if self.min_provisioned_throughput is not None: + body['min_provisioned_throughput'] = self.min_provisioned_throughput if self.name is not None: body['name'] = self.name if self.scale_to_zero_enabled is not None: body['scale_to_zero_enabled'] = self.scale_to_zero_enabled if self.workload_size is not None: body['workload_size'] = self.workload_size @@ -1459,6 +1452,8 @@ def from_dict(cls, d: Dict[str, any]) -> ServedEntityInput: environment_vars=d.get('environment_vars', None), external_model=_from_dict(d, 'external_model', ExternalModel), instance_profile_arn=d.get('instance_profile_arn', None), + max_provisioned_throughput=d.get('max_provisioned_throughput', None), + min_provisioned_throughput=d.get('min_provisioned_throughput', None), name=d.get('name', None), scale_to_zero_enabled=d.get('scale_to_zero_enabled', None), workload_size=d.get('workload_size', None), @@ -1502,6 +1497,12 @@ class ServedEntityOutput: instance_profile_arn: Optional[str] = None """ARN of the instance profile that the served entity uses to access AWS resources.""" + max_provisioned_throughput: Optional[int] = None + """The maximum tokens per second that the endpoint can scale up to.""" + + min_provisioned_throughput: Optional[int] = None + """The minimum tokens per second that the endpoint can scale down to.""" + name: Optional[str] = None """The name of the served entity.""" @@ -1538,6 +1539,10 @@ def as_dict(self) -> dict: if self.external_model: body['external_model'] = self.external_model.as_dict() if self.foundation_model: body['foundation_model'] = self.foundation_model.as_dict() if self.instance_profile_arn is not None: body['instance_profile_arn'] = self.instance_profile_arn + if self.max_provisioned_throughput is not None: + body['max_provisioned_throughput'] = self.max_provisioned_throughput + if self.min_provisioned_throughput is not None: + body['min_provisioned_throughput'] = self.min_provisioned_throughput if self.name is not None: body['name'] = self.name if self.scale_to_zero_enabled is not None: body['scale_to_zero_enabled'] = self.scale_to_zero_enabled if self.state: body['state'] = self.state.as_dict() @@ -1556,6 +1561,8 @@ def from_dict(cls, d: Dict[str, any]) -> ServedEntityOutput: external_model=_from_dict(d, 'external_model', ExternalModel), foundation_model=_from_dict(d, 'foundation_model', FoundationModel), instance_profile_arn=d.get('instance_profile_arn', None), + max_provisioned_throughput=d.get('max_provisioned_throughput', None), + min_provisioned_throughput=d.get('min_provisioned_throughput', None), name=d.get('name', None), scale_to_zero_enabled=d.get('scale_to_zero_enabled', None), state=_from_dict(d, 'state', ServedModelState), diff --git a/databricks/sdk/service/sharing.py b/databricks/sdk/service/sharing.py index d367cc4db..ce1a6675a 100755 --- a/databricks/sdk/service/sharing.py +++ b/databricks/sdk/service/sharing.py @@ -1380,7 +1380,7 @@ class UpdateCleanRoom: comment: Optional[str] = None """User-provided free-form text description.""" - name_arg: Optional[str] = None + name: Optional[str] = None """The name of the clean room.""" owner: Optional[str] = None @@ -1391,7 +1391,7 @@ def as_dict(self) -> dict: body = {} if self.catalog_updates: body['catalog_updates'] = [v.as_dict() for v in self.catalog_updates] if self.comment is not None: body['comment'] = self.comment - if self.name_arg is not None: body['name_arg'] = self.name_arg + if self.name is not None: body['name'] = self.name if self.owner is not None: body['owner'] = self.owner return body @@ -1400,7 +1400,7 @@ def from_dict(cls, d: Dict[str, any]) -> UpdateCleanRoom: """Deserializes the UpdateCleanRoom from a dictionary.""" return cls(catalog_updates=_repeated_dict(d, 'catalog_updates', CleanRoomCatalogUpdate), comment=d.get('comment', None), - name_arg=d.get('name_arg', None), + name=d.get('name', None), owner=d.get('owner', None)) @@ -1580,12 +1580,12 @@ def create(self, res = self._api.do('POST', '/api/2.1/unity-catalog/clean-rooms', body=body, headers=headers) return CleanRoomInfo.from_dict(res) - def delete(self, name_arg: str): + def delete(self, name: str): """Delete a clean room. Deletes a data object clean room from the metastore. The caller must be an owner of the clean room. - :param name_arg: str + :param name: str The name of the clean room. @@ -1593,15 +1593,15 @@ def delete(self, name_arg: str): headers = {'Accept': 'application/json', } - self._api.do('DELETE', f'/api/2.1/unity-catalog/clean-rooms/{name_arg}', headers=headers) + self._api.do('DELETE', f'/api/2.1/unity-catalog/clean-rooms/{name}', headers=headers) - def get(self, name_arg: str, *, include_remote_details: Optional[bool] = None) -> CleanRoomInfo: + def get(self, name: str, *, include_remote_details: Optional[bool] = None) -> CleanRoomInfo: """Get a clean room. Gets a data object clean room from the metastore. The caller must be a metastore admin or the owner of the clean room. - :param name_arg: str + :param name: str The name of the clean room. :param include_remote_details: bool (optional) Whether to include remote details (central) on the clean room. @@ -1613,10 +1613,7 @@ def get(self, name_arg: str, *, include_remote_details: Optional[bool] = None) - if include_remote_details is not None: query['include_remote_details'] = include_remote_details headers = {'Accept': 'application/json', } - res = self._api.do('GET', - f'/api/2.1/unity-catalog/clean-rooms/{name_arg}', - query=query, - headers=headers) + res = self._api.do('GET', f'/api/2.1/unity-catalog/clean-rooms/{name}', query=query, headers=headers) return CleanRoomInfo.from_dict(res) def list(self, @@ -1655,7 +1652,7 @@ def list(self, query['page_token'] = json['next_page_token'] def update(self, - name_arg: str, + name: str, *, catalog_updates: Optional[List[CleanRoomCatalogUpdate]] = None, comment: Optional[str] = None, @@ -1676,7 +1673,7 @@ def update(self, Table removals through **update** do not require additional privileges. - :param name_arg: str + :param name: str The name of the clean room. :param catalog_updates: List[:class:`CleanRoomCatalogUpdate`] (optional) Array of shared data object updates. @@ -1693,10 +1690,7 @@ def update(self, if owner is not None: body['owner'] = owner headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } - res = self._api.do('PATCH', - f'/api/2.1/unity-catalog/clean-rooms/{name_arg}', - body=body, - headers=headers) + res = self._api.do('PATCH', f'/api/2.1/unity-catalog/clean-rooms/{name}', body=body, headers=headers) return CleanRoomInfo.from_dict(res) diff --git a/databricks/sdk/service/sql.py b/databricks/sdk/service/sql.py index c5da129e6..292c428c9 100755 --- a/databricks/sdk/service/sql.py +++ b/databricks/sdk/service/sql.py @@ -2007,6 +2007,36 @@ def from_dict(cls, d: Dict[str, any]) -> ListWarehousesResponse: return cls(warehouses=_repeated_dict(d, 'warehouses', EndpointInfo)) +@dataclass +class MultiValuesOptions: + """If specified, allows multiple values to be selected for this parameter. Only applies to dropdown + list and query-based dropdown list parameters.""" + + prefix: Optional[str] = None + """Character that prefixes each selected parameter value.""" + + separator: Optional[str] = None + """Character that separates each selected parameter value. Defaults to a comma.""" + + suffix: Optional[str] = None + """Character that suffixes each selected parameter value.""" + + def as_dict(self) -> dict: + """Serializes the MultiValuesOptions into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.prefix is not None: body['prefix'] = self.prefix + if self.separator is not None: body['separator'] = self.separator + if self.suffix is not None: body['suffix'] = self.suffix + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> MultiValuesOptions: + """Deserializes the MultiValuesOptions from a dictionary.""" + return cls(prefix=d.get('prefix', None), + separator=d.get('separator', None), + suffix=d.get('suffix', None)) + + class ObjectType(Enum): """A singular noun object type.""" @@ -2063,9 +2093,20 @@ class OwnableObjectType(Enum): @dataclass class Parameter: + enum_options: Optional[str] = None + """List of valid parameter values, newline delimited. Only applies for dropdown list parameters.""" + + multi_values_options: Optional[MultiValuesOptions] = None + """If specified, allows multiple values to be selected for this parameter. Only applies to dropdown + list and query-based dropdown list parameters.""" + name: Optional[str] = None """The literal parameter marker that appears between double curly braces in the query text.""" + query_id: Optional[str] = None + """The UUID of the query that provides the parameter values. Only applies for query-based dropdown + list parameters.""" + title: Optional[str] = None """The text displayed in a parameter picking widget.""" @@ -2078,7 +2119,10 @@ class Parameter: def as_dict(self) -> dict: """Serializes the Parameter into a dictionary suitable for use as a JSON request body.""" body = {} + if self.enum_options is not None: body['enumOptions'] = self.enum_options + if self.multi_values_options: body['multiValuesOptions'] = self.multi_values_options.as_dict() if self.name is not None: body['name'] = self.name + if self.query_id is not None: body['queryId'] = self.query_id if self.title is not None: body['title'] = self.title if self.type is not None: body['type'] = self.type.value if self.value: body['value'] = self.value @@ -2087,7 +2131,10 @@ def as_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, any]) -> Parameter: """Deserializes the Parameter from a dictionary.""" - return cls(name=d.get('name', None), + return cls(enum_options=d.get('enumOptions', None), + multi_values_options=_from_dict(d, 'multiValuesOptions', MultiValuesOptions), + name=d.get('name', None), + query_id=d.get('queryId', None), title=d.get('title', None), type=_enum(d, 'type', ParameterType), value=d.get('value', None)) @@ -2097,7 +2144,9 @@ class ParameterType(Enum): """Parameters can have several different types.""" DATETIME = 'datetime' + ENUM = 'enum' NUMBER = 'number' + QUERY = 'query' TEXT = 'text' diff --git a/databricks/sdk/version.py b/databricks/sdk/version.py index db7a41602..2f15b8cd3 100644 --- a/databricks/sdk/version.py +++ b/databricks/sdk/version.py @@ -1 +1 @@ -__version__ = '0.19.1' +__version__ = '0.20.0' diff --git a/docs/dbdataclasses/catalog.rst b/docs/dbdataclasses/catalog.rst index 6b5dba6af..3401589f1 100644 --- a/docs/dbdataclasses/catalog.rst +++ b/docs/dbdataclasses/catalog.rst @@ -80,6 +80,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ContinuousUpdateStatus + :members: + :undoc-members: + .. autoclass:: CreateCatalog :members: :undoc-members: @@ -176,6 +180,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: FailedStatus + :members: + :undoc-members: + .. autoclass:: ForeignKeyConstraint :members: :undoc-members: @@ -308,6 +316,18 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: OnlineTable + :members: + :undoc-members: + +.. autoclass:: OnlineTableSpec + :members: + :undoc-members: + +.. autoclass:: OnlineTableStatus + :members: + :undoc-members: + .. autoclass:: PermissionsChange :members: :undoc-members: @@ -316,6 +336,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: PipelineProgress + :members: + :undoc-members: + .. autoclass:: PrimaryKeyConstraint :members: :undoc-members: @@ -328,6 +352,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ProvisioningStatus + :members: + :undoc-members: + .. autoclass:: RegisteredModelAlias :members: :undoc-members: @@ -384,6 +412,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: TriggeredUpdateStatus + :members: + :undoc-members: + .. autoclass:: UpdateCatalog :members: :undoc-members: @@ -456,6 +488,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ViewData + :members: + :undoc-members: + .. autoclass:: VolumeInfo :members: :undoc-members: diff --git a/docs/dbdataclasses/files.rst b/docs/dbdataclasses/files.rst index ee2410600..6999ab8c3 100644 --- a/docs/dbdataclasses/files.rst +++ b/docs/dbdataclasses/files.rst @@ -36,6 +36,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: GetMetadataResponse + :members: + :undoc-members: + .. autoclass:: ListDirectoryResponse :members: :undoc-members: diff --git a/docs/dbdataclasses/jobs.rst b/docs/dbdataclasses/jobs.rst index 7c9576cf5..0fa7f0051 100644 --- a/docs/dbdataclasses/jobs.rst +++ b/docs/dbdataclasses/jobs.rst @@ -376,6 +376,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: TableTriggerConfiguration + :members: + :undoc-members: + .. autoclass:: Task :members: :undoc-members: @@ -392,14 +396,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: -.. autoclass:: TriggerEvaluation - :members: - :undoc-members: - -.. autoclass:: TriggerHistory - :members: - :undoc-members: - .. autoclass:: TriggerInfo :members: :undoc-members: diff --git a/docs/dbdataclasses/serving.rst b/docs/dbdataclasses/serving.rst index b75c0f4c8..c6186e5b9 100644 --- a/docs/dbdataclasses/serving.rst +++ b/docs/dbdataclasses/serving.rst @@ -108,10 +108,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: -.. autoclass:: ExternalModelConfig - :members: - :undoc-members: - .. autoclass:: ExternalModelUsageElement :members: :undoc-members: diff --git a/docs/dbdataclasses/sql.rst b/docs/dbdataclasses/sql.rst index 8cf367a47..ebd5d22d9 100644 --- a/docs/dbdataclasses/sql.rst +++ b/docs/dbdataclasses/sql.rst @@ -144,6 +144,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: MultiValuesOptions + :members: + :undoc-members: + .. autoclass:: OdbcParams :members: :undoc-members: diff --git a/docs/workspace/catalog/connections.rst b/docs/workspace/catalog/connections.rst index 5ed7ad429..ba38188b4 100644 --- a/docs/workspace/catalog/connections.rst +++ b/docs/workspace/catalog/connections.rst @@ -40,7 +40,7 @@ }) # cleanup - w.connections.delete(name_arg=conn_create.name) + w.connections.delete(name=conn_create.name) Create a connection. @@ -65,19 +65,19 @@ :returns: :class:`ConnectionInfo` - .. py:method:: delete(name_arg: str) + .. py:method:: delete(name: str) Delete a connection. Deletes the connection that matches the supplied name. - :param name_arg: str + :param name: str The name of the connection to be deleted. - .. py:method:: get(name_arg: str) -> ConnectionInfo + .. py:method:: get(name: str) -> ConnectionInfo Usage: @@ -103,7 +103,7 @@ f'sdk-{time.time_ns()}', }) - conn_update = w.connections.update(name_arg=conn_create.name, + conn_update = w.connections.update(name=conn_create.name, options={ "host": "%s-fake-workspace.cloud.databricks.com" % (f'sdk-{time.time_ns()}'), @@ -113,16 +113,16 @@ f'sdk-{time.time_ns()}', }) - conn = w.connections.get(name_arg=conn_update.name) + conn = w.connections.get(name=conn_update.name) # cleanup - w.connections.delete(name_arg=conn_create.name) + w.connections.delete(name=conn_create.name) Get a connection. Gets a connection from it's name. - :param name_arg: str + :param name: str Name of the connection. :returns: :class:`ConnectionInfo` @@ -148,7 +148,7 @@ :returns: Iterator over :class:`ConnectionInfo` - .. py:method:: update(name_arg: str, options: Dict[str, str] [, new_name: Optional[str], owner: Optional[str]]) -> ConnectionInfo + .. py:method:: update(name: str, options: Dict[str, str] [, new_name: Optional[str], owner: Optional[str]]) -> ConnectionInfo Usage: @@ -174,7 +174,7 @@ f'sdk-{time.time_ns()}', }) - conn_update = w.connections.update(name_arg=conn_create.name, + conn_update = w.connections.update(name=conn_create.name, options={ "host": "%s-fake-workspace.cloud.databricks.com" % (f'sdk-{time.time_ns()}'), @@ -185,13 +185,13 @@ }) # cleanup - w.connections.delete(name_arg=conn_create.name) + w.connections.delete(name=conn_create.name) Update a connection. Updates the connection that matches the supplied name. - :param name_arg: str + :param name: str Name of the connection. :param options: Dict[str,str] A map of key-value properties attached to the securable. diff --git a/docs/workspace/catalog/index.rst b/docs/workspace/catalog/index.rst index f9139c572..d61c95cf6 100644 --- a/docs/workspace/catalog/index.rst +++ b/docs/workspace/catalog/index.rst @@ -16,6 +16,7 @@ Configure data governance with Unity Catalog for metastores, catalogs, schemas, lakehouse_monitors metastores model_versions + online_tables registered_models schemas storage_credentials diff --git a/docs/workspace/catalog/lakehouse_monitors.rst b/docs/workspace/catalog/lakehouse_monitors.rst index e4162b09a..f353a8dc1 100644 --- a/docs/workspace/catalog/lakehouse_monitors.rst +++ b/docs/workspace/catalog/lakehouse_monitors.rst @@ -186,7 +186,7 @@ :returns: :class:`MonitorRefreshInfo` - .. py:method:: update(full_name: str, assets_dir: str, output_schema_name: str [, baseline_table_name: Optional[str], custom_metrics: Optional[List[MonitorCustomMetric]], data_classification_config: Optional[MonitorDataClassificationConfig], inference_log: Optional[MonitorInferenceLogProfileType], notifications: Optional[List[MonitorNotificationsConfig]], schedule: Optional[MonitorCronSchedule], slicing_exprs: Optional[List[str]], snapshot: Optional[Any], time_series: Optional[MonitorTimeSeriesProfileType]]) -> MonitorInfo + .. py:method:: update(full_name: str, output_schema_name: str [, baseline_table_name: Optional[str], custom_metrics: Optional[List[MonitorCustomMetric]], data_classification_config: Optional[MonitorDataClassificationConfig], inference_log: Optional[MonitorInferenceLogProfileType], notifications: Optional[List[MonitorNotificationsConfig]], schedule: Optional[MonitorCronSchedule], slicing_exprs: Optional[List[str]], snapshot: Optional[Any], time_series: Optional[MonitorTimeSeriesProfileType]]) -> MonitorInfo Update a table monitor. @@ -204,8 +204,6 @@ :param full_name: str Full name of the table. - :param assets_dir: str - The directory to store monitoring assets (e.g. dashboard, metric tables). :param output_schema_name: str Schema where output metric tables are created. :param baseline_table_name: str (optional) diff --git a/docs/workspace/catalog/online_tables.rst b/docs/workspace/catalog/online_tables.rst new file mode 100644 index 000000000..164832b0f --- /dev/null +++ b/docs/workspace/catalog/online_tables.rst @@ -0,0 +1,47 @@ +``w.online_tables``: Online Tables +================================== +.. currentmodule:: databricks.sdk.service.catalog + +.. py:class:: OnlineTablesAPI + + Online tables provide lower latency and higher QPS access to data from Delta tables. + + .. py:method:: create( [, name: Optional[str], spec: Optional[OnlineTableSpec]]) -> OnlineTable + + Create an Online Table. + + Create a new Online Table. + + :param name: str (optional) + Full three-part (catalog, schema, table) name of the table. + :param spec: :class:`OnlineTableSpec` (optional) + Specification of the online table. + + :returns: :class:`OnlineTable` + + + .. py:method:: delete(name: str) + + Delete an Online Table. + + Delete an online table. Warning: This will delete all the data in the online table. If the source + Delta table was deleted or modified since this Online Table was created, this will lose the data + forever! + + :param name: str + Full three-part (catalog, schema, table) name of the table. + + + + + .. py:method:: get(name: str) -> OnlineTable + + Get an Online Table. + + Get information about an existing online table and its status. + + :param name: str + Full three-part (catalog, schema, table) name of the table. + + :returns: :class:`OnlineTable` + \ No newline at end of file diff --git a/docs/workspace/catalog/volumes.rst b/docs/workspace/catalog/volumes.rst index e8da5a906..04a5bc4fb 100644 --- a/docs/workspace/catalog/volumes.rst +++ b/docs/workspace/catalog/volumes.rst @@ -52,7 +52,7 @@ w.external_locations.delete(name=external_location.name) w.schemas.delete(full_name=created_schema.full_name) w.catalogs.delete(name=created_catalog.name, force=True) - w.volumes.delete(full_name_arg=created_volume.full_name) + w.volumes.delete(name=created_volume.full_name) Create a Volume. @@ -87,7 +87,7 @@ :returns: :class:`VolumeInfo` - .. py:method:: delete(full_name_arg: str) + .. py:method:: delete(name: str) Delete a Volume. @@ -97,13 +97,13 @@ also be the owner or have the **USE_CATALOG** privilege on the parent catalog and the **USE_SCHEMA** privilege on the parent schema. - :param full_name_arg: str + :param name: str The three-level (fully qualified) name of the volume - .. py:method:: list(catalog_name: str, schema_name: str) -> Iterator[VolumeInfo] + .. py:method:: list(catalog_name: str, schema_name: str [, max_results: Optional[int], page_token: Optional[str]]) -> Iterator[VolumeInfo] Usage: @@ -128,7 +128,7 @@ List Volumes. - Gets an array of all volumes for the current metastore under the parent catalog and schema. + Gets an array of volumes for the current metastore under the parent catalog and schema. The returned volumes are filtered based on the privileges of the calling user. For example, the metastore admin is able to list all the volumes. A regular user needs to be the owner or have the @@ -142,11 +142,26 @@ The identifier of the catalog :param schema_name: str The identifier of the schema + :param max_results: int (optional) + Maximum number of volumes to return (page length). + + If not set, the page length is set to a server configured value (10000, as of 1/29/2024). - when set + to a value greater than 0, the page length is the minimum of this value and a server configured + value (10000, as of 1/29/2024); - when set to 0, the page length is set to a server configured value + (10000, as of 1/29/2024) (recommended); - when set to a value less than 0, an invalid parameter + error is returned; + + Note: this parameter controls only the maximum number of volumes to return. The actual number of + volumes returned in a page may be smaller than this value, including 0, even if there are more + pages. + :param page_token: str (optional) + Opaque token returned by a previous request. It must be included in the request to retrieve the next + page of results (pagination). :returns: Iterator over :class:`VolumeInfo` - .. py:method:: read(full_name_arg: str) -> VolumeInfo + .. py:method:: read(name: str) -> VolumeInfo Usage: @@ -182,14 +197,14 @@ storage_location=external_location.url, volume_type=catalog.VolumeType.EXTERNAL) - loaded_volume = w.volumes.read(full_name_arg=created_volume.full_name) + loaded_volume = w.volumes.read(name=created_volume.full_name) # cleanup w.storage_credentials.delete(name=storage_credential.name) w.external_locations.delete(name=external_location.name) w.schemas.delete(full_name=created_schema.full_name) w.catalogs.delete(name=created_catalog.name, force=True) - w.volumes.delete(full_name_arg=created_volume.full_name) + w.volumes.delete(name=created_volume.full_name) Get a Volume. @@ -199,13 +214,13 @@ volume. For the latter case, the caller must also be the owner or have the **USE_CATALOG** privilege on the parent catalog and the **USE_SCHEMA** privilege on the parent schema. - :param full_name_arg: str + :param name: str The three-level (fully qualified) name of the volume :returns: :class:`VolumeInfo` - .. py:method:: update(full_name_arg: str [, comment: Optional[str], new_name: Optional[str], owner: Optional[str]]) -> VolumeInfo + .. py:method:: update(name: str [, comment: Optional[str], new_name: Optional[str], owner: Optional[str]]) -> VolumeInfo Usage: @@ -241,16 +256,16 @@ storage_location=external_location.url, volume_type=catalog.VolumeType.EXTERNAL) - loaded_volume = w.volumes.read(full_name_arg=created_volume.full_name) + loaded_volume = w.volumes.read(name=created_volume.full_name) - _ = w.volumes.update(full_name_arg=loaded_volume.full_name, comment="Updated volume comment") + _ = w.volumes.update(name=loaded_volume.full_name, comment="Updated volume comment") # cleanup w.storage_credentials.delete(name=storage_credential.name) w.external_locations.delete(name=external_location.name) w.schemas.delete(full_name=created_schema.full_name) w.catalogs.delete(name=created_catalog.name, force=True) - w.volumes.delete(full_name_arg=created_volume.full_name) + w.volumes.delete(name=created_volume.full_name) Update a Volume. @@ -262,7 +277,7 @@ Currently only the name, the owner or the comment of the volume could be updated. - :param full_name_arg: str + :param name: str The three-level (fully qualified) name of the volume :param comment: str (optional) The comment attached to the volume diff --git a/docs/workspace/jobs/jobs.rst b/docs/workspace/jobs/jobs.rst index 54a42c979..f57921e46 100644 --- a/docs/workspace/jobs/jobs.rst +++ b/docs/workspace/jobs/jobs.rst @@ -233,9 +233,9 @@ :param timeout_seconds: int (optional) An optional timeout applied to each run of this job. A value of `0` means no timeout. :param trigger: :class:`TriggerSettings` (optional) - Trigger settings for the job. Can be used to trigger a run when new files arrive in an external - location. The default behavior is that the job runs only when triggered by clicking “Run Now” in - the Jobs UI or sending an API request to `runNow`. + A configuration to trigger a run when certain conditions are met. The default behavior is that the + job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to + `runNow`. :param webhook_notifications: :class:`WebhookNotifications` (optional) A collection of system notification IDs to notify when runs of this job begin or complete. @@ -658,8 +658,9 @@ in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param job_parameters: Dict[str,str] (optional) Job-level parameters used in the run. for example `"param": "overriding_val"` :param latest_repair_id: int (optional) @@ -674,13 +675,13 @@ notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param pipeline_params: :class:`PipelineParams` (optional) :param python_named_params: Dict[str,str] (optional) A map from keys to values for jobs with Python wheel task, for example `"python_named_params": @@ -691,7 +692,7 @@ would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -699,7 +700,7 @@ returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param rerun_all_failed_tasks: bool (optional) If true, repair all failed tasks. Only one of `rerun_tasks` or `rerun_all_failed_tasks` can be used. :param rerun_dependent_tasks: bool (optional) @@ -714,7 +715,7 @@ in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -722,7 +723,7 @@ returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param sql_params: Dict[str,str] (optional) A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. @@ -852,8 +853,9 @@ in conjunction with notebook_params. The JSON representation of this field (for example `{"jar_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables](/jobs.html"#parameter-variables") to set parameters containing - information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. + + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param job_parameters: Dict[str,str] (optional) Job-level parameters used in the run. for example `"param": "overriding_val"` :param notebook_params: Dict[str,str] (optional) @@ -865,13 +867,13 @@ notebook_params cannot be specified in conjunction with jar_params. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. The JSON representation of this field (for example `{"notebook_params":{"name":"john doe","age":"35"}}`) cannot exceed 10,000 bytes. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables [dbutils.widgets.get]: https://docs.databricks.com/dev-tools/databricks-utils.html + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param pipeline_params: :class:`PipelineParams` (optional) :param python_named_params: Dict[str,str] (optional) A map from keys to values for jobs with Python wheel task, for example `"python_named_params": @@ -882,7 +884,7 @@ would overwrite the parameters specified in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs. + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -890,7 +892,7 @@ returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param queue: :class:`QueueSettings` (optional) The queue settings of the run. :param spark_submit_params: List[str] (optional) @@ -900,7 +902,7 @@ in job setting. The JSON representation of this field (for example `{"python_params":["john doe","35"]}`) cannot exceed 10,000 bytes. - Use [Task parameter variables] to set parameters containing information about job runs + Use [task parameter variables] such as `{{job.id}}` to pass context about job runs. Important @@ -908,7 +910,7 @@ returns an error. Examples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis. - [Task parameter variables]: https://docs.databricks.com/jobs.html#parameter-variables + [task parameter variables]: https://docs.databricks.com/workflows/jobs/parameter-value-references.html :param sql_params: Dict[str,str] (optional) A map from keys to values for jobs with SQL task, for example `"sql_params": {"name": "john doe", "age": "35"}`. The SQL alert task does not support custom parameters. diff --git a/docs/workspace/sharing/clean_rooms.rst b/docs/workspace/sharing/clean_rooms.rst index 827b39f0d..186b3b3dd 100644 --- a/docs/workspace/sharing/clean_rooms.rst +++ b/docs/workspace/sharing/clean_rooms.rst @@ -26,26 +26,26 @@ :returns: :class:`CleanRoomInfo` - .. py:method:: delete(name_arg: str) + .. py:method:: delete(name: str) Delete a clean room. Deletes a data object clean room from the metastore. The caller must be an owner of the clean room. - :param name_arg: str + :param name: str The name of the clean room. - .. py:method:: get(name_arg: str [, include_remote_details: Optional[bool]]) -> CleanRoomInfo + .. py:method:: get(name: str [, include_remote_details: Optional[bool]]) -> CleanRoomInfo Get a clean room. Gets a data object clean room from the metastore. The caller must be a metastore admin or the owner of the clean room. - :param name_arg: str + :param name: str The name of the clean room. :param include_remote_details: bool (optional) Whether to include remote details (central) on the clean room. @@ -72,7 +72,7 @@ :returns: Iterator over :class:`CleanRoomInfo` - .. py:method:: update(name_arg: str [, catalog_updates: Optional[List[CleanRoomCatalogUpdate]], comment: Optional[str], owner: Optional[str]]) -> CleanRoomInfo + .. py:method:: update(name: str [, catalog_updates: Optional[List[CleanRoomCatalogUpdate]], comment: Optional[str], owner: Optional[str]]) -> CleanRoomInfo Update a clean room. @@ -90,7 +90,7 @@ Table removals through **update** do not require additional privileges. - :param name_arg: str + :param name: str The name of the clean room. :param catalog_updates: List[:class:`CleanRoomCatalogUpdate`] (optional) Array of shared data object updates. diff --git a/tests/integration/test_files.py b/tests/integration/test_files.py index 5820a80ae..85a0203d1 100644 --- a/tests/integration/test_files.py +++ b/tests/integration/test_files.py @@ -221,9 +221,8 @@ def test_files_api_upload_download(ucws, random): f = io.BytesIO(b"some text data") target_file = f'/Volumes/main/{schema}/{volume}/filesit-{random()}.txt' w.files.upload(target_file, f) - # TODO: Enable after generating with the latest spec - # m = w.files.get_metadata(target_file) - # assert m.content_type == 'application/octet-stream' + m = w.files.get_metadata(target_file) + assert m.content_type == 'application/octet-stream' with w.files.download(target_file).contents as f: assert f.read() == b"some text data"