diff --git a/dlt/destinations/impl/clickhouse/clickhouse.py b/dlt/destinations/impl/clickhouse/clickhouse.py index b6f23ee221..603215d889 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse.py +++ b/dlt/destinations/impl/clickhouse/clickhouse.py @@ -61,11 +61,13 @@ class ClickHouseLoadJob(RunnableLoadJob, HasFollowupJobs): def __init__( self, file_path: str, + config: ClickHouseClientConfiguration, staging_credentials: Optional[CredentialsConfiguration] = None, ) -> None: super().__init__(file_path) self._job_client: "ClickHouseClient" = None self._staging_credentials = staging_credentials + self._config = config def run(self) -> None: client = self._job_client.sql_client @@ -138,7 +140,9 @@ def run(self) -> None: ) bucket_http_url = convert_storage_to_http_scheme( - bucket_url, endpoint=self._staging_credentials.endpoint_url + bucket_url, + endpoint=self._staging_credentials.endpoint_url, + use_https=self._config.staging_use_https, ) access_key_id = self._staging_credentials.aws_access_key_id secret_access_key = self._staging_credentials.aws_secret_access_key @@ -255,6 +259,7 @@ def create_load_job( ) -> LoadJob: return super().create_load_job(table, file_path, load_id, restore) or ClickHouseLoadJob( file_path, + config=self.config, staging_credentials=( self.config.staging_config.credentials if self.config.staging_config else None ), diff --git a/dlt/destinations/impl/clickhouse/configuration.py b/dlt/destinations/impl/clickhouse/configuration.py index cba909eb9a..7acfc08885 100644 --- a/dlt/destinations/impl/clickhouse/configuration.py +++ b/dlt/destinations/impl/clickhouse/configuration.py @@ -77,6 +77,8 @@ class ClickHouseClientConfiguration(DestinationClientDwhWithStagingConfiguration """The default table engine to use. Defaults to 'merge_tree'. Other implemented options are 'shared_merge_tree' and 'replicated_merge_tree'.""" dataset_sentinel_table_name: str = "dlt_sentinel_table" """Special table to mark dataset as existing""" + staging_use_https: bool = True + """Connect to the staging buckets via https""" __config_gen_annotations__: ClassVar[List[str]] = [ "dataset_table_separator",