dlt-hub · rudolfix · Oct 15, 2024 · Oct 10, 2024 · Oct 11, 2024 · Oct 12, 2024
diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml
@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"athena\"]"

diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml
@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"athena\"]"

diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml
@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_clickhouse.yml b/.github/workflows/test_destination_clickhouse.yml
@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml
@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_dremio.yml b/.github/workflows/test_destination_dremio.yml
@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_lancedb.yml b/.github/workflows/test_destination_lancedb.yml
@@ -16,7 +16,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_motherduck.yml b/.github/workflows/test_destination_motherduck.yml
@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml
@@ -18,7 +18,7 @@ env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml
@@ -16,7 +16,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml
@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml
@@ -23,7 +23,7 @@ env:
   TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
   TESTS__R2_ENDPOINT_URL: https://9830548e4e4b582989be0811f2a0a97f.r2.cloudflarestorage.com
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   # Test redshift and filesystem with all buckets

diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml
@@ -15,7 +15,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml
@@ -18,7 +18,7 @@ env:
   # NOTE: this workflow can't use github secrets!
   # DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\", \"qdrant\"]"

diff --git a/.github/workflows/test_local_sources.yml b/.github/workflows/test_local_sources.yml
@@ -15,7 +15,7 @@ concurrency:
 
 env:
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

diff --git a/.github/workflows/test_pyarrow17.yml b/.github/workflows/test_pyarrow17.yml
@@ -18,7 +18,7 @@ env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
@@ -70,7 +70,7 @@ jobs:
 
       - name: Upgrade pyarrow
         run: poetry run pip install pyarrow==17.0.0
-      
+
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 

diff --git a/.github/workflows/test_sqlalchemy_destinations.yml b/.github/workflows/test_sqlalchemy_destinations.yml
@@ -18,7 +18,7 @@ env:
   # NOTE: this workflow can't use github secrets!
   # DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"sqlalchemy\"]"

diff --git a/Makefile b/Makefile
@@ -107,7 +107,7 @@ test-build-images: build-library
 	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
 	# docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
 
-preprocess-docs: 
+preprocess-docs:
 	# run docs preprocessing to run a few checks and ensure examples can be parsed
 	cd docs/website && npm i && npm run preprocess-docs
 

diff --git a/dlt/cli/deploy_command_helpers.py b/dlt/cli/deploy_command_helpers.py
@@ -16,12 +16,12 @@
 from dlt.common import git
 from dlt.common.configuration.exceptions import LookupTrace, ConfigFieldMissingException
 from dlt.common.configuration.providers import (
-    ConfigTomlProvider,
+    CONFIG_TOML,
     EnvironProvider,
     StringTomlProvider,
 )
 from dlt.common.git import get_origin, get_repo, Repo
-from dlt.common.configuration.specs.run_configuration import get_default_pipeline_name
+from dlt.common.configuration.specs.runtime_configuration import get_default_pipeline_name
 from dlt.common.typing import StrAny
 from dlt.common.reflection.utils import evaluate_node_literal
 from dlt.common.pipeline import LoadInfo, TPipelineState, get_dlt_repos_dir
@@ -71,7 +71,6 @@ def __init__(
         self.working_directory: str
         self.state: TPipelineState
 
-        self.config_prov = ConfigTomlProvider()
         self.env_prov = EnvironProvider()
         self.envs: List[LookupTrace] = []
         self.secret_envs: List[LookupTrace] = []
@@ -190,7 +189,7 @@ def _update_envs(self, trace: PipelineTrace) -> None:
                 # fmt.echo(f"{resolved_value.key}:{resolved_value.value}{type(resolved_value.value)} in {resolved_value.sections} is SECRET")
             else:
                 # move all config values that are not in config.toml into env
-                if resolved_value.provider_name != self.config_prov.name:
+                if resolved_value.provider_name != CONFIG_TOML:
                     self.envs.append(
                         LookupTrace(
                             self.env_prov.name,

diff --git a/dlt/cli/init_command.py b/dlt/cli/init_command.py
@@ -25,7 +25,7 @@
 from dlt.sources import SourceReference
 
 import dlt.reflection.names as n
-from dlt.reflection.script_inspector import inspect_pipeline_script
+from dlt.reflection.script_inspector import import_pipeline_script
 
 from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection
 from dlt.cli import utils
@@ -452,7 +452,7 @@ def init_command(
     )
 
     # inspect the script
-    inspect_pipeline_script(
+    import_pipeline_script(
         source_configuration.storage.storage_path,
         source_configuration.storage.to_relative_path(source_configuration.src_pipeline_script),
         ignore_missing_imports=True,

diff --git a/dlt/cli/telemetry_command.py b/dlt/cli/telemetry_command.py
@@ -3,12 +3,12 @@
 
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.providers.toml import ConfigTomlProvider
-from dlt.common.configuration.specs import RunConfiguration
+from dlt.common.configuration.specs import RuntimeConfiguration
 
 from dlt.cli import echo as fmt
 from dlt.cli.utils import get_telemetry_status
 from dlt.cli.config_toml_writer import WritableConfigValue, write_values
-from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext
+from dlt.common.configuration.specs import PluggableRunContext
 from dlt.common.runtime.anon_tracker import get_anonymous_id
 
 DLT_TELEMETRY_DOCS_URL = "https://dlthub.com/docs/reference/telemetry"
@@ -23,23 +23,24 @@ def telemetry_status_command() -> None:
 
 
 def change_telemetry_status_command(enabled: bool) -> None:
+    from dlt.common.runtime import run_context
+
     # value to write
     telemetry_value = [
-        WritableConfigValue("dlthub_telemetry", bool, enabled, (RunConfiguration.__section__,))
+        WritableConfigValue("dlthub_telemetry", bool, enabled, (RuntimeConfiguration.__section__,))
     ]
     # write local config
     # TODO: use designated (main) config provider (for non secret values) ie. taken from run context
-    config = ConfigTomlProvider(add_global_config=False)
+    run_ctx = run_context.current()
+    config = ConfigTomlProvider(run_ctx.settings_dir)
     if not config.is_empty:
         write_values(config._config_toml, telemetry_value, overwrite_existing=True)
         config.write_toml()
 
     # write global config
-    from dlt.common.runtime import run_context
-
-    global_path = run_context.current().global_dir
+    global_path = run_ctx.global_dir
     os.makedirs(global_path, exist_ok=True)
-    config = ConfigTomlProvider(settings_dir=global_path, add_global_config=False)
+    config = ConfigTomlProvider(settings_dir=global_path)
     write_values(config._config_toml, telemetry_value, overwrite_existing=True)
     config.write_toml()
 
@@ -48,5 +49,4 @@ def change_telemetry_status_command(enabled: bool) -> None:
     else:
         fmt.echo("Telemetry switched %s" % fmt.bold("OFF"))
     # reload config providers
-    if ConfigProvidersContext in Container():
-        del Container()[ConfigProvidersContext]
+    Container()[PluggableRunContext].reload_providers()
diff --git a/dlt/cli/utils.py b/dlt/cli/utils.py
@@ -5,7 +5,7 @@
 from dlt.common.reflection.utils import set_ast_parents
 from dlt.common.typing import TFun
 from dlt.common.configuration import resolve_configuration
-from dlt.common.configuration.specs import RunConfiguration
+from dlt.common.configuration.specs import RuntimeConfiguration
 from dlt.common.runtime.telemetry import with_telemetry
 from dlt.common.runtime import run_context
 
@@ -60,7 +60,7 @@ def track_command(command: str, track_before: bool, *args: str) -> Callable[[TFu
 
 
 def get_telemetry_status() -> bool:
-    c = resolve_configuration(RunConfiguration())
+    c = resolve_configuration(RuntimeConfiguration())
     return c.dlthub_telemetry
 
 

diff --git a/dlt/common/configuration/accessors.py b/dlt/common/configuration/accessors.py
@@ -6,7 +6,7 @@
 from dlt.common.configuration.providers.provider import ConfigProvider
 from dlt.common.configuration.specs import BaseConfiguration, is_base_configuration_inner_hint
 from dlt.common.configuration.utils import deserialize_value, log_traces, auto_cast
-from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext
+from dlt.common.configuration.specs import PluggableRunContext
 from dlt.common.typing import AnyType, ConfigValue, SecretValue, TSecretValue
 
 TConfigAny = TypeVar("TConfigAny", bound=Any)
@@ -54,7 +54,7 @@ def writable_provider(self) -> ConfigProvider:
         pass
 
     def _get_providers_from_context(self) -> Sequence[ConfigProvider]:
-        return Container()[ConfigProvidersContext].providers
+        return Container()[PluggableRunContext].providers.providers
 
     def _get_value(self, field: str, type_hint: Type[Any] = None) -> Tuple[Any, List[LookupTrace]]:
         # get default hint type, in case of dlt.secrets it it TSecretValue
@@ -85,7 +85,7 @@ def register_provider(provider: ConfigProvider) -> None:
         """Registers `provider` to participate in the configuration resolution. `provider`
         is added after all existing providers and will be used if all others do not resolve.
         """
-        Container()[ConfigProvidersContext].add_provider(provider)
+        Container()[PluggableRunContext].providers.add_provider(provider)
 
 
 class _ConfigAccessor(_Accessor):