Merge branch 'devel' into docs/how-to-deploy-using-modal

.github/workflows/test_destinations.yml

@@ -77,11 +77,14 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline -E deltalake
+        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli -E filesystem --with sentry-sdk --with pipeline -E deltalake
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
+      - name: clear duckdb secrets and cache
+        run: rm -rf ~/.duckdb
+
       - run: |
           poetry run pytest tests/load --ignore tests/load/sources -m "essential"
         name: Run essential tests Linux

.github/workflows/test_doc_snippets.yml

@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker compose -f ".github/weaviate-compose.yml" up -d
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/test_local_destinations.yml

@@ -73,7 +73,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker compose -f ".github/weaviate-compose.yml" up -d
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -122,7 +122,7 @@ jobs:
 
       - name: Stop weaviate
         if: always()
-        run: docker compose -f ".github/weaviate-compose.yml" down -v
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" down -v
 
       - name: Stop SFTP server
         if: always()

.github/workflows/test_pyarrow17.yml

@@ -65,14 +65,18 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-pyarrow17
 
       - name: Install dependencies
-        run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E gs -E s3 -E az
+        run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E duckdb -E filesystem -E gs -E s3 -E az
+
 
       - name: Upgrade pyarrow
         run: poetry run pip install pyarrow==17.0.0
-
+      
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
+      - name: clear duckdb secrets and cache
+        run: rm -rf ~/.duckdb
+
       - name: Run needspyarrow17 tests Linux
         run: |
           poetry run pytest tests/libs -m "needspyarrow17"

.github/workflows/test_sqlalchemy_destinations.yml

@@ -94,6 +94,3 @@ jobs:
       # always run full suite, also on branches
       - run: poetry run pytest tests/load -x --ignore tests/load/sources
         name: Run tests Linux
-        env:
-          DESTINATION__SQLALCHEMY_MYSQL__CREDENTIALS: mysql://root:[email protected]:3306/dlt_data  # Use root cause we need to create databases
-          DESTINATION__SQLALCHEMY_SQLITE__CREDENTIALS: sqlite:///_storage/dl_data.sqlite

Makefile

@@ -110,4 +110,11 @@ test-build-images: build-library
 
 preprocess-docs: 
 	# run docs preprocessing to run a few checks and ensure examples can be parsed
-	cd docs/website && npm i && npm run preprocess-docs
+	cd docs/website && npm i && npm run preprocess-docs
+
+start-test-containers:
+	docker compose -f "tests/load/dremio/docker-compose.yml" up -d
+	docker compose -f "tests/load/postgres/docker-compose.yml" up -d
+	docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
+	docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d
+	docker compose -f "tests/load/sqlalchemy/docker-compose.yml" up -d

dlt/__init__.py

@@ -22,7 +22,7 @@
 
 from dlt.version import __version__
 from dlt.common.configuration.accessors import config, secrets
-from dlt.common.typing import TSecretValue as _TSecretValue
+from dlt.common.typing import TSecretValue as _TSecretValue, TSecretStrValue as _TSecretStrValue
 from dlt.common.configuration.specs import CredentialsConfiguration as _CredentialsConfiguration
 from dlt.common.pipeline import source_state as state
 from dlt.common.schema import Schema
@@ -50,10 +50,12 @@
 TSecretValue = _TSecretValue
 "When typing source/resource function arguments it indicates that a given argument is a secret and should be taken from dlt.secrets."
 
+TSecretStrValue = _TSecretStrValue
+"When typing source/resource function arguments it indicates that a given argument is a secret STRING and should be taken from dlt.secrets."
+
 TCredentials = _CredentialsConfiguration
 "When typing source/resource function arguments it indicates that a given argument represents credentials and should be taken from dlt.secrets. Credentials may be a string, dictionary or any other type."
 
-
 __all__ = [
     "__version__",
     "config",
@@ -78,3 +80,12 @@
     "sources",
     "destinations",
 ]
+
+# verify that no injection context was created
+from dlt.common.configuration.container import Container as _Container
+
+assert (
+    _Container._INSTANCE is None
+), "Injection container should not be initialized during initial import"
+# create injection container
+_Container()

dlt/cli/config_toml_writer.py

@@ -104,6 +104,7 @@ def write_spec(toml_table: TOMLTable, config: BaseConfiguration, overwrite_exist
 def write_values(
     toml: TOMLContainer, values: Iterable[WritableConfigValue], overwrite_existing: bool
 ) -> None:
+    # TODO: decouple writers from a particular object model ie. TOML
     for value in values:
         toml_table: TOMLTable = toml  # type: ignore
         for section in value.sections:

dlt/cli/deploy_command.py

@@ -5,7 +5,6 @@
 from importlib.metadata import version as pkg_version
 
 from dlt.common.configuration.providers import SECRETS_TOML, SECRETS_TOML_KEY
-from dlt.common.configuration.paths import make_dlt_settings_path
 from dlt.common.configuration.utils import serialize_value
 from dlt.common.git import is_dirty
 
@@ -210,7 +209,7 @@ def _echo_instructions(self, *args: Optional[Any]) -> None:
             fmt.echo(
                 "1. Add the following secret values (typically stored in %s): \n%s\nin %s"
                 % (
-                    fmt.bold(make_dlt_settings_path(SECRETS_TOML)),
+                    fmt.bold(utils.make_dlt_settings_path(SECRETS_TOML)),
                     fmt.bold(
                         "\n".join(
                             self.env_prov.get_key_name(s_v.key, *s_v.sections)
@@ -368,7 +367,7 @@ def _echo_instructions(self, *args: Optional[Any]) -> None:
                     "3. Add the following secret values (typically stored in %s): \n%s\n%s\nin"
                     " ENVIRONMENT VARIABLES using Google Composer UI"
                     % (
-                        fmt.bold(make_dlt_settings_path(SECRETS_TOML)),
+                        fmt.bold(utils.make_dlt_settings_path(SECRETS_TOML)),
                         fmt.bold(
                             "\n".join(
                                 self.env_prov.get_key_name(s_v.key, *s_v.sections)

dlt/cli/deploy_command_helpers.py

@@ -15,8 +15,11 @@
 
 from dlt.common import git
 from dlt.common.configuration.exceptions import LookupTrace, ConfigFieldMissingException
-from dlt.common.configuration.providers import ConfigTomlProvider, EnvironProvider
-from dlt.common.configuration.providers.toml import BaseDocProvider, StringTomlProvider
+from dlt.common.configuration.providers import (
+    ConfigTomlProvider,
+    EnvironProvider,
+    StringTomlProvider,
+)
 from dlt.common.git import get_origin, get_repo, Repo
 from dlt.common.configuration.specs.run_configuration import get_default_pipeline_name
 from dlt.common.typing import StrAny
@@ -242,7 +245,7 @@ def _display_missing_secret_info(self) -> None:
         )
 
     def _lookup_secret_value(self, trace: LookupTrace) -> Any:
-        return dlt.secrets[BaseDocProvider.get_key_name(trace.key, *trace.sections)]
+        return dlt.secrets[StringTomlProvider.get_key_name(trace.key, *trace.sections)]
 
     def _echo_envs(self) -> None:
         for v in self.envs: