From 142ed76ae90d333b76553c7fa012d31b97ec6b3b Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Wed, 9 Oct 2024 18:06:21 +0200 Subject: [PATCH 1/3] allows to pass run_dir to RunContext init --- dlt/common/runtime/run_context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dlt/common/runtime/run_context.py b/dlt/common/runtime/run_context.py index f8e7920577..bd799bbfe0 100644 --- a/dlt/common/runtime/run_context.py +++ b/dlt/common/runtime/run_context.py @@ -19,6 +19,9 @@ class RunContext(SupportsRunContext): CONTEXT_NAME: ClassVar[str] = "dlt" + def __init__(self, run_dir: str = "."): + self._init_run_dir = run_dir + @property def global_dir(self) -> str: return self.data_dir @@ -26,7 +29,7 @@ def global_dir(self) -> str: @property def run_dir(self) -> str: """The default run dir is the current working directory but may be overridden by DLT_PROJECT_DIR env variable.""" - return os.environ.get(known_env.DLT_PROJECT_DIR, ".") + return os.environ.get(known_env.DLT_PROJECT_DIR, self._init_run_dir) @property def settings_dir(self) -> str: From 0772e1f6e9a0a758f9ce3c7320e374d6b350109b Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Wed, 9 Oct 2024 18:06:45 +0200 Subject: [PATCH 2/3] uses uv to install deps in Venv if found --- dlt/common/known_env.py | 3 +++ dlt/common/runners/venv.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dlt/common/known_env.py b/dlt/common/known_env.py index 7ac36d252d..9c2028a28b 100644 --- a/dlt/common/known_env.py +++ b/dlt/common/known_env.py @@ -23,3 +23,6 @@ DLT_JSON_TYPED_PUA_START = "DLT_JSON_TYPED_PUA_START" """Start of the unicode block within the PUA used to encode types in typed json""" + +DLT_PIP_TOOL = "DLT_PIP_TOOL" +"""Pip tool used to install deps in Venv""" diff --git a/dlt/common/runners/venv.py b/dlt/common/runners/venv.py index b59456dcc2..5b892aeaf6 100644 --- a/dlt/common/runners/venv.py +++ b/dlt/common/runners/venv.py @@ -4,8 +4,9 @@ import venv import types import subprocess -from typing import Any, List, Type +from typing import Any, ClassVar, List, Type +from dlt.common import known_env from dlt.common.exceptions import CannotInstallDependencies, VenvNotFound @@ -22,6 +23,8 @@ def post_setup(self, context: types.SimpleNamespace) -> None: class Venv: """Creates and wraps the Python Virtual Environment to allow for code execution""" + PIP_TOOL: ClassVar[str] = os.environ.get(known_env.DLT_PIP_TOOL, None) + def __init__(self, context: types.SimpleNamespace, current: bool = False) -> None: """Please use `Venv.create`, `Venv.restore` or `Venv.restore_current` methods to create Venv instance""" self.context = context @@ -119,8 +122,17 @@ def add_dependencies(self, dependencies: List[str] = None) -> None: @staticmethod def _install_deps(context: types.SimpleNamespace, dependencies: List[str]) -> None: - cmd = [context.env_exe, "-Im", "pip", "install"] - # cmd = ["uv", "pip", "install", "--python", context.env_exe] + if Venv.PIP_TOOL is None: + # autodetect tool + import shutil + + Venv.PIP_TOOL = "uv" if shutil.which("uv") else "pip" + + if Venv.PIP_TOOL == "uv": + cmd = ["uv", "pip", "install", "--python", context.env_exe] + else: + cmd = [context.env_exe, "-Im", Venv.PIP_TOOL, "install"] + try: subprocess.check_output(cmd + dependencies, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: From 5f0114f70dd688e0604293808d73bed1ed547ce4 Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Wed, 9 Oct 2024 18:07:02 +0200 Subject: [PATCH 3/3] removes semver deprecations --- dlt/common/storages/exceptions.py | 14 ++++++------- dlt/common/storages/normalize_storage.py | 6 +++--- dlt/common/storages/versioned_storage.py | 20 +++++++++---------- dlt/common/warnings.py | 14 ++++++------- .../common/storages/test_versioned_storage.py | 8 +++----- 5 files changed, 28 insertions(+), 34 deletions(-) diff --git a/dlt/common/storages/exceptions.py b/dlt/common/storages/exceptions.py index 028491dd9c..098bf7afce 100644 --- a/dlt/common/storages/exceptions.py +++ b/dlt/common/storages/exceptions.py @@ -13,9 +13,9 @@ class NoMigrationPathException(StorageException): def __init__( self, storage_path: str, - initial_version: semver.VersionInfo, - migrated_version: semver.VersionInfo, - target_version: semver.VersionInfo, + initial_version: semver.Version, + migrated_version: semver.Version, + target_version: semver.Version, ) -> None: self.storage_path = storage_path self.initial_version = initial_version @@ -31,8 +31,8 @@ class WrongStorageVersionException(StorageException): def __init__( self, storage_path: str, - initial_version: semver.VersionInfo, - target_version: semver.VersionInfo, + initial_version: semver.Version, + target_version: semver.Version, ) -> None: self.storage_path = storage_path self.initial_version = initial_version @@ -46,8 +46,8 @@ class StorageMigrationError(StorageException): def __init__( self, storage_path: str, - from_version: semver.VersionInfo, - target_version: semver.VersionInfo, + from_version: semver.Version, + target_version: semver.Version, info: str, ) -> None: self.storage_path = storage_path diff --git a/dlt/common/storages/normalize_storage.py b/dlt/common/storages/normalize_storage.py index 2b90b7c088..0416c18c1c 100644 --- a/dlt/common/storages/normalize_storage.py +++ b/dlt/common/storages/normalize_storage.py @@ -3,7 +3,7 @@ import semver from typing import ClassVar, Sequence -from semver import VersionInfo +from semver import Version from dlt.common.configuration import with_config, known_sections from dlt.common.configuration.accessors import config @@ -57,7 +57,7 @@ def list_files_to_normalize_sorted(self) -> Sequence[str]: ] ) - def migrate_storage(self, from_version: VersionInfo, to_version: VersionInfo) -> None: + def migrate_storage(self, from_version: Version, to_version: Version) -> None: if from_version == "1.0.0" and from_version < to_version: # get files in storage if len(self.list_files_to_normalize_sorted()) > 0: @@ -69,5 +69,5 @@ def migrate_storage(self, from_version: VersionInfo, to_version: VersionInfo) -> " Storage will not migrate automatically duo to possible data loss. Delete the" " files or normalize it with dlt 0.3.x", ) - from_version = semver.VersionInfo.parse("1.0.1") + from_version = semver.Version.parse("1.0.1") self._save_version(from_version) diff --git a/dlt/common/storages/versioned_storage.py b/dlt/common/storages/versioned_storage.py index 8e9a3eb88d..450f0c3a4b 100644 --- a/dlt/common/storages/versioned_storage.py +++ b/dlt/common/storages/versioned_storage.py @@ -10,10 +10,10 @@ class VersionedStorage: VERSION_FILE = ".version" def __init__( - self, version: Union[semver.VersionInfo, str], is_owner: bool, storage: FileStorage + self, version: Union[semver.Version, str], is_owner: bool, storage: FileStorage ) -> None: if isinstance(version, str): - version = semver.VersionInfo.parse(version) + version = semver.Version.parse(version) self.storage = storage # read current version if self.storage.has_file(VersionedStorage.VERSION_FILE): @@ -43,28 +43,26 @@ def __init__( self._save_version(version) else: raise WrongStorageVersionException( - storage.storage_path, semver.VersionInfo.parse("0.0.0"), version + storage.storage_path, semver.Version.parse("0.0.0"), version ) - def migrate_storage( - self, from_version: semver.VersionInfo, to_version: semver.VersionInfo - ) -> None: + def migrate_storage(self, from_version: semver.Version, to_version: semver.Version) -> None: # migration example: # # semver lib supports comparing both to string and other semvers # if from_version == "1.0.0" and from_version < to_version: # # do migration # # save migrated version - # from_version = semver.VersionInfo.parse("1.1.0") + # from_version = semver.Version.parse("1.1.0") # self._save_version(from_version) pass @property - def version(self) -> semver.VersionInfo: + def version(self) -> semver.Version: return self._load_version() - def _load_version(self) -> semver.VersionInfo: + def _load_version(self) -> semver.Version: version_str = self.storage.load(VersionedStorage.VERSION_FILE) - return semver.VersionInfo.parse(version_str) + return semver.Version.parse(version_str) - def _save_version(self, version: semver.VersionInfo) -> None: + def _save_version(self, version: semver.Version) -> None: self.storage.save(VersionedStorage.VERSION_FILE, str(version)) diff --git a/dlt/common/warnings.py b/dlt/common/warnings.py index 95d5a19f08..243a8e2dfa 100644 --- a/dlt/common/warnings.py +++ b/dlt/common/warnings.py @@ -6,7 +6,7 @@ from dlt.version import __version__ -VersionString = typing.Union[str, semver.VersionInfo] +VersionString = typing.Union[str, semver.Version] class DltDeprecationWarning(DeprecationWarning): @@ -30,14 +30,12 @@ def __init__( ) -> None: super().__init__(message, *args) self.message = message.rstrip(".") - self.since = ( - since if isinstance(since, semver.VersionInfo) else semver.parse_version_info(since) - ) + self.since = since if isinstance(since, semver.Version) else semver.Version.parse(since) if expected_due: expected_due = ( expected_due - if isinstance(expected_due, semver.VersionInfo) - else semver.parse_version_info(expected_due) + if isinstance(expected_due, semver.Version) + else semver.Version.parse(expected_due) ) # we deprecate across major version since 1.0.0 self.expected_due = expected_due if expected_due is not None else self.since.bump_major() @@ -50,7 +48,7 @@ def __str__(self) -> str: class Dlt04DeprecationWarning(DltDeprecationWarning): - V04 = semver.parse_version_info("0.4.0") + V04 = semver.Version.parse("0.4.0") def __init__(self, message: str, *args: typing.Any, expected_due: VersionString = None) -> None: super().__init__( @@ -59,7 +57,7 @@ def __init__(self, message: str, *args: typing.Any, expected_due: VersionString class Dlt100DeprecationWarning(DltDeprecationWarning): - V100 = semver.parse_version_info("1.0.0") + V100 = semver.Version.parse("1.0.0") def __init__(self, message: str, *args: typing.Any, expected_due: VersionString = None) -> None: super().__init__( diff --git a/tests/common/storages/test_versioned_storage.py b/tests/common/storages/test_versioned_storage.py index 2859c7662c..28fc964e1d 100644 --- a/tests/common/storages/test_versioned_storage.py +++ b/tests/common/storages/test_versioned_storage.py @@ -9,15 +9,13 @@ class MigratedStorage(VersionedStorage): - def migrate_storage( - self, from_version: semver.VersionInfo, to_version: semver.VersionInfo - ) -> None: + def migrate_storage(self, from_version: semver.Version, to_version: semver.Version) -> None: # migration example: if from_version == "1.0.0" and from_version < to_version: - from_version = semver.VersionInfo.parse("1.1.0") + from_version = semver.Version.parse("1.1.0") self._save_version(from_version) if from_version == "1.1.0" and from_version < to_version: - from_version = semver.VersionInfo.parse("1.2.0") + from_version = semver.Version.parse("1.2.0") self._save_version(from_version)