From e38fb896ca8f45c792b571e003c5f1fccebae6d6 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Aug 2024 21:16:59 +0300 Subject: [PATCH 01/10] feat: use container label to download specific module version --- lean/commands/backtest.py | 19 +++++++++++-------- lean/components/cloud/module_manager.py | 22 ++++++++++++++++------ lean/constants.py | 3 +++ lean/models/json_module.py | 16 +++++++++++++--- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index 9c0836d2..6f412467 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -16,7 +16,7 @@ from click import command, option, argument, Choice from lean.click import LeanCommand, PathParameter -from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH +from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH, CONTAINER_LABEL_LEAN_VERSION_NAME from lean.container import container, Logger from lean.models.utils import DebuggingMethod from lean.models.cli import cli_data_downloaders, cli_addon_modules @@ -359,21 +359,24 @@ def backtest(project: Path, organization_id = container.organization_manager.try_get_working_organization_id() paths_to_mount = None + cli_config_manager = container.cli_config_manager + project_config_manager = container.project_config_manager + + project_config = project_config_manager.get_project_config(algorithm_file.parent) + engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None)) + + container_module_version = container.docker_manager.get_image_label(engine_image, + CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, cli_data_downloaders, kwargs, logger, environment_name) - data_provider.ensure_module_installed(organization_id) + data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) - cli_config_manager = container.cli_config_manager - project_config_manager = container.project_config_manager - - project_config = project_config_manager.get_project_config(algorithm_file.parent) - engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None)) - if str(engine_image) != DEFAULT_ENGINE_IMAGE: logger.warn(f'A custom engine image: "{engine_image}" is being used!') diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index 505557b7..20a91379 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -37,14 +37,18 @@ def __init__(self, logger: Logger, api_client: APIClient, http_client: HTTPClien self._installed_product_ids: Set[int] = set() self._installed_packages: Dict[int, List[NuGetPackage]] = {} - def install_module(self, product_id: int, organization_id: str) -> None: - """Installs a module into the global modules directory. + def install_module(self, product_id: int, organization_id: str, module_version: str = None) -> None: + """ + Installs a module into the global modules' directory. - If an outdated version is already installed, it is automatically updated. - If the organization does not have a subscription for the given module, an error is raised. + If an outdated version is already installed, it is automatically updated. If a specific version + is provided and is different from the installed version, it will be updated. If the organization + does not have a subscription for the given module, an error is raised. - :param product_id: the product id of the module to download - :param organization_id: the id of the organization that has a license for the module + Args: + product_id (int): The product id of the module to download. + organization_id (str): The id of the organization that has a license for the module. + module_version (str, optional): The specific version of the module to install. If None, installs the latest version. """ if product_id in self._installed_product_ids: return @@ -57,8 +61,14 @@ def install_module(self, product_id: int, organization_id: str) -> None: if package.name not in packages_to_download or package.version > packages_to_download[package.name].version: packages_to_download[package.name] = package + self._logger.debug(f'module_manager.install_module: {package.version}') + if module_version and package.version.split('.')[-1] == module_version: + self._logger.debug(f'module_manager.install_module:' + f'Found requested version {module_version} for module {product_id}.') + break for package in packages_to_download.values(): + self._logger.info(f'install_module.package: {package}') self._download_file(product_id, organization_id, package) self._installed_product_ids.add(product_id) diff --git a/lean/constants.py b/lean/constants.py index f2fe3482..ada2dbc1 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -26,6 +26,9 @@ DEFAULT_LEAN_STRICT_PYTHON_VERSION = f"{DEFAULT_LEAN_PYTHON_VERSION}.7" DEFAULT_LEAN_DOTNET_FRAMEWORK = "net6.0" +# Label name used in Docker containers to specify the version of Lean being used +CONTAINER_LABEL_LEAN_VERSION_NAME = "lean_version" + # The path to the root python directory in docker image DOCKER_PYTHON_SITE_PACKAGES_PATH = "/root/.local/lib/python{LEAN_PYTHON_VERSION}/site-packages" diff --git a/lean/models/json_module.py b/lean/models/json_module.py index b95173fe..e3440d02 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -246,11 +246,21 @@ def get_paths_to_mount(self) -> Dict[str, str]: if (isinstance(config, PathParameterUserInput) and self._check_if_config_passes_filters(config, all_for_platform_type=False))} - def ensure_module_installed(self, organization_id: str) -> None: + def ensure_module_installed(self, organization_id: str, module_version: str = None) -> None: + """ + Ensures that the specified module is installed. If the module is not installed, it will be installed. + + Args: + organization_id (str): The ID of the organization where the module should be installed. + module_version (str, optional): The version of the module to install. If not provided, + the latest version will be installed. + + Returns: + None + """ if not self._is_module_installed and self._installs: container.logger.debug(f"JsonModule.ensure_module_installed(): installing module {self}: {self._product_id}") - container.module_manager.install_module( - self._product_id, organization_id) + container.module_manager.install_module(self._product_id, organization_id, module_version) self._is_module_installed = True def get_default(self, lean_config: Dict[str, Any], key: str, environment_name: str, logger: Logger): From 3e5a4f1ff19eb8970cf7752236b7a1eaa91f10a8 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Aug 2024 21:43:16 +0300 Subject: [PATCH 02/10] feat: module version in deploy/download/optimize/research --- lean/commands/data/download.py | 20 ++++++++++++-------- lean/commands/live/deploy.py | 19 +++++++++++-------- lean/commands/optimize.py | 6 ++++-- lean/commands/research.py | 29 ++++++++++++++++------------- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 77bd4474..d85d9486 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -18,7 +18,7 @@ from click import command, option, confirm, pass_context, Context, Choice, prompt from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name -from lean.constants import DEFAULT_ENGINE_IMAGE +from lean.constants import DEFAULT_ENGINE_IMAGE, CONTAINER_LABEL_LEAN_VERSION_NAME from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType from lean.models.click_options import get_configs_for_options, options_from_json @@ -675,13 +675,6 @@ def download(ctx: Context, logger = container.logger lean_config = container.lean_config_manager.get_complete_lean_config(None, None, None) - data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), - cli_data_downloaders, kwargs, logger, interactive=True) - data_downloader_provider.ensure_module_installed(organization.id) - container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) - # mounting additional data_downloader config files - paths_to_mount = data_downloader_provider.get_paths_to_mount() - engine_image = container.cli_config_manager.get_engine_image(image) if str(engine_image) != DEFAULT_ENGINE_IMAGE: @@ -690,6 +683,17 @@ def download(ctx: Context, container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) + container_module_version = container.docker_manager.get_image_label(engine_image, + CONTAINER_LABEL_LEAN_VERSION_NAME, + "Unknown") + + data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), + cli_data_downloaders, kwargs, logger, interactive=True) + data_downloader_provider.ensure_module_installed(organization.id, container_module_version) + container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) + # mounting additional data_downloader config files + paths_to_mount = data_downloader_provider.get_paths_to_mount() + downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 3e6bc852..10c3a9ee 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -16,7 +16,7 @@ from click import option, argument, Choice from lean.click import LeanCommand, PathParameter from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format -from lean.constants import DEFAULT_ENGINE_IMAGE +from lean.constants import DEFAULT_ENGINE_IMAGE, CONTAINER_LABEL_LEAN_VERSION_NAME from lean.container import container from lean.models.cli import (cli_brokerages, cli_data_queue_handlers, cli_data_downloaders, cli_addon_modules, cli_history_provider) @@ -271,23 +271,26 @@ def deploy(project: Path, kwargs, logger, interactive=True, environment_name=environment_name)) + project_config_manager = container.project_config_manager + cli_config_manager = container.cli_config_manager + + project_config = project_config_manager.get_project_config(algorithm_file.parent) + engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None)) + + container_module_version = container.docker_manager.get_image_label(engine_image, + CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + organization_id = container.organization_manager.try_get_working_organization_id() paths_to_mount = {} for module in (data_provider_live_instances + [data_downloader_instances, brokerage_instance] + history_providers_instances): - module.ensure_module_installed(organization_id) + module.ensure_module_installed(organization_id, container_module_version) paths_to_mount.update(module.get_paths_to_mount()) if not lean_config["environments"][environment_name]["live-mode"]: raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - project_config_manager = container.project_config_manager - cli_config_manager = container.cli_config_manager - - project_config = project_config_manager.get_project_config(algorithm_file.parent) - engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None)) - container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) _start_iqconnect_if_necessary(lean_config, environment_name) diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 7df8065c..031358a5 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -19,7 +19,7 @@ from lean.click import LeanCommand, PathParameter, ensure_options from lean.components.docker.lean_runner import LeanRunner -from lean.constants import DEFAULT_ENGINE_IMAGE +from lean.constants import DEFAULT_ENGINE_IMAGE, CONTAINER_LABEL_LEAN_VERSION_NAME from lean.container import container from lean.models.api import QCParameter, QCBacktest from lean.models.click_options import options_from_json, get_configs_for_options @@ -307,10 +307,12 @@ def optimize(project: Path, paths_to_mount = None + container_module_version = container.docker_manager.get_image_label(engine_image, CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, cli_data_downloaders, kwargs, logger, environment_name) - data_provider.ensure_module_installed(organization_id) + data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() diff --git a/lean/commands/research.py b/lean/commands/research.py index 6596e87c..42aa44d5 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -16,7 +16,7 @@ from click import command, argument, option, Choice from lean.click import LeanCommand, PathParameter from lean.components.docker.lean_runner import LeanRunner -from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH +from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH, CONTAINER_LABEL_LEAN_VERSION_NAME from lean.container import container from lean.models.cli import cli_data_downloaders from lean.components.util.name_extraction import convert_to_class_name @@ -113,13 +113,27 @@ def research(project: Path, if download_data: data_provider_historical = "QuantConnect" + project_config_manager = container.project_config_manager + cli_config_manager = container.cli_config_manager + + project_config = project_config_manager.get_project_config(algorithm_file.parent) + research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None)) + + container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update) + + container_module_version = container.docker_manager.get_image_label(research_image, + CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + + if str(research_image) != DEFAULT_RESEARCH_IMAGE: + logger.warn(f'A custom research image: "{research_image}" is being used!') + paths_to_mount = None if data_provider_historical is not None: organization_id = container.organization_manager.try_get_working_organization_id() data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, cli_data_downloaders, kwargs, logger, environment_name) - data_provider.ensure_module_installed(organization_id) + data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) @@ -131,17 +145,6 @@ def research(project: Path, for key, value in extra_config: lean_config[key] = value - project_config_manager = container.project_config_manager - cli_config_manager = container.cli_config_manager - - project_config = project_config_manager.get_project_config(algorithm_file.parent) - research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None)) - - container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update) - - if str(research_image) != DEFAULT_RESEARCH_IMAGE: - logger.warn(f'A custom research image: "{research_image}" is being used!') - run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, temp_manager.create_temporary_directory(), From 875d400705aaa298c1edd5dfebb22f87cbd963d4 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Aug 2024 22:55:36 +0300 Subject: [PATCH 03/10] feat: change Unknown to None in get lean version name --- lean/commands/backtest.py | 2 +- lean/commands/data/download.py | 3 +-- lean/commands/live/deploy.py | 2 +- lean/commands/optimize.py | 3 ++- lean/commands/research.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index 6f412467..779b07fd 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -366,7 +366,7 @@ def backtest(project: Path, engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None)) container_module_version = container.docker_manager.get_image_label(engine_image, - CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + CONTAINER_LABEL_LEAN_VERSION_NAME, None) if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index d85d9486..8cc88ef0 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -684,8 +684,7 @@ def download(ctx: Context, container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) container_module_version = container.docker_manager.get_image_label(engine_image, - CONTAINER_LABEL_LEAN_VERSION_NAME, - "Unknown") + CONTAINER_LABEL_LEAN_VERSION_NAME, None) data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), cli_data_downloaders, kwargs, logger, interactive=True) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 10c3a9ee..ad600fae 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -278,7 +278,7 @@ def deploy(project: Path, engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None)) container_module_version = container.docker_manager.get_image_label(engine_image, - CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + CONTAINER_LABEL_LEAN_VERSION_NAME, None) organization_id = container.organization_manager.try_get_working_organization_id() paths_to_mount = {} diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 031358a5..cd361121 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -307,7 +307,8 @@ def optimize(project: Path, paths_to_mount = None - container_module_version = container.docker_manager.get_image_label(engine_image, CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + container_module_version = container.docker_manager.get_image_label(engine_image, + CONTAINER_LABEL_LEAN_VERSION_NAME, None) if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, diff --git a/lean/commands/research.py b/lean/commands/research.py index 42aa44d5..604c92a4 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -122,7 +122,7 @@ def research(project: Path, container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update) container_module_version = container.docker_manager.get_image_label(research_image, - CONTAINER_LABEL_LEAN_VERSION_NAME, "Unknown") + CONTAINER_LABEL_LEAN_VERSION_NAME, None) if str(research_image) != DEFAULT_RESEARCH_IMAGE: logger.warn(f'A custom research image: "{research_image}" is being used!') From b53b936864a73070c7792652ca8dcf081517b220 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 13 Aug 2024 23:40:55 +0300 Subject: [PATCH 04/10] remove: extra log in module manager --- lean/components/cloud/module_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index 20a91379..22c98d04 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -61,7 +61,6 @@ def install_module(self, product_id: int, organization_id: str, module_version: if package.name not in packages_to_download or package.version > packages_to_download[package.name].version: packages_to_download[package.name] = package - self._logger.debug(f'module_manager.install_module: {package.version}') if module_version and package.version.split('.')[-1] == module_version: self._logger.debug(f'module_manager.install_module:' f'Found requested version {module_version} for module {product_id}.') From 612f7464c38eaf9eae05bfcbff51dcf4fcafdff6 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Aug 2024 16:42:20 +0300 Subject: [PATCH 05/10] refactor: independent collection on specific module version and replace with actual download one --- lean/components/cloud/module_manager.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index 22c98d04..a53cd606 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -55,16 +55,21 @@ def install_module(self, product_id: int, organization_id: str, module_version: module_files = self._api_client.modules.list_files(product_id, organization_id) packages_to_download: Dict[str, NuGetPackage] = {} + packages_to_download_specific_version: Dict[str, NuGetPackage] = {} for file_name in module_files: package = NuGetPackage.parse(file_name) if package.name not in packages_to_download or package.version > packages_to_download[package.name].version: packages_to_download[package.name] = package - if module_version and package.version.split('.')[-1] == module_version: + if module_version and package.version.split('.')[-1] <= module_version: self._logger.debug(f'module_manager.install_module:' f'Found requested version {module_version} for module {product_id}.') - break + packages_to_download_specific_version[package.name] = package + + # Replace version packages based on module_version if available + for package_name, package_specific_version in packages_to_download_specific_version.items(): + packages_to_download[package_name] = package_specific_version for package in packages_to_download.values(): self._logger.info(f'install_module.package: {package}') From 93c8fda150bb081f54f4c8b63817d01713586b57 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Aug 2024 16:57:05 +0300 Subject: [PATCH 06/10] remove: extra logging row --- lean/components/cloud/module_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index a53cd606..6fb488c8 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -72,7 +72,6 @@ def install_module(self, product_id: int, organization_id: str, module_version: packages_to_download[package_name] = package_specific_version for package in packages_to_download.values(): - self._logger.info(f'install_module.package: {package}') self._download_file(product_id, organization_id, package) self._installed_product_ids.add(product_id) From c50f92b285a6cc33bd85f6c28676104edfcd0879 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Aug 2024 17:35:41 +0300 Subject: [PATCH 07/10] feat: specific log message if package has not found --- lean/components/cloud/module_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index 6fb488c8..5ee660d5 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -63,8 +63,6 @@ def install_module(self, product_id: int, organization_id: str, module_version: if package.name not in packages_to_download or package.version > packages_to_download[package.name].version: packages_to_download[package.name] = package if module_version and package.version.split('.')[-1] <= module_version: - self._logger.debug(f'module_manager.install_module:' - f'Found requested version {module_version} for module {product_id}.') packages_to_download_specific_version[package.name] = package # Replace version packages based on module_version if available @@ -72,6 +70,9 @@ def install_module(self, product_id: int, organization_id: str, module_version: packages_to_download[package_name] = package_specific_version for package in packages_to_download.values(): + if module_version and package.version.split('.')[-1] != module_version: + self._logger.debug(f'Package "{package.name}" does not have the specified version {module_version}. ' + f'Using available version {package.version} instead.') self._download_file(product_id, organization_id, package) self._installed_product_ids.add(product_id) From 2e763fb9eb117b61a774af1c10d344d6df42f393 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Aug 2024 23:22:28 +0300 Subject: [PATCH 08/10] remove: duplication of downloading modules in build_and_config_modules refactor: use module_version like required parameter --- lean/components/util/json_modules_handler.py | 1 - lean/models/json_module.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index 6657db2c..bf449cd2 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -33,7 +33,6 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso for target_module_name in target_modules: module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties, logger, environment_name) - module.ensure_module_installed(organization_id) lean_config["environments"][environment_name].update(module.get_settings()) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index e3440d02..6003af34 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -246,13 +246,13 @@ def get_paths_to_mount(self) -> Dict[str, str]: if (isinstance(config, PathParameterUserInput) and self._check_if_config_passes_filters(config, all_for_platform_type=False))} - def ensure_module_installed(self, organization_id: str, module_version: str = None) -> None: + def ensure_module_installed(self, organization_id: str, module_version: str) -> None: """ Ensures that the specified module is installed. If the module is not installed, it will be installed. Args: organization_id (str): The ID of the organization where the module should be installed. - module_version (str, optional): The version of the module to install. If not provided, + module_version (str): The version of the module to install. If not provided, the latest version will be installed. Returns: From b53e1ccd029a209b6996bf5c40cfd523579f3e30 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 14 Aug 2024 23:49:32 +0300 Subject: [PATCH 09/10] refactor: sort packages returns from API by version to will be sure that we return last one version --- lean/components/cloud/module_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index 5ee660d5..232f8ae3 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -53,15 +53,21 @@ def install_module(self, product_id: int, organization_id: str, module_version: if product_id in self._installed_product_ids: return + # Retrieve the list of module files for the specified product and organization module_files = self._api_client.modules.list_files(product_id, organization_id) + # Dictionaries to store the latest packages to download and specific version packages packages_to_download: Dict[str, NuGetPackage] = {} packages_to_download_specific_version: Dict[str, NuGetPackage] = {} - for file_name in module_files: - package = NuGetPackage.parse(file_name) + # Parse the module files into NuGetPackage objects and sort them by version + packages = [NuGetPackage.parse(file_name) for file_name in module_files] + sorted_packages = sorted(packages, key=lambda p: p.version) + for package in sorted_packages: + # Store the latest version of each package if package.name not in packages_to_download or package.version > packages_to_download[package.name].version: packages_to_download[package.name] = package + # If a specific version is requested, keep track of the highest version <= module_version if module_version and package.version.split('.')[-1] <= module_version: packages_to_download_specific_version[package.name] = package From 6072a0344cdfe8cf99b11cba3f21e04fd6bcba4e Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 15 Aug 2024 16:44:05 +0300 Subject: [PATCH 10/10] remove: optional type parameter in install_module --- lean/components/cloud/module_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lean/components/cloud/module_manager.py b/lean/components/cloud/module_manager.py index 232f8ae3..e54f1963 100644 --- a/lean/components/cloud/module_manager.py +++ b/lean/components/cloud/module_manager.py @@ -37,7 +37,7 @@ def __init__(self, logger: Logger, api_client: APIClient, http_client: HTTPClien self._installed_product_ids: Set[int] = set() self._installed_packages: Dict[int, List[NuGetPackage]] = {} - def install_module(self, product_id: int, organization_id: str, module_version: str = None) -> None: + def install_module(self, product_id: int, organization_id: str, module_version: str) -> None: """ Installs a module into the global modules' directory. @@ -48,7 +48,7 @@ def install_module(self, product_id: int, organization_id: str, module_version: Args: product_id (int): The product id of the module to download. organization_id (str): The id of the organization that has a license for the module. - module_version (str, optional): The specific version of the module to install. If None, installs the latest version. + module_version (str): The specific version of the module to install. If None, installs the latest version. """ if product_id in self._installed_product_ids: return