From efed71c0646b07f5d115d67a8556a18a9bcc7d4e Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 14 Nov 2024 15:46:51 +0200 Subject: [PATCH 01/21] feat: CharlesSchwab in Readme --- README.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 91abe5f0..52c1318a 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Options: -d, --detach Run the backtest in a detached Docker container and return immediately --debug [pycharm|ptvsd|debugpy|vsdbg|rider|local-platform] Enable a certain debugging method (see --help for more information) - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -219,6 +219,8 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used + --charles-schwab-account-number TEXT + The CharlesSchwab account number --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect --data-purchase-limit INTEGER The maximum amount of QCC to spend on downloading data during the backtest when using @@ -349,9 +351,9 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT --notify-insights. Options: - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit|TradeStation|Alpaca] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit|TradeStation|Alpaca|CharlesSchwab] The brokerage to use - --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Polygon|IEX|CoinApi|Bybit|TradeStation|Alpaca] + --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Polygon|IEX|CoinApi|Bybit|TradeStation|Alpaca|CharlesSchwab] The live data provider to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -442,6 +444,8 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used + --charles-schwab-account-number TEXT + The CharlesSchwab account number --polygon-api-key TEXT Your Polygon.io API Key --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] @@ -852,7 +856,7 @@ Usage: lean data download [OPTIONS] https://www.quantconnect.com/datasets Options: - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] The name of the downloader data provider. --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -921,6 +925,8 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used + --charles-schwab-account-number TEXT + The CharlesSchwab account number --dataset TEXT The name of the dataset to download non-interactively --overwrite Overwrite existing local data -y, --yes Automatically confirm payment confirmation prompts @@ -1289,11 +1295,11 @@ Options: --environment TEXT The environment to use --output DIRECTORY Directory to store results in (defaults to PROJECT/live/TIMESTAMP) -d, --detach Run the live deployment in a detached Docker container and return immediately - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit|TradeStation|Alpaca] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit|TradeStation|Alpaca|CharlesSchwab] The brokerage to use - --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|ThetaData|Custom data only|Bybit|TradeStation|Alpaca] + --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|ThetaData|Custom data only|Bybit|TradeStation|Alpaca|CharlesSchwab] The live data provider to use - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Bybit|TradeStation|Alpaca] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Bybit|TradeStation|Alpaca|CharlesSchwab] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -1399,6 +1405,8 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used + --charles-schwab-account-number TEXT + The CharlesSchwab account number --ib-enable-delayed-streaming-data BOOLEAN Whether delayed data may be used when your algorithm subscribes to a security you don't have a market data subscription for (Optional). @@ -1730,7 +1738,7 @@ Options: --parameter ... The 'parameter min max step' pairs configuring the parameters to optimize --constraint TEXT The 'statistic operator value' pairs configuring the constraints of the optimization - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] Update the Lean configuration file to retrieve data from the given historical provider --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect @@ -1812,6 +1820,8 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used + --charles-schwab-account-number TEXT + The CharlesSchwab account number --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) --verbose Enable debug logging --help Show this message and exit. @@ -1989,7 +1999,7 @@ Usage: lean research [OPTIONS] PROJECT Options: --port INTEGER The port to run Jupyter Lab on (defaults to 8888) - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -2058,6 +2068,8 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used + --charles-schwab-account-number TEXT + The CharlesSchwab account number --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect --data-purchase-limit INTEGER The maximum amount of QCC to spend on downloading data during the research session From 8c2d4e4ae608d864fb8d439e9424db0963458923 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 14 Nov 2024 16:26:44 +0200 Subject: [PATCH 02/21] remove: obsolete TDAmeritrade --- README.md | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 52c1318a..87c08a67 100644 --- a/README.md +++ b/README.md @@ -351,9 +351,9 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT --notify-insights. Options: - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit|TradeStation|Alpaca|CharlesSchwab] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|Bybit|TradeStation|Alpaca|CharlesSchwab] The brokerage to use - --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Polygon|IEX|CoinApi|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|Polygon|IEX|CoinApi|Bybit|TradeStation|Alpaca|CharlesSchwab] The live data provider to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -429,11 +429,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --tdameritrade-api-key TEXT Your TDAmeritrade API key - --tdameritrade-access-token TEXT - Your TDAmeritrade OAuth Access Token - --tdameritrade-account-number TEXT - Your TDAmeritrade account number --bybit-api-key TEXT Your Bybit API key --bybit-api-secret TEXT Your Bybit API secret --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5] @@ -1295,9 +1290,9 @@ Options: --environment TEXT The environment to use --output DIRECTORY Directory to store results in (defaults to PROJECT/live/TIMESTAMP) -d, --detach Run the live deployment in a detached Docker container and return immediately - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit|TradeStation|Alpaca|CharlesSchwab] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|Bybit|TradeStation|Alpaca|CharlesSchwab] The brokerage to use - --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|ThetaData|Custom data only|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|IQFeed|Polygon|IEX|CoinApi|ThetaData|Custom data only|Bybit|TradeStation|Alpaca|CharlesSchwab] The live data provider to use --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Bybit|TradeStation|Alpaca|CharlesSchwab] Update the Lean configuration file to retrieve data from the given historical provider @@ -1388,11 +1383,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --tdameritrade-api-key TEXT Your TDAmeritrade API key - --tdameritrade-access-token TEXT - Your TDAmeritrade OAuth Access Token - --tdameritrade-account-number TEXT - Your TDAmeritrade account number --bybit-api-key TEXT Your Bybit API key --bybit-api-secret TEXT Your Bybit API secret --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5] From d2c1bac0505b13933b9562212451588004e2d31e Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 18 Dec 2024 02:08:03 +0200 Subject: [PATCH 03/21] feat: add project_id to authorize in auth0_client --- lean/commands/backtest.py | 8 +++--- lean/commands/cloud/live/deploy.py | 14 ++++++---- lean/commands/data/download.py | 7 +++-- lean/commands/live/deploy.py | 27 +++++++++++--------- lean/commands/optimize.py | 9 ++++--- lean/commands/research.py | 7 ++--- lean/components/api/auth0_client.py | 5 ++-- lean/components/util/auth0_helper.py | 5 ++-- lean/components/util/json_modules_handler.py | 23 +++++++++-------- lean/container.py | 14 ++++++++++ lean/models/json_module.py | 5 +++- 11 files changed, 81 insertions(+), 43 deletions(-) diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index 82f53efd..383cb3c2 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -17,7 +17,7 @@ from lean.click import LeanCommand, PathParameter from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH -from lean.container import container, Logger +from lean.container import container, Logger, get_project_id from lean.models.utils import DebuggingMethod from lean.models.cli import cli_data_downloaders, cli_addon_modules from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name @@ -362,9 +362,11 @@ def backtest(project: Path, engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, algorithm_file.parent) + project_id = get_project_id(project_config) + if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, environment_name) + cli_data_downloaders, kwargs, logger, project_id, environment_name) data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() @@ -394,7 +396,7 @@ def backtest(project: Path, # Configure addon modules build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, - kwargs, logger, environment_name, container_module_version) + kwargs, logger, environment_name, container_module_version, project_id) lean_runner = container.lean_runner lean_runner.run_lean(lean_config, diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 4821cbb3..a551ea58 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -245,7 +245,7 @@ def deploy(project: str, ensure_options(["brokerage", "node", "auto_restart", "notify_order_events", "notify_insights"]) brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cloud_brokerages, - kwargs, logger) + kwargs, logger, cloud_project.projectId) notify_methods = [] if notify_emails is not None: for config in notify_emails.split(","): @@ -287,11 +287,13 @@ def deploy(project: str, else: # let the user choose the brokerage brokerage_instance = interactive_config_build(lean_config, cloud_brokerages, logger, kwargs, show_secrets, - "Select a brokerage", multiple=False) + "Select a brokerage", multiple=False, + project_id=cloud_project.projectId) notify_order_events, notify_insights, notify_methods = _configure_notifications(logger) auto_restart = _configure_auto_restart(logger) - cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project) + cash_balance_option, holdings_option, last_cash, last_holdings = ( + get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project)) if cash_balance_option != LiveInitialStateInput.NotSupported: live_cash_balance = _configure_initial_cash_interactively(logger, cash_balance_option, last_cash) if holdings_option != LiveInitialStateInput.NotSupported: @@ -303,13 +305,15 @@ def deploy(project: str, # the user sent the live data provider to use for data_provider in data_provider_live: data_provider_instance = non_interactive_config_build_for_name(lean_config, data_provider, - cloud_data_queue_handlers, kwargs, logger) + cloud_data_queue_handlers, kwargs, logger, + cloud_project.projectId) live_data_provider_settings.update({data_provider_instance.get_id(): data_provider_instance.get_settings()}) else: # let's ask the user which live data providers to use data_feed_instances = interactive_config_build(lean_config, cloud_data_queue_handlers, logger, kwargs, - show_secrets, "Select a live data feed", multiple=True) + show_secrets, "Select a live data feed", + multiple=True, project_id=cloud_project.projectId) for data_feed in data_feed_instances: settings = data_feed.get_settings() diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 1db52c39..328370bf 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -19,7 +19,7 @@ from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name from lean.constants import DEFAULT_ENGINE_IMAGE -from lean.container import container +from lean.container import container, get_project_id from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType from lean.models.click_options import get_configs_for_options, options_from_json from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform,OptionResult, Product @@ -677,8 +677,11 @@ def download(ctx: Context, engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update) + project_id = get_project_id(project_config) + data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), - cli_data_downloaders, kwargs, logger, interactive=True) + cli_data_downloaders, kwargs, logger, interactive=True, + project_id=project_id) data_downloader_provider.ensure_module_installed(organization.id, container_module_version) container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) # mounting additional data_downloader config files diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index a7d41f83..f3010826 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -17,7 +17,7 @@ from lean.click import LeanCommand, PathParameter from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format from lean.constants import DEFAULT_ENGINE_IMAGE -from lean.container import container +from lean.container import container, get_project_id from lean.models.cli import (cli_brokerages, cli_data_queue_handlers, cli_data_downloaders, cli_addon_modules, cli_history_provider) from lean.models.errors import MoreInfoError @@ -194,6 +194,11 @@ def deploy(project: Path, project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(Path(project)) + engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, + algorithm_file.parent) + + project_id = get_project_id(project_config) + if output is None: output = algorithm_file.parent / "live" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") @@ -238,21 +243,22 @@ def deploy(project: Path, if brokerage: # user provided brokerage, check all arguments were provided brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cli_brokerages, kwargs, - logger, environment_name) + logger, project_id, environment_name) else: # let the user choose the brokerage brokerage_instance = interactive_config_build(lean_config, cli_brokerages, logger, kwargs, show_secrets, "Select a brokerage", multiple=False, - environment_name=environment_name) + project_id=project_id, environment_name=environment_name) if data_provider_live and len(data_provider_live) > 0: for data_feed_name in data_provider_live: data_feed = non_interactive_config_build_for_name(lean_config, data_feed_name, cli_data_queue_handlers, - kwargs, logger, environment_name) + kwargs, logger, project_id, environment_name) data_provider_live_instances.append(data_feed) else: data_provider_live_instances = interactive_config_build(lean_config, cli_data_queue_handlers, logger, kwargs, - show_secrets, "Select a live data feed", multiple=True, + show_secrets, "Select a live data feed", + multiple=True, project_id=project_id, environment_name=environment_name) # based on the live data providers we set up the history providers @@ -260,7 +266,7 @@ def deploy(project: Path, if data_provider_historical is None: data_provider_historical = "Local" data_downloader_instances = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, + cli_data_downloaders, kwargs, logger, project_id, environment_name) if history_providers is None or len(history_providers) == 0: history_providers = _get_history_provider_name(data_provider_live) @@ -268,11 +274,8 @@ def deploy(project: Path, if history_provider in ["BrokerageHistoryProvider", "SubscriptionDataReaderHistoryProvider"]: continue history_providers_instances.append(config_build_for_name(lean_config, history_provider, cli_history_provider, - kwargs, logger, interactive=True, - environment_name=environment_name)) - - engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, - algorithm_file.parent) + kwargs, logger, True, project_id, + environment_name)) organization_id = container.organization_manager.try_get_working_organization_id() paths_to_mount = {} @@ -341,7 +344,7 @@ def deploy(project: Path, # Configure addon modules build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, - kwargs, logger, environment_name, container_module_version) + kwargs, logger, environment_name, container_module_version, project_id) if container.platform_manager.is_host_arm(): if "InteractiveBrokersBrokerage" in lean_config["environments"][environment_name]["live-mode-brokerage"] \ diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 8b686bee..189a3278 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -20,7 +20,7 @@ from lean.click import LeanCommand, PathParameter, ensure_options from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_ENGINE_IMAGE -from lean.container import container +from lean.container import container, get_project_id from lean.models.api import QCParameter, QCBacktest from lean.models.click_options import options_from_json, get_configs_for_options from lean.models.cli import cli_data_downloaders, cli_addon_modules @@ -298,9 +298,12 @@ def optimize(project: Path, paths_to_mount = None + project_id = get_project_id(project_config) + if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, environment_name) + cli_data_downloaders, kwargs, logger, project_id, + environment_name) data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() @@ -328,7 +331,7 @@ def optimize(project: Path, # Configure addon modules build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, - kwargs, logger, environment_name, container_module_version) + kwargs, logger, environment_name, container_module_version, project_id) run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach, engine_image, paths_to_mount) diff --git a/lean/commands/research.py b/lean/commands/research.py index 70061cfd..57012ef3 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -17,7 +17,7 @@ from lean.click import LeanCommand, PathParameter from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH -from lean.container import container +from lean.container import container, get_project_id from lean.models.cli import cli_data_downloaders from lean.components.util.name_extraction import convert_to_class_name from lean.components.util.json_modules_handler import non_interactive_config_build_for_name @@ -121,13 +121,14 @@ def research(project: Path, research_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, algorithm_file.parent, False) - + project_id = get_project_id(project_config) paths_to_mount = None if data_provider_historical is not None: organization_id = container.organization_manager.try_get_working_organization_id() data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, environment_name) + cli_data_downloaders, kwargs, logger, project_id, + environment_name) data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() diff --git a/lean/components/api/auth0_client.py b/lean/components/api/auth0_client.py index f1350c1f..0f23ab41 100644 --- a/lean/components/api/auth0_client.py +++ b/lean/components/api/auth0_client.py @@ -52,15 +52,16 @@ def read(self, brokerage_id: str) -> QCAuth0Authorization: return QCAuth0Authorization(authorization=None) @staticmethod - def authorize(brokerage_id: str, logger: Logger) -> None: + def authorize(brokerage_id: str, project_id: str, logger: Logger) -> None: """Starts the authorization process for a brokerage. :param brokerage_id: the id of the brokerage to start the authorization process for + :param project_id: The local or cloud project_id :param logger: the logger instance to use """ from webbrowser import open - full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}" + full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}&projectId={project_id}" logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.") logger.info(full_url) open(full_url) diff --git a/lean/components/util/auth0_helper.py b/lean/components/util/auth0_helper.py index b5c453d4..a8b4cb0c 100644 --- a/lean/components/util/auth0_helper.py +++ b/lean/components/util/auth0_helper.py @@ -16,11 +16,12 @@ from lean.components.util.logger import Logger -def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger) -> QCAuth0Authorization: +def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: str, logger: Logger) -> QCAuth0Authorization: """Gets the authorization data for a brokerage, authorizing if necessary. :param auth0_client: An instance of Auth0Client, containing methods to interact with live/auth0/* API endpoints. :param brokerage_id: The ID of the brokerage to get the authorization data for. + :param project_id: The local or cloud project_id. :param logger: An instance of Logger, handling all output printing. :return: The authorization data for the specified brokerage. """ @@ -31,7 +32,7 @@ def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logg return data start_time = time() - auth0_client.authorize(brokerage_id, logger) + auth0_client.authorize(brokerage_id, project_id, logger) # keep checking for new data every 5 seconds for 7 minutes while time() - start_time < 420: diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index 5b59ba47..8d96d564 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -19,7 +19,7 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[JsonModule], organization_id: str, lean_config: Dict[str, Any], properties: Dict[str, Any], logger: Logger, - environment_name: str, module_version: str): + environment_name: str, module_version: str, project_id: str): """Builds and configures the given modules :param target_modules: the requested modules @@ -30,10 +30,11 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso :param logger: the logger instance :param environment_name: the environment name to use :param module_version: The version of the module to install. If not provided, the latest version will be installed. + :param project_id: The cloud or local project_id """ for target_module_name in target_modules: module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties, - logger, environment_name) + logger, project_id, environment_name) # Ensures extra modules (not brokerage or data feeds) are installed. module.ensure_module_installed(organization_id, module_version) lean_config["environments"][environment_name].update(module.get_settings()) @@ -41,9 +42,9 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso def non_interactive_config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger, - environment_name: str = None) -> JsonModule: + project_id: str, environment_name: str = None) -> JsonModule: return config_build_for_name(lean_config, target_module_name, module_list, properties, logger, interactive=False, - environment_name=environment_name) + project_id=project_id, environment_name=environment_name) def find_module(target_module_name: str, module_list: List[JsonModule], logger: Logger) -> JsonModule: @@ -78,18 +79,18 @@ def find_module(target_module_name: str, module_list: List[JsonModule], logger: def config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule], - properties: Dict[str, Any], logger: Logger, interactive: bool, + properties: Dict[str, Any], logger: Logger, interactive: bool, project_id: str, environment_name: str = None) -> JsonModule: target_module = find_module(target_module_name, module_list, logger) - target_module.config_build(lean_config, logger, interactive=interactive, properties=properties, - environment_name=environment_name) + target_module.config_build(lean_config, logger, interactive=interactive, project_id=project_id, + properties=properties, environment_name=environment_name) _update_settings(logger, environment_name, target_module, lean_config) return target_module def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], logger: Logger, user_provided_options: Dict[str, Any], show_secrets: bool, select_message: str, - multiple: bool, environment_name: str = None) -> [JsonModule]: + multiple: bool, project_id: str, environment_name: str = None) -> [JsonModule]: """Interactively configures the brokerage to use. :param lean_config: the LEAN configuration that should be used @@ -99,6 +100,7 @@ def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], :param show_secrets: whether to show secrets on input :param select_message: the user facing selection message :param multiple: true if multiple selections are allowed + :param project_id: The local or cloud project_id. :param environment_name: the target environment name :return: the brokerage the user configured """ @@ -112,8 +114,9 @@ def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], modules.append(module) for module in modules: - module.config_build(lean_config, logger, interactive=True, properties=user_provided_options, - hide_input=not show_secrets, environment_name=environment_name) + module.config_build(lean_config, logger, interactive=True, project_id=project_id, + properties=user_provided_options, hide_input=not show_secrets, + environment_name=environment_name) _update_settings(logger, environment_name, module, lean_config) if multiple: return modules diff --git a/lean/container.py b/lean/container.py index 234f126d..6b042c3c 100644 --- a/lean/container.py +++ b/lean/container.py @@ -47,6 +47,20 @@ from lean.models.docker import DockerImage +def get_project_id(project_config: Storage): + """ + Retrieves the ID from the project configuration. + + Args: + project_config (dict): A dictionary containing project configuration with potential keys 'cloud-id' and 'local-id'. + + Returns: + str: The 'cloud-id' if it exists, otherwise the 'local-id'. + If neither is found, returns None. + """ + return project_config.get("cloud-id") or project_config.get("local-id") + + class Container: def __init__(self): diff --git a/lean/models/json_module.py b/lean/models/json_module.py index c07d1777..285f6d55 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -179,6 +179,7 @@ def config_build(self, lean_config: Dict[str, Any], logger: Logger, interactive: bool, + project_id: str, properties: Dict[str, Any] = {}, hide_input: bool = False, environment_name: str = None) -> 'JsonModule': @@ -187,6 +188,7 @@ def config_build(self, :param lean_config: the Lean configuration dict to read defaults from :param logger: the logger to use :param interactive: true if running in interactive mode + :param project_id: The local or cloud project_id. :param properties: the properties that passed as options :param hide_input: whether to hide secrets inputs :param environment_name: the target environment name @@ -219,7 +221,8 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), logger) + auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), + project_id, logger) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From aeded8800dfa990bd6ae53af30d2b9821fecdfe1 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 19 Dec 2024 14:55:23 +0200 Subject: [PATCH 04/21] feat: use default project id like 0 if configuration is none --- lean/container.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lean/container.py b/lean/container.py index 6b042c3c..392e06e7 100644 --- a/lean/container.py +++ b/lean/container.py @@ -58,6 +58,9 @@ def get_project_id(project_config: Storage): str: The 'cloud-id' if it exists, otherwise the 'local-id'. If neither is found, returns None. """ + if not project_config: + container.logger.debug("get_project_id: Project configuration is missing. Using default project ID: '0'") + return '0' return project_config.get("cloud-id") or project_config.get("local-id") From 4de30a4476734b2b0824363738c5cf2144f2650d Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 00:14:08 +0200 Subject: [PATCH 05/21] Revert "feat: use default project id like 0 if configuration is none" This reverts commit ebaa82b5a1980155c03f122cfc22055c4d55d5d8. --- lean/container.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lean/container.py b/lean/container.py index 392e06e7..6b042c3c 100644 --- a/lean/container.py +++ b/lean/container.py @@ -58,9 +58,6 @@ def get_project_id(project_config: Storage): str: The 'cloud-id' if it exists, otherwise the 'local-id'. If neither is found, returns None. """ - if not project_config: - container.logger.debug("get_project_id: Project configuration is missing. Using default project ID: '0'") - return '0' return project_config.get("cloud-id") or project_config.get("local-id") From 259f317590212aab36b4cb936bdcc2ccdd21287c Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 00:15:20 +0200 Subject: [PATCH 06/21] Revert "feat: add project_id to authorize in auth0_client" This reverts commit cf0d69125686fde2939caabbf71d7f30d055dd05. --- lean/commands/backtest.py | 8 +++--- lean/commands/cloud/live/deploy.py | 14 ++++------ lean/commands/data/download.py | 7 ++--- lean/commands/live/deploy.py | 27 +++++++++----------- lean/commands/optimize.py | 9 +++---- lean/commands/research.py | 7 +++-- lean/components/api/auth0_client.py | 5 ++-- lean/components/util/auth0_helper.py | 5 ++-- lean/components/util/json_modules_handler.py | 23 ++++++++--------- lean/container.py | 14 ---------- lean/models/json_module.py | 5 +--- 11 files changed, 43 insertions(+), 81 deletions(-) diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index 383cb3c2..82f53efd 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -17,7 +17,7 @@ from lean.click import LeanCommand, PathParameter from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH -from lean.container import container, Logger, get_project_id +from lean.container import container, Logger from lean.models.utils import DebuggingMethod from lean.models.cli import cli_data_downloaders, cli_addon_modules from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name @@ -362,11 +362,9 @@ def backtest(project: Path, engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, algorithm_file.parent) - project_id = get_project_id(project_config) - if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, project_id, environment_name) + cli_data_downloaders, kwargs, logger, environment_name) data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() @@ -396,7 +394,7 @@ def backtest(project: Path, # Configure addon modules build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, - kwargs, logger, environment_name, container_module_version, project_id) + kwargs, logger, environment_name, container_module_version) lean_runner = container.lean_runner lean_runner.run_lean(lean_config, diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index a551ea58..4821cbb3 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -245,7 +245,7 @@ def deploy(project: str, ensure_options(["brokerage", "node", "auto_restart", "notify_order_events", "notify_insights"]) brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cloud_brokerages, - kwargs, logger, cloud_project.projectId) + kwargs, logger) notify_methods = [] if notify_emails is not None: for config in notify_emails.split(","): @@ -287,13 +287,11 @@ def deploy(project: str, else: # let the user choose the brokerage brokerage_instance = interactive_config_build(lean_config, cloud_brokerages, logger, kwargs, show_secrets, - "Select a brokerage", multiple=False, - project_id=cloud_project.projectId) + "Select a brokerage", multiple=False) notify_order_events, notify_insights, notify_methods = _configure_notifications(logger) auto_restart = _configure_auto_restart(logger) - cash_balance_option, holdings_option, last_cash, last_holdings = ( - get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project)) + cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project) if cash_balance_option != LiveInitialStateInput.NotSupported: live_cash_balance = _configure_initial_cash_interactively(logger, cash_balance_option, last_cash) if holdings_option != LiveInitialStateInput.NotSupported: @@ -305,15 +303,13 @@ def deploy(project: str, # the user sent the live data provider to use for data_provider in data_provider_live: data_provider_instance = non_interactive_config_build_for_name(lean_config, data_provider, - cloud_data_queue_handlers, kwargs, logger, - cloud_project.projectId) + cloud_data_queue_handlers, kwargs, logger) live_data_provider_settings.update({data_provider_instance.get_id(): data_provider_instance.get_settings()}) else: # let's ask the user which live data providers to use data_feed_instances = interactive_config_build(lean_config, cloud_data_queue_handlers, logger, kwargs, - show_secrets, "Select a live data feed", - multiple=True, project_id=cloud_project.projectId) + show_secrets, "Select a live data feed", multiple=True) for data_feed in data_feed_instances: settings = data_feed.get_settings() diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 328370bf..1db52c39 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -19,7 +19,7 @@ from lean.click import LeanCommand, ensure_options from lean.components.util.json_modules_handler import config_build_for_name from lean.constants import DEFAULT_ENGINE_IMAGE -from lean.container import container, get_project_id +from lean.container import container from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType from lean.models.click_options import get_configs_for_options, options_from_json from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform,OptionResult, Product @@ -677,11 +677,8 @@ def download(ctx: Context, engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update) - project_id = get_project_id(project_config) - data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), - cli_data_downloaders, kwargs, logger, interactive=True, - project_id=project_id) + cli_data_downloaders, kwargs, logger, interactive=True) data_downloader_provider.ensure_module_installed(organization.id, container_module_version) container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) # mounting additional data_downloader config files diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index f3010826..a7d41f83 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -17,7 +17,7 @@ from lean.click import LeanCommand, PathParameter from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format from lean.constants import DEFAULT_ENGINE_IMAGE -from lean.container import container, get_project_id +from lean.container import container from lean.models.cli import (cli_brokerages, cli_data_queue_handlers, cli_data_downloaders, cli_addon_modules, cli_history_provider) from lean.models.errors import MoreInfoError @@ -194,11 +194,6 @@ def deploy(project: Path, project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(Path(project)) - engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, - algorithm_file.parent) - - project_id = get_project_id(project_config) - if output is None: output = algorithm_file.parent / "live" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") @@ -243,22 +238,21 @@ def deploy(project: Path, if brokerage: # user provided brokerage, check all arguments were provided brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cli_brokerages, kwargs, - logger, project_id, environment_name) + logger, environment_name) else: # let the user choose the brokerage brokerage_instance = interactive_config_build(lean_config, cli_brokerages, logger, kwargs, show_secrets, "Select a brokerage", multiple=False, - project_id=project_id, environment_name=environment_name) + environment_name=environment_name) if data_provider_live and len(data_provider_live) > 0: for data_feed_name in data_provider_live: data_feed = non_interactive_config_build_for_name(lean_config, data_feed_name, cli_data_queue_handlers, - kwargs, logger, project_id, environment_name) + kwargs, logger, environment_name) data_provider_live_instances.append(data_feed) else: data_provider_live_instances = interactive_config_build(lean_config, cli_data_queue_handlers, logger, kwargs, - show_secrets, "Select a live data feed", - multiple=True, project_id=project_id, + show_secrets, "Select a live data feed", multiple=True, environment_name=environment_name) # based on the live data providers we set up the history providers @@ -266,7 +260,7 @@ def deploy(project: Path, if data_provider_historical is None: data_provider_historical = "Local" data_downloader_instances = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, project_id, + cli_data_downloaders, kwargs, logger, environment_name) if history_providers is None or len(history_providers) == 0: history_providers = _get_history_provider_name(data_provider_live) @@ -274,8 +268,11 @@ def deploy(project: Path, if history_provider in ["BrokerageHistoryProvider", "SubscriptionDataReaderHistoryProvider"]: continue history_providers_instances.append(config_build_for_name(lean_config, history_provider, cli_history_provider, - kwargs, logger, True, project_id, - environment_name)) + kwargs, logger, interactive=True, + environment_name=environment_name)) + + engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, + algorithm_file.parent) organization_id = container.organization_manager.try_get_working_organization_id() paths_to_mount = {} @@ -344,7 +341,7 @@ def deploy(project: Path, # Configure addon modules build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, - kwargs, logger, environment_name, container_module_version, project_id) + kwargs, logger, environment_name, container_module_version) if container.platform_manager.is_host_arm(): if "InteractiveBrokersBrokerage" in lean_config["environments"][environment_name]["live-mode-brokerage"] \ diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 189a3278..8b686bee 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -20,7 +20,7 @@ from lean.click import LeanCommand, PathParameter, ensure_options from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_ENGINE_IMAGE -from lean.container import container, get_project_id +from lean.container import container from lean.models.api import QCParameter, QCBacktest from lean.models.click_options import options_from_json, get_configs_for_options from lean.models.cli import cli_data_downloaders, cli_addon_modules @@ -298,12 +298,9 @@ def optimize(project: Path, paths_to_mount = None - project_id = get_project_id(project_config) - if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, project_id, - environment_name) + cli_data_downloaders, kwargs, logger, environment_name) data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() @@ -331,7 +328,7 @@ def optimize(project: Path, # Configure addon modules build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, - kwargs, logger, environment_name, container_module_version, project_id) + kwargs, logger, environment_name, container_module_version) run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach, engine_image, paths_to_mount) diff --git a/lean/commands/research.py b/lean/commands/research.py index 57012ef3..70061cfd 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -17,7 +17,7 @@ from lean.click import LeanCommand, PathParameter from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH -from lean.container import container, get_project_id +from lean.container import container from lean.models.cli import cli_data_downloaders from lean.components.util.name_extraction import convert_to_class_name from lean.components.util.json_modules_handler import non_interactive_config_build_for_name @@ -121,14 +121,13 @@ def research(project: Path, research_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update, algorithm_file.parent, False) - project_id = get_project_id(project_config) + paths_to_mount = None if data_provider_historical is not None: organization_id = container.organization_manager.try_get_working_organization_id() data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, - cli_data_downloaders, kwargs, logger, project_id, - environment_name) + cli_data_downloaders, kwargs, logger, environment_name) data_provider.ensure_module_installed(organization_id, container_module_version) container.lean_config_manager.set_properties(data_provider.get_settings()) paths_to_mount = data_provider.get_paths_to_mount() diff --git a/lean/components/api/auth0_client.py b/lean/components/api/auth0_client.py index 0f23ab41..f1350c1f 100644 --- a/lean/components/api/auth0_client.py +++ b/lean/components/api/auth0_client.py @@ -52,16 +52,15 @@ def read(self, brokerage_id: str) -> QCAuth0Authorization: return QCAuth0Authorization(authorization=None) @staticmethod - def authorize(brokerage_id: str, project_id: str, logger: Logger) -> None: + def authorize(brokerage_id: str, logger: Logger) -> None: """Starts the authorization process for a brokerage. :param brokerage_id: the id of the brokerage to start the authorization process for - :param project_id: The local or cloud project_id :param logger: the logger instance to use """ from webbrowser import open - full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}&projectId={project_id}" + full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}" logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.") logger.info(full_url) open(full_url) diff --git a/lean/components/util/auth0_helper.py b/lean/components/util/auth0_helper.py index a8b4cb0c..b5c453d4 100644 --- a/lean/components/util/auth0_helper.py +++ b/lean/components/util/auth0_helper.py @@ -16,12 +16,11 @@ from lean.components.util.logger import Logger -def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: str, logger: Logger) -> QCAuth0Authorization: +def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger) -> QCAuth0Authorization: """Gets the authorization data for a brokerage, authorizing if necessary. :param auth0_client: An instance of Auth0Client, containing methods to interact with live/auth0/* API endpoints. :param brokerage_id: The ID of the brokerage to get the authorization data for. - :param project_id: The local or cloud project_id. :param logger: An instance of Logger, handling all output printing. :return: The authorization data for the specified brokerage. """ @@ -32,7 +31,7 @@ def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: return data start_time = time() - auth0_client.authorize(brokerage_id, project_id, logger) + auth0_client.authorize(brokerage_id, logger) # keep checking for new data every 5 seconds for 7 minutes while time() - start_time < 420: diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index 8d96d564..5b59ba47 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -19,7 +19,7 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[JsonModule], organization_id: str, lean_config: Dict[str, Any], properties: Dict[str, Any], logger: Logger, - environment_name: str, module_version: str, project_id: str): + environment_name: str, module_version: str): """Builds and configures the given modules :param target_modules: the requested modules @@ -30,11 +30,10 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso :param logger: the logger instance :param environment_name: the environment name to use :param module_version: The version of the module to install. If not provided, the latest version will be installed. - :param project_id: The cloud or local project_id """ for target_module_name in target_modules: module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties, - logger, project_id, environment_name) + logger, environment_name) # Ensures extra modules (not brokerage or data feeds) are installed. module.ensure_module_installed(organization_id, module_version) lean_config["environments"][environment_name].update(module.get_settings()) @@ -42,9 +41,9 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso def non_interactive_config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger, - project_id: str, environment_name: str = None) -> JsonModule: + environment_name: str = None) -> JsonModule: return config_build_for_name(lean_config, target_module_name, module_list, properties, logger, interactive=False, - project_id=project_id, environment_name=environment_name) + environment_name=environment_name) def find_module(target_module_name: str, module_list: List[JsonModule], logger: Logger) -> JsonModule: @@ -79,18 +78,18 @@ def find_module(target_module_name: str, module_list: List[JsonModule], logger: def config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule], - properties: Dict[str, Any], logger: Logger, interactive: bool, project_id: str, + properties: Dict[str, Any], logger: Logger, interactive: bool, environment_name: str = None) -> JsonModule: target_module = find_module(target_module_name, module_list, logger) - target_module.config_build(lean_config, logger, interactive=interactive, project_id=project_id, - properties=properties, environment_name=environment_name) + target_module.config_build(lean_config, logger, interactive=interactive, properties=properties, + environment_name=environment_name) _update_settings(logger, environment_name, target_module, lean_config) return target_module def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], logger: Logger, user_provided_options: Dict[str, Any], show_secrets: bool, select_message: str, - multiple: bool, project_id: str, environment_name: str = None) -> [JsonModule]: + multiple: bool, environment_name: str = None) -> [JsonModule]: """Interactively configures the brokerage to use. :param lean_config: the LEAN configuration that should be used @@ -100,7 +99,6 @@ def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], :param show_secrets: whether to show secrets on input :param select_message: the user facing selection message :param multiple: true if multiple selections are allowed - :param project_id: The local or cloud project_id. :param environment_name: the target environment name :return: the brokerage the user configured """ @@ -114,9 +112,8 @@ def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], modules.append(module) for module in modules: - module.config_build(lean_config, logger, interactive=True, project_id=project_id, - properties=user_provided_options, hide_input=not show_secrets, - environment_name=environment_name) + module.config_build(lean_config, logger, interactive=True, properties=user_provided_options, + hide_input=not show_secrets, environment_name=environment_name) _update_settings(logger, environment_name, module, lean_config) if multiple: return modules diff --git a/lean/container.py b/lean/container.py index 6b042c3c..234f126d 100644 --- a/lean/container.py +++ b/lean/container.py @@ -47,20 +47,6 @@ from lean.models.docker import DockerImage -def get_project_id(project_config: Storage): - """ - Retrieves the ID from the project configuration. - - Args: - project_config (dict): A dictionary containing project configuration with potential keys 'cloud-id' and 'local-id'. - - Returns: - str: The 'cloud-id' if it exists, otherwise the 'local-id'. - If neither is found, returns None. - """ - return project_config.get("cloud-id") or project_config.get("local-id") - - class Container: def __init__(self): diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 285f6d55..c07d1777 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -179,7 +179,6 @@ def config_build(self, lean_config: Dict[str, Any], logger: Logger, interactive: bool, - project_id: str, properties: Dict[str, Any] = {}, hide_input: bool = False, environment_name: str = None) -> 'JsonModule': @@ -188,7 +187,6 @@ def config_build(self, :param lean_config: the Lean configuration dict to read defaults from :param logger: the logger to use :param interactive: true if running in interactive mode - :param project_id: The local or cloud project_id. :param properties: the properties that passed as options :param hide_input: whether to hide secrets inputs :param environment_name: the target environment name @@ -221,8 +219,7 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), - project_id, logger) + auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), logger) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From 0828c0e2e4a8fb8ba6d911a6f475b5cf44b70ebc Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 00:22:29 +0200 Subject: [PATCH 07/21] feat: add project_id to authorize in auth0_client --- lean/components/api/auth0_client.py | 5 +++-- lean/components/util/auth0_helper.py | 5 +++-- lean/container.py | 20 ++++++++++++++++++++ lean/models/json_module.py | 7 ++++++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/lean/components/api/auth0_client.py b/lean/components/api/auth0_client.py index f1350c1f..0f23ab41 100644 --- a/lean/components/api/auth0_client.py +++ b/lean/components/api/auth0_client.py @@ -52,15 +52,16 @@ def read(self, brokerage_id: str) -> QCAuth0Authorization: return QCAuth0Authorization(authorization=None) @staticmethod - def authorize(brokerage_id: str, logger: Logger) -> None: + def authorize(brokerage_id: str, project_id: str, logger: Logger) -> None: """Starts the authorization process for a brokerage. :param brokerage_id: the id of the brokerage to start the authorization process for + :param project_id: The local or cloud project_id :param logger: the logger instance to use """ from webbrowser import open - full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}" + full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}&projectId={project_id}" logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.") logger.info(full_url) open(full_url) diff --git a/lean/components/util/auth0_helper.py b/lean/components/util/auth0_helper.py index b5c453d4..a8b4cb0c 100644 --- a/lean/components/util/auth0_helper.py +++ b/lean/components/util/auth0_helper.py @@ -16,11 +16,12 @@ from lean.components.util.logger import Logger -def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger) -> QCAuth0Authorization: +def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: str, logger: Logger) -> QCAuth0Authorization: """Gets the authorization data for a brokerage, authorizing if necessary. :param auth0_client: An instance of Auth0Client, containing methods to interact with live/auth0/* API endpoints. :param brokerage_id: The ID of the brokerage to get the authorization data for. + :param project_id: The local or cloud project_id. :param logger: An instance of Logger, handling all output printing. :return: The authorization data for the specified brokerage. """ @@ -31,7 +32,7 @@ def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logg return data start_time = time() - auth0_client.authorize(brokerage_id, logger) + auth0_client.authorize(brokerage_id, project_id, logger) # keep checking for new data every 5 seconds for 7 minutes while time() - start_time < 420: diff --git a/lean/container.py b/lean/container.py index 234f126d..9590a0a2 100644 --- a/lean/container.py +++ b/lean/container.py @@ -207,6 +207,26 @@ def manage_docker_image(self, image: Optional[str], update: bool, no_update: boo return engine_image, container_module_version, project_config + def get_project_id(self, project_directory: Path) -> str: + """ + Retrieves the project ID from the configuration. + + Args: + project_directory (Path): The directory of the project. + + Returns: + str: Returns the 'cloud-id' if available, otherwise the 'local-id'. + If neither is found nor the configuration is missing, returns '0'. + """ + project_config = self.project_config_manager.get_project_config(project_directory) + + if not project_config: + container.logger.debug( + f"Project configuration for {project_directory} is missing. Using default project ID: '0'") + return '0' + + return project_config.get("cloud-id") or project_config.get("local-id") + container = Container() container.data_downloader.update_database_files() diff --git a/lean/models/json_module.py b/lean/models/json_module.py index c07d1777..eda63a58 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -11,7 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from enum import Enum +from pathlib import Path from typing import Any, Dict, List, Type from click import get_current_context @@ -219,7 +221,10 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), logger) + project_id = container.get_project_id( + Path.cwd() / os.path.splitext(lean_config.get('algorithm-location'))[0]) + auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), + project_id, logger) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From 18ce5bda03c8a73c29043453296c3e966c829a73 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 16:11:57 +0200 Subject: [PATCH 08/21] refactor: set project_id in config_manager --- lean/components/api/auth0_client.py | 10 ++++--- lean/components/config/lean_config_manager.py | 1 + .../config/project_config_manager.py | 15 +++++++++++ lean/components/util/auth0_helper.py | 6 ++--- lean/container.py | 20 -------------- lean/models/json_module.py | 26 ++++++++++++++++--- 6 files changed, 49 insertions(+), 29 deletions(-) diff --git a/lean/components/api/auth0_client.py b/lean/components/api/auth0_client.py index 0f23ab41..313c7528 100644 --- a/lean/components/api/auth0_client.py +++ b/lean/components/api/auth0_client.py @@ -52,16 +52,20 @@ def read(self, brokerage_id: str) -> QCAuth0Authorization: return QCAuth0Authorization(authorization=None) @staticmethod - def authorize(brokerage_id: str, project_id: str, logger: Logger) -> None: + def authorize(brokerage_id: str, logger: Logger, project_id: str = None) -> None: """Starts the authorization process for a brokerage. :param brokerage_id: the id of the brokerage to start the authorization process for - :param project_id: The local or cloud project_id :param logger: the logger instance to use + :param project_id: The local or cloud project_id """ from webbrowser import open - full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}&projectId={project_id}" + full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}" + + if project_id: + full_url += f"&projectId={project_id}" + logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.") logger.info(full_url) open(full_url) diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index aa26ab16..538c007a 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -260,6 +260,7 @@ def get_complete_lean_config(self, if algorithm_file and len(algorithm_file.name) > 0: config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) + config["project-id"] = self._project_config_manager.get_cloud_id(algorithm_file.parent) if algorithm_file.name.endswith(".py"): config["algorithm-type-name"] = algorithm_file.name.split(".")[0] diff --git a/lean/components/config/project_config_manager.py b/lean/components/config/project_config_manager.py index 1a9aeb80..dd0b7609 100644 --- a/lean/components/config/project_config_manager.py +++ b/lean/components/config/project_config_manager.py @@ -68,6 +68,21 @@ def get_local_id(self, project_directory: Path) -> int: return project_id + def get_cloud_id(self, project_directory: Path) -> int: + """ + Retrieves the project ID from the configuration. + + Args: + project_directory (Path): The directory of the project. + + Returns: + str: Returns the 'cloud-id' if available, otherwise the 'local-id'. + If neither is found nor the configuration is missing, returns '0'. + """ + project_config = self.get_project_config(project_directory) + + return project_config.get("cloud-id", default=0) + def get_latest_live_directory(self, project_directory: Path) -> Path: """Returns the path of the latest live directory. diff --git a/lean/components/util/auth0_helper.py b/lean/components/util/auth0_helper.py index a8b4cb0c..929d6e1f 100644 --- a/lean/components/util/auth0_helper.py +++ b/lean/components/util/auth0_helper.py @@ -16,13 +16,13 @@ from lean.components.util.logger import Logger -def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: str, logger: Logger) -> QCAuth0Authorization: +def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger, project_id: str = None) -> QCAuth0Authorization: """Gets the authorization data for a brokerage, authorizing if necessary. :param auth0_client: An instance of Auth0Client, containing methods to interact with live/auth0/* API endpoints. :param brokerage_id: The ID of the brokerage to get the authorization data for. - :param project_id: The local or cloud project_id. :param logger: An instance of Logger, handling all output printing. + :param project_id: The local or cloud project_id. :return: The authorization data for the specified brokerage. """ from time import time, sleep @@ -32,7 +32,7 @@ def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: return data start_time = time() - auth0_client.authorize(brokerage_id, project_id, logger) + auth0_client.authorize(brokerage_id, logger, project_id) # keep checking for new data every 5 seconds for 7 minutes while time() - start_time < 420: diff --git a/lean/container.py b/lean/container.py index 9590a0a2..234f126d 100644 --- a/lean/container.py +++ b/lean/container.py @@ -207,26 +207,6 @@ def manage_docker_image(self, image: Optional[str], update: bool, no_update: boo return engine_image, container_module_version, project_config - def get_project_id(self, project_directory: Path) -> str: - """ - Retrieves the project ID from the configuration. - - Args: - project_directory (Path): The directory of the project. - - Returns: - str: Returns the 'cloud-id' if available, otherwise the 'local-id'. - If neither is found nor the configuration is missing, returns '0'. - """ - project_config = self.project_config_manager.get_project_config(project_directory) - - if not project_config: - container.logger.debug( - f"Project configuration for {project_directory} is missing. Using default project ID: '0'") - return '0' - - return project_config.get("cloud-id") or project_config.get("local-id") - container = Container() container.data_downloader.update_database_files() diff --git a/lean/models/json_module.py b/lean/models/json_module.py index eda63a58..7d6f501f 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -177,6 +177,27 @@ def convert_variable_to_lean_key(self, variable_key: str) -> str: """ return variable_key.replace('_', '-') + def validate_project_id(self, project_id: int, display_name: str) -> str: + """ + Validates the given project ID and raises an exception for specific conditions. + + Args: + project_id (int): The project ID to validate. + display_name (str): The display name of the module. + + Returns: + int: The validated project ID. + + Raises: + NotImplementedError: If the project ID is 0 and the display name is "CharlesSchwab". + """ + if project_id == 0 and display_name.strip().lower() == "CharlesSchwab".lower(): + raise NotImplementedError( + "Cloud ID is missing from the project configuration for the module 'CharlesSchwabBrokerage'. " + "Please ensure the project is synchronized with the cloud by pushing or pulling the project." + ) + return str(project_id) + def config_build(self, lean_config: Dict[str, Any], logger: Logger, @@ -221,10 +242,9 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - project_id = container.get_project_id( - Path.cwd() / os.path.splitext(lean_config.get('algorithm-location'))[0]) + project_id = self.validate_project_id(lean_config.get("project-id"), self._display_name.lower()) auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), - project_id, logger) + logger, project_id) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From 168e375fdc8720484c9cfbfbb3cd6fb904fb478b Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 16:13:27 +0200 Subject: [PATCH 09/21] remove: not used imports --- lean/models/json_module.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 7d6f501f..347d11ea 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -11,9 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from enum import Enum -from pathlib import Path from typing import Any, Dict, List, Type from click import get_current_context From e0e6b544fe86ea141d9ec6b7d57eb3c3d7c24db4 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 16:50:43 +0200 Subject: [PATCH 10/21] refactor: use negative local id if cloud id is not provided --- lean/components/api/auth0_client.py | 4 ++-- .../config/project_config_manager.py | 2 +- lean/components/util/auth0_helper.py | 2 +- lean/models/json_module.py | 24 +------------------ 4 files changed, 5 insertions(+), 27 deletions(-) diff --git a/lean/components/api/auth0_client.py b/lean/components/api/auth0_client.py index 313c7528..b55bda37 100644 --- a/lean/components/api/auth0_client.py +++ b/lean/components/api/auth0_client.py @@ -52,7 +52,7 @@ def read(self, brokerage_id: str) -> QCAuth0Authorization: return QCAuth0Authorization(authorization=None) @staticmethod - def authorize(brokerage_id: str, logger: Logger, project_id: str = None) -> None: + def authorize(brokerage_id: str, logger: Logger, project_id: int) -> None: """Starts the authorization process for a brokerage. :param brokerage_id: the id of the brokerage to start the authorization process for @@ -63,7 +63,7 @@ def authorize(brokerage_id: str, logger: Logger, project_id: str = None) -> Non full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}" - if project_id: + if project_id > 0: full_url += f"&projectId={project_id}" logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.") diff --git a/lean/components/config/project_config_manager.py b/lean/components/config/project_config_manager.py index dd0b7609..5c8d1595 100644 --- a/lean/components/config/project_config_manager.py +++ b/lean/components/config/project_config_manager.py @@ -81,7 +81,7 @@ def get_cloud_id(self, project_directory: Path) -> int: """ project_config = self.get_project_config(project_directory) - return project_config.get("cloud-id", default=0) + return project_config.get("cloud-id") or -project_config.get("local-id") def get_latest_live_directory(self, project_directory: Path) -> Path: """Returns the path of the latest live directory. diff --git a/lean/components/util/auth0_helper.py b/lean/components/util/auth0_helper.py index 929d6e1f..8450a7be 100644 --- a/lean/components/util/auth0_helper.py +++ b/lean/components/util/auth0_helper.py @@ -16,7 +16,7 @@ from lean.components.util.logger import Logger -def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger, project_id: str = None) -> QCAuth0Authorization: +def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger, project_id: int) -> QCAuth0Authorization: """Gets the authorization data for a brokerage, authorizing if necessary. :param auth0_client: An instance of Auth0Client, containing methods to interact with live/auth0/* API endpoints. diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 347d11ea..dfca26d3 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -175,27 +175,6 @@ def convert_variable_to_lean_key(self, variable_key: str) -> str: """ return variable_key.replace('_', '-') - def validate_project_id(self, project_id: int, display_name: str) -> str: - """ - Validates the given project ID and raises an exception for specific conditions. - - Args: - project_id (int): The project ID to validate. - display_name (str): The display name of the module. - - Returns: - int: The validated project ID. - - Raises: - NotImplementedError: If the project ID is 0 and the display name is "CharlesSchwab". - """ - if project_id == 0 and display_name.strip().lower() == "CharlesSchwab".lower(): - raise NotImplementedError( - "Cloud ID is missing from the project configuration for the module 'CharlesSchwabBrokerage'. " - "Please ensure the project is synchronized with the cloud by pushing or pulling the project." - ) - return str(project_id) - def config_build(self, lean_config: Dict[str, Any], logger: Logger, @@ -240,9 +219,8 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - project_id = self.validate_project_id(lean_config.get("project-id"), self._display_name.lower()) auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), - logger, project_id) + logger, lean_config.get("project-id")) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From be9d33facb03d9c94b2f0e6047e32cf74691e3c4 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 20 Dec 2024 17:09:51 +0200 Subject: [PATCH 11/21] rename: get project id method refactor: return auth url with any project_id --- lean/components/api/auth0_client.py | 5 +---- lean/components/config/lean_config_manager.py | 2 +- lean/components/config/project_config_manager.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lean/components/api/auth0_client.py b/lean/components/api/auth0_client.py index b55bda37..1cc0b996 100644 --- a/lean/components/api/auth0_client.py +++ b/lean/components/api/auth0_client.py @@ -61,10 +61,7 @@ def authorize(brokerage_id: str, logger: Logger, project_id: int) -> None: """ from webbrowser import open - full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}" - - if project_id > 0: - full_url += f"&projectId={project_id}" + full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}&projectId={project_id}" logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.") logger.info(full_url) diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 538c007a..8d367141 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -260,7 +260,7 @@ def get_complete_lean_config(self, if algorithm_file and len(algorithm_file.name) > 0: config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) - config["project-id"] = self._project_config_manager.get_cloud_id(algorithm_file.parent) + config["project-id"] = self._project_config_manager.get_project_id_from_project_config(algorithm_file.parent) if algorithm_file.name.endswith(".py"): config["algorithm-type-name"] = algorithm_file.name.split(".")[0] diff --git a/lean/components/config/project_config_manager.py b/lean/components/config/project_config_manager.py index 5c8d1595..75c80c69 100644 --- a/lean/components/config/project_config_manager.py +++ b/lean/components/config/project_config_manager.py @@ -68,7 +68,7 @@ def get_local_id(self, project_directory: Path) -> int: return project_id - def get_cloud_id(self, project_directory: Path) -> int: + def get_project_id_from_project_config(self, project_directory: Path) -> int: """ Retrieves the project ID from the configuration. From bb45cfe7ee8df20983838c7e01507135917d0015 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 23 Dec 2024 21:57:15 +0200 Subject: [PATCH 12/21] refactor: get project id refactor: Readme --- README.md | 54 +++++++++++-------- .../config/project_config_manager.py | 23 ++++++-- lean/models/json_module.py | 3 +- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 87c08a67..0bd82267 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Options: -d, --detach Run the backtest in a detached Docker container and return immediately --debug [pycharm|ptvsd|debugpy|vsdbg|rider|local-platform] Enable a certain debugging method (see --help for more information) - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|CharlesSchwab|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -178,6 +178,10 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier + --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- + cli/live-trading/brokerages/charles-schwab) + --charles-schwab-account-number TEXT + The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary --iqfeed-username TEXT Your IQFeed username --iqfeed-password TEXT Your IQFeed password @@ -219,8 +223,6 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used - --charles-schwab-account-number TEXT - The CharlesSchwab account number --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect --data-purchase-limit INTEGER The maximum amount of QCC to spend on downloading data during the backtest when using @@ -351,9 +353,9 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT --notify-insights. Options: - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|Bybit|TradeStation|Alpaca|CharlesSchwab] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|CharlesSchwab|Bybit|TradeStation|Alpaca] The brokerage to use - --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|Polygon|IEX|CoinApi|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|CharlesSchwab|Polygon|IEX|CoinApi|Bybit|TradeStation|Alpaca] The live data provider to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -429,6 +431,10 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier + --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- + cli/live-trading/brokerages/charles-schwab) + --charles-schwab-account-number TEXT + The CharlesSchwab account number --bybit-api-key TEXT Your Bybit API key --bybit-api-secret TEXT Your Bybit API secret --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5] @@ -439,8 +445,6 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used - --charles-schwab-account-number TEXT - The CharlesSchwab account number --polygon-api-key TEXT Your Polygon.io API Key --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] @@ -851,7 +855,7 @@ Usage: lean data download [OPTIONS] https://www.quantconnect.com/datasets Options: - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|CharlesSchwab|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] The name of the downloader data provider. --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -879,6 +883,10 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier + --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- + cli/live-trading/brokerages/charles-schwab) + --charles-schwab-account-number TEXT + The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary --iqfeed-username TEXT Your IQFeed username --iqfeed-password TEXT Your IQFeed password @@ -920,8 +928,6 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used - --charles-schwab-account-number TEXT - The CharlesSchwab account number --dataset TEXT The name of the dataset to download non-interactively --overwrite Overwrite existing local data -y, --yes Automatically confirm payment confirmation prompts @@ -1290,11 +1296,11 @@ Options: --environment TEXT The environment to use --output DIRECTORY Directory to store results in (defaults to PROJECT/live/TIMESTAMP) -d, --detach Run the live deployment in a detached Docker container and return immediately - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|Bybit|TradeStation|Alpaca|CharlesSchwab] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|CharlesSchwab|Bybit|TradeStation|Alpaca] The brokerage to use - --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|IQFeed|Polygon|IEX|CoinApi|ThetaData|Custom data only|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|CharlesSchwab|IQFeed|Polygon|IEX|CoinApi|ThetaData|Custom data only|Bybit|TradeStation|Alpaca] The live data provider to use - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|CharlesSchwab|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Bybit|TradeStation|Alpaca] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -1383,6 +1389,10 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier + --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- + cli/live-trading/brokerages/charles-schwab) + --charles-schwab-account-number TEXT + The CharlesSchwab account number --bybit-api-key TEXT Your Bybit API key --bybit-api-secret TEXT Your Bybit API secret --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5] @@ -1395,8 +1405,6 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used - --charles-schwab-account-number TEXT - The CharlesSchwab account number --ib-enable-delayed-streaming-data BOOLEAN Whether delayed data may be used when your algorithm subscribes to a security you don't have a market data subscription for (Optional). @@ -1728,7 +1736,7 @@ Options: --parameter ... The 'parameter min max step' pairs configuring the parameters to optimize --constraint TEXT The 'statistic operator value' pairs configuring the constraints of the optimization - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|CharlesSchwab|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] Update the Lean configuration file to retrieve data from the given historical provider --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect @@ -1769,6 +1777,10 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier + --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- + cli/live-trading/brokerages/charles-schwab) + --charles-schwab-account-number TEXT + The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary --iqfeed-username TEXT Your IQFeed username --iqfeed-password TEXT Your IQFeed password @@ -1810,8 +1822,6 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used - --charles-schwab-account-number TEXT - The CharlesSchwab account number --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) --verbose Enable debug logging --help Show this message and exit. @@ -1989,7 +1999,7 @@ Usage: lean research [OPTIONS] PROJECT Options: --port INTEGER The port to run Jupyter Lab on (defaults to 8888) - --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca|CharlesSchwab] + --data-provider-historical [Interactive Brokers|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Kraken|CharlesSchwab|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|ThetaData|QuantConnect|Local|Terminal Link|Bybit|TradeStation|Alpaca] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -2017,6 +2027,10 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier + --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- + cli/live-trading/brokerages/charles-schwab) + --charles-schwab-account-number TEXT + The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary --iqfeed-username TEXT Your IQFeed username --iqfeed-password TEXT Your IQFeed password @@ -2058,8 +2072,6 @@ Options: The TradeStation account Id --alpaca-environment [live|paper] Whether Live or Paper environment should be used - --charles-schwab-account-number TEXT - The CharlesSchwab account number --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect --data-purchase-limit INTEGER The maximum amount of QCC to spend on downloading data during the research session diff --git a/lean/components/config/project_config_manager.py b/lean/components/config/project_config_manager.py index 75c80c69..e66c2875 100644 --- a/lean/components/config/project_config_manager.py +++ b/lean/components/config/project_config_manager.py @@ -70,18 +70,31 @@ def get_local_id(self, project_directory: Path) -> int: def get_project_id_from_project_config(self, project_directory: Path) -> int: """ - Retrieves the project ID from the configuration. + Resolves the project ID from the configuration. Args: - project_directory (Path): The directory of the project. + project_directory (Path): The directory of the project. If None, + it indicates the directory is unavailable. Returns: - str: Returns the 'cloud-id' if available, otherwise the 'local-id'. - If neither is found nor the configuration is missing, returns '0'. + int: Returns the 'cloud-id' if available. + If 'cloud-id' is missing, returns the negative of 'local-id'. + If neither is found nor if project_directory is None, returns -1. """ + if project_directory is None: + return -1 + project_config = self.get_project_config(project_directory) - return project_config.get("cloud-id") or -project_config.get("local-id") + cloud_id = project_config.get("cloud-id") + if cloud_id is not None: + return cloud_id + + local_id = project_config.get("local-id") + if local_id is not None: + return -local_id # Local ID must be negative. + + return -1 # Return -1 if no valid IDs are found def get_latest_live_directory(self, project_directory: Path) -> Path: """Returns the path of the latest live directory. diff --git a/lean/models/json_module.py b/lean/models/json_module.py index dfca26d3..efb8b0ed 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -219,8 +219,9 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): + project_id = next((config._value for config in self._lean_configs if config._id == "project-id"), -1) auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), - logger, lean_config.get("project-id")) + logger, project_id) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From 75dad5809fe3403cfc461e479c4907af0a28f018 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 23 Dec 2024 22:21:37 +0200 Subject: [PATCH 13/21] refactor: Readme --- README.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 0bd82267..37065b01 100644 --- a/README.md +++ b/README.md @@ -178,8 +178,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- - cli/live-trading/brokerages/charles-schwab) + --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -431,8 +430,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- - cli/live-trading/brokerages/charles-schwab) + --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --bybit-api-key TEXT Your Bybit API key @@ -883,8 +881,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- - cli/live-trading/brokerages/charles-schwab) + --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -1389,8 +1386,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- - cli/live-trading/brokerages/charles-schwab) + --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --bybit-api-key TEXT Your Bybit API key @@ -1777,8 +1773,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- - cli/live-trading/brokerages/charles-schwab) + --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -2027,8 +2022,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT This is the ID of your cloud project. (https://www.quantconnect.com/docs/v2/lean- - cli/live-trading/brokerages/charles-schwab) + --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary From f38c7797f69d46625c8050999c664feb965af4f7 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 23 Dec 2024 22:40:52 +0200 Subject: [PATCH 14/21] feat: set project id always in default configs --- lean/components/config/lean_config_manager.py | 3 ++- lean/models/json_module.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 8d367141..3606592b 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -245,6 +245,8 @@ def get_complete_lean_config(self, "job-user-id": self._cli_config_manager.user_id.get_value(default="0"), "api-access-token": self._cli_config_manager.api_token.get_value(default=""), "job-organization-id": get_organization(config), + "project-id": self._project_config_manager + .get_project_id_from_project_config(algorithm_file.parent if algorithm_file else None), "ib-host": "127.0.0.1", "ib-port": "4002", @@ -260,7 +262,6 @@ def get_complete_lean_config(self, if algorithm_file and len(algorithm_file.name) > 0: config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) - config["project-id"] = self._project_config_manager.get_project_id_from_project_config(algorithm_file.parent) if algorithm_file.name.endswith(".py"): config["algorithm-type-name"] = algorithm_file.name.split(".")[0] diff --git a/lean/models/json_module.py b/lean/models/json_module.py index efb8b0ed..16ef131d 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -219,7 +219,9 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - project_id = next((config._value for config in self._lean_configs if config._id == "project-id"), -1) + project_id = next((config._value for config in self._lean_configs if config._id == "project-id"), + lean_config["project-id"]) + logger.debug(f'project_id: {project_id}') auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), logger, project_id) logger.debug(f'auth: {auth_authorizations}') From e2e8c2b30a06712c157d4cc9ef8d79046350df47 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 23 Dec 2024 23:36:34 +0200 Subject: [PATCH 15/21] feat: validate config value on empty --- lean/components/config/lean_config_manager.py | 4 ++-- lean/models/json_module.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 3606592b..609df996 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -245,8 +245,6 @@ def get_complete_lean_config(self, "job-user-id": self._cli_config_manager.user_id.get_value(default="0"), "api-access-token": self._cli_config_manager.api_token.get_value(default=""), "job-organization-id": get_organization(config), - "project-id": self._project_config_manager - .get_project_id_from_project_config(algorithm_file.parent if algorithm_file else None), "ib-host": "127.0.0.1", "ib-port": "4002", @@ -262,6 +260,8 @@ def get_complete_lean_config(self, if algorithm_file and len(algorithm_file.name) > 0: config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) + config["project-id"] = (self._project_config_manager + .get_project_id_from_project_config(algorithm_file.parent)) if algorithm_file.name.endswith(".py"): config["algorithm-type-name"] = algorithm_file.name.split(".")[0] diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 16ef131d..be968cf8 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -219,8 +219,8 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - project_id = next((config._value for config in self._lean_configs if config._id == "project-id"), - lean_config["project-id"]) + project_id = next((config._value for config in self._lean_configs + if config._id == "project-id" and config._value != ""), lean_config["project-id"]) logger.debug(f'project_id: {project_id}') auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), logger, project_id) From 4c4569ae57dfb3517b65e9763924abfa606bd5e8 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 24 Dec 2024 00:17:22 +0200 Subject: [PATCH 16/21] refactor: description of project-id in Readme --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 37065b01..3f9c58e5 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID is required for OAuth process. + --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -430,7 +430,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --bybit-api-key TEXT Your Bybit API key @@ -881,7 +880,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID is required for OAuth process. + --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -1386,7 +1385,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID is required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --bybit-api-key TEXT Your Bybit API key @@ -1422,6 +1420,7 @@ Options: Your ThetaData subscription price plan --alpaca-api-key TEXT Your Alpaca Api Key --alpaca-api-secret TEXT Your Alpaca Api Secret + --project-id TEXT The cloud project ID may be required for OAuth process. --factset-auth-config-file FILE The path to the FactSet authentication configuration file --alpha-vantage-api-key TEXT Your Alpha Vantage Api Key @@ -1773,7 +1772,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID is required for OAuth process. + --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -2022,7 +2021,7 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID is required for OAuth process. + --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary From d7255de2a935df3ae2d6446fca247ca7812cf2ea Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 24 Dec 2024 15:56:16 +0200 Subject: [PATCH 17/21] feat: new 'require_project_id' property in AuthConfiguration feat: prompt user to write project id if it required feat: use project_id always -1 refactor: remove extra project-id from CharlesSchwab parameters in Readme --- README.md | 11 +++++---- lean/components/config/lean_config_manager.py | 4 ++-- lean/models/configuration.py | 1 + lean/models/json_module.py | 23 +++++++++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3f9c58e5..61c175a1 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -220,6 +219,7 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id + --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --download-data Update the Lean configuration file to download data from the QuantConnect API, alias @@ -440,6 +440,7 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id + --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --polygon-api-key TEXT Your Polygon.io API Key @@ -880,7 +881,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -922,6 +922,7 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id + --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --dataset TEXT The name of the dataset to download non-interactively @@ -1397,6 +1398,7 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id + --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --ib-enable-delayed-streaming-data BOOLEAN @@ -1420,7 +1422,6 @@ Options: Your ThetaData subscription price plan --alpaca-api-key TEXT Your Alpaca Api Key --alpaca-api-secret TEXT Your Alpaca Api Secret - --project-id TEXT The cloud project ID may be required for OAuth process. --factset-auth-config-file FILE The path to the FactSet authentication configuration file --alpha-vantage-api-key TEXT Your Alpha Vantage Api Key @@ -1772,7 +1773,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -1814,6 +1814,7 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id + --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) @@ -2021,7 +2022,6 @@ Options: --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] Your Kraken Verification Tier - --project-id TEXT The cloud project ID may be required for OAuth process. --charles-schwab-account-number TEXT The CharlesSchwab account number --iqfeed-iqconnect TEXT The path to the IQConnect binary @@ -2063,6 +2063,7 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id + --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --download-data Update the Lean configuration file to download data from the QuantConnect API, alias diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 609df996..59859dd0 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -245,6 +245,8 @@ def get_complete_lean_config(self, "job-user-id": self._cli_config_manager.user_id.get_value(default="0"), "api-access-token": self._cli_config_manager.api_token.get_value(default=""), "job-organization-id": get_organization(config), + "project-id": self._project_config_manager.get_project_id_from_project_config( + algorithm_file.parent if algorithm_file else None), "ib-host": "127.0.0.1", "ib-port": "4002", @@ -260,8 +262,6 @@ def get_complete_lean_config(self, if algorithm_file and len(algorithm_file.name) > 0: config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) - config["project-id"] = (self._project_config_manager - .get_project_id_from_project_config(algorithm_file.parent)) if algorithm_file.name.endswith(".py"): config["algorithm-type-name"] = algorithm_file.name.split(".")[0] diff --git a/lean/models/configuration.py b/lean/models/configuration.py index b3cd23d7..8f60e663 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -400,6 +400,7 @@ class AuthConfiguration(InternalInputUserInput): def __init__(self, config_json_object): super().__init__(config_json_object) + self.require_project_id = config_json_object.get("require_project_id", False) def factory(config_json_object) -> 'AuthConfiguration': """Creates an instance of the child classes. diff --git a/lean/models/json_module.py b/lean/models/json_module.py index be968cf8..beb18efc 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -14,7 +14,7 @@ from enum import Enum from typing import Any, Dict, List, Type -from click import get_current_context +from click import get_current_context, prompt from click.core import ParameterSource from lean.components.util.auth0_helper import get_authorization @@ -175,6 +175,19 @@ def convert_variable_to_lean_key(self, variable_key: str) -> str: """ return variable_key.replace('_', '-') + def get_project_id(self, default_project_id: int, require_project_id: bool) -> int: + """Retrieve the project ID, prompting the user if required and default is invalid. + + :param default_project_id: The default project ID to use. + :param require_project_id: Flag to determine if prompting is necessary. + :return: A valid project ID. + """ + project_id = default_project_id + if require_project_id and project_id <= 0: + project_id = prompt("Please enter any cloud project ID to proceed with Auth0 authentication", + -1, show_default=False) + return project_id + def config_build(self, lean_config: Dict[str, Any], logger: Logger, @@ -219,11 +232,11 @@ def config_build(self, logger.debug(f"skipping configuration '{configuration._id}': no choices available.") continue elif isinstance(configuration, AuthConfiguration): - project_id = next((config._value for config in self._lean_configs - if config._id == "project-id" and config._value != ""), lean_config["project-id"]) - logger.debug(f'project_id: {project_id}') + lean_config["project-id"] = self.get_project_id(lean_config["project-id"], + configuration.require_project_id) + logger.debug(f'project_id: {lean_config["project-id"]}') auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), - logger, project_id) + logger, lean_config["project-id"]) logger.debug(f'auth: {auth_authorizations}') configuration._value = auth_authorizations.get_authorization_config_without_account() for inner_config in self._lean_configs: From 81428e7c2626393bd56bfb2d330707b7550b7049 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 24 Dec 2024 16:03:01 +0200 Subject: [PATCH 18/21] refactor: carry out prompt import into get_project_id --- lean/models/json_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index beb18efc..dd6ab486 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -14,7 +14,7 @@ from enum import Enum from typing import Any, Dict, List, Type -from click import get_current_context, prompt +from click import get_current_context from click.core import ParameterSource from lean.components.util.auth0_helper import get_authorization @@ -182,6 +182,7 @@ def get_project_id(self, default_project_id: int, require_project_id: bool) -> i :param require_project_id: Flag to determine if prompting is necessary. :return: A valid project ID. """ + from click import prompt project_id = default_project_id if require_project_id and project_id <= 0: project_id = prompt("Please enter any cloud project ID to proceed with Auth0 authentication", From ad54c9af4883ee2ed6e0685fbe70a8965bca697e Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 24 Dec 2024 16:25:10 +0200 Subject: [PATCH 19/21] refactor: remove extra project id from Readme --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 61c175a1..f4a169d2 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,6 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id - --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --download-data Update the Lean configuration file to download data from the QuantConnect API, alias @@ -440,7 +439,6 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id - --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --polygon-api-key TEXT Your Polygon.io API Key @@ -922,7 +920,6 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id - --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --dataset TEXT The name of the dataset to download non-interactively @@ -1398,7 +1395,6 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id - --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --ib-enable-delayed-streaming-data BOOLEAN @@ -1814,7 +1810,6 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id - --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) @@ -2063,7 +2058,6 @@ Options: Whether Live or Paper environment should be used --trade-station-account-id TEXT The TradeStation account Id - --project-id TEXT The cloud project ID may be required for OAuth process. --alpaca-environment [live|paper] Whether Live or Paper environment should be used --download-data Update the Lean configuration file to download data from the QuantConnect API, alias From 98307de18fdd3878ee8279869bae7ded5261d396 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 24 Dec 2024 16:58:26 +0200 Subject: [PATCH 20/21] refactor: use int type implicitly --- lean/models/json_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index dd6ab486..e998c2f1 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -183,7 +183,7 @@ def get_project_id(self, default_project_id: int, require_project_id: bool) -> i :return: A valid project ID. """ from click import prompt - project_id = default_project_id + project_id: int = default_project_id if require_project_id and project_id <= 0: project_id = prompt("Please enter any cloud project ID to proceed with Auth0 authentication", -1, show_default=False) From 67d879cf839dbc20c12d76dc94ef4915cffeb755 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 24 Dec 2024 17:14:17 +0200 Subject: [PATCH 21/21] fix: underscore in "require-project-id" param --- lean/models/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/models/configuration.py b/lean/models/configuration.py index 8f60e663..50d7ce1d 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -400,7 +400,7 @@ class AuthConfiguration(InternalInputUserInput): def __init__(self, config_json_object): super().__init__(config_json_object) - self.require_project_id = config_json_object.get("require_project_id", False) + self.require_project_id = config_json_object.get("require-project-id", False) def factory(config_json_object) -> 'AuthConfiguration': """Creates an instance of the child classes.