Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

LEAN CLI CharlesSchwab Brokerage support #517

Merged
Merged
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
efed71c
feat: CharlesSchwab in Readme
Romazes Nov 14, 2024
8c2d4e4
remove: obsolete TDAmeritrade
Romazes Nov 14, 2024
d2c1bac
feat: add project_id to authorize in auth0_client
Romazes Dec 18, 2024
aeded88
feat: use default project id like 0 if configuration is none
Romazes Dec 19, 2024
4de30a4
Revert "feat: use default project id like 0 if configuration is none"
Romazes Dec 19, 2024
259f317
Revert "feat: add project_id to authorize in auth0_client"
Romazes Dec 19, 2024
0828c0e
feat: add project_id to authorize in auth0_client
Romazes Dec 19, 2024
18ce5bd
refactor: set project_id in config_manager
Romazes Dec 20, 2024
168e375
remove: not used imports
Romazes Dec 20, 2024
e0e6b54
refactor: use negative local id if cloud id is not provided
Romazes Dec 20, 2024
be9d33f
rename: get project id method
Romazes Dec 20, 2024
bb45cfe
refactor: get project id
Romazes Dec 23, 2024
75dad58
refactor: Readme
Romazes Dec 23, 2024
f38c779
feat: set project id always in default configs
Romazes Dec 23, 2024
e2e8c2b
feat: validate config value on empty
Romazes Dec 23, 2024
4c4569a
refactor: description of project-id in Readme
Romazes Dec 23, 2024
d7255de
feat: new 'require_project_id' property in AuthConfiguration
Romazes Dec 24, 2024
81428e7
refactor: carry out prompt import into get_project_id
Romazes Dec 24, 2024
ad54c9a
refactor: remove extra project id from Readme
Romazes Dec 24, 2024
98307de
refactor: use int type implicitly
Romazes Dec 24, 2024
67d879c
fix: underscore in "require-project-id" param
Romazes Dec 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert "feat: add project_id to authorize in auth0_client"
This reverts commit cf0d691.
Romazes committed Dec 20, 2024
commit 259f317590212aab36b4cb936bdcc2ccdd21287c
8 changes: 3 additions & 5 deletions lean/commands/backtest.py
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@

from lean.click import LeanCommand, PathParameter
from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH
from lean.container import container, Logger, get_project_id
from lean.container import container, Logger
from lean.models.utils import DebuggingMethod
from lean.models.cli import cli_data_downloaders, cli_addon_modules
from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name
@@ -362,11 +362,9 @@ def backtest(project: Path,
engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update,
algorithm_file.parent)

project_id = get_project_id(project_config)

if data_provider_historical is not None:
data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
cli_data_downloaders, kwargs, logger, project_id, environment_name)
cli_data_downloaders, kwargs, logger, environment_name)
data_provider.ensure_module_installed(organization_id, container_module_version)
container.lean_config_manager.set_properties(data_provider.get_settings())
paths_to_mount = data_provider.get_paths_to_mount()
@@ -396,7 +394,7 @@ def backtest(project: Path,

# Configure addon modules
build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
kwargs, logger, environment_name, container_module_version, project_id)
kwargs, logger, environment_name, container_module_version)

lean_runner = container.lean_runner
lean_runner.run_lean(lean_config,
14 changes: 5 additions & 9 deletions lean/commands/cloud/live/deploy.py
Original file line number Diff line number Diff line change
@@ -245,7 +245,7 @@ def deploy(project: str,
ensure_options(["brokerage", "node", "auto_restart", "notify_order_events", "notify_insights"])

brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cloud_brokerages,
kwargs, logger, cloud_project.projectId)
kwargs, logger)
notify_methods = []
if notify_emails is not None:
for config in notify_emails.split(","):
@@ -287,13 +287,11 @@ def deploy(project: str,
else:
# let the user choose the brokerage
brokerage_instance = interactive_config_build(lean_config, cloud_brokerages, logger, kwargs, show_secrets,
"Select a brokerage", multiple=False,
project_id=cloud_project.projectId)
"Select a brokerage", multiple=False)

notify_order_events, notify_insights, notify_methods = _configure_notifications(logger)
auto_restart = _configure_auto_restart(logger)
cash_balance_option, holdings_option, last_cash, last_holdings = (
get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project))
cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project)
if cash_balance_option != LiveInitialStateInput.NotSupported:
live_cash_balance = _configure_initial_cash_interactively(logger, cash_balance_option, last_cash)
if holdings_option != LiveInitialStateInput.NotSupported:
@@ -305,15 +303,13 @@ def deploy(project: str,
# the user sent the live data provider to use
for data_provider in data_provider_live:
data_provider_instance = non_interactive_config_build_for_name(lean_config, data_provider,
cloud_data_queue_handlers, kwargs, logger,
cloud_project.projectId)
cloud_data_queue_handlers, kwargs, logger)

live_data_provider_settings.update({data_provider_instance.get_id(): data_provider_instance.get_settings()})
else:
# let's ask the user which live data providers to use
data_feed_instances = interactive_config_build(lean_config, cloud_data_queue_handlers, logger, kwargs,
show_secrets, "Select a live data feed",
multiple=True, project_id=cloud_project.projectId)
show_secrets, "Select a live data feed", multiple=True)
for data_feed in data_feed_instances:
settings = data_feed.get_settings()

7 changes: 2 additions & 5 deletions lean/commands/data/download.py
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@
from lean.click import LeanCommand, ensure_options
from lean.components.util.json_modules_handler import config_build_for_name
from lean.constants import DEFAULT_ENGINE_IMAGE
from lean.container import container, get_project_id
from lean.container import container
from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType
from lean.models.click_options import get_configs_for_options, options_from_json
from lean.models.data import Dataset, DataFile, DatasetDateOption, DatasetTextOption, DatasetTextOptionTransform,OptionResult, Product
@@ -677,11 +677,8 @@ def download(ctx: Context,

engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update)

project_id = get_project_id(project_config)

data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(),
cli_data_downloaders, kwargs, logger, interactive=True,
project_id=project_id)
cli_data_downloaders, kwargs, logger, interactive=True)
data_downloader_provider.ensure_module_installed(organization.id, container_module_version)
container.lean_config_manager.set_properties(data_downloader_provider.get_settings())
# mounting additional data_downloader config files
27 changes: 12 additions & 15 deletions lean/commands/live/deploy.py
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
from lean.click import LeanCommand, PathParameter
from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format
from lean.constants import DEFAULT_ENGINE_IMAGE
from lean.container import container, get_project_id
from lean.container import container
from lean.models.cli import (cli_brokerages, cli_data_queue_handlers, cli_data_downloaders,
cli_addon_modules, cli_history_provider)
from lean.models.errors import MoreInfoError
@@ -194,11 +194,6 @@ def deploy(project: Path,
project_manager = container.project_manager
algorithm_file = project_manager.find_algorithm_file(Path(project))

engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update,
algorithm_file.parent)

project_id = get_project_id(project_config)

if output is None:
output = algorithm_file.parent / "live" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

@@ -243,39 +238,41 @@ def deploy(project: Path,
if brokerage:
# user provided brokerage, check all arguments were provided
brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cli_brokerages, kwargs,
logger, project_id, environment_name)
logger, environment_name)
else:
# let the user choose the brokerage
brokerage_instance = interactive_config_build(lean_config, cli_brokerages, logger, kwargs, show_secrets,
"Select a brokerage", multiple=False,
project_id=project_id, environment_name=environment_name)
environment_name=environment_name)

if data_provider_live and len(data_provider_live) > 0:
for data_feed_name in data_provider_live:
data_feed = non_interactive_config_build_for_name(lean_config, data_feed_name, cli_data_queue_handlers,
kwargs, logger, project_id, environment_name)
kwargs, logger, environment_name)
data_provider_live_instances.append(data_feed)
else:
data_provider_live_instances = interactive_config_build(lean_config, cli_data_queue_handlers, logger, kwargs,
show_secrets, "Select a live data feed",
multiple=True, project_id=project_id,
show_secrets, "Select a live data feed", multiple=True,
environment_name=environment_name)

# based on the live data providers we set up the history providers
data_provider_live = [provider.get_name() for provider in data_provider_live_instances]
if data_provider_historical is None:
data_provider_historical = "Local"
data_downloader_instances = non_interactive_config_build_for_name(lean_config, data_provider_historical,
cli_data_downloaders, kwargs, logger, project_id,
cli_data_downloaders, kwargs, logger,
environment_name)
if history_providers is None or len(history_providers) == 0:
history_providers = _get_history_provider_name(data_provider_live)
for history_provider in history_providers:
if history_provider in ["BrokerageHistoryProvider", "SubscriptionDataReaderHistoryProvider"]:
continue
history_providers_instances.append(config_build_for_name(lean_config, history_provider, cli_history_provider,
kwargs, logger, True, project_id,
environment_name))
kwargs, logger, interactive=True,
environment_name=environment_name))

engine_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update,
algorithm_file.parent)

organization_id = container.organization_manager.try_get_working_organization_id()
paths_to_mount = {}
@@ -344,7 +341,7 @@ def deploy(project: Path,

# Configure addon modules
build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
kwargs, logger, environment_name, container_module_version, project_id)
kwargs, logger, environment_name, container_module_version)

if container.platform_manager.is_host_arm():
if "InteractiveBrokersBrokerage" in lean_config["environments"][environment_name]["live-mode-brokerage"] \
9 changes: 3 additions & 6 deletions lean/commands/optimize.py
Original file line number Diff line number Diff line change
@@ -20,7 +20,7 @@
from lean.click import LeanCommand, PathParameter, ensure_options
from lean.components.docker.lean_runner import LeanRunner
from lean.constants import DEFAULT_ENGINE_IMAGE
from lean.container import container, get_project_id
from lean.container import container
from lean.models.api import QCParameter, QCBacktest
from lean.models.click_options import options_from_json, get_configs_for_options
from lean.models.cli import cli_data_downloaders, cli_addon_modules
@@ -298,12 +298,9 @@ def optimize(project: Path,

paths_to_mount = None

project_id = get_project_id(project_config)

if data_provider_historical is not None:
data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
cli_data_downloaders, kwargs, logger, project_id,
environment_name)
cli_data_downloaders, kwargs, logger, environment_name)
data_provider.ensure_module_installed(organization_id, container_module_version)
container.lean_config_manager.set_properties(data_provider.get_settings())
paths_to_mount = data_provider.get_paths_to_mount()
@@ -331,7 +328,7 @@ def optimize(project: Path,

# Configure addon modules
build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
kwargs, logger, environment_name, container_module_version, project_id)
kwargs, logger, environment_name, container_module_version)

run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach,
engine_image, paths_to_mount)
7 changes: 3 additions & 4 deletions lean/commands/research.py
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
from lean.click import LeanCommand, PathParameter
from lean.components.docker.lean_runner import LeanRunner
from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
from lean.container import container, get_project_id
from lean.container import container
from lean.models.cli import cli_data_downloaders
from lean.components.util.name_extraction import convert_to_class_name
from lean.components.util.json_modules_handler import non_interactive_config_build_for_name
@@ -121,14 +121,13 @@ def research(project: Path,
research_image, container_module_version, project_config = container.manage_docker_image(image, update, no_update,
algorithm_file.parent,
False)
project_id = get_project_id(project_config)

paths_to_mount = None

if data_provider_historical is not None:
organization_id = container.organization_manager.try_get_working_organization_id()
data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
cli_data_downloaders, kwargs, logger, project_id,
environment_name)
cli_data_downloaders, kwargs, logger, environment_name)
data_provider.ensure_module_installed(organization_id, container_module_version)
container.lean_config_manager.set_properties(data_provider.get_settings())
paths_to_mount = data_provider.get_paths_to_mount()
5 changes: 2 additions & 3 deletions lean/components/api/auth0_client.py
Original file line number Diff line number Diff line change
@@ -52,16 +52,15 @@ def read(self, brokerage_id: str) -> QCAuth0Authorization:
return QCAuth0Authorization(authorization=None)

@staticmethod
def authorize(brokerage_id: str, project_id: str, logger: Logger) -> None:
def authorize(brokerage_id: str, logger: Logger) -> None:
"""Starts the authorization process for a brokerage.
:param brokerage_id: the id of the brokerage to start the authorization process for
:param project_id: The local or cloud project_id
:param logger: the logger instance to use
"""
from webbrowser import open

full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}&projectId={project_id}"
full_url = f"{API_BASE_URL}live/auth0/authorize?brokerage={brokerage_id}"
logger.info(f"Please open the following URL in your browser to authorize the LEAN CLI.")
logger.info(full_url)
open(full_url)
5 changes: 2 additions & 3 deletions lean/components/util/auth0_helper.py
Original file line number Diff line number Diff line change
@@ -16,12 +16,11 @@
from lean.components.util.logger import Logger


def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id: str, logger: Logger) -> QCAuth0Authorization:
def get_authorization(auth0_client: Auth0Client, brokerage_id: str, logger: Logger) -> QCAuth0Authorization:
"""Gets the authorization data for a brokerage, authorizing if necessary.
:param auth0_client: An instance of Auth0Client, containing methods to interact with live/auth0/* API endpoints.
:param brokerage_id: The ID of the brokerage to get the authorization data for.
:param project_id: The local or cloud project_id.
:param logger: An instance of Logger, handling all output printing.
:return: The authorization data for the specified brokerage.
"""
@@ -32,7 +31,7 @@ def get_authorization(auth0_client: Auth0Client, brokerage_id: str, project_id:
return data

start_time = time()
auth0_client.authorize(brokerage_id, project_id, logger)
auth0_client.authorize(brokerage_id, logger)

# keep checking for new data every 5 seconds for 7 minutes
while time() - start_time < 420:
23 changes: 10 additions & 13 deletions lean/components/util/json_modules_handler.py
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@

def build_and_configure_modules(target_modules: List[str], module_list: List[JsonModule], organization_id: str,
lean_config: Dict[str, Any], properties: Dict[str, Any], logger: Logger,
environment_name: str, module_version: str, project_id: str):
environment_name: str, module_version: str):
"""Builds and configures the given modules
:param target_modules: the requested modules
@@ -30,21 +30,20 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso
:param logger: the logger instance
:param environment_name: the environment name to use
:param module_version: The version of the module to install. If not provided, the latest version will be installed.
:param project_id: The cloud or local project_id
"""
for target_module_name in target_modules:
module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties,
logger, project_id, environment_name)
logger, environment_name)
# Ensures extra modules (not brokerage or data feeds) are installed.
module.ensure_module_installed(organization_id, module_version)
lean_config["environments"][environment_name].update(module.get_settings())


def non_interactive_config_build_for_name(lean_config: Dict[str, Any], target_module_name: str,
module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger,
project_id: str, environment_name: str = None) -> JsonModule:
environment_name: str = None) -> JsonModule:
return config_build_for_name(lean_config, target_module_name, module_list, properties, logger, interactive=False,
project_id=project_id, environment_name=environment_name)
environment_name=environment_name)


def find_module(target_module_name: str, module_list: List[JsonModule], logger: Logger) -> JsonModule:
@@ -79,18 +78,18 @@ def find_module(target_module_name: str, module_list: List[JsonModule], logger:


def config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, module_list: List[JsonModule],
properties: Dict[str, Any], logger: Logger, interactive: bool, project_id: str,
properties: Dict[str, Any], logger: Logger, interactive: bool,
environment_name: str = None) -> JsonModule:
target_module = find_module(target_module_name, module_list, logger)
target_module.config_build(lean_config, logger, interactive=interactive, project_id=project_id,
properties=properties, environment_name=environment_name)
target_module.config_build(lean_config, logger, interactive=interactive, properties=properties,
environment_name=environment_name)
_update_settings(logger, environment_name, target_module, lean_config)
return target_module


def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], logger: Logger,
user_provided_options: Dict[str, Any], show_secrets: bool, select_message: str,
multiple: bool, project_id: str, environment_name: str = None) -> [JsonModule]:
multiple: bool, environment_name: str = None) -> [JsonModule]:
"""Interactively configures the brokerage to use.
:param lean_config: the LEAN configuration that should be used
@@ -100,7 +99,6 @@ def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule],
:param show_secrets: whether to show secrets on input
:param select_message: the user facing selection message
:param multiple: true if multiple selections are allowed
:param project_id: The local or cloud project_id.
:param environment_name: the target environment name
:return: the brokerage the user configured
"""
@@ -114,9 +112,8 @@ def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule],
modules.append(module)

for module in modules:
module.config_build(lean_config, logger, interactive=True, project_id=project_id,
properties=user_provided_options, hide_input=not show_secrets,
environment_name=environment_name)
module.config_build(lean_config, logger, interactive=True, properties=user_provided_options,
hide_input=not show_secrets, environment_name=environment_name)
_update_settings(logger, environment_name, module, lean_config)
if multiple:
return modules
14 changes: 0 additions & 14 deletions lean/container.py
Original file line number Diff line number Diff line change
@@ -47,20 +47,6 @@
from lean.models.docker import DockerImage


def get_project_id(project_config: Storage):
"""
Retrieves the ID from the project configuration.
Args:
project_config (dict): A dictionary containing project configuration with potential keys 'cloud-id' and 'local-id'.
Returns:
str: The 'cloud-id' if it exists, otherwise the 'local-id'.
If neither is found, returns None.
"""
return project_config.get("cloud-id") or project_config.get("local-id")


class Container:

def __init__(self):
5 changes: 1 addition & 4 deletions lean/models/json_module.py
Original file line number Diff line number Diff line change
@@ -179,7 +179,6 @@ def config_build(self,
lean_config: Dict[str, Any],
logger: Logger,
interactive: bool,
project_id: str,
properties: Dict[str, Any] = {},
hide_input: bool = False,
environment_name: str = None) -> 'JsonModule':
@@ -188,7 +187,6 @@ def config_build(self,
:param lean_config: the Lean configuration dict to read defaults from
:param logger: the logger to use
:param interactive: true if running in interactive mode
:param project_id: The local or cloud project_id.
:param properties: the properties that passed as options
:param hide_input: whether to hide secrets inputs
:param environment_name: the target environment name
@@ -221,8 +219,7 @@ def config_build(self,
logger.debug(f"skipping configuration '{configuration._id}': no choices available.")
continue
elif isinstance(configuration, AuthConfiguration):
auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(),
project_id, logger)
auth_authorizations = get_authorization(container.api_client.auth0, self._display_name.lower(), logger)
logger.debug(f'auth: {auth_authorizations}')
configuration._value = auth_authorizations.get_authorization_config_without_account()
for inner_config in self._lean_configs: