From 6da29ea76a43efd4884065c3358b1faf7cffefbd Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 12 Jul 2024 20:48:23 +0300 Subject: [PATCH 1/3] (feat) update the backtesting router --- routers/manage_backtesting.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/routers/manage_backtesting.py b/routers/manage_backtesting.py index 2460d3e..800fbf2 100644 --- a/routers/manage_backtesting.py +++ b/routers/manage_backtesting.py @@ -4,7 +4,11 @@ from hummingbot.data_feed.candles_feed.candles_factory import CandlesFactory from pydantic import BaseModel -from services.backtesting_engine import BacktestingEngine, DirectionalTradingBacktesting, MarketMakingBacktesting +from hummingbot.strategy_v2.backtesting.backtesting_engine_base import BacktestingEngineBase +from hummingbot.strategy_v2.backtesting.controllers_backtesting.directional_trading_backtesting import DirectionalTradingBacktesting +from hummingbot.strategy_v2.backtesting.controllers_backtesting.market_making_backtesting import MarketMakingBacktesting + +from config import CONTROLLERS_MODULE, CONTROLLERS_PATH router = APIRouter(tags=["Market Backtesting"]) candles_factory = CandlesFactory() @@ -18,8 +22,8 @@ class BacktestingConfig(BaseModel): - start_time: int = 1672542000000 # 2023-01-01 00:00:00 - end_time: int = 1672628400000 # 2023-01-01 23:59:00 + start_time: int = 1672542000 # 2023-01-01 00:00:00 + end_time: int = 1672628400 # 2023-01-01 23:59:00 backtesting_resolution: str = "1m" trade_cost: float = 0.0006 config: Union[Dict, str] @@ -29,9 +33,16 @@ class BacktestingConfig(BaseModel): async def run_backtesting(backtesting_config: BacktestingConfig): try: if isinstance(backtesting_config.config, str): - controller_config = BacktestingEngine.get_controller_config_instance_from_yml(backtesting_config.config) + controller_config = BacktestingEngineBase.get_controller_config_instance_from_yml( + config_path=backtesting_config.config, + controllers_conf_dir_path=CONTROLLERS_PATH, + controllers_module=CONTROLLERS_MODULE + ) else: - controller_config = BacktestingEngine.get_controller_config_instance_from_dict(backtesting_config.config) + controller_config = BacktestingEngineBase.get_controller_config_instance_from_dict( + config_data=backtesting_config.config, + controllers_module=CONTROLLERS_MODULE + ) backtesting_engine = BACKTESTING_ENGINES.get(controller_config.controller_type) if not backtesting_engine: raise ValueError(f"Backtesting engine for controller type {controller_config.controller_type} not found.") From f787ab0618c24289fef17589f576ee23fc438787 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 12 Jul 2024 20:48:35 +0300 Subject: [PATCH 2/3] (feat) update market data to use candles directly --- routers/manage_market_data.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/routers/manage_market_data.py b/routers/manage_market_data.py index f768f1b..91e86c1 100644 --- a/routers/manage_market_data.py +++ b/routers/manage_market_data.py @@ -15,8 +15,8 @@ class HistoricalCandlesConfig(BaseModel): connector_name: str = "binance_perpetual" trading_pair: str = "BTC-USDT" interval: str = "3m" - start_time: int = 1672542000000 # 2023-01-01 00:00:00 - end_time: int = 1672628400000 # 2023-01-01 23:59:00 + start_time: int = 1672542000 + end_time: int = 1672628400 @router.post("/real-time-candles") @@ -43,22 +43,6 @@ async def get_historical_candles(config: HistoricalCandlesConfig): interval=config.interval ) candles = candles_factory.get_candle(candles_config) - all_candles = [] - current_start_time = config.start_time - - while current_start_time <= config.end_time: - fetched_candles = await candles.fetch_candles(start_time=current_start_time) - if fetched_candles.size < 1: - break - - all_candles.append(fetched_candles) - last_timestamp = fetched_candles[-1][0] # Assuming the first column is the timestamp - current_start_time = int(last_timestamp) - - final_candles = np.concatenate(all_candles, axis=0) if all_candles else np.array([]) - candles_df = pd.DataFrame(final_candles, columns=candles.columns) - candles_df.drop_duplicates(subset=["timestamp"], inplace=True) - candles_df["timestamp"] = candles_df["timestamp"] // 1e3 - return candles_df + return await candles.get_historical_candles(config=config) except Exception as e: return {"error": str(e)} From e91e51b99088fd8c10665072b4fecdf5004aaea8 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 12 Jul 2024 20:48:50 +0300 Subject: [PATCH 3/3] (feat) remove custom backtesting engine --- services/backtesting_engine.py | 93 ---------------------------------- 1 file changed, 93 deletions(-) delete mode 100644 services/backtesting_engine.py diff --git a/services/backtesting_engine.py b/services/backtesting_engine.py deleted file mode 100644 index 051746e..0000000 --- a/services/backtesting_engine.py +++ /dev/null @@ -1,93 +0,0 @@ -from typing import List, Dict -import os - -import pandas as pd -from decimal import Decimal -import yaml -import importlib -import inspect - -from hummingbot.strategy_v2.backtesting.backtesting_engine_base import BacktestingEngineBase -from hummingbot.exceptions import InvalidController -from hummingbot.strategy_v2.controllers import ControllerConfigBase, MarketMakingControllerConfigBase, DirectionalTradingControllerConfigBase - -import config - - -class BacktestingEngine(BacktestingEngineBase): - - @classmethod - def load_controller_config(cls, config_path: str) -> Dict: - full_path = os.path.join(config.CONTROLLERS_PATH, config_path) - with open(full_path, 'r') as file: - config_data = yaml.safe_load(file) - return config_data - - @classmethod - def get_controller_config_instance_from_yml(cls, config_path: str) -> ControllerConfigBase: - config_data = cls.load_controller_config(config_path) - return cls.get_controller_config_instance_from_dict(config_data) - - @classmethod - def get_controller_config_instance_from_dict(cls, config_data: Dict) -> ControllerConfigBase: - controller_type = config_data.get('controller_type') - controller_name = config_data.get('controller_name') - - if not controller_type or not controller_name: - raise ValueError(f"Missing controller_type or controller_name") - - module_path = f"{config.CONTROLLERS_MODULE}.{controller_type}.{controller_name}" - module = importlib.import_module(module_path) - - config_class = next((member for member_name, member in inspect.getmembers(module) - if inspect.isclass(member) and member not in [ControllerConfigBase, - MarketMakingControllerConfigBase, - DirectionalTradingControllerConfigBase] - and (issubclass(member, ControllerConfigBase))), None) - if not config_class: - raise InvalidController(f"No configuration class found in the module {controller_name}.") - - return config_class(**config_data) - - def prepare_market_data(self) -> pd.DataFrame: - """ - Prepares market data by merging candle data with strategy features, filling missing values. - - Returns: - pd.DataFrame: The prepared market data with necessary features. - """ - backtesting_candles = self.controller.market_data_provider.get_candles_df( - connector_name=self.controller.config.connector_name, - trading_pair=self.controller.config.trading_pair, - interval=self.backtesting_resolution - ).add_suffix("_bt") - - if "features" not in self.controller.processed_data: - backtesting_candles["reference_price"] = backtesting_candles["close_bt"] - backtesting_candles["spread_multiplier"] = 1 - backtesting_candles["signal"] = 0 - else: - backtesting_candles = pd.merge_asof(backtesting_candles, self.controller.processed_data["features"], - left_on="timestamp_bt", right_on="timestamp", - direction="backward") - backtesting_candles["timestamp"] = backtesting_candles["timestamp_bt"] // 1e3 - backtesting_candles["open"] = backtesting_candles["open_bt"] - backtesting_candles["high"] = backtesting_candles["high_bt"] - backtesting_candles["low"] = backtesting_candles["low_bt"] - backtesting_candles["close"] = backtesting_candles["close_bt"] - backtesting_candles["volume"] = backtesting_candles["volume_bt"] - # TODO: Apply this changes in the Base class to avoid code duplication - backtesting_candles.dropna(inplace=True) - self.controller.processed_data["features"] = backtesting_candles - return backtesting_candles - - -class MarketMakingBacktesting(BacktestingEngine): - def update_processed_data(self, row: pd.Series): - self.controller.processed_data["reference_price"] = Decimal(row["reference_price"]) - self.controller.processed_data["spread_multiplier"] = Decimal(row["spread_multiplier"]) - - -class DirectionalTradingBacktesting(BacktestingEngine): - def update_processed_data(self, row: pd.Series): - self.controller.processed_data["signal"] = row["signal"]