From 58b8962744089894353b5f7592d7636976b002d1 Mon Sep 17 00:00:00 2001 From: Robert Grzesik Date: Sat, 18 Nov 2023 23:31:28 -0500 Subject: [PATCH 1/3] bug fix for yahoo get last price (it was getting the price from the day before!) --- lumibot/data_sources/yahoo_data.py | 46 ++++++++------------- setup.py | 2 +- tests/backtest/test_yahoo.py | 64 ++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 31 deletions(-) create mode 100644 tests/backtest/test_yahoo.py diff --git a/lumibot/data_sources/yahoo_data.py b/lumibot/data_sources/yahoo_data.py index 09474ab2c..a5fae34b6 100644 --- a/lumibot/data_sources/yahoo_data.py +++ b/lumibot/data_sources/yahoo_data.py @@ -54,14 +54,7 @@ def _append_data(self, asset, data): return data def _pull_source_symbol_bars( - self, - asset, - length, - timestep=MIN_TIMESTEP, - timeshift=None, - quote=None, - exchange=None, - include_after_hours=True + self, asset, length, timestep=MIN_TIMESTEP, timeshift=None, quote=None, exchange=None, include_after_hours=True ): if exchange is not None: logging.warning( @@ -69,9 +62,7 @@ def _pull_source_symbol_bars( ) if quote is not None: - logging.warning( - f"quote is not implemented for YahooData, but {quote} was passed as the quote" - ) + logging.warning(f"quote is not implemented for YahooData, but {quote} was passed as the quote") self._parse_source_timestep(timestep, reverse=True) if asset in self._data_store: @@ -111,14 +102,10 @@ def _pull_source_bars( """pull broker bars for a list assets""" if quote is not None: - logging.warning( - f"quote is not implemented for YahooData, but {quote} was passed as the quote" - ) + logging.warning(f"quote is not implemented for YahooData, but {quote} was passed as the quote") self._parse_source_timestep(timestep, reverse=True) - missing_assets = [ - asset.symbol for asset in assets if asset not in self._data_store - ] + missing_assets = [asset.symbol for asset in assets if asset not in self._data_store] if missing_assets: dfs = YahooHelper.get_symbols_data(missing_assets, auto_adjust=self.auto_adjust) @@ -127,16 +114,12 @@ def _pull_source_bars( result = {} for asset in assets: - result[asset] = self._pull_source_symbol_bars( - asset, length, timestep=timestep, timeshift=timeshift - ) + result[asset] = self._pull_source_symbol_bars(asset, length, timestep=timestep, timeshift=timeshift) return result def _parse_source_symbol_bars(self, response, asset, quote=None, length=None): if quote is not None: - logging.warning( - f"quote is not implemented for YahooData, but {quote} was passed as the quote" - ) + logging.warning(f"quote is not implemented for YahooData, but {quote} was passed as the quote") bars = Bars(response, self.SOURCE, asset, raw=response) return bars @@ -146,9 +129,8 @@ def get_last_price(self, asset, timestep=None, quote=None, exchange=None, **kwar if timestep is None: timestep = self.get_timestep() - bars = self.get_historical_prices( - asset, 1, timestep=timestep, quote=quote # , timeshift=timedelta(days=-1) - ) + bars = self.get_historical_prices(asset, 1, timestep=timestep, quote=quote, timeshift=timedelta(days=-1)) + if isinstance(bars, float): return bars elif bars is None: @@ -171,9 +153,13 @@ def get_chains(self, asset): >>> expirations = spy.options >>> chain_data = spy.option_chain() """ - raise NotImplementedError("Lumibot YahooData does not support historical options data. If you need this " - "feature, please use a different data source.") + raise NotImplementedError( + "Lumibot YahooData does not support historical options data. If you need this " + "feature, please use a different data source." + ) def get_strikes(self, asset): - raise NotImplementedError("Lumibot YahooData does not support historical options data. If you need this " - "feature, please use a different data source.") + raise NotImplementedError( + "Lumibot YahooData does not support historical options data. If you need this " + "feature, please use a different data source." + ) diff --git a/setup.py b/setup.py index 71d46822b..10c0be321 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lumibot", - version="2.9.2", + version="2.9.3", author="Robert Grzesik", author_email="rob@lumiwealth.com", description="Backtesting and Trading Library, Made by Lumiwealth", diff --git a/tests/backtest/test_yahoo.py b/tests/backtest/test_yahoo.py new file mode 100644 index 000000000..7197ebf98 --- /dev/null +++ b/tests/backtest/test_yahoo.py @@ -0,0 +1,64 @@ +import datetime + +from lumibot.backtesting import BacktestingBroker, YahooDataBacktesting +from lumibot.strategies import Strategy +from lumibot.traders import Trader + + +class YahooPriceTest(Strategy): + parameters = { + "symbol": "SPY", # The symbol to trade + } + + def initialize(self): + # There is only one trading operation per day + # No need to sleep between iterations + self.sleeptime = "1D" + + def on_trading_iteration(self): + # Get the parameters + symbol = self.parameters["symbol"] + + # Get the datetime + self.dt = self.get_datetime() + + # Get the last price + self.last_price = self.get_last_price(symbol) + + +class TestYahooBacktestFull: + def test_yahoo_last_price(self): + """ + Test Polygon REST Client with Lumibot Backtesting and real API calls to Polygon. Using the Amazon stock + which only has options expiring on Fridays. This test will buy 10 shares of Amazon and 1 option contract + in the historical 2023-08-04 period (in the past!). + """ + # Parameters: True = Live Trading | False = Backtest + # trade_live = False + backtesting_start = datetime.datetime(2023, 11, 1) + backtesting_end = datetime.datetime(2023, 11, 2) + + data_source = YahooDataBacktesting( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + ) + + broker = BacktestingBroker(data_source=data_source) + + poly_strat_obj = YahooPriceTest( + broker=broker, + backtesting_start=backtesting_start, + backtesting_end=backtesting_end, + ) + + trader = Trader(logfile="", backtest=True) + trader.add_strategy(poly_strat_obj) + results = trader.run_all(show_plot=False, show_tearsheet=False, save_tearsheet=False) + + assert results + + last_price = poly_strat_obj.last_price + # Round to 2 decimal places + last_price = round(last_price, 2) + + assert last_price == 416.18 # This is the correct price for 2023-11-01 (the open price) From f420d34d53b6cd7e04a4ceed3c6bc865e0c85d03 Mon Sep 17 00:00:00 2001 From: Robert Grzesik Date: Sat, 18 Nov 2023 23:38:30 -0500 Subject: [PATCH 2/3] formatting --- tests/backtest/test_example_strategies.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/backtest/test_example_strategies.py b/tests/backtest/test_example_strategies.py index 9e5341ea5..51ff410f7 100644 --- a/tests/backtest/test_example_strategies.py +++ b/tests/backtest/test_example_strategies.py @@ -4,10 +4,8 @@ from lumibot.backtesting import YahooDataBacktesting from lumibot.example_strategies.stock_bracket import StockBracket from lumibot.example_strategies.stock_buy_and_hold import BuyAndHold -from lumibot.example_strategies.stock_diversified_leverage import \ - DiversifiedLeverage -from lumibot.example_strategies.stock_limit_and_trailing_stops import \ - LimitAndTrailingStop +from lumibot.example_strategies.stock_diversified_leverage import DiversifiedLeverage +from lumibot.example_strategies.stock_limit_and_trailing_stops import LimitAndTrailingStop from lumibot.example_strategies.stock_oco import StockOco # Global parameters @@ -179,20 +177,21 @@ def test_limit_and_trailing_stops(self): assert filled_limit_orders.iloc[1]["filled_quantity"] == 100 # Get all the filled trailing stop orders - filled_trailing_stop_orders = trades_df[(trades_df["status"] == "fill") - & (trades_df["type"] == "trailing_stop")] + filled_trailing_stop_orders = trades_df[ + (trades_df["status"] == "fill") & (trades_df["type"] == "trailing_stop") + ] # Check if we have an order with a rounded price of 2 decimals of 400.45 and a quantity of 50 order1 = filled_trailing_stop_orders[ - (round(filled_trailing_stop_orders["price"], 2) == 400.45) & ( - filled_trailing_stop_orders["filled_quantity"] == 50) + (round(filled_trailing_stop_orders["price"], 2) == 400.45) + & (filled_trailing_stop_orders["filled_quantity"] == 50) ] assert len(order1) == 1 # Check if we have an order with a price of 399.30 and a quantity of 100 order2 = filled_trailing_stop_orders[ - (round(filled_trailing_stop_orders["price"], 2) == 399.30) & ( - filled_trailing_stop_orders["filled_quantity"] == 100) + (round(filled_trailing_stop_orders["price"], 2) == 399.30) + & (filled_trailing_stop_orders["filled_quantity"] == 100) ] assert len(order2) == 1 From 1e53517270ed3bda0a7f74fafde64da1e4977b43 Mon Sep 17 00:00:00 2001 From: Robert Grzesik Date: Sun, 19 Nov 2023 00:57:32 -0500 Subject: [PATCH 3/3] fixed tests --- lumibot/data_sources/yahoo_data.py | 2 +- .../example_strategies/stock_buy_and_hold.py | 1 + tests/backtest/test_example_strategies.py | 20 +++++++++---------- tests/backtest/test_yahoo.py | 8 +++----- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/lumibot/data_sources/yahoo_data.py b/lumibot/data_sources/yahoo_data.py index a5fae34b6..d7bc804f4 100644 --- a/lumibot/data_sources/yahoo_data.py +++ b/lumibot/data_sources/yahoo_data.py @@ -3,7 +3,6 @@ from decimal import Decimal import numpy - from lumibot.data_sources import DataSourceBacktesting from lumibot.entities import Asset, Bars from lumibot.tools import YahooHelper @@ -129,6 +128,7 @@ def get_last_price(self, asset, timestep=None, quote=None, exchange=None, **kwar if timestep is None: timestep = self.get_timestep() + # Use -1 timeshift to get the price for the current bar (otherwise gets yesterdays prices) bars = self.get_historical_prices(asset, 1, timestep=timestep, quote=quote, timeshift=timedelta(days=-1)) if isinstance(bars, float): diff --git a/lumibot/example_strategies/stock_buy_and_hold.py b/lumibot/example_strategies/stock_buy_and_hold.py index 08ed9acb2..0e9e9ca35 100644 --- a/lumibot/example_strategies/stock_buy_and_hold.py +++ b/lumibot/example_strategies/stock_buy_and_hold.py @@ -46,6 +46,7 @@ def on_trading_iteration(self): if is_live: from credentials import ALPACA_CONFIG + from lumibot.brokers import Alpaca from lumibot.traders import Trader diff --git a/tests/backtest/test_example_strategies.py b/tests/backtest/test_example_strategies.py index 51ff410f7..da82b5574 100644 --- a/tests/backtest/test_example_strategies.py +++ b/tests/backtest/test_example_strategies.py @@ -104,9 +104,9 @@ def test_stock_buy_and_hold(self): assert isinstance(strat_obj, BuyAndHold) # Check that the results are correct - assert round(results["cagr"] * 100, 1) == 155.7 - assert round(results["volatility"] * 100, 1) == 7.0 - assert round(results["total_return"] * 100, 1) == 0.5 + assert round(results["cagr"] * 100, 1) == 2857.5 + assert round(results["volatility"] * 100, 1) == 11.2 + assert round(results["total_return"] * 100, 1) == 1.9 assert round(results["max_drawdown"]["drawdown"] * 100, 1) == 0.0 def test_stock_diversified_leverage(self): @@ -133,9 +133,9 @@ def test_stock_diversified_leverage(self): assert isinstance(strat_obj, DiversifiedLeverage) # Check that the results are correct - assert round(results["cagr"] * 100, 1) == 2907.9 - assert round(results["volatility"] * 100, 0) == 25 - assert round(results["total_return"] * 100, 1) == 1.9 + assert round(results["cagr"] * 100, 1) == 1235709.3 + assert round(results["volatility"] * 100, 0) == 20.0 + assert round(results["total_return"] * 100, 1) == 5.3 assert round(results["max_drawdown"]["drawdown"] * 100, 1) == 0.0 def test_limit_and_trailing_stops(self): @@ -196,7 +196,7 @@ def test_limit_and_trailing_stops(self): assert len(order2) == 1 # Check that the results are correct - assert round(results["cagr"] * 100, 1) == 75 - assert round(results["volatility"] * 100, 1) == 11.3 - assert round(results["total_return"] * 100, 1) == 0.9 - assert round(results["max_drawdown"]["drawdown"] * 100, 1) == 0.7 + assert round(results["cagr"] * 100, 1) == 54.8 + assert round(results["volatility"] * 100, 1) == 6.2 + assert round(results["total_return"] * 100, 1) == 0.7 + assert round(results["max_drawdown"]["drawdown"] * 100, 1) == 0.2 diff --git a/tests/backtest/test_yahoo.py b/tests/backtest/test_yahoo.py index 7197ebf98..1212b0750 100644 --- a/tests/backtest/test_yahoo.py +++ b/tests/backtest/test_yahoo.py @@ -12,7 +12,6 @@ class YahooPriceTest(Strategy): def initialize(self): # There is only one trading operation per day - # No need to sleep between iterations self.sleeptime = "1D" def on_trading_iteration(self): @@ -29,9 +28,8 @@ def on_trading_iteration(self): class TestYahooBacktestFull: def test_yahoo_last_price(self): """ - Test Polygon REST Client with Lumibot Backtesting and real API calls to Polygon. Using the Amazon stock - which only has options expiring on Fridays. This test will buy 10 shares of Amazon and 1 option contract - in the historical 2023-08-04 period (in the past!). + Test the YahooDataBacktesting class by running a backtest and checking that the strategy object is returned + along with the correct results """ # Parameters: True = Live Trading | False = Backtest # trade_live = False @@ -61,4 +59,4 @@ def test_yahoo_last_price(self): # Round to 2 decimal places last_price = round(last_price, 2) - assert last_price == 416.18 # This is the correct price for 2023-11-01 (the open price) + assert last_price == 419.20 # This is the correct price for 2023-11-01 (the open price)