From 1474a8d4868097d17b2f24cf018d0d618e8bd154 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Thu, 14 Dec 2023 01:17:45 +0400 Subject: [PATCH 01/11] TODOs shouldnt be in this PR --- TODOs.md | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 TODOs.md diff --git a/TODOs.md b/TODOs.md deleted file mode 100644 index f3f236022..000000000 --- a/TODOs.md +++ /dev/null @@ -1,35 +0,0 @@ -# Features & refactoring - -### `cache.py` - -- probably to be dropped; should use `_loader_*` and `_storer_*` from `data.py` - -### `forecast.py` - -- cache logic needs improvement, not easily exposable to third-parties now with `dataclass.__hash__` - - drop decorator - - drop dataclass - - cache IO logic should be managed by forecaster not by simulator, could be done by `initialize_estimator`; maybe enough to just - define it in the base class of forecasters -- improve names of internal methods, clean them (lots of stuff can be re-used at universe change, ...) -- generalize the mean estimator: - - use same code for `past_returns`, `past_returns**2`, `past_volumes`, ... - - add rolling window option, should be in `pd.Timedelta` - - add exponential moving avg, should be in half-life `pd.Timedelta` -- add same extras to the covariance estimator -- goal: make this module crystal clear; third-party ML models should use it (at least for caching) - -### `estimator.py` - -- `DataEstimator` needs refactoring, too long and complex methods - - - -### Development & testing -- add extra pylint checkers: - - code complexity -- consider removing downloaded data from `test_simulator.py`, so only `test_data.py` requires internet - -## Documentation - -## Examples From 38dd79e6384419203052e6cdf891d1aa6635a3b1 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Thu, 28 Dec 2023 12:38:24 +0400 Subject: [PATCH 02/11] merge overwrote the change --- cvxportfolio/estimator.py | 58 ++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/cvxportfolio/estimator.py b/cvxportfolio/estimator.py index e52491b0a..ac14ebbcf 100644 --- a/cvxportfolio/estimator.py +++ b/cvxportfolio/estimator.py @@ -91,18 +91,56 @@ def current_value(self): """ return self._current_value - def values_in_time_recursive(self, **kwargs): - """Evaluate recursively on sub-estimators. + + # pylint: disable=useless-type-doc,useless-param-doc + def values_in_time( + self, t, current_weights, current_portfolio_value, + past_returns, past_volumes, current_prices, + mpo_step=None, cache=None, **kwargs): + """Evaluate estimator at current time, possibly return current value. + + This method is usually the most important for Estimator classes. + It is called at each point in a back-test with all data of the current + state. Sub-estimators are evaluated first, in a depth-first recursive + tree fashion (defined in :meth:`values_in_time_recursive`). The + signature differs slightly between different estimators, see below. + + :param t: Current timestamp. + :type t: pandas.Timestamp + :param current_weights: Current allocation weights. + :type current_weights: pandas.Series + :param current_portfolio_value: Current total value of the portfolio + in cash units. + :type current_portfolio_value: float + :param past_returns: Past market returns (including cash). + :type past_returns: pandas.DataFrame + :param past_volumes: Past market volumes, or None if not available. + :type past_volumes: pandas.DataFrame or None + :param current_prices: Current (open) prices, or None if not available. + :type current_prices: pandas.Series or None + :param mpo_step: For :class:`cvxportfolio.MultiPeriodOptimization` + which step in future planning this estimator is at: 0 is for + the current step (:class:`cvxportfolio.SinglePeriodOptimization`), + 1 is for day ahead, .... Defaults to ``None`` if unused. + :type mpo_step: int, optional + :param cache: Cache or workspace shared between all elements of an + estimator tree, currently only used by + :class:`cvxportfolio.MultiPeriodOptimization` (and derived + classes). It's useful to avoid re-computing expensive + things like covariance estimates at different MPO steps. + Defaults to ``None`` if unused. + :type cache: dict, optional + :param kwargs: Reserved for future new features. + :type kwargs: dict - This function is called by Simulator classes on Policy classes - returning the current trades list. Policy classes, if they - contain internal estimators, should declare them as attributes - and call this base function (via `super()`) before they do their - internal computation. CvxpyExpression estimators should instead - define this method to update their Cvxpy parameters. + :returns: Current value of the estimator. + :rtype: object or None + """ + # we don't raise NotImplementedError because this is called + # on classes that don't re-define it - Once we finalize the interface all parameters will be listed - here. + def values_in_time_recursive(self, **kwargs): + """Evaluate recursively on sub-estimators. :param kwargs: Various parameters that are passed to all elements contained in a policy object. From ac8f3872b6794c107f51914cf5e79fb8395002a2 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Thu, 28 Dec 2023 14:54:53 +0400 Subject: [PATCH 03/11] added finalize_estimator to base class --- cvxportfolio/costs.py | 10 ++++- cvxportfolio/estimator.py | 88 +++++++++++++++++++++++++-------------- cvxportfolio/policies.py | 16 +++++++ cvxportfolio/risks.py | 9 ++++ cvxportfolio/simulator.py | 3 ++ pyproject.toml | 3 ++ 6 files changed, 97 insertions(+), 32 deletions(-) diff --git a/cvxportfolio/costs.py b/cvxportfolio/costs.py index d62325a67..8a71ebc0b 100644 --- a/cvxportfolio/costs.py +++ b/cvxportfolio/costs.py @@ -142,7 +142,7 @@ def __mul__(self, other): [el * other for el in self.multipliers]) def initialize_estimator_recursive(self, universe, trading_calendar): - """Iterate over constituent costs. + """Initialize iterating over constituent costs. :param universe: Trading universe, including cash. :type universe: pandas.Index @@ -152,6 +152,14 @@ def initialize_estimator_recursive(self, universe, trading_calendar): _ = [el.initialize_estimator_recursive(universe, trading_calendar) for el in self.costs] + def finalize_estimator_recursive(self, **kwargs): + """Finalize iterating over constituent costs. + + :param kwargs: Arguments. + :type kwargs: dict + """ + _ = [el.finalize_estimator_recursive(**kwargs) for el in self.costs] + def values_in_time_recursive(self, **kwargs): """Iterate over constituent costs. diff --git a/cvxportfolio/estimator.py b/cvxportfolio/estimator.py index ac14ebbcf..2bf21dadb 100644 --- a/cvxportfolio/estimator.py +++ b/cvxportfolio/estimator.py @@ -39,46 +39,70 @@ class Estimator: :meth:`values_in_time_recursive`. """ - # pylint: disable=useless-type-doc,useless-param-doc - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, trading_calendar, **kwargs): """Initialize estimator instance with universe and trading times. - This function is called whenever the trading universe changes. + This method is called at the start of an online execution, or, in a + back-test, at its start and whenever the trading universe changes. It provides the instance with the current trading universe and a :class:`pandas.DatetimeIndex` representing the current and future trading calendar, *i.e.*, the times at which the estimator will be - evaluated. The instance uses these to appropriately initialize any - internal object, such as Cvxpy parameters, to the right size (as - implied by the universe). Also, especially for multi-period - optimization and similar policies, awareness of the future trading - calendar is essential to, *e.g.*, plan in advance. + evaluated, or a best guess of it. The instance uses these to + appropriately initialize any internal object, such as Cvxpy parameters, + to the right size (as implied by the universe). Also, especially for + multi-period optimization and similar policies, awareness of the future + trading calendar is essential to, *e.g.*, plan in advance. :param universe: Trading universe, including cash. :type universe: pandas.Index :param trading_calendar: Future (including current) trading calendar. :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Reserved for future expansion. + :type kwargs: dict """ # we don't raise NotImplementedError because this is called # on classes that don't re-define it - def initialize_estimator_recursive(self, universe, trading_calendar): + def initialize_estimator_recursive(self, **kwargs): """Recursively initialize all estimators in a policy. - :param universe: Names of assets to be traded. - :type universe: pandas.Index - :param trading_calendar: Times at which the estimator is - expected to be evaluated. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Parameters sent down an estimator tree to inizialize it. + :type kwargs: dict """ - # pylint: disable=arguments-differ for _, subestimator in self.__dict__.items(): if hasattr(subestimator, "initialize_estimator_recursive"): - subestimator.initialize_estimator_recursive( - universe=universe, trading_calendar=trading_calendar) + subestimator.initialize_estimator_recursive(**kwargs) if hasattr(self, "initialize_estimator"): - self.initialize_estimator(universe=universe, - trading_calendar=trading_calendar) + self.initialize_estimator(**kwargs) + + def finalize_estimator(self, **kwargs): + """Finalize estimator instance (currently unused). + + This method is called at the end of an online execution, or, in a + back-test, whenever the trading universe changes (before calling + :meth:`initialize_estimator` with the new universe) and at its end. + We aren't currently using in the rest of the library but we plan to + move the caching logic in it. + + :param kwargs: Reserved for future expansion. + :type kwargs: dict + """ + + # we don't raise NotImplementedError because this is called + # on classes that don't re-define it + + def finalize_estimator_recursive(self, **kwargs): + """Recursively finalize all estimators in a policy. + + :param kwargs: Parameters sent down an estimator tree to finalize it. + :type kwargs: dict + """ + for _, subestimator in self.__dict__.items(): + if hasattr(subestimator, "finalize_estimator_recursive"): + subestimator.finalize_estimator_recursive(**kwargs) + if hasattr(self, "finalize_estimator"): + self.initialize_estimator(**kwargs) _current_value = None @@ -91,18 +115,17 @@ def current_value(self): """ return self._current_value - - # pylint: disable=useless-type-doc,useless-param-doc + # pylint: disable=too-many-arguments def values_in_time( self, t, current_weights, current_portfolio_value, - past_returns, past_volumes, current_prices, + past_returns, past_volumes, current_prices, mpo_step=None, cache=None, **kwargs): """Evaluate estimator at current time, possibly return current value. This method is usually the most important for Estimator classes. It is called at each point in a back-test with all data of the current state. Sub-estimators are evaluated first, in a depth-first recursive - tree fashion (defined in :meth:`values_in_time_recursive`). The + tree fashion (defined in :meth:`values_in_time_recursive`). The signature differs slightly between different estimators, see below. :param t: Current timestamp. @@ -124,13 +147,13 @@ def values_in_time( 1 is for day ahead, .... Defaults to ``None`` if unused. :type mpo_step: int, optional :param cache: Cache or workspace shared between all elements of an - estimator tree, currently only used by - :class:`cvxportfolio.MultiPeriodOptimization` (and derived - classes). It's useful to avoid re-computing expensive - things like covariance estimates at different MPO steps. - Defaults to ``None`` if unused. + estimator tree, currently only used by + :class:`cvxportfolio.MultiPeriodOptimization` (and derived + classes). It's useful to avoid re-computing expensive things like + covariance estimates at different MPO steps. Defaults to ``None`` + if unused. :type cache: dict, optional - :param kwargs: Reserved for future new features. + :param kwargs: Reserved for future expansion. :type kwargs: dict :returns: Current value of the estimator. @@ -155,6 +178,7 @@ def values_in_time_recursive(self, **kwargs): if hasattr(subestimator, "values_in_time_recursive"): subestimator.values_in_time_recursive(**kwargs) if hasattr(self, "values_in_time"): + # pylint: disable=assignment-from-no-return self._current_value = self.values_in_time(**kwargs) return self.current_value return None @@ -225,7 +249,7 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): """ raise NotImplementedError - +# pylint: disable=too-many-arguments class DataEstimator(Estimator): """Estimator of point-in-time values from internal data. @@ -292,13 +316,15 @@ def __init__( self._ignore_shape_check = ignore_shape_check self.parameter = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, trading_calendar, **kwargs): """Initialize with current universe. :param universe: Trading universe, including cash. :type universe: pandas.Index :param trading_calendar: Future (including current) trading calendar. :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Unused arguments. + :type kwargs: dict """ self._universe_maybe_noncash = \ diff --git a/cvxportfolio/policies.py b/cvxportfolio/policies.py index 928da5a13..cc2f026ed 100644 --- a/cvxportfolio/policies.py +++ b/cvxportfolio/policies.py @@ -112,6 +112,9 @@ def execute(self, h, market_data, t=None): current_weights=w, current_portfolio_value=v, current_prices=current_prices) + # this could be optional, currently unused) + self.finalize_estimator_recursive() + z = w_plus - w u = z * v @@ -728,6 +731,19 @@ def initialize_estimator_recursive(self, universe, trading_calendar): self._compile_to_cvxpy() + def finalize_estimator_recursive(self, **kwargs): + """Finalize all objects in this policy's estimator tree. + + :param kwargs: Arguments. + :type kwargs: dict + """ + for obj in self.objective: + obj.finalize_estimator_recursive(**kwargs) + for constr_at_lag in self.constraints: + for constr in constr_at_lag: + constr.finalize_estimator_recursive(**kwargs) + self.benchmark.finalize_estimator_recursive(**kwargs) + def values_in_time_recursive( self, t, current_weights, current_portfolio_value, past_returns, past_volumes, diff --git a/cvxportfolio/risks.py b/cvxportfolio/risks.py index c4fc3d379..3b14a9e2e 100644 --- a/cvxportfolio/risks.py +++ b/cvxportfolio/risks.py @@ -444,6 +444,15 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): for risk in self.riskmodels] return cp.max(cp.hstack(risks)) + def finalize_estimator_recursive(self, **kwargs): + """Finalize object. + + :param kwargs: Arguments. + :type kwargs: dict + """ + for risk in self.riskmodels: + risk.finalize_estimator_recursive(**kwargs) + # Aliases class FullSigma(FullCovariance): diff --git a/cvxportfolio/simulator.py b/cvxportfolio/simulator.py index 0b630788e..5d9632920 100644 --- a/cvxportfolio/simulator.py +++ b/cvxportfolio/simulator.py @@ -293,6 +293,9 @@ def _get_initialized_policy(self, orig_policy, universe, trading_calendar): return policy def _finalize_policy(self, policy, universe): + + policy.finalize_estimator_recursive() # currently unused + if hasattr(policy, '_cache'): logger.info('Storing cache from policy to disk...') _store_cache( diff --git a/pyproject.toml b/pyproject.toml index 8128f6703..8cb735581 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,9 @@ accept-no-raise-doc = false accept-no-return-doc = false accept-no-yields-doc = false +[tool.pylint.'MESSAGE CONTROL'] # check updates for the spelling of this +enable=["useless-suppression"] # flag useless pylint pragmas + [tool.coverage.report] fail_under = 99 From d587a825bd18d2c47d4cc65a958c0e11d66457b2 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Thu, 28 Dec 2023 15:28:44 +0400 Subject: [PATCH 04/11] changed base methods signatures; need deep review and fix tests --- cvxportfolio/constraints.py | 18 +++++++++--------- cvxportfolio/costs.py | 25 +++++++++++-------------- cvxportfolio/estimator.py | 2 +- cvxportfolio/forecast.py | 24 +++++++++--------------- cvxportfolio/policies.py | 25 ++++++++++++++----------- cvxportfolio/returns.py | 20 +++++++++----------- cvxportfolio/risks.py | 36 +++++++++++++++++------------------- cvxportfolio/simulator.py | 1 + 8 files changed, 71 insertions(+), 80 deletions(-) diff --git a/cvxportfolio/constraints.py b/cvxportfolio/constraints.py index e732c7a1e..51f74313e 100644 --- a/cvxportfolio/constraints.py +++ b/cvxportfolio/constraints.py @@ -231,13 +231,13 @@ def __init__(self, window=250, #benchmark=MarketBenchmark # self.benchmark = benchmark self.market_vector = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize parameter with size of universe. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.market_vector = cp.Parameter(len(universe)-1) @@ -383,13 +383,13 @@ def __init__(self, asset, periods): self._low = None self._high = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize internal parameters. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self._index = (universe.get_loc if hasattr( universe, 'get_loc') else universe.index)(self.asset) @@ -639,13 +639,13 @@ def __init__(self, limit, times): self.limit = None self.trading_calendar = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, trading_calendar, **kwargs): """Initialize estimator instance with updated trading_calendar. - :param universe: Trading universe, including cash. - :type universe: pandas.Index :param trading_calendar: Future (including current) trading calendar. :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.trading_calendar = trading_calendar self.limit = cp.Parameter() diff --git a/cvxportfolio/costs.py b/cvxportfolio/costs.py index 8a71ebc0b..f21b63613 100644 --- a/cvxportfolio/costs.py +++ b/cvxportfolio/costs.py @@ -141,21 +141,18 @@ def __mul__(self, other): return CombinedCosts(self.costs, [el * other for el in self.multipliers]) - def initialize_estimator_recursive(self, universe, trading_calendar): + def initialize_estimator_recursive(self, **kwargs): """Initialize iterating over constituent costs. - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: All parameters passed to :meth:`initialize_estimator`. + :type kwargs: dict """ - _ = [el.initialize_estimator_recursive(universe, trading_calendar) - for el in self.costs] + _ = [el.initialize_estimator_recursive(**kwargs) for el in self.costs] def finalize_estimator_recursive(self, **kwargs): """Finalize iterating over constituent costs. - :param kwargs: Arguments. + :param kwargs: All parameters passed to :meth:`finalize_estimator`. :type kwargs: dict """ _ = [el.finalize_estimator_recursive(**kwargs) for el in self.costs] @@ -414,7 +411,7 @@ def __init__(self, short_fees=None, long_fees=None, dividends=None, dividends) self.periods_per_year = periods_per_year - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize cvxpy parameters. We don't use the parameter from @@ -423,8 +420,8 @@ def initialize_estimator(self, universe, trading_calendar): :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ if self.short_fees is not None: @@ -608,13 +605,13 @@ def __init__(self, a=None, pershare_cost=None, b=0., window_sigma_est=None, self.first_term_multiplier = None self.second_term_multiplier = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize cvxpy parameters. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ if self.a is not None or self.pershare_cost is not None: self.first_term_multiplier = cp.Parameter( diff --git a/cvxportfolio/estimator.py b/cvxportfolio/estimator.py index 2bf21dadb..dcd6418c4 100644 --- a/cvxportfolio/estimator.py +++ b/cvxportfolio/estimator.py @@ -102,7 +102,7 @@ def finalize_estimator_recursive(self, **kwargs): if hasattr(subestimator, "finalize_estimator_recursive"): subestimator.finalize_estimator_recursive(**kwargs) if hasattr(self, "finalize_estimator"): - self.initialize_estimator(**kwargs) + self.finalize_estimator(**kwargs) _current_value = None diff --git a/cvxportfolio/forecast.py b/cvxportfolio/forecast.py index 03a86e7fd..f43d2e8b4 100644 --- a/cvxportfolio/forecast.py +++ b/cvxportfolio/forecast.py @@ -129,13 +129,11 @@ def __post_init__(self): self._last_counts = None self._last_sum = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, **kwargs): """Re-initialize whenever universe changes. - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.__post_init__() @@ -189,13 +187,11 @@ def __post_init__(self): self._last_counts = None self._last_sum = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, **kwargs): """Re-initialize whenever universe changes. - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.__post_init__() @@ -422,13 +418,11 @@ def __post_init__(self): self._last_sum_matrix = None self._joint_mean = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, **kwargs): """Re-initialize whenever universe changes. - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.__post_init__() diff --git a/cvxportfolio/policies.py b/cvxportfolio/policies.py index cc2f026ed..59547e100 100644 --- a/cvxportfolio/policies.py +++ b/cvxportfolio/policies.py @@ -103,6 +103,7 @@ def execute(self, h, market_data, t=None): h = h[past_returns.columns] w = h / v + # consider adding caching logic here self.initialize_estimator_recursive( universe=past_returns.columns, trading_calendar=trading_calendar[trading_calendar >= t]) @@ -273,13 +274,15 @@ def __init__(self, targets): self.targets = targets self.trading_days = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, trading_calendar, **kwargs): """Initialize policy instance with updated trading_calendar. :param universe: Trading universe, including cash. :type universe: pandas.Index :param trading_calendar: Future (including current) trading calendar. :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.trading_days = trading_calendar @@ -441,13 +444,13 @@ def __init__(self, leverage=1.): # values_in_time_recursive self.target_weights = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize this estimator. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ target_weights = pd.Series(1., universe) target_weights.iloc[-1] = 0 @@ -694,7 +697,7 @@ def _compile_and_check_constraint(constr, i): + " mis-specified custom term.", self.__class__.__name__) # pylint: disable=useless-type-doc,useless-param-doc - def initialize_estimator_recursive(self, universe, trading_calendar): + def initialize_estimator_recursive(self, universe, **kwargs): """Initialize the policy object with the trading universe. We redefine the recursive version of :meth:`initialize_estimator` @@ -702,20 +705,20 @@ def initialize_estimator_recursive(self, universe, trading_calendar): :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ for obj in self.objective: - obj.initialize_estimator_recursive( - universe=universe, trading_calendar=trading_calendar) + obj.initialize_estimator_recursive(universe=universe, **kwargs) for constr_at_lag in self.constraints: for constr in constr_at_lag: constr.initialize_estimator_recursive( - universe=universe, trading_calendar=trading_calendar) + universe=universe, **kwargs) self.benchmark.initialize_estimator_recursive( - universe=universe, trading_calendar=trading_calendar) + universe=universe, **kwargs) + self.w_bm = cp.Parameter(len(universe)) self.w_current = cp.Parameter(len(universe)) diff --git a/cvxportfolio/returns.py b/cvxportfolio/returns.py index b85f36e0f..582c33880 100644 --- a/cvxportfolio/returns.py +++ b/cvxportfolio/returns.py @@ -53,13 +53,11 @@ def __init__(self, cash_returns=None): cash_returns, compile_parameter=True) self._cash_return_parameter = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, **kwargs): """Initialize model. - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self._cash_return_parameter = (cp.Parameter() if self.cash_returns is None else self.cash_returns.parameter) @@ -158,13 +156,13 @@ def __init__(self, r_hat=HistoricalMeanReturn, decay=1.): self.decay = decay self._r_hat_parameter = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize model with universe size. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self._r_hat_parameter = cp.Parameter(len(universe)-1) @@ -227,13 +225,13 @@ def __init__(self, deltas=HistoricalMeanError): self.deltas = DataEstimator(deltas) self._deltas_parameter = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize model with universe size. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self._deltas_parameter = cp.Parameter(len(universe)-1, nonneg=True) diff --git a/cvxportfolio/risks.py b/cvxportfolio/risks.py index 3b14a9e2e..69be4b9e4 100644 --- a/cvxportfolio/risks.py +++ b/cvxportfolio/risks.py @@ -70,13 +70,13 @@ def __init__(self, Sigma=HistoricalFactorizedCovariance): self.Sigma = DataEstimator(Sigma) self._sigma_sqrt = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self._sigma_sqrt = cp.Parameter((len(universe)-1, len(universe)-1)) @@ -130,13 +130,13 @@ def __init__(self, sigma_squares=HistoricalVariance): self.sigma_squares = DataEstimator(sigma_squares) - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.sigmas_parameter = cp.Parameter( len(universe)-1, nonneg=True) # +self.kelly)) @@ -184,15 +184,15 @@ def __init__(self, sigma_squares=HistoricalVariance): sigma_squares = sigma_squares() self.sigma_squares = DataEstimator(sigma_squares) - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ - self.sigmas_parameter = cp.Parameter(len(universe)-1) # +self.kelly)) + self.sigmas_parameter = cp.Parameter(len(universe)-1) def values_in_time(self, **kwargs): """Update parameters of risk model. @@ -314,13 +314,13 @@ def __init__(self, F=None, d=None, Sigma_F=None, num_factors=1, self._fit = False self.idyosync_sqrt_parameter = None - def initialize_estimator(self, universe, trading_calendar): + def initialize_estimator(self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ self.idyosync_sqrt_parameter = cp.Parameter(len(universe)-1) if self._fit: @@ -406,16 +406,14 @@ class WorstCaseRisk(Cost): def __init__(self, riskmodels): self.riskmodels = riskmodels - def initialize_estimator_recursive(self, universe, trading_calendar): + def initialize_estimator_recursive(self, **kwargs): """Initialize risk model with universe and trading times. - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param trading_calendar: Future (including current) trading calendar. - :type trading_calendar: pandas.DatetimeIndex + :param kwargs: Arguments to :meth:`initialize_estimator`. + :type kwargs: dict """ for risk in self.riskmodels: - risk.initialize_estimator_recursive(universe, trading_calendar) + risk.initialize_estimator_recursive(**kwargs) def values_in_time_recursive(self, **kwargs): """Update parameters of constituent risk models. diff --git a/cvxportfolio/simulator.py b/cvxportfolio/simulator.py index 5d9632920..e70fdcda3 100644 --- a/cvxportfolio/simulator.py +++ b/cvxportfolio/simulator.py @@ -277,6 +277,7 @@ def _get_initialized_policy(self, orig_policy, universe, trading_calendar): policy = copy.deepcopy(orig_policy) + # caching will be handled here policy.initialize_estimator_recursive( universe=universe, trading_calendar=trading_calendar) From 07c9976b6f5df722e5fe7d7b9416c3546b13cbe6 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Mon, 8 Jan 2024 17:20:45 +0400 Subject: [PATCH 05/11] cleanup, now fix tests --- cvxportfolio/constraints.py | 24 ++++++++++----- cvxportfolio/costs.py | 40 ++++++++++++++----------- cvxportfolio/errors.py | 2 +- cvxportfolio/estimator.py | 9 +++--- cvxportfolio/forecast.py | 21 ++++++++----- cvxportfolio/policies.py | 60 ++++++++++++++++++++----------------- cvxportfolio/returns.py | 37 ++++++++--------------- cvxportfolio/risks.py | 33 +++++++++++++------- 8 files changed, 126 insertions(+), 100 deletions(-) diff --git a/cvxportfolio/constraints.py b/cvxportfolio/constraints.py index 02699a9f8..dcc307dfb 100644 --- a/cvxportfolio/constraints.py +++ b/cvxportfolio/constraints.py @@ -233,7 +233,8 @@ def __init__(self, window=250, #benchmark=MarketBenchmark # self.benchmark = benchmark self.market_vector = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize parameter with size of universe. :param universe: Trading universe, including cash. @@ -243,7 +244,8 @@ def initialize_estimator(self, universe, **kwargs): """ self.market_vector = cp.Parameter(len(universe)-1) - def values_in_time(self, past_volumes, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, past_volumes, **kwargs): """Update parameter with current market weights and covariance. :param past_volumes: Past market volumes, in units of value. @@ -315,7 +317,8 @@ def __init__(self, volumes, max_fraction_of_volumes=0.05): max_fraction_of_volumes, compile_parameter=True) self.portfolio_value = cp.Parameter(nonneg=True) - def values_in_time(self, current_portfolio_value, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, current_portfolio_value, **kwargs): """Update parameter with current portfolio value. :param current_portfolio_value: Current total value of the portfolio. @@ -385,7 +388,8 @@ def __init__(self, asset, periods): self._low = None self._high = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize internal parameters. :param universe: Trading universe, including cash. @@ -398,7 +402,8 @@ def initialize_estimator(self, universe, **kwargs): self._low = cp.Parameter() self._high = cp.Parameter() - def values_in_time(self, t, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, t, **kwargs): """Update parameters, if necessary by imposing no-trade. :param t: Current time. @@ -481,7 +486,8 @@ def __init__(self, c_min): self.c_min = DataEstimator(c_min) self.rhs = cp.Parameter() - def values_in_time(self, current_portfolio_value, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, current_portfolio_value, **kwargs): """Update parameter with current portfolio value. :param current_portfolio_value: Current total value of the portfolio. @@ -712,7 +718,8 @@ def __init__(self, limit, times): self.limit = None self.trading_calendar = None - def initialize_estimator(self, trading_calendar, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, trading_calendar, **kwargs): """Initialize estimator instance with updated trading_calendar. :param trading_calendar: Future (including current) trading calendar. @@ -723,7 +730,8 @@ def initialize_estimator(self, trading_calendar, **kwargs): self.trading_calendar = trading_calendar self.limit = cp.Parameter() - def values_in_time(self, t, mpo_step, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, t, mpo_step, **kwargs): """If target period is in sight activate constraint. :param t: Current time. diff --git a/cvxportfolio/costs.py b/cvxportfolio/costs.py index f21b63613..0bee6ca77 100644 --- a/cvxportfolio/costs.py +++ b/cvxportfolio/costs.py @@ -40,7 +40,7 @@ "HcostModel"] -class Cost(CvxpyExpressionEstimator): +class Cost(CvxpyExpressionEstimator): # pylint: disable=abstract-method """Base class for cost objects (and also risks). Here there is some logic used to implement the algebraic operations. @@ -147,7 +147,8 @@ def initialize_estimator_recursive(self, **kwargs): :param kwargs: All parameters passed to :meth:`initialize_estimator`. :type kwargs: dict """ - _ = [el.initialize_estimator_recursive(**kwargs) for el in self.costs] + for el in self.costs: + el.initialize_estimator_recursive(**kwargs) def finalize_estimator_recursive(self, **kwargs): """Finalize iterating over constituent costs. @@ -155,18 +156,20 @@ def finalize_estimator_recursive(self, **kwargs): :param kwargs: All parameters passed to :meth:`finalize_estimator`. :type kwargs: dict """ - _ = [el.finalize_estimator_recursive(**kwargs) for el in self.costs] + for el in self.costs: + el.finalize_estimator_recursive(**kwargs) def values_in_time_recursive(self, **kwargs): - """Iterate over constituent costs. + """Evaluate estimators by iterating over constituent costs. :param kwargs: All parameters passed to :meth:`values_in_time`. :type kwargs: dict """ - _ = [el.values_in_time_recursive(**kwargs) for el in self.costs] + for el in self.costs: + el.values_in_time_recursive(**kwargs) def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): - """Iterate over constituent costs. + """Compile cost by iterating over constituent costs. :param w_plus: Post-trade weights. :type w_plus: cvxpy.Variable @@ -252,9 +255,8 @@ class SoftConstraint(Cost): def __init__(self, constraint): self.constraint = constraint - # pylint: disable=inconsistent-return-statements - # because we catch the exception - def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): + def compile_to_cvxpy( # pylint: disable=inconsistent-return-statements + self, w_plus, z, w_plus_minus_w_bm): """Compile cost to cvxpy expression. :param w_plus: Post-trade weights. @@ -411,7 +413,8 @@ def __init__(self, short_fees=None, long_fees=None, dividends=None, dividends) self.periods_per_year = periods_per_year - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize cvxpy parameters. We don't use the parameter from @@ -435,7 +438,8 @@ def initialize_estimator(self, universe, **kwargs): if self.dividends is not None: self._dividends_parameter = cp.Parameter(len(universe) - 1) - def values_in_time(self, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, past_returns, **kwargs): """Update cvxpy parameters. We compute the estimate of periods per year from past returns @@ -605,7 +609,8 @@ def __init__(self, a=None, pershare_cost=None, b=0., window_sigma_est=None, self.first_term_multiplier = None self.second_term_multiplier = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize cvxpy parameters. :param universe: Trading universe, including cash. @@ -620,9 +625,9 @@ def initialize_estimator(self, universe, **kwargs): self.second_term_multiplier = cp.Parameter( len(universe)-1, nonneg=True) - def values_in_time( - self, current_portfolio_value, past_returns, past_volumes, - current_prices, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, current_portfolio_value, past_returns, past_volumes, + current_prices, **kwargs): """Update cvxpy parameters. :raises SyntaxError: If the prices are missing from the market data. @@ -676,8 +681,9 @@ def values_in_time( volume_est) ** ( (2 if self.exponent is None else self.exponent) - 1) - def simulate(self, t, u, past_returns, current_returns, current_volumes, - current_prices, **kwargs): + def simulate( + self, t, u, past_returns, current_returns, current_volumes, + current_prices, **kwargs): """Simulate transaction cost in cash units. :raises SyntaxError: If the market returns are not available in the diff --git a/cvxportfolio/errors.py b/cvxportfolio/errors.py index 86003eddc..b14fca3f5 100644 --- a/cvxportfolio/errors.py +++ b/cvxportfolio/errors.py @@ -19,7 +19,7 @@ 'ConvexSpecificationError', 'ConvexityError'] -class DataError(Exception): +class DataError(ValueError): """Base class for exception related to data.""" diff --git a/cvxportfolio/estimator.py b/cvxportfolio/estimator.py index dcd6418c4..b25c10811 100644 --- a/cvxportfolio/estimator.py +++ b/cvxportfolio/estimator.py @@ -165,8 +165,8 @@ def values_in_time( def values_in_time_recursive(self, **kwargs): """Evaluate recursively on sub-estimators. - :param kwargs: Various parameters that are passed to all elements - contained in a policy object. + :param kwargs: All parameters to :meth:`values_in_time` that are passed + to all elements contained in a policy object. :type kwargs: dict :returns: The current value evaluated by this instance, if it @@ -323,7 +323,7 @@ def initialize_estimator(self, universe, trading_calendar, **kwargs): :type universe: pandas.Index :param trading_calendar: Future (including current) trading calendar. :type trading_calendar: pandas.DatetimeIndex - :param kwargs: Unused arguments. + :param kwargs: Other unused arguments to :meth:`initialize_estimator`. :type kwargs: dict """ @@ -486,7 +486,8 @@ def _internal_values_in_time(self, t, **kwargs): # if data is scalar or numpy return self.value_checker(self._universe_subselect(self.data)) - def values_in_time(self, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, **kwargs): """Obtain value of `self.data` at time t or right before. :param kwargs: All parameters passed to :meth:`values_in_time`. diff --git a/cvxportfolio/forecast.py b/cvxportfolio/forecast.py index f43d2e8b4..b375dfedd 100644 --- a/cvxportfolio/forecast.py +++ b/cvxportfolio/forecast.py @@ -129,7 +129,8 @@ def __post_init__(self): self._last_counts = None self._last_sum = None - def initialize_estimator(self, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, **kwargs): """Re-initialize whenever universe changes. :param kwargs: Unused arguments to :meth:`initialize_estimator`. @@ -137,7 +138,8 @@ def initialize_estimator(self, **kwargs): """ self.__post_init__() - def values_in_time(self, t, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, t, past_returns, **kwargs): """Obtain current value of the mean returns. :param t: Current time. @@ -187,7 +189,8 @@ def __post_init__(self): self._last_counts = None self._last_sum = None - def initialize_estimator(self, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, **kwargs): """Re-initialize whenever universe changes. :param kwargs: Unused arguments to :meth:`initialize_estimator`. @@ -195,7 +198,8 @@ def initialize_estimator(self, **kwargs): """ self.__post_init__() - def values_in_time(self, t, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, t, past_returns, **kwargs): """Obtain current value either by update or from scratch. :param t: Current time. @@ -360,7 +364,8 @@ def build_low_rank_model(rets, num_factors=10, iters=10, svd='numpy'): return F.values, idyosyncratic.values @online_cache - def values_in_time(self, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, past_returns, **kwargs): """Current low-rank model, also cached. :param past_returns: Past market returns (including cash). @@ -418,7 +423,8 @@ def __post_init__(self): self._last_sum_matrix = None self._joint_mean = None - def initialize_estimator(self, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, **kwargs): """Re-initialize whenever universe changes. :param kwargs: Unused arguments to :meth:`initialize_estimator`. @@ -464,7 +470,8 @@ def _online_update(self, t, past_returns): self._joint_mean += last_ret @online_cache - def values_in_time(self, t, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, t, past_returns, **kwargs): """Obtain current value of the covariance estimate. :param t: Current time. diff --git a/cvxportfolio/policies.py b/cvxportfolio/policies.py index 1e0883aef..d39288a9d 100644 --- a/cvxportfolio/policies.py +++ b/cvxportfolio/policies.py @@ -73,7 +73,7 @@ def execute(self, h, market_data, t=None): :type t: pandas.Timestamp or None :raises cvxportfolio.errors.DataError: Holdings vector sum to a - negative value. + negative value or don't match the market data server's universe. :returns: u, t, shares_traded :rtype: pandas.Series, pandas.Timestamp, pandas.Series @@ -96,7 +96,7 @@ def execute(self, h, market_data, t=None): past_returns, _, past_volumes, _, current_prices = market_data.serve(t) if sorted(h.index) != sorted(past_returns.columns): - raise ValueError( + raise DataError( "Holdings provided don't match the universe" " implied by the market data server.") @@ -131,7 +131,8 @@ def execute(self, h, market_data, t=None): class Hold(Policy): """Hold initial portfolio, don't trade.""" - def values_in_time(self, current_weights, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, current_weights, **kwargs): """Return current_weights. :param current_weights: Current weights. @@ -151,7 +152,8 @@ class AllCash(Policy): :class:`MultiPeriodOptimization` policies. """ - def values_in_time(self, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, past_returns, **kwargs): """Return all cash weights. :param past_returns: Past market returns (used to infer universe). @@ -169,7 +171,8 @@ def values_in_time(self, past_returns, **kwargs): class MarketBenchmark(Policy): """Allocation weighted by last year's total market volumes.""" - def values_in_time(self, past_returns, past_volumes, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, past_returns, past_volumes, **kwargs): """Return market benchmark weights. :param past_returns: Past market returns (used to infer universe with @@ -226,7 +229,8 @@ def __init__(self, signal, num_long=1, num_short=1, target_leverage=1.): self.signal = DataEstimator(signal) self.target_leverage = DataEstimator(target_leverage) - def values_in_time(self, current_weights, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, current_weights, **kwargs): """Get allocation weights. :param current_weights: Current allocation weights. @@ -274,11 +278,10 @@ def __init__(self, targets): self.targets = targets self.trading_days = None - def initialize_estimator(self, trading_calendar, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, trading_calendar, **kwargs): """Initialize policy instance with updated trading_calendar. - :param universe: Trading universe, including cash. - :type universe: pandas.Index :param trading_calendar: Future (including current) trading calendar. :type trading_calendar: pandas.DatetimeIndex :param kwargs: Other unused arguments to :meth:`initialize_estimator`. @@ -286,13 +289,14 @@ def initialize_estimator(self, trading_calendar, **kwargs): """ self.trading_days = trading_calendar - def values_in_time(self, t, current_weights, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, t, current_weights, **kwargs): """Get current allocation weights. - :param current_weights: Current allocation weights. - :type current_weights: pandas.Series :param t: Current time. :type t: pandas.Timestamp + :param current_weights: Current allocation weights. + :type current_weights: pandas.Series :param kwargs: Unused arguments to :meth:`values_in_time`. :type kwargs: dict @@ -347,17 +351,18 @@ def __init__(self, trades_weights): self.trades_weights = DataEstimator( trades_weights, data_includes_cash=True) - def values_in_time_recursive(self, t, current_weights, **kwargs): + def values_in_time_recursive( # pylint: disable=arguments-differ + self, t, current_weights, **kwargs): """Get current allocation weights. We redefine the recursive version of :meth:`values_in_time` because we catch an exception thrown by a sub-estimator. - :param current_weights: Current allocation weights. - :type current_weights: pandas.Series :param t: Current time. :type t: pandas.Timestamp - :param kwargs: Unused arguments to :meth:`values_in_time_recursive`. + :param current_weights: Current allocation weights. + :type current_weights: pandas.Series + :param kwargs: Unused arguments to :meth:`values_in_time`. :type kwargs: dict :returns: Allocation weights. @@ -398,16 +403,17 @@ def __init__(self, target_weights): self.target_weights = DataEstimator( target_weights, data_includes_cash=True) - def values_in_time_recursive(self, t, current_weights, **kwargs): + def values_in_time_recursive( # pylint: disable=arguments-differ + self, t, current_weights, **kwargs): """Get current allocation weights. We redefine the recursive version of :meth:`values_in_time` because we catch an exception thrown by a sub-estimator. - :param current_weights: Current allocation weights. - :type current_weights: pandas.Series :param t: Current time. :type t: pandas.Timestamp + :param current_weights: Current allocation weights. + :type current_weights: pandas.Series :param kwargs: Unused arguments to :meth:`values_in_time_recursive`. :type kwargs: dict @@ -444,7 +450,8 @@ def __init__(self, leverage=1.): # values_in_time_recursive self.target_weights = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize this estimator. :param universe: Trading universe, including cash. @@ -518,7 +525,8 @@ def __init__(self, target, tracking_error): self.target = DataEstimator(target) self.tracking_error = DataEstimator(tracking_error) - def values_in_time(self, current_weights, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, current_weights, **kwargs): """Get target allocation weights. :param current_weights: Current allocation weights. @@ -697,8 +705,8 @@ def _compile_and_check_constraint(constr, i): + " cvxportfolio terms and is probably due to a" + " mis-specified custom term.", self.__class__.__name__) - # pylint: disable=useless-type-doc,useless-param-doc - def initialize_estimator_recursive(self, universe, **kwargs): + def initialize_estimator_recursive( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize the policy object with the trading universe. We redefine the recursive version of :meth:`initialize_estimator` @@ -720,7 +728,6 @@ def initialize_estimator_recursive(self, universe, **kwargs): self.benchmark.initialize_estimator_recursive( universe=universe, **kwargs) - self._w_bm = cp.Parameter(len(universe)) self._w_current = cp.Parameter(len(universe)) @@ -749,9 +756,8 @@ def finalize_estimator_recursive(self, **kwargs): constr.finalize_estimator_recursive(**kwargs) self.benchmark.finalize_estimator_recursive(**kwargs) - # pylint: disable=arguments-differ - def values_in_time_recursive( - self, t, current_weights, current_portfolio_value, **kwargs): + def values_in_time_recursive( # pylint: disable=arguments-differ + self, t, current_weights, current_portfolio_value, **kwargs): """Update all cvxpy parameters, solve, and return allocation weights. We redefine the recursive version of :meth:`values_in_time` diff --git a/cvxportfolio/returns.py b/cvxportfolio/returns.py index 582c33880..8b943daff 100644 --- a/cvxportfolio/returns.py +++ b/cvxportfolio/returns.py @@ -53,7 +53,8 @@ def __init__(self, cash_returns=None): cash_returns, compile_parameter=True) self._cash_return_parameter = None - def initialize_estimator(self, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, **kwargs): """Initialize model. :param kwargs: Unused arguments to :meth:`initialize_estimator`. @@ -62,7 +63,8 @@ def initialize_estimator(self, **kwargs): self._cash_return_parameter = (cp.Parameter() if self.cash_returns is None else self.cash_returns.parameter) - def values_in_time(self, past_returns, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, past_returns, **kwargs): """Update cash return parameter as last cash return. :param past_returns: Past market returns. @@ -156,7 +158,8 @@ def __init__(self, r_hat=HistoricalMeanReturn, decay=1.): self.decay = decay self._r_hat_parameter = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize model with universe size. :param universe: Trading universe, including cash. @@ -166,7 +169,8 @@ def initialize_estimator(self, universe, **kwargs): """ self._r_hat_parameter = cp.Parameter(len(universe)-1) - def values_in_time(self, mpo_step=0, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, mpo_step=0, **kwargs): """Update returns parameter knowing which MPO step we're at. :param mpo_step: MPO step, 0 is current. @@ -222,26 +226,7 @@ def __init__(self, deltas=HistoricalMeanError): if isinstance(deltas, type): deltas = deltas() - self.deltas = DataEstimator(deltas) - self._deltas_parameter = None - - def initialize_estimator(self, universe, **kwargs): - """Initialize model with universe size. - - :param universe: Trading universe, including cash. - :type universe: pandas.Index - :param kwargs: Other unused arguments to :meth:`initialize_estimator`. - :type kwargs: dict - """ - self._deltas_parameter = cp.Parameter(len(universe)-1, nonneg=True) - - def values_in_time(self, **kwargs): - """Update returns forecast error parameters. - - :param kwargs: All parameters to :meth:`values_in_time`. - :type kwargs: dict - """ - self._deltas_parameter.value = self.deltas.current_value + self.deltas = DataEstimator(deltas, compile_parameter=True) def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): """Compile to cvxpy expression. @@ -257,4 +242,6 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): :returns: Cvxpy expression representing the risk model. :rtype: cvxpy.expression """ - return cp.abs(w_plus_minus_w_bm[:-1]).T @ self._deltas_parameter + return cp.sum( + cp.multiply( + cp.abs(w_plus_minus_w_bm[:-1]).T, self.deltas.parameter)) diff --git a/cvxportfolio/risks.py b/cvxportfolio/risks.py index 69be4b9e4..e811c8ecb 100644 --- a/cvxportfolio/risks.py +++ b/cvxportfolio/risks.py @@ -70,7 +70,8 @@ def __init__(self, Sigma=HistoricalFactorizedCovariance): self.Sigma = DataEstimator(Sigma) self._sigma_sqrt = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. @@ -80,7 +81,8 @@ def initialize_estimator(self, universe, **kwargs): """ self._sigma_sqrt = cp.Parameter((len(universe)-1, len(universe)-1)) - def values_in_time(self, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, **kwargs): """Update parameters of risk model. :param kwargs: All parameters to :meth:`values_in_time`. @@ -130,7 +132,8 @@ def __init__(self, sigma_squares=HistoricalVariance): self.sigma_squares = DataEstimator(sigma_squares) - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. @@ -141,7 +144,8 @@ def initialize_estimator(self, universe, **kwargs): self.sigmas_parameter = cp.Parameter( len(universe)-1, nonneg=True) # +self.kelly)) - def values_in_time(self, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, **kwargs): """Update parameters of risk model. :param kwargs: All parameters to :meth:`values_in_time`. @@ -184,7 +188,8 @@ def __init__(self, sigma_squares=HistoricalVariance): sigma_squares = sigma_squares() self.sigma_squares = DataEstimator(sigma_squares) - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. @@ -194,7 +199,8 @@ def initialize_estimator(self, universe, **kwargs): """ self.sigmas_parameter = cp.Parameter(len(universe)-1) - def values_in_time(self, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, **kwargs): """Update parameters of risk model. :param kwargs: All parameters to :meth:`values_in_time`. @@ -314,7 +320,8 @@ def __init__(self, F=None, d=None, Sigma_F=None, num_factors=1, self._fit = False self.idyosync_sqrt_parameter = None - def initialize_estimator(self, universe, **kwargs): + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): """Initialize risk model with universe and trading times. :param universe: Trading universe, including cash. @@ -335,7 +342,8 @@ def initialize_estimator(self, universe, **kwargs): self.factor_exposures_parameter = cp.Parameter( self.F.parameter.shape) - def values_in_time(self, **kwargs): + def values_in_time( # pylint: disable=arguments-differ + self, **kwargs): """Update internal parameters. :param kwargs: All parameters to :meth:`values_in_time`. @@ -406,7 +414,8 @@ class WorstCaseRisk(Cost): def __init__(self, riskmodels): self.riskmodels = riskmodels - def initialize_estimator_recursive(self, **kwargs): + def initialize_estimator_recursive( # pylint: disable=arguments-differ + self, **kwargs): """Initialize risk model with universe and trading times. :param kwargs: Arguments to :meth:`initialize_estimator`. @@ -415,7 +424,8 @@ def initialize_estimator_recursive(self, **kwargs): for risk in self.riskmodels: risk.initialize_estimator_recursive(**kwargs) - def values_in_time_recursive(self, **kwargs): + def values_in_time_recursive( # pylint: disable=arguments-differ + self, **kwargs): """Update parameters of constituent risk models. :param kwargs: All parameters to :meth:`values_in_time`. @@ -442,7 +452,8 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): for risk in self.riskmodels] return cp.max(cp.hstack(risks)) - def finalize_estimator_recursive(self, **kwargs): + def finalize_estimator_recursive( # pylint: disable=arguments-differ + self, **kwargs): """Finalize object. :param kwargs: Arguments. From e84b08d94cc7ac7d68ea940d1334d0299e2b601a Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Mon, 8 Jan 2024 18:59:42 +0400 Subject: [PATCH 06/11] fixed tests, only final check --- cvxportfolio/returns.py | 27 ++++++++++++--- cvxportfolio/risks.py | 9 ++--- cvxportfolio/tests/test_constraints.py | 44 +++++++++++++++++------ cvxportfolio/tests/test_estimator.py | 29 ++++++++-------- cvxportfolio/tests/test_policies.py | 48 +++++++++++++++++++++----- docs/estimators.rst | 14 ++++++++ 6 files changed, 127 insertions(+), 44 deletions(-) diff --git a/cvxportfolio/returns.py b/cvxportfolio/returns.py index 8b943daff..ee627bb0a 100644 --- a/cvxportfolio/returns.py +++ b/cvxportfolio/returns.py @@ -226,7 +226,28 @@ def __init__(self, deltas=HistoricalMeanError): if isinstance(deltas, type): deltas = deltas() - self.deltas = DataEstimator(deltas, compile_parameter=True) + self.deltas = DataEstimator(deltas) + self._deltas_parameter = None + + def initialize_estimator( # pylint: disable=arguments-differ + self, universe, **kwargs): + """Initialize model with universe size. + + :param universe: Trading universe, including cash. + :type universe: pandas.Index + :param kwargs: Unused arguments to :meth:`initialize_estimator`. + :type kwargs: pandas.DatetimeIndex + """ + self._deltas_parameter = cp.Parameter(len(universe)-1, nonneg=True) + + def values_in_time( # pylint: disable=arguments-differ + self, **kwargs): + """Update returns forecast error parameters. + + :param kwargs: All arguments to :meth:`values_in_time`. + :type kwargs: dict + """ + self._deltas_parameter.value = self.deltas.current_value def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): """Compile to cvxpy expression. @@ -242,6 +263,4 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): :returns: Cvxpy expression representing the risk model. :rtype: cvxpy.expression """ - return cp.sum( - cp.multiply( - cp.abs(w_plus_minus_w_bm[:-1]).T, self.deltas.parameter)) + return cp.abs(w_plus_minus_w_bm[:-1]).T @ self._deltas_parameter diff --git a/cvxportfolio/risks.py b/cvxportfolio/risks.py index e811c8ecb..2bd47fa02 100644 --- a/cvxportfolio/risks.py +++ b/cvxportfolio/risks.py @@ -414,8 +414,7 @@ class WorstCaseRisk(Cost): def __init__(self, riskmodels): self.riskmodels = riskmodels - def initialize_estimator_recursive( # pylint: disable=arguments-differ - self, **kwargs): + def initialize_estimator_recursive(self, **kwargs): """Initialize risk model with universe and trading times. :param kwargs: Arguments to :meth:`initialize_estimator`. @@ -424,8 +423,7 @@ def initialize_estimator_recursive( # pylint: disable=arguments-differ for risk in self.riskmodels: risk.initialize_estimator_recursive(**kwargs) - def values_in_time_recursive( # pylint: disable=arguments-differ - self, **kwargs): + def values_in_time_recursive(self, **kwargs): """Update parameters of constituent risk models. :param kwargs: All parameters to :meth:`values_in_time`. @@ -452,8 +450,7 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): for risk in self.riskmodels] return cp.max(cp.hstack(risks)) - def finalize_estimator_recursive( # pylint: disable=arguments-differ - self, **kwargs): + def finalize_estimator_recursive(self, **kwargs): """Finalize object. :param kwargs: Arguments. diff --git a/cvxportfolio/tests/test_constraints.py b/cvxportfolio/tests/test_constraints.py index b6544ddaf..9f014a00d 100644 --- a/cvxportfolio/tests/test_constraints.py +++ b/cvxportfolio/tests/test_constraints.py @@ -21,6 +21,12 @@ import cvxportfolio as cvx from cvxportfolio.tests import CvxportfolioTest +VALUES_IN_TIME_DUMMY_KWARGS = { + 'current_weights': None, + 'past_returns': None, + 'current_prices': None, + 'past_volumes': None +} class TestConstraints(CvxportfolioTest): """Test Cvxportfolio constraint objects.""" @@ -28,11 +34,14 @@ class TestConstraints(CvxportfolioTest): def _build_constraint(self, constraint, t=None): """Initialize constraint, build expression, and point it to time.""" constraint.initialize_estimator_recursive( - self.returns.columns, self.returns.index) + universe=self.returns.columns, trading_calendar=self.returns.index) cvxpy_expression = constraint.compile_to_cvxpy( self.w_plus, self.z, self.w_plus_minus_w_bm) - constraint.values_in_time_recursive(t=pd.Timestamp( - "2020-01-01") if t is None else t, current_portfolio_value=1000) + constraint.values_in_time_recursive( + t=pd.Timestamp("2020-01-01") if t is None else t, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS + ) return cvxpy_expression def test_long_only(self): @@ -74,10 +83,12 @@ def test_min_cash(self): self.w_plus.value = np.zeros(self.N) self.w_plus.value[-1] = 1 model.values_in_time_recursive(t=pd.Timestamp( - "2020-01-01"), current_portfolio_value=10001) + "2020-01-01"), current_portfolio_value=10001, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertTrue(cons.value()) model.values_in_time_recursive(t=pd.Timestamp( - "2020-01-01"), current_portfolio_value=9999) + "2020-01-01"), current_portfolio_value=9999, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertFalse(cons.value()) def test_dollar_neutral(self): @@ -123,7 +134,9 @@ def test_leverage_limit_in_time(self): tmp[-1] = -3 self.w_plus.value = tmp self.assertTrue(cons.value()) - model.values_in_time_recursive(t=self.returns.index[2]) + model.values_in_time_recursive(t=self.returns.index[2], + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertFalse(cons.value()) def test_max_weights(self): @@ -158,7 +171,9 @@ def test_max_weights(self): tmp[-1] = -3 self.w_plus.value = tmp self.assertTrue(cons.value()) - model.values_in_time_recursive(t=self.returns.index[2]) + model.values_in_time_recursive(t=self.returns.index[2], + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertFalse(cons.value()) def test_min_weights(self): @@ -190,7 +205,9 @@ def test_min_weights(self): tmp[-1] = -3 self.w_plus.value = tmp self.assertTrue(cons.value()) - model.values_in_time_recursive(t=self.returns.index[2]) + model.values_in_time_recursive(t=self.returns.index[2], + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertFalse(cons.value()) def test_max_bm_dev(self): @@ -225,7 +242,9 @@ def test_max_bm_dev(self): tmp[-1] = -3 self.w_plus_minus_w_bm.value = tmp self.assertTrue(cons.value()) - model.values_in_time_recursive(t=self.returns.index[2]) + model.values_in_time_recursive(t=self.returns.index[2], + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertFalse(cons.value()) def test_min_bm_dev(self): @@ -257,7 +276,9 @@ def test_min_bm_dev(self): tmp[-1] = -3 self.w_plus_minus_w_bm.value = tmp self.assertTrue(cons.value()) - model.values_in_time_recursive(t=self.returns.index[2]) + model.values_in_time_recursive(t=self.returns.index[2], + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertFalse(cons.value()) def test_factor_max_limit(self): @@ -388,7 +409,8 @@ def test_participation_rate(self): model = cvx.ParticipationRateLimit( self.volumes, max_fraction_of_volumes=0.1) model.initialize_estimator_recursive( - self.returns.columns, self.returns.index) + universe=self.returns.columns, + trading_calendar=self.returns.index) cons = model.compile_to_cvxpy( self.w_plus, self.z, self.w_plus_minus_w_bm) model.values_in_time_recursive(t=t, current_portfolio_value=value) diff --git a/cvxportfolio/tests/test_estimator.py b/cvxportfolio/tests/test_estimator.py index 75a4f29e1..f61f2a63e 100644 --- a/cvxportfolio/tests/test_estimator.py +++ b/cvxportfolio/tests/test_estimator.py @@ -166,7 +166,7 @@ def test_series_notime_assetselect(self): data = pd.Series(range(len(universe)), index=universe) estimator = DataEstimator(data, data_includes_cash=True) estimator.initialize_estimator_recursive( - universe, trading_calendar=[t]) + universe=universe, trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data.values) @@ -174,29 +174,30 @@ def test_series_notime_assetselect(self): data = pd.Series(range(len(universe)), index=universe) estimator = DataEstimator(data) estimator.initialize_estimator_recursive( - universe, trading_calendar=[t]) + universe=universe, trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data.values[:2]) # shuffled universe estimator = DataEstimator(data.iloc[::-1]) estimator.initialize_estimator_recursive( - universe, trading_calendar=[t]) + universe=universe, trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data.values[:2]) # wrong universe data = pd.Series(range(len(universe)), index=universe) estimator = DataEstimator(data) - estimator.initialize_estimator_recursive(['d', 'e', 'f'], - trading_calendar=[t]) + estimator.initialize_estimator_recursive( + universe=['d', 'e', 'f'], trading_calendar=[t]) with self.assertRaises(MissingAssetsError): result = estimator.values_in_time_recursive(t=t) # selection of universe data = pd.Series(range(len(universe)), index=universe) estimator = DataEstimator(data, data_includes_cash=True) - estimator.initialize_estimator_recursive(['b'], trading_calendar=[t]) + estimator.initialize_estimator_recursive( + universe=['b'], trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data.values[1]) @@ -207,30 +208,30 @@ def test_ndarray_assetselect(self): # with universe of size 2 estimator = DataEstimator(data, data_includes_cash=True) - estimator.initialize_estimator_recursive(['a', 'b'], - trading_calendar=[t]) + estimator.initialize_estimator_recursive( + universe=['a', 'b'], trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data) # with universe of size 3 estimator = DataEstimator(data, data_includes_cash=True) - estimator.initialize_estimator_recursive(['a', 'b', 'c'], - trading_calendar=[t]) + estimator.initialize_estimator_recursive( + universe=['a', 'b', 'c'], trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data) # error with universe of size 4 estimator = DataEstimator(data, data_includes_cash=True) - estimator.initialize_estimator_recursive(['a', 'b', 'c', 'd'], - trading_calendar=[t]) + estimator.initialize_estimator_recursive( + universe=['a', 'b', 'c', 'd'], trading_calendar=[t]) with self.assertRaises(MissingAssetsError): result = estimator.values_in_time_recursive(t=t) # all ok if skipping check estimator = DataEstimator(data, data_includes_cash=True, ignore_shape_check=True) - estimator.initialize_estimator_recursive(['a', 'b', 'c', 'd'], - trading_calendar=[t]) + estimator.initialize_estimator_recursive( + universe=['a', 'b', 'c', 'd'], trading_calendar=[t]) result = estimator.values_in_time_recursive(t=t) assert np.all(result == data) diff --git a/cvxportfolio/tests/test_policies.py b/cvxportfolio/tests/test_policies.py index cf630f517..b368cc01e 100644 --- a/cvxportfolio/tests/test_policies.py +++ b/cvxportfolio/tests/test_policies.py @@ -24,6 +24,11 @@ from cvxportfolio.forecast import HistoricalFactorizedCovariance from cvxportfolio.tests import CvxportfolioTest +VALUES_IN_TIME_DUMMY_KWARGS = { + 'past_returns': None, + 'current_prices': None, + 'past_volumes': None +} class TestPolicies(CvxportfolioTest): """Test trading policies.""" @@ -150,11 +155,19 @@ def test_fixed_trade(self): policy = cvx.FixedTrades(fixed_trades) t = self.returns.index[123] w = pd.Series(0., self.returns.columns) - wplus = policy.values_in_time_recursive(t=t, current_weights=w) + wplus = policy.values_in_time_recursive( + t=t, current_weights=w, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.all(wplus-w == fixed_trades.loc[t])) w = wplus-w t = pd.Timestamp('1900-01-01') - wplus = policy.values_in_time_recursive(t=t, current_weights=w) + wplus = policy.values_in_time_recursive( + t=t, current_weights=w, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.all(wplus-w == 0.)) def test_fixed_weights(self): @@ -169,16 +182,24 @@ def test_fixed_weights(self): policy = cvx.FixedWeights(fixed_weights) t = self.returns.index[123] wplus = policy.values_in_time_recursive( - t=t, current_weights=pd.Series(0., self.returns.columns)) + t=t, current_weights=pd.Series(0., self.returns.columns), + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.all(wplus == fixed_weights.loc[t])) t = self.returns.index[111] wplus = policy.values_in_time_recursive( - t=t, current_weights=fixed_weights.iloc[110]) + t=t, current_weights=fixed_weights.iloc[110], + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS) self.assertTrue(np.allclose( wplus, fixed_weights.loc[t])) t = pd.Timestamp('1900-01-01') - wplus1 = policy.values_in_time_recursive(t=t, current_weights=wplus) + wplus1 = policy.values_in_time_recursive(t=t, current_weights=wplus, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.all(wplus1 == wplus)) def test_periodic_rebalance(self): @@ -196,23 +217,32 @@ def test_periodic_rebalance(self): self.returns.shape[1]), self.returns.columns) wplus = policy.values_in_time_recursive( - t=rebalancing_times[0], current_weights=init) + t=rebalancing_times[0], current_weights=init, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.allclose(wplus, target)) wplus = policy.values_in_time_recursive( - t=rebalancing_times[0] + pd.Timedelta('1d'), current_weights=init) + t=rebalancing_times[0] + pd.Timedelta('1d'), current_weights=init, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.allclose(wplus, init)) def test_uniform(self): """Test uniform allocation.""" pol = cvx.Uniform() pol.initialize_estimator_recursive( - self.returns.columns, self.returns.index) + universe=self.returns.columns, trading_calendar=self.returns.index) init = pd.Series(np.random.randn( self.returns.shape[1]), self.returns.columns) wplus = pol.values_in_time_recursive( - t=self.returns.index[123], current_weights=init) + t=self.returns.index[123], current_weights=init, + current_portfolio_value=1000, + **VALUES_IN_TIME_DUMMY_KWARGS, + ) self.assertTrue(np.allclose( wplus[:-1], np.ones(self.returns.shape[1]-1)/(self.returns.shape[1]-1))) diff --git a/docs/estimators.rst b/docs/estimators.rst index f1e428fc9..172c10c01 100644 --- a/docs/estimators.rst +++ b/docs/estimators.rst @@ -10,7 +10,21 @@ Estimators .. autoclass:: Estimator + .. automethod:: initialize_estimator + + .. automethod:: initialize_estimator_recursive + + .. automethod:: values_in_time + + .. automethod:: values_in_time_recursive + + .. automethod:: finalize_estimator + + .. automethod:: finalize_estimator_recursive + .. autoclass:: CvxpyExpressionEstimator + .. automethod:: compile_to_cvxpy + .. autoclass:: DataEstimator From db8304fa6dd2cc686bcf11bb77571e26e0f7e88d Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Tue, 9 Jan 2024 12:49:01 +0400 Subject: [PATCH 07/11] minor (unrelated) fix to worstcaserisk, tests pass and full coverage of new code --- Makefile | 2 +- cvxportfolio/costs.py | 6 +++++- cvxportfolio/estimator.py | 4 ++++ cvxportfolio/risks.py | 20 ++++++++++++++++---- cvxportfolio/tests/test_simulator.py | 22 ++++++++++++++++++++-- 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 0fd25df87..427883a9a 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ test: ## run tests w/ cov report # $(BINDIR)/bandit $(PROJECT)/*.py $(TESTS)/*.py lint: ## run linter - $(BINDIR)/pylint $(PROJECT) $(EXTRA_SCRIPTS) $(EXAMPLES) + $(BINDIR)/pylint $(PROJECT) # $(EXTRA_SCRIPTS) $(EXAMPLES) $(BINDIR)/diff-quality --violations=pylint --config-file pyproject.toml docs: ## build docs diff --git a/cvxportfolio/costs.py b/cvxportfolio/costs.py index 0bee6ca77..d5cab8c2f 100644 --- a/cvxportfolio/costs.py +++ b/cvxportfolio/costs.py @@ -128,6 +128,8 @@ def __init__(self, costs, multipliers): "You can only sum cost instances to other cost instances.") self.costs = costs self.multipliers = multipliers + # this is changed by WorstCaseRisk before compiling to Cvxpy + self.DO_CONVEXITY_CHECK = True def __add__(self, other): """Add other (combined) cost to self.""" @@ -194,8 +196,10 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): cost.compile_to_cvxpy(w_plus, z, w_plus_minus_w_bm) if not add.is_dcp(): raise ConvexSpecificationError(cost * multiplier) - if not add.is_concave(): + if self.DO_CONVEXITY_CHECK and (not add.is_concave()): raise ConvexityError(cost * multiplier) + if (not self.DO_CONVEXITY_CHECK) and add.is_concave(): + raise ConvexityError(-cost * multiplier) expression += add return expression diff --git a/cvxportfolio/estimator.py b/cvxportfolio/estimator.py index b25c10811..71fe45cb5 100644 --- a/cvxportfolio/estimator.py +++ b/cvxportfolio/estimator.py @@ -85,6 +85,8 @@ def finalize_estimator(self, **kwargs): We aren't currently using in the rest of the library but we plan to move the caching logic in it. + .. versionadded:: 1.1.0 + :param kwargs: Reserved for future expansion. :type kwargs: dict """ @@ -95,6 +97,8 @@ def finalize_estimator(self, **kwargs): def finalize_estimator_recursive(self, **kwargs): """Recursively finalize all estimators in a policy. + .. versionadded:: 1.1.0 + :param kwargs: Parameters sent down an estimator tree to finalize it. :type kwargs: dict """ diff --git a/cvxportfolio/risks.py b/cvxportfolio/risks.py index 2bd47fa02..8bde56a30 100644 --- a/cvxportfolio/risks.py +++ b/cvxportfolio/risks.py @@ -399,13 +399,17 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): class WorstCaseRisk(Cost): """Select the most restrictive risk model for each value of the allocation. - vector. - Given a list of risk models, penalize the portfolio allocation by the one with highest risk value at the solution point. If uncertain about which risk model to use this procedure can be an easy solution. + :Example: + + >>> risk_model = cvx.WorstCaseRisk( + [cvx.FullCovariance(), + cvx.DiagonalCovariance() + 0.25 * cvx.RiskForecastError()]) + :param riskmodels: risk model instances on which to compute the worst-case risk. :type riskmodels: list @@ -446,8 +450,16 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): :returns: Cvxpy expression representing the risk model. :rtype: cvxpy.expression """ - risks = [risk.compile_to_cvxpy(w_plus, z, w_plus_minus_w_bm) - for risk in self.riskmodels] + risks = [] + for risk in self.riskmodels: + # this is needed if user provides individual risk terms + # that are composed objects (CombinedCost) + # it will check concavity instead of convexity + risk.DO_CONVEXITY_CHECK = False + risks.append(risk.compile_to_cvxpy(w_plus, z, w_plus_minus_w_bm)) + # we also change it back in case the user is sharing the instance + risk.DO_CONVEXITY_CHECK = True + return cp.max(cp.hstack(risks)) def finalize_estimator_recursive(self, **kwargs): diff --git a/cvxportfolio/tests/test_simulator.py b/cvxportfolio/tests/test_simulator.py index e53cfba7c..c1ae7f3d8 100644 --- a/cvxportfolio/tests/test_simulator.py +++ b/cvxportfolio/tests/test_simulator.py @@ -433,9 +433,11 @@ def test_backtest(self): """Test simple back-test.""" pol = cvx.SinglePeriodOptimization( cvx.ReturnsForecast() - cvx.ReturnsForecastError() - - .5 * cvx.FullCovariance(), + - .5 * cvx.WorstCaseRisk( + [cvx.FullCovariance(), + cvx.DiagonalCovariance() + .25 * cvx.DiagonalCovariance()]), [cvx.LeverageLimit(1)], verbose=True, - solver=self.default_qp_solver) + solver=self.default_socp_solver) sim = cvx.MarketSimulator( market_data=self.market_data_4, base_location=self.datadir) result = sim.backtest(pol, pd.Timestamp( @@ -443,6 +445,22 @@ def test_backtest(self): print(result) + def test_wrong_worstcase(self): + """Test wrong worst-case convexity.""" + pol = cvx.SinglePeriodOptimization( + cvx.ReturnsForecast() - cvx.ReturnsForecastError() + - .5 * cvx.WorstCaseRisk( + [-cvx.FullCovariance(), + cvx.DiagonalCovariance() + .25 * cvx.DiagonalCovariance()]), + [cvx.LeverageLimit(1)], verbose=True, + solver=self.default_socp_solver) + sim = cvx.MarketSimulator( + market_data=self.market_data_4, base_location=self.datadir) + + with self.assertRaises(ConvexityError): + sim.backtest(pol, pd.Timestamp( + '2023-01-01'), pd.Timestamp('2023-04-20')) + def test_backtest_changing_universe(self): """Test back-test with changing universe.""" sim = cvx.MarketSimulator( From f738e7a8391c60dec124dc1b8ef516b120cc27f4 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Tue, 9 Jan 2024 12:59:22 +0400 Subject: [PATCH 08/11] trying debug github workflow --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09635d025..514ef183d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,6 +87,7 @@ jobs: coverage lcov - name: Coveralls GitHub Action + if: ${{ github.event_name == 'push'}} uses: coverallsapp/github-action@v1 with: path-to-lcov: coverage.lcov From 118e59bfb3f9f15f1903eb3d56a347bb9eeeb242 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Tue, 9 Jan 2024 14:15:12 +0400 Subject: [PATCH 09/11] minor --- Makefile | 2 +- bumpversion.py | 54 +++++++++++++++++++++++++------------------ cvxportfolio/costs.py | 6 ++--- cvxportfolio/risks.py | 4 ++-- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 427883a9a..90308b786 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ test: ## run tests w/ cov report # $(BINDIR)/bandit $(PROJECT)/*.py $(TESTS)/*.py lint: ## run linter - $(BINDIR)/pylint $(PROJECT) # $(EXTRA_SCRIPTS) $(EXAMPLES) + $(BINDIR)/pylint $(PROJECT) $(EXTRA_SCRIPTS) # $(EXAMPLES) $(BINDIR)/diff-quality --violations=pylint --config-file pyproject.toml docs: ## build docs diff --git a/bumpversion.py b/bumpversion.py index 23cec6dd8..a10902b7d 100644 --- a/bumpversion.py +++ b/bumpversion.py @@ -34,7 +34,7 @@ def findversion(root='.'): We use the first __init__.py with a __version__ that we find. :param root: Root folder of the project. - :type root: Pathlib.Path or str + :type root: pathlib.Path or str :raises ValueError: No version found. @@ -64,6 +64,13 @@ def replaceversion(new_version, version, root='.'): """Replace version number. Skip [env, venv, .*]. We replace in all __init__.py, conf.py, setup.py, and pyproject.toml + + :param new_version: New version. + :type new_version: str + :param version: Old version. + :type version: str + :param root: Root folder of the project. + :type root: pathlib.Path or str """ p = Path(root) @@ -73,14 +80,14 @@ def replaceversion(new_version, version, root='.'): 'pyproject.toml']: lines = [] - with open(fname, 'rt') as fin: + with open(fname, 'rt', encoding="utf-8") as fin: for line in fin: lines.append(line.replace(version, new_version)) - with open(fname, "wt") as fout: + with open(fname, "wt", encoding="utf-8") as fout: for line in lines: fout.write(line) - subprocess.run(['git', 'add', str(fname)]) + subprocess.run(['git', 'add', str(fname)], check=False) if fname.is_dir(): if not (fname.name in ['env', 'venv'] or fname.name[0] == '.'): @@ -91,28 +98,29 @@ def replaceversion(new_version, version, root='.'): while True: print('[revision/minor/major]') - what = input() - if what in ['revision', 'minor', 'major']: + WHAT = input() + if WHAT in ['revision', 'minor', 'major']: break - version = findversion() + VERSION = findversion() - x, y, z = [int(el) for el in version.split('.')] - if what == 'revision': - z += 1 - if what == 'minor': - y += 1 - z = 0 - if what == 'major': - x += 1 - y = 0 - z = 0 - new_version = f"{x}.{y}.{z}" + X, Y, Z = [int(el) for el in VERSION.split('.')] + if WHAT == 'revision': + Z += 1 + if WHAT == 'minor': + Y += 1 + Z = 0 + if WHAT == 'major': + X += 1 + Y = 0 + Z = 0 + NEW_VERSION = f"{X}.{Y}.{Z}" - print(new_version) + print(NEW_VERSION) - replaceversion(new_version, version) + replaceversion(NEW_VERSION, VERSION) subprocess.run(['git', 'commit', '--no-verify', '-em', - f"version {new_version}\n"]) - subprocess.run(['git', 'tag', new_version]) - subprocess.run(['git', 'push', '--no-verify', 'origin', new_version]) + f"version {NEW_VERSION}\n"], check=False) + subprocess.run(['git', 'tag', NEW_VERSION], check=False) + subprocess.run( + ['git', 'push', '--no-verify', 'origin', NEW_VERSION], check=False) diff --git a/cvxportfolio/costs.py b/cvxportfolio/costs.py index d5cab8c2f..d16577f97 100644 --- a/cvxportfolio/costs.py +++ b/cvxportfolio/costs.py @@ -129,7 +129,7 @@ def __init__(self, costs, multipliers): self.costs = costs self.multipliers = multipliers # this is changed by WorstCaseRisk before compiling to Cvxpy - self.DO_CONVEXITY_CHECK = True + self.do_convexity_check = True def __add__(self, other): """Add other (combined) cost to self.""" @@ -196,9 +196,9 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): cost.compile_to_cvxpy(w_plus, z, w_plus_minus_w_bm) if not add.is_dcp(): raise ConvexSpecificationError(cost * multiplier) - if self.DO_CONVEXITY_CHECK and (not add.is_concave()): + if self.do_convexity_check and (not add.is_concave()): raise ConvexityError(cost * multiplier) - if (not self.DO_CONVEXITY_CHECK) and add.is_concave(): + if (not self.do_convexity_check) and add.is_concave(): raise ConvexityError(-cost * multiplier) expression += add return expression diff --git a/cvxportfolio/risks.py b/cvxportfolio/risks.py index 8bde56a30..4b9d23e62 100644 --- a/cvxportfolio/risks.py +++ b/cvxportfolio/risks.py @@ -455,10 +455,10 @@ def compile_to_cvxpy(self, w_plus, z, w_plus_minus_w_bm): # this is needed if user provides individual risk terms # that are composed objects (CombinedCost) # it will check concavity instead of convexity - risk.DO_CONVEXITY_CHECK = False + risk.do_convexity_check = False risks.append(risk.compile_to_cvxpy(w_plus, z, w_plus_minus_w_bm)) # we also change it back in case the user is sharing the instance - risk.DO_CONVEXITY_CHECK = True + risk.do_convexity_check = True return cp.max(cp.hstack(risks)) From edda2e383be36c0105de36bf3c4ef65adef63bb8 Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Tue, 9 Jan 2024 14:45:43 +0400 Subject: [PATCH 10/11] fixed minor ordering issue of combinedcost as visible through __repr__ --- cvxportfolio/costs.py | 14 ++++++++++++-- cvxportfolio/hyperparameters.py | 16 ++++++++++------ cvxportfolio/tests/test_hyperparameters.py | 10 +++++----- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/cvxportfolio/costs.py b/cvxportfolio/costs.py index d16577f97..296e44eb7 100644 --- a/cvxportfolio/costs.py +++ b/cvxportfolio/costs.py @@ -63,7 +63,7 @@ def __add__(self, other): by summing over costs. """ if isinstance(other, CombinedCosts): - return other + self + return other.__radd__(self) return CombinedCosts([self, other], [1.0, 1.0]) def __rmul__(self, other): @@ -132,17 +132,27 @@ def __init__(self, costs, multipliers): self.do_convexity_check = True def __add__(self, other): - """Add other (combined) cost to self.""" + """Add self to other (combined) cost.""" if isinstance(other, CombinedCosts): return CombinedCosts(self.costs + other.costs, self.multipliers + other.multipliers) return CombinedCosts(self.costs + [other], self.multipliers + [1.0]) + def __radd__(self, other): + """Add other (combined) cost to self.""" + if isinstance(other, CombinedCosts): + return other + self # pragma: no cover + return CombinedCosts([other] + self.costs, [1.0] + self.multipliers) + def __mul__(self, other): """Multiply by constant.""" return CombinedCosts(self.costs, [el * other for el in self.multipliers]) + def __neg__(self): + """Take negative of cost.""" + return self * -1 + def initialize_estimator_recursive(self, **kwargs): """Initialize iterating over constituent costs. diff --git a/cvxportfolio/hyperparameters.py b/cvxportfolio/hyperparameters.py index f1ec87966..26c557eec 100644 --- a/cvxportfolio/hyperparameters.py +++ b/cvxportfolio/hyperparameters.py @@ -129,21 +129,25 @@ def __repr__(self): expressions in general. It's not perfect but readable. """ + # TODO this gives wrong string repr with nested expressions like + # ``-(cvx.Gamma() * -3 * (cvx.Gamma() - cvx.Gamma()))`` + # internal algebra is correct, though + def _minus_repr(obj): rawrepr = str(obj).lstrip() if rawrepr[0] == '-': - return ' +' + rawrepr[1:].lstrip() + return ' + ' + rawrepr[1:].lstrip() if rawrepr[0] == '+': - return ' -' + rawrepr[1:].lstrip() # pragma: no cover - return ' -' + rawrepr + return ' - ' + rawrepr[1:].lstrip() # pragma: no cover + return ' - ' + rawrepr def _plus_repr(obj): rawrepr = str(obj).lstrip() if rawrepr[0] == '-': - return ' -' + rawrepr[1:].lstrip() + return ' - ' + rawrepr[1:].lstrip() if rawrepr[0] == '+': - return ' +' + rawrepr[1:].lstrip() - return ' +' + str(obj) + return ' + ' + rawrepr[1:].lstrip() + return ' + ' + rawrepr result = '' diff --git a/cvxportfolio/tests/test_hyperparameters.py b/cvxportfolio/tests/test_hyperparameters.py index 599e8a984..3772adb5f 100644 --- a/cvxportfolio/tests/test_hyperparameters.py +++ b/cvxportfolio/tests/test_hyperparameters.py @@ -44,11 +44,11 @@ def test_repr(self): - cvx.Gamma() * cvx.StocksTransactionCost() self.assertTrue(str(obj) == - '-Gamma(current_value=1.0) * ' - + 'FullCovariance(Sigma=HistoricalFactorizedCovariance(kelly=True))' - + ' + ReturnsForecast(r_hat=HistoricalMeanReturn(), decay=1.0)' - + '-Gamma(current_value=1.0) * StocksTransactionCost(a=0.0,' - + ' pershare_cost=0.005, b=1.0, exponent=1.5)') + 'ReturnsForecast(r_hat=HistoricalMeanReturn(), decay=1.0)' + + '- Gamma(current_value=1.0) * FullCovariance(' + + 'Sigma=HistoricalFactorizedCovariance(kelly=True))' + + '- Gamma(current_value=1.0) * StocksTransactionCost(' + + 'a=0.0, pershare_cost=0.005, b=1.0, exponent=1.5)') print(cvx.Gamma() * cvx.Gamma()) print(cvx.Gamma() - cvx.Gamma()) From b38f3a4a6f609738f07a1b503f5ed9b9e5f5113b Mon Sep 17 00:00:00 2001 From: Enzo Busseti Date: Tue, 9 Jan 2024 15:13:00 +0400 Subject: [PATCH 11/11] roadmap.rst; ready to merge --- TODOs_ROADMAP.rst | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/TODOs_ROADMAP.rst b/TODOs_ROADMAP.rst index 18636a88d..d8d3553bb 100644 --- a/TODOs_ROADMAP.rst +++ b/TODOs_ROADMAP.rst @@ -52,15 +52,29 @@ their planned release. - [ ] ``initialize_estimator`` could get optional market data partial signature for caching. Default None, no incompatible change. - - [ ] Could get a ``finalize_estimator`` method used for storing + - [X] Could get a ``finalize_estimator`` method used for storing data, like risk models on disk, doesn't need arguments; it can use the partial signature got above. No incompatible change. ``cvxportfolio.data`` -------------------------- +- [ ] Improve ``YahooFinance`` data cleaning. Idea is to factor it in a + base ``OpenLowHighCloseVolume`` class, which should flexible enough to + accommodate adjusted closes (i.e., with backwards dividend adjustments like + YF), total returns like other data sources, or neither for non-stocks assets. + This would implement all data cleaning process as sequence of small steps + in separate methods, with good logging. It would also implement data quality + check in the ``preload`` method to give feedback to the user. PR #125 +- [ ] Factor ``data.py`` in ``data/`` submodule. PR #125 + ``cvxportfolio.simulator`` -------------------------- +- [ ] Make ``BackTestResult`` interface methods with ``MarketSimulator`` + public. It probably should do a context manager b/c logging code in + ``BackTestResult`` does cleanup of loggers at the end, to ensure all right + in case back-test fails. +- [ ] Move caching logic out of it; see above. ``cvxportfolio.risks`` ---------------------- @@ -95,7 +109,7 @@ Optimization policies compatibility, it doesn't if we don't give defaults (so exceptions are raised all the way to the caller), but then it's extra complication (more arguments). Consider for ``2.0.0``. -- [ ] Improve ``__repr__`` method, now hard to read. Target ``1.1.1``. +- [X] Improve ``__repr__`` method, now hard to read. Target ``1.1.0``. ``cvxportfolio.constraints`` ---------------------------- @@ -106,8 +120,6 @@ Optimization policies ``cvxportfolio.result`` ----------------------- -- [ ] Make ``BackTestResult`` interface methods with ``MarketSimulator`` - public. - [ ] Add a ``bankruptcy`` property (boolean). Amend ``sharpe_ratio`` and other aggregate statistics (as best as possible) to return ``-np.inf`` if back-test ended in backruptcy. This is needed specifically for @@ -122,7 +134,7 @@ Optimization policies Other ----- -- [ ] Exceptions are not too good, probably ``cvxportfolio.DataError`` should +- [X] Exceptions are not too good, probably ``cvxportfolio.DataError`` should be ``ValueError``, .... Research this, one option is to simply derive from built-ins (``class DataError(ValueError): pass``), .... No compatibility breaks. @@ -140,8 +152,8 @@ Documentation ------------- - [ ] Improve examples section, also how "Hello world" is mentioned in readme. -- [ ] Manual. -- [ ] Quickstart, probably to merge into manual. +- [ ] Manual. PR #124 +- [ ] Quickstart, probably to merge into manual. PR #124 Examples --------