BUG: InconsistencyError Multiple destroyers of CGemv{no_inplace}.0

[mvds314]

Describe the issue:

I get the following error when setting up a hierarchical model for a Bayesian spline:

Multiple destroyers of CGemv{no_inplace}.0

I tried to simplify to the most simple setup I could find

I am completely clueless, as the error only seems to happen when I increase the number of knots to a somewhat larger number.

Reproduceable code example:

import numpy as np
import pymc as pm
import pytensor.tensor as pt

# Simulated data of some spline
N = 500
np.random.seed(42)
x = np.linspace(0, 10, N)
true_knots = [3, 7]
y = np.piecewise(
    x,
    [x <= 3, (x > 3) & (x <= 7), x > 7],
    [lambda x: 0.5 * x, lambda x: 1.5 + 0.2 * (x - 3), lambda x: 2.3 - 0.1 * (x - 7)],
)
y += np.random.normal(0, 0.2, size=len(x))  # Add noise

# Artificial groups
group = np.random.choice([1, 2, 3], size=N)

# Try to fit another spline with knots at:
# many knots leads to InconsistencyError error
knots = np.linspace(0, 10, num=50)
# Few knots seems to work
# knots = [0, 1, 2, 7, 9]
n_knots = len(knots)
groups = np.unique(group).tolist()

with pm.Model() as hinge_model_flex:
    # Hyperprior for beta0
    sigma_beta0 = pm.HalfNormal("sigma_beta0", sigma=10)

    # Global priors
    sigma = pm.HalfCauchy("sigma", beta=1)

    # Create likelihood per group
    for gr in groups:
        idx = group == gr

        # Define hinge basis using scan
        def hinge_term(knot):
            return pt.maximum(0, x[idx] - knot)

        hinge_terms = [hinge_term(knot) for knot in knots]

        # beta for initial slope
        beta0 = pm.HalfNormal(f"beta0_{gr}", sigma=sigma_beta0)

        z = pm.Normal(f"z_{gr}", mu=0, sigma=2, shape=n_knots)
        delta_factors = pm.Deterministic(f"delta_factors_{gr}", pt.special.softmax(z))
        slope_factors = 1 - pm.Deterministic(f"beta_factors_{gr}", pt.cumsum(delta_factors[:-1]))
        spline_slopes = pm.Deterministic(
            f"spline_slopes_{gr}",
            pt.stack([beta0] + [beta0 * slope_factors[i] for i in range(n_knots - 1)]),
        )
        beta = pm.Deterministic(f"beta_{gr}", pt.concatenate(([beta0], pt.diff(spline_slopes))))

        # Combine basis terms
        X = pt.stack([hinge_terms[i] for i in range(n_knots)], axis=1)

        mu = pt.dot(X, beta)

        # Likelihood
        y_obs = pm.Normal(f"y_obs_{gr}", mu=mu, sigma=sigma, observed=y[idx])

    # Sampling
    map_res = pm.find_MAP()

Error message:

---------------------------------------------------------------------------
InconsistencyError                        Traceback (most recent call last)
File c:\....\untitled14.py:73
     70     y_obs = pm.Normal(f"y_obs_{gr}", mu=mu, sigma=sigma, observed=y[idx])
     72 # Sampling
---> 73 map_res = pm.find_MAP()

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\tuning\starting.py:151, in find_MAP(start, vars, method, return_raw, include_transformed, progressbar, progressbar_theme, maxeval, model, seed, *args, **kwargs)
    146 rvs = [model.values_to_rvs[vars_dict[name]] for name, _, _, _ in x0.point_map_info]
    147 try:
    148     # This might be needed for calls to `dlogp_func`
    149     # start_map_info = tuple((v.name, v.shape, v.dtype) for v in vars)
    150     compiled_dlogp_func = DictToArrayBijection.mapf(
--> 151         model.compile_dlogp(rvs, jacobian=False), start
    152     )
    153     dlogp_func = lambda x: compiled_dlogp_func(RaveledVars(x, x0.point_map_info))  # noqa: E731
    154     compute_gradient = True

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\model\core.py:620, in Model.compile_dlogp(self, vars, jacobian, **compile_kwargs)
    604 def compile_dlogp(
    605     self,
    606     vars: Variable | Sequence[Variable] | None = None,
    607     jacobian: bool = True,
    608     **compile_kwargs,
    609 ) -> PointFunc:
    610     """Compiled log probability density gradient function.
    611 
    612     Parameters
   (...)
    618         Whether to include jacobian terms in logprob graph. Defaults to True.
    619     """
--> 620     return self.compile_fn(self.dlogp(vars=vars, jacobian=jacobian), **compile_kwargs)

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\model\core.py:1649, in Model.compile_fn(self, outs, inputs, mode, point_fn, **kwargs)
   1646     inputs = inputvars(outs)
   1648 with self:
-> 1649     fn = compile(
   1650         inputs,
   1651         outs,
   1652         allow_input_downcast=True,
   1653         accept_inplace=True,
   1654         mode=mode,
   1655         **kwargs,
   1656     )
   1658 if point_fn:
   1659     return PointFunc(fn)

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\pytensorf.py:947, in compile(inputs, outputs, random_seed, mode, **kwargs)
    945 opt_qry = mode.provided_optimizer.including("random_make_inplace", check_parameter_opt)
    946 mode = Mode(linker=mode.linker, optimizer=opt_qry)
--> 947 pytensor_function = pytensor.function(
    948     inputs,
    949     outputs,
    950     updates={**rng_updates, **kwargs.pop("updates", {})},
    951     mode=mode,
    952     **kwargs,
    953 )
    954 return pytensor_function

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\__init__.py:332, in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, trust_input)
    321     fn = orig_function(
    322         inputs,
    323         outputs,
   (...)
    327         trust_input=trust_input,
    328     )
    329 else:
    330     # note: pfunc will also call orig_function -- orig_function is
    331     #      a choke point that all compilation must pass through
--> 332     fn = pfunc(
    333         params=inputs,
    334         outputs=outputs,
    335         mode=mode,
    336         updates=updates,
    337         givens=givens,
    338         no_default_updates=no_default_updates,
    339         accept_inplace=accept_inplace,
    340         name=name,
    341         rebuild_strict=rebuild_strict,
    342         allow_input_downcast=allow_input_downcast,
    343         on_unused_input=on_unused_input,
    344         profile=profile,
    345         output_keys=output_keys,
    346         trust_input=trust_input,
    347     )
    348 return fn

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\pfunc.py:466, in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys, fgraph, trust_input)
    452     profile = ProfileStats(message=profile)
    454 inputs, cloned_outputs = construct_pfunc_ins_and_outs(
    455     params,
    456     outputs,
   (...)
    463     fgraph=fgraph,
    464 )
--> 466 return orig_function(
    467     inputs,
    468     cloned_outputs,
    469     mode,
    470     accept_inplace=accept_inplace,
    471     name=name,
    472     profile=profile,
    473     on_unused_input=on_unused_input,
    474     output_keys=output_keys,
    475     fgraph=fgraph,
    476     trust_input=trust_input,
    477 )

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\types.py:1833, in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys, fgraph, trust_input)
   1820     m = Maker(
   1821         inputs,
   1822         outputs,
   (...)
   1830         trust_input=trust_input,
   1831     )
   1832     with config.change_flags(compute_test_value="off"):
-> 1833         fn = m.create(defaults)
   1834 finally:
   1835     if profile and fn:

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\types.py:1717, in FunctionMaker.create(self, input_storage, storage_map)
   1714 start_import_time = pytensor.link.c.cmodule.import_time
   1716 with config.change_flags(traceback__limit=config.traceback__compile_limit):
-> 1717     _fn, _i, _o = self.linker.make_thunk(
   1718         input_storage=input_storage_lists, storage_map=storage_map
   1719     )
   1721 end_linker = time.perf_counter()
   1723 linker_time = end_linker - start_linker

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\link\basic.py:245, in LocalLinker.make_thunk(self, input_storage, output_storage, storage_map, **kwargs)
    238 def make_thunk(
    239     self,
    240     input_storage: Optional["InputStorageType"] = None,
   (...)
    243     **kwargs,
    244 ) -> tuple["BasicThunkType", "InputStorageType", "OutputStorageType"]:
--> 245     return self.make_all(
    246         input_storage=input_storage,
    247         output_storage=output_storage,
    248         storage_map=storage_map,
    249     )[:3]

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\link\vm.py:1207, in VMLinker.make_all(self, profiler, input_storage, output_storage, storage_map)
   1199 def make_all(
   1200     self,
   1201     profiler=None,
   (...)
   1204     storage_map=None,
   1205 ):
   1206     fgraph = self.fgraph
-> 1207     order = self.schedule(fgraph)
   1209     input_storage, output_storage, storage_map = map_storage(
   1210         fgraph, order, input_storage, output_storage, storage_map
   1211     )
   1212     compute_map = {}

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\link\basic.py:228, in Linker.schedule(self, fgraph)
    226 if callable(self._scheduler):
    227     return self._scheduler(fgraph)
--> 228 return fgraph.toposort()

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\fg.py:760, in FunctionGraph.toposort(self)
    756 if len(self.apply_nodes) < 2:
    757     # No sorting is necessary
    758     return list(self.apply_nodes)
--> 760 return io_toposort(self.inputs, self.outputs, self.orderings())

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\fg.py:784, in FunctionGraph.orderings(self)
    782 for feature in self._features:
    783     if hasattr(feature, "orderings"):
--> 784         orderings = feature.orderings(self)
    785         if not isinstance(orderings, dict):
    786             raise TypeError(
    787                 "Non-deterministic return value from "
    788                 + str(feature.orderings)
    789                 + ". Nondeterministic object is "
    790                 + str(orderings)
    791             )

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\destroyhandler.py:683, in DestroyHandler.orderings(self, fgraph, ordered)
    677 rval = {}
    679 if self.destroyers:
    680     # BUILD DATA STRUCTURES
    681     # CHECK for multiple destructions during construction of variables
--> 683     droot, impact, __ignore = self.refresh_droot_impact()
    685     # check for destruction of constants
    686     illegal_destroy = [
    687         r
    688         for r in droot
    689         if getattr(r.tag, "indestructible", False) or isinstance(r, Constant)
    690     ]

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\destroyhandler.py:431, in DestroyHandler.refresh_droot_impact(self)
    425 """
    426 Makes sure self.droot, self.impact, and self.root_destroyer are up to
    427 date, and returns them (see docstrings for these properties above).
    428 
    429 """
    430 if self.stale_droot:
--> 431     self.droot, self.impact, self.root_destroyer = _build_droot_impact(self)
    432     self.stale_droot = False
    433 return self.droot, self.impact, self.root_destroyer

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\destroyhandler.py:200, in _build_droot_impact(destroy_handler)
    197 input_root = r
    199 if input_root in droot:
--> 200     raise InconsistencyError(f"Multiple destroyers of {input_root}")
    201 droot[input_root] = input_root
    202 root_destroyer[input_root] = app

InconsistencyError: Multiple destroyers of CGemv{no_inplace}.0

PyMC version information:

PYMC 5.22.0
Pytensor 2.30.3
Winpython 3.13.3

Context for the issue:

No response

[ricardoV94]

I can reproduce, sounds like a PyTensor bug. Perhaps something breaks when the graph is too large during rewrites.

In the meantime just wanted to say that you probably want to think of a way you can vectorize your model, instead of defining separate variables / likelihood for each group. It should scale much better:

NOTE: UNTESTED CODE AND LOGIC, LIKELY MESSED SOMETHING UP IN THE REWRITE

import numpy as np
import pymc as pm
import pytensor.tensor as pt

# Simulated data of some spline
N = 500
np.random.seed(42)
x = np.linspace(0, 10, N)
true_knots = [3, 7]
y = np.piecewise(
    x,
    [x <= 3, (x > 3) & (x <= 7), x > 7],
    [lambda x: 0.5 * x, lambda x: 1.5 + 0.2 * (x - 3), lambda x: 2.3 - 0.1 * (x - 7)],
)
y += np.random.normal(0, 0.2, size=len(x))  # Add noise

# Artificial groups
group = np.random.choice([1, 2, 3], size=N)

# Try to fit another spline with knots at:
# many knots leads to InconsistencyError error
knots = np.linspace(0, 10, num=25)
# Few knots seems to work
# knots = [0, 1, 2, 7, 9]
n_knots = len(knots)
groups = np.unique(group).tolist()

with pm.Model() as model:
    sigma = pm.HalfCauchy("sigma", beta=1)
    sigma_beta0 = pm.HalfNormal("sigma_beta0", sigma=10)
    beta0 = pm.HalfNormal("beta_0", sigma=sigma_beta0, shape=len(groups))
    z = pm.Normal(f"z", mu=0, sigma=2, shape=(len(groups), n_knots))

    X = pt.maximum(0, x[:, None] - knots[None, :])  # (n, knots)
    delta_factors =  pt.special.softmax(z, axis=-1) # (groups, knots)
    slope_factors = 1 - pt.cumsum(delta_factors[:, :-1], axis=1) # (groups, knots-1)
    spline_slopes = pt.join(-1, beta0[:, None], beta0[:, None] * slope_factors) # (groups, knots-1)
    beta = pt.join(-1, beta0[:, None], pt.diff(spline_slopes, axis=-1))  # (groups, knots)

    for i, gr in enumerate(groups):
        # (n_i, knots) @ (knots) -> (n_i)
        mu_i = pt.matvec(X[group == gr], beta[i])  
        y_obs = pm.Normal(f"y_obs_{gr}", mu_i, sigma=sigma, observed=y[group == gr])
        
    # If you ve many groups this may be better
    # ((n, knots) * (n, knots)).sum(-1) = (n,)
    # mu = (X * beta[group]).sum(-1)
    # y_obs = pm.Normal("y_obs", mu=mu, sigma=sigma, observed=y)

model.compile_logp()(model.initial_point())

The logp at the initial point matches, which is the first sanity check I would do to make sure I didn't mess up.

[ricardoV94]

I'll move this issue to PyTensor. Thanks for reporting :)