BUG: InconsistencyError Multiple destroyers of CGemv{no_inplace}.0

[mvds314]

Describe the issue:

I get the following error when setting up a hierarchical model for a Bayesian spline:

Multiple destroyers of CGemv{no_inplace}.0

I tried to simplify to the most simple setup I could find

I am completely clueless, as the error only seems to happen when I increase the number of knots to a somewhat larger number.

Reproduceable code example:

import numpy as np
import pymc as pm
import pytensor.tensor as pt

# Simulated data of some spline
N = 500
np.random.seed(42)
x = np.linspace(0, 10, N)
true_knots = [3, 7]
y = np.piecewise(
    x,
    [x <= 3, (x > 3) & (x <= 7), x > 7],
    [lambda x: 0.5 * x, lambda x: 1.5 + 0.2 * (x - 3), lambda x: 2.3 - 0.1 * (x - 7)],
)
y += np.random.normal(0, 0.2, size=len(x))  # Add noise

# Artificial groups
group = np.random.choice([1, 2, 3], size=N)

# Try to fit another spline with knots at:
# many knots leads to InconsistencyError error
knots = np.linspace(0, 10, num=50)
# Few knots seems to work
# knots = [0, 1, 2, 7, 9]
n_knots = len(knots)
groups = np.unique(group).tolist()

with pm.Model() as hinge_model_flex:
    # Hyperprior for beta0
    sigma_beta0 = pm.HalfNormal("sigma_beta0", sigma=10)

    # Global priors
    sigma = pm.HalfCauchy("sigma", beta=1)

    # Create likelihood per group
    for gr in groups:
        idx = group == gr

        # Define hinge basis using scan
        def hinge_term(knot):
            return pt.maximum(0, x[idx] - knot)

        hinge_terms = [hinge_term(knot) for knot in knots]

        # beta for initial slope
        beta0 = pm.HalfNormal(f"beta0_{gr}", sigma=sigma_beta0)

        z = pm.Normal(f"z_{gr}", mu=0, sigma=2, shape=n_knots)
        delta_factors = pm.Deterministic(f"delta_factors_{gr}", pt.special.softmax(z))
        slope_factors = 1 - pm.Deterministic(f"beta_factors_{gr}", pt.cumsum(delta_factors[:-1]))
        spline_slopes = pm.Deterministic(
            f"spline_slopes_{gr}",
            pt.stack([beta0] + [beta0 * slope_factors[i] for i in range(n_knots - 1)]),
        )
        beta = pm.Deterministic(f"beta_{gr}", pt.concatenate(([beta0], pt.diff(spline_slopes))))

        # Combine basis terms
        X = pt.stack([hinge_terms[i] for i in range(n_knots)], axis=1)

        mu = pt.dot(X, beta)

        # Likelihood
        y_obs = pm.Normal(f"y_obs_{gr}", mu=mu, sigma=sigma, observed=y[idx])

    # Sampling
    map_res = pm.find_MAP()

Error message:

---------------------------------------------------------------------------
InconsistencyError                        Traceback (most recent call last)
File c:\....\untitled14.py:73
     70     y_obs = pm.Normal(f"y_obs_{gr}", mu=mu, sigma=sigma, observed=y[idx])
     72 # Sampling
---> 73 map_res = pm.find_MAP()

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\tuning\starting.py:151, in find_MAP(start, vars, method, return_raw, include_transformed, progressbar, progressbar_theme, maxeval, model, seed, *args, **kwargs)
    146 rvs = [model.values_to_rvs[vars_dict[name]] for name, _, _, _ in x0.point_map_info]
    147 try:
    148     # This might be needed for calls to `dlogp_func`
    149     # start_map_info = tuple((v.name, v.shape, v.dtype) for v in vars)
    150     compiled_dlogp_func = DictToArrayBijection.mapf(
--> 151         model.compile_dlogp(rvs, jacobian=False), start
    152     )
    153     dlogp_func = lambda x: compiled_dlogp_func(RaveledVars(x, x0.point_map_info))  # noqa: E731
    154     compute_gradient = True

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\model\core.py:620, in Model.compile_dlogp(self, vars, jacobian, **compile_kwargs)
    604 def compile_dlogp(
    605     self,
    606     vars: Variable | Sequence[Variable] | None = None,
    607     jacobian: bool = True,
    608     **compile_kwargs,
    609 ) -> PointFunc:
    610     """Compiled log probability density gradient function.
    611 
    612     Parameters
   (...)
    618         Whether to include jacobian terms in logprob graph. Defaults to True.
    619     """
--> 620     return self.compile_fn(self.dlogp(vars=vars, jacobian=jacobian), **compile_kwargs)

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\model\core.py:1649, in Model.compile_fn(self, outs, inputs, mode, point_fn, **kwargs)
   1646     inputs = inputvars(outs)
   1648 with self:
-> 1649     fn = compile(
   1650         inputs,
   1651         outs,
   1652         allow_input_downcast=True,
   1653         accept_inplace=True,
   1654         mode=mode,
   1655         **kwargs,
   1656     )
   1658 if point_fn:
   1659     return PointFunc(fn)

File C:\...\WPy64-31330\python\Lib\site-packages\pymc\pytensorf.py:947, in compile(inputs, outputs, random_seed, mode, **kwargs)
    945 opt_qry = mode.provided_optimizer.including("random_make_inplace", check_parameter_opt)
    946 mode = Mode(linker=mode.linker, optimizer=opt_qry)
--> 947 pytensor_function = pytensor.function(
    948     inputs,
    949     outputs,
    950     updates={**rng_updates, **kwargs.pop("updates", {})},
    951     mode=mode,
    952     **kwargs,
    953 )
    954 return pytensor_function

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\__init__.py:332, in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, trust_input)
    321     fn = orig_function(
    322         inputs,
    323         outputs,
   (...)
    327         trust_input=trust_input,
    328     )
    329 else:
    330     # note: pfunc will also call orig_function -- orig_function is
    331     #      a choke point that all compilation must pass through
--> 332     fn = pfunc(
    333         params=inputs,
    334         outputs=outputs,
    335         mode=mode,
    336         updates=updates,
    337         givens=givens,
    338         no_default_updates=no_default_updates,
    339         accept_inplace=accept_inplace,
    340         name=name,
    341         rebuild_strict=rebuild_strict,
    342         allow_input_downcast=allow_input_downcast,
    343         on_unused_input=on_unused_input,
    344         profile=profile,
    345         output_keys=output_keys,
    346         trust_input=trust_input,
    347     )
    348 return fn

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\pfunc.py:466, in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys, fgraph, trust_input)
    452     profile = ProfileStats(message=profile)
    454 inputs, cloned_outputs = construct_pfunc_ins_and_outs(
    455     params,
    456     outputs,
   (...)
    463     fgraph=fgraph,
    464 )
--> 466 return orig_function(
    467     inputs,
    468     cloned_outputs,
    469     mode,
    470     accept_inplace=accept_inplace,
    471     name=name,
    472     profile=profile,
    473     on_unused_input=on_unused_input,
    474     output_keys=output_keys,
    475     fgraph=fgraph,
    476     trust_input=trust_input,
    477 )

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\types.py:1833, in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys, fgraph, trust_input)
   1820     m = Maker(
   1821         inputs,
   1822         outputs,
   (...)
   1830         trust_input=trust_input,
   1831     )
   1832     with config.change_flags(compute_test_value="off"):
-> 1833         fn = m.create(defaults)
   1834 finally:
   1835     if profile and fn:

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\compile\function\types.py:1717, in FunctionMaker.create(self, input_storage, storage_map)
   1714 start_import_time = pytensor.link.c.cmodule.import_time
   1716 with config.change_flags(traceback__limit=config.traceback__compile_limit):
-> 1717     _fn, _i, _o = self.linker.make_thunk(
   1718         input_storage=input_storage_lists, storage_map=storage_map
   1719     )
   1721 end_linker = time.perf_counter()
   1723 linker_time = end_linker - start_linker

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\link\basic.py:245, in LocalLinker.make_thunk(self, input_storage, output_storage, storage_map, **kwargs)
    238 def make_thunk(
    239     self,
    240     input_storage: Optional["InputStorageType"] = None,
   (...)
    243     **kwargs,
    244 ) -> tuple["BasicThunkType", "InputStorageType", "OutputStorageType"]:
--> 245     return self.make_all(
    246         input_storage=input_storage,
    247         output_storage=output_storage,
    248         storage_map=storage_map,
    249     )[:3]

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\link\vm.py:1207, in VMLinker.make_all(self, profiler, input_storage, output_storage, storage_map)
   1199 def make_all(
   1200     self,
   1201     profiler=None,
   (...)
   1204     storage_map=None,
   1205 ):
   1206     fgraph = self.fgraph
-> 1207     order = self.schedule(fgraph)
   1209     input_storage, output_storage, storage_map = map_storage(
   1210         fgraph, order, input_storage, output_storage, storage_map
   1211     )
   1212     compute_map = {}

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\link\basic.py:228, in Linker.schedule(self, fgraph)
    226 if callable(self._scheduler):
    227     return self._scheduler(fgraph)
--> 228 return fgraph.toposort()

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\fg.py:760, in FunctionGraph.toposort(self)
    756 if len(self.apply_nodes) < 2:
    757     # No sorting is necessary
    758     return list(self.apply_nodes)
--> 760 return io_toposort(self.inputs, self.outputs, self.orderings())

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\fg.py:784, in FunctionGraph.orderings(self)
    782 for feature in self._features:
    783     if hasattr(feature, "orderings"):
--> 784         orderings = feature.orderings(self)
    785         if not isinstance(orderings, dict):
    786             raise TypeError(
    787                 "Non-deterministic return value from "
    788                 + str(feature.orderings)
    789                 + ". Nondeterministic object is "
    790                 + str(orderings)
    791             )

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\destroyhandler.py:683, in DestroyHandler.orderings(self, fgraph, ordered)
    677 rval = {}
    679 if self.destroyers:
    680     # BUILD DATA STRUCTURES
    681     # CHECK for multiple destructions during construction of variables
--> 683     droot, impact, __ignore = self.refresh_droot_impact()
    685     # check for destruction of constants
    686     illegal_destroy = [
    687         r
    688         for r in droot
    689         if getattr(r.tag, "indestructible", False) or isinstance(r, Constant)
    690     ]

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\destroyhandler.py:431, in DestroyHandler.refresh_droot_impact(self)
    425 """
    426 Makes sure self.droot, self.impact, and self.root_destroyer are up to
    427 date, and returns them (see docstrings for these properties above).
    428 
    429 """
    430 if self.stale_droot:
--> 431     self.droot, self.impact, self.root_destroyer = _build_droot_impact(self)
    432     self.stale_droot = False
    433 return self.droot, self.impact, self.root_destroyer

File C:\...\WPy64-31330\python\Lib\site-packages\pytensor\graph\destroyhandler.py:200, in _build_droot_impact(destroy_handler)
    197 input_root = r
    199 if input_root in droot:
--> 200     raise InconsistencyError(f"Multiple destroyers of {input_root}")
    201 droot[input_root] = input_root
    202 root_destroyer[input_root] = app

InconsistencyError: Multiple destroyers of CGemv{no_inplace}.0

PyMC version information:

PYMC 5.22.0
Pytensor 2.30.3
Winpython 3.13.3

Context for the issue:

No response

[ricardoV94]

I can reproduce, sounds like a PyTensor bug. Perhaps something breaks when the graph is too large during rewrites.

In the meantime just wanted to say that you probably want to think of a way you can vectorize your model, instead of defining separate variables / likelihood for each group. It should scale much better:

NOTE: UNTESTED CODE AND LOGIC, LIKELY MESSED SOMETHING UP IN THE REWRITE

import numpy as np
import pymc as pm
import pytensor.tensor as pt

# Simulated data of some spline
N = 500
np.random.seed(42)
x = np.linspace(0, 10, N)
true_knots = [3, 7]
y = np.piecewise(
    x,
    [x <= 3, (x > 3) & (x <= 7), x > 7],
    [lambda x: 0.5 * x, lambda x: 1.5 + 0.2 * (x - 3), lambda x: 2.3 - 0.1 * (x - 7)],
)
y += np.random.normal(0, 0.2, size=len(x))  # Add noise

# Artificial groups
group = np.random.choice([1, 2, 3], size=N)

# Try to fit another spline with knots at:
# many knots leads to InconsistencyError error
knots = np.linspace(0, 10, num=25)
# Few knots seems to work
# knots = [0, 1, 2, 7, 9]
n_knots = len(knots)
groups = np.unique(group).tolist()

with pm.Model() as model:
    sigma = pm.HalfCauchy("sigma", beta=1)
    sigma_beta0 = pm.HalfNormal("sigma_beta0", sigma=10)
    beta0 = pm.HalfNormal("beta_0", sigma=sigma_beta0, shape=len(groups))
    z = pm.Normal(f"z", mu=0, sigma=2, shape=(len(groups), n_knots))

    X = pt.maximum(0, x[:, None] - knots[None, :])  # (n, knots)
    delta_factors =  pt.special.softmax(z, axis=-1) # (groups, knots)
    slope_factors = 1 - pt.cumsum(delta_factors[:, :-1], axis=1) # (groups, knots-1)
    spline_slopes = pt.join(-1, beta0[:, None], beta0[:, None] * slope_factors) # (groups, knots-1)
    beta = pt.join(-1, beta0[:, None], pt.diff(spline_slopes, axis=-1))  # (groups, knots)

    for i, gr in enumerate(groups):
        # (n_i, knots) @ (knots) -> (n_i)
        mu_i = pt.matvec(X[group == gr], beta[i])  
        y_obs = pm.Normal(f"y_obs_{gr}", mu_i, sigma=sigma, observed=y[group == gr])
        
    # If you ve many groups this may be better
    # ((n, knots) * (n, knots)).sum(-1) = (n,)
    # mu = (X * beta[group]).sum(-1)
    # y_obs = pm.Normal("y_obs", mu=mu, sigma=sigma, observed=y)

model.compile_logp()(model.initial_point())

The logp at the initial point matches, which is the first sanity check I would do to make sure I didn't mess up.

[ricardoV94]

I'll move this issue to PyTensor. Thanks for reporting :)

[velochy]

I am also hitting this error (it is still present on pytensor 2.31.3), and also only for bigger models, so in my case it is hard to actually abstract a reasonably readable minimum example.

Any progress on this? Or any pointers on what might be causing it so I could work around it?

In our case, more vectorization is not really a solution as we are fitting a model with different shaped inputs and outputs at each step - but it already shows up with 7 categorical inputs and 2 categorical outputs with ~1000 observations so we are not talking huge models in absolute.

[velochy]

There is some good news on this bug, it seems.
It shows up starting a minor version 2.30.3 i.e. 2.30.2 does not yet throw this error.
Considering it is a minor version bump, I'm hoping that considerably narrows down the list of potential culprits.
Also - currently, 2.30.2 is not that long ago to roll back to.

[ricardoV94]

Can you set pytensor.config.optimizer_verbose=True just before the relevant function is compiled and share the output?

[velochy]

Sure. It is very verbose, however, so I'm attaching them as files.

pt_ex.log is the log from the reproducing example above
pt.log is the log from my own use case

[ricardoV94]

Here is a direct pytensor test. It's still too convoluted, because it seems to depend on the scale of the graph / order of inputs:

import pytensor.tensor as pt
from pytensor.graph import rewrite_graph
from pytensor import function

n_knots = 20
sigma_beta, beta_0, beta_1, beta_2 = pt.scalars("sigma_beta", "beta_0", "beta_1", "beta_2")
z_0, z_1, z_2 = (pt.vector(name, shape=(n_knots,)) for name in ("z_0", "z_1", "z_2"))

cost = 0
spline_slopes = pt.stack([beta_0 * z_0[i] for i in range(n_knots)])
beta = pt.concatenate(([0], pt.diff(spline_slopes)))
cost += (beta_0 / sigma_beta) ** 2 + beta.sum() ** 2

spline_slopes = pt.stack([beta_1 * z_1[i] for i in range(n_knots)])
beta = pt.concatenate(([0], pt.diff(spline_slopes)))
cost += (beta_1 / sigma_beta) ** 2 + beta.sum() ** 2

spline_slopes = pt.stack([beta_2 * z_2[i] for i in range(n_knots)])
beta = pt.concatenate(([0], pt.diff(spline_slopes)))
cost += (beta_2 / sigma_beta) ** 2 + z_2.sum() ** 2 + beta.sum() ** 2

cost = rewrite_graph(cost, include=("canonicalize",))

all_vars = [sigma_beta, beta_0, z_0, beta_1, z_1, beta_2, z_2]
grad_vars = [beta_0, z_0, beta_1, z_1, beta_2, z_2, sigma_beta]
grad = [pt.atleast_1d(pt.grad(cost, v)) for v in grad_vars]
fn = function(all_vars, grad)

Some rewrites call fgraph.has_destroyers to check if an input is destroyed and that triggers an InconsistencyError that should have been caught earlier. I'm don't think this is related to 2.30.3 but that this PR made the error more common, because it also calls has_destroyers: a0abe86

I can trigger the error before that PR as well

[ricardoV94]

I think this is the culprit:

pytensor/pytensor/tensor/rewriting/elemwise.py

Lines 124 to 131 in ff09268

	check_each_change = config.tensor__insert_inplace_optimizer_validate_nb
	if check_each_change == -1:
	if len(fgraph.apply_nodes) > 500:
	check_each_change = 10
	else:
	check_each_change = 1
	nb_change_no_validate = 0

[ricardoV94]

@velochy can you check if setting the pytensor config flag tensor__insert_inplace_optimizer_validate_nb=1 solves the error on your end?

[velochy]

It does :)