From 5824e63bc9eae350fa0edd7230f4a44b71d2f1af Mon Sep 17 00:00:00 2001 From: ldeluigi <44567586+ldeluigi@users.noreply.github.com> Date: Sun, 5 Jan 2025 01:05:49 +0100 Subject: [PATCH] Improve logging for variant generator errors --- .../test_combo_graph.py | 28 ++++++++----------- backend/spellbook/variants/combo_graph.py | 16 +++-------- .../spellbook/variants/variants_generator.py | 14 ++++++---- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/backend/spellbook/tests/test_variants_generation/test_combo_graph.py b/backend/spellbook/tests/test_variants_generation/test_combo_graph.py index 76189841..f648528b 100644 --- a/backend/spellbook/tests/test_variants_generation/test_combo_graph.py +++ b/backend/spellbook/tests/test_variants_generation/test_combo_graph.py @@ -42,40 +42,34 @@ def test_combo_nodes(self): self.assertTrue(all(c.item.status in (Combo.Status.GENERATOR, Combo.Status.UTILITY) for c in combo_graph.bnodes.values())) def test_variant_limit(self): - combo_graph = Graph(Data(), log=lambda _: None, variant_limit=0) + combo_graph = Graph(Data(), variant_limit=0) with self.assertNumQueries(0): self.assertRaises(Graph.GraphError, lambda: combo_graph.variants(self.b2_id)) - combo_graph = Graph(Data(), log=lambda _: None, variant_limit=1) + combo_graph = Graph(Data(), variant_limit=1) with self.assertNumQueries(0): self.assertRaises(Graph.GraphError, lambda: combo_graph.variants(self.b2_id)) - combo_graph = Graph(Data(), log=lambda _: None, variant_limit=20) + combo_graph = Graph(Data(), variant_limit=20) with self.assertNumQueries(0): self.assertEqual(len(list(combo_graph.results(combo_graph.variants(self.b2_id)))), 3) - def test_default_log(self): - def test(): - combo_graph = Graph(Data(), variant_limit=0) - list(combo_graph.results(combo_graph.variants(self.b2_id))) - self.assertRaises(Exception, test) - def test_card_limit(self): self.maxDiff = None - combo_graph = Graph(Data(), log=lambda _: None, card_limit=0) + combo_graph = Graph(Data(), card_limit=0) with self.assertNumQueries(0): self.assertCountEqual(combo_graph.results(combo_graph.variants(self.b2_id)), []) - combo_graph = Graph(Data(), log=lambda _: None, card_limit=1) + combo_graph = Graph(Data(), card_limit=1) with self.assertNumQueries(0): self.assertCountEqual(combo_graph.results(combo_graph.variants(self.b2_id)), []) - combo_graph = Graph(Data(), log=lambda _: None, card_limit=2) + combo_graph = Graph(Data(), card_limit=2) with self.assertNumQueries(0): self.assertCountEqual(combo_graph.results(combo_graph.variants(self.b2_id)), []) - combo_graph = Graph(Data(), log=lambda _: None, card_limit=3) + combo_graph = Graph(Data(), card_limit=3) with self.assertNumQueries(0): self.assertEqual(len(list(combo_graph.results(combo_graph.variants(self.b2_id)))), 1) - combo_graph = Graph(Data(), log=lambda _: None, card_limit=4) + combo_graph = Graph(Data(), card_limit=4) with self.assertNumQueries(0): self.assertEqual(len(list(combo_graph.results(combo_graph.variants(self.b2_id)))), 2) - combo_graph = Graph(Data(), log=lambda _: None, card_limit=5) + combo_graph = Graph(Data(), card_limit=5) with self.assertNumQueries(0): self.assertEqual(len(list(combo_graph.results(combo_graph.variants(self.b2_id)))), 3) @@ -86,10 +80,10 @@ def test_allow_multiple_copies(self): assert card_needed is not None card_needed.quantity = 2 card_needed.save() - combo_graph = Graph(Data(), log=lambda _: None, allow_multiple_copies=False) + combo_graph = Graph(Data(), allow_multiple_copies=False) with self.assertNumQueries(0): self.assertEqual(len(list(combo_graph.results(combo_graph.variants(self.b2_id)))), 2) - combo_graph = Graph(Data(), log=lambda _: None, allow_multiple_copies=True) + combo_graph = Graph(Data(), allow_multiple_copies=True) with self.assertNumQueries(0): self.assertEqual(len(list(combo_graph.results(combo_graph.variants(self.b2_id)))), 3) diff --git a/backend/spellbook/variants/combo_graph.py b/backend/spellbook/variants/combo_graph.py index a24bce0f..5fb5187f 100644 --- a/backend/spellbook/variants/combo_graph.py +++ b/backend/spellbook/variants/combo_graph.py @@ -1,4 +1,4 @@ -from typing import Mapping, Iterable, Callable, Generic, TypeVar +from typing import Mapping, Iterable, Generic, TypeVar from math import prod from collections import deque, defaultdict from multiset import FrozenMultiset @@ -201,11 +201,9 @@ class GraphError(Exception): def __init__(self, data: Data, - log=None, card_limit=5, variant_limit=10000, allow_multiple_copies=False): - self.logger: Callable[[str], None] = log if log is not None else lambda msg: self._error(msg) self.card_limit = card_limit self.variant_limit = variant_limit self.allow_multiple_copies = allow_multiple_copies @@ -368,9 +366,7 @@ def _combo_nodes_down(self, combo: ComboNode) -> VariantSet: variant_set = self._feature_with_attribute_matchers_nodes_down(f) variant_count_estimate = len(variant_set) * q if variant_count_estimate > self.variant_limit: - msg = f'{q} x Feature "{f.item}" has too many variants, approx. {variant_count_estimate}' - self.logger(msg) - raise Graph.GraphError(msg) + raise Graph.GraphError(f'{q} x Feature "{f.item}" has too many variants, approx. {variant_count_estimate}') variant_set = variant_set ** q if self.filter is not None: variant_set = variant_set.filter(self.filter.cards, self.filter.templates) @@ -382,9 +378,7 @@ def _combo_nodes_down(self, combo: ComboNode) -> VariantSet: variant_sets: list[VariantSet] = card_variant_sets + template_variant_sets + needed_features_variant_sets variant_count_estimate = prod(len(vs) for vs in variant_sets) if variant_count_estimate > self.variant_limit: - msg = f'Combo {combo.item} has too many variants, approx. {variant_count_estimate}' - self.logger(msg) - raise Graph.GraphError(msg) + raise Graph.GraphError(f'Combo {combo.item} has too many variants, approx. {variant_count_estimate}') combo.variant_set = VariantSet.and_sets(variant_sets, limit=self.card_limit, allow_multiple_copies=self.allow_multiple_copies) combo.state = NodeState.VISITED return combo.variant_set @@ -421,9 +415,7 @@ def _feature_with_attributes_nodes_down(self, feature: FeatureWithAttributesNode variant_sets = card_variant_sets + produced_combos_variant_sets variant_count_estimate = sum(len(vs) for vs in variant_sets) if variant_count_estimate > self.variant_limit: - msg = f'Feature "{feature.item}" has too many variants, approx. {variant_count_estimate}' - self.logger(msg) - raise Graph.GraphError(msg) + raise Graph.GraphError(f'Feature "{feature.item}" has too many variants, approx. {variant_count_estimate}') feature.variant_set = VariantSet.or_sets(variant_sets, limit=self.card_limit, allow_multiple_copies=self.allow_multiple_copies) feature.state = NodeState.VISITED return feature.variant_set diff --git a/backend/spellbook/variants/variants_generator.py b/backend/spellbook/variants/variants_generator.py index ffe2f40f..af667452 100644 --- a/backend/spellbook/variants/variants_generator.py +++ b/backend/spellbook/variants/variants_generator.py @@ -49,7 +49,6 @@ def get_variants_from_graph(data: Data, single_combo: int | None, job: Job | Non variant_limit = LOWER_VARIANT_LIMIT graph = Graph( data, - log=lambda msg: log_into_job(job, msg), card_limit=card_limit, variant_limit=variant_limit, allow_multiple_copies=allows_multiple_copies, @@ -61,9 +60,9 @@ def get_variants_from_graph(data: Data, single_combo: int | None, job: Job | Non for combo in combos: try: variant_set = graph.variants(combo.id) - except Graph.GraphError as e: - log_into_job(job, f'Error while processing combo {combo.id}:') - raise e + except Graph.GraphError: + log_into_job(job, f'Error while computing all variants for generator combo {combo} with ID {combo.id}') + raise variant_sets.append((combo, variant_set)) if len(variant_set) > 50 or index % log_count == 0 or index == total - 1: log_into_job(job, f'{index + 1}/{total} combos processed (just processed combo {combo.id})') @@ -73,7 +72,12 @@ def get_variants_from_graph(data: Data, single_combo: int | None, job: Job | Non for combo, variant_set in variant_sets: if len(variant_set) > 50: log_into_job(job, f'About to process results for combo {combo.id} ({index + 1}/{total}) with {len(variant_set)} variants...') - for variant in graph.results(variant_set): + try: + variants = graph.results(variant_set) + except Graph.GraphError: + log_into_job(job, f'Error while computing all results for generator combo {combo} with ID {combo.id}') + raise + for variant in variants: cards_ids = variant.cards templates_ids = variant.templates id = id_from_cards_and_templates_ids(cards_ids.distinct_elements(), templates_ids.distinct_elements())