adding tests and cleanup for additional nutrition info/migratrion ser…

…vices

frontend/lang/messages/en-US.json

@@ -511,7 +511,6 @@
     "saturated-fat-content": "Saturated fat",
     "save-recipe-before-use": "Save recipe before use",
     "section-title": "Section Title",
-    "serving-size": "Serving size",
     "servings": "Servings",
     "share-recipe-message": "I wanted to share my {0} recipe with you.",
     "show-nutrition-values": "Show Nutrition Values",

mealie/services/migrations/myrecipebox.py

@@ -12,6 +12,18 @@
 from ._migration_base import BaseMigrator
 from .utils.migration_helpers import scrape_image, split_by_line_break, split_by_semicolon
 
+nutrition_map = {
+    "carbohydrate": "carbohydrateContent",
+    "protein": "proteinContent",
+    "fat": "fatContent",
+    "saturatedfat": "saturatedFatContent",
+    "transfat": "transFatContent",
+    "sodium": "sodiumContent",
+    "fiber": "fiberContent",
+    "sugar": "sugarContent",
+    "unsaturatedfat": "unsaturatedFatContent",
+}
+
 
 class MyRecipeBoxMigrator(BaseMigrator):
     def __init__(self, **kwargs):
@@ -53,22 +65,26 @@ def parse_time(self, time: Any) -> str | None:
         except Exception:
             return None
 
-    def parse_nutrition(self, input: Any) -> dict | None:
-        if not input or not isinstance(input, str):
+    def parse_nutrition(self, input_: Any) -> dict | None:
+        if not input_ or not isinstance(input_, str):
             return None
 
         nutrition = {}
 
-        vals = [x.strip() for x in input.split(",") if x]
+        vals = (x.strip() for x in input_.split("\n") if x)
         for val in vals:
             try:
-                key, value = val.split(":", maxsplit=1)
+                key, value = (x.strip() for x in val.split(":", maxsplit=1))
+
                 if not (key and value):
                     continue
+
+                key = nutrition_map.get(key.lower(), key)
+
             except ValueError:
                 continue
 
-            nutrition[key.strip()] = value.strip()
+            nutrition[key] = value
 
         return cleaner.clean_nutrition(nutrition) if nutrition else None
 

mealie/services/migrations/plantoeat.py

@@ -37,6 +37,19 @@ def get_value_as_string_or_none(dictionary: dict, key: str):
         return None
 
 
+nutrition_map = {
+    "Calories": "calories",
+    "Fat": "fatContent",
+    "Saturated Fat": "saturatedFatContent",
+    "Cholesterol": "cholesterolContent",
+    "Sodium": "sodiumContent",
+    "Sugar": "sugarContent",
+    "Carbohydrate": "carbohydrateContent",
+    "Fiber": "fiberContent",
+    "Protein": "proteinContent",
+}
+
+
 class PlanToEatMigrator(BaseMigrator):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -63,18 +76,7 @@ def __init__(self, **kwargs):
 
     def _parse_recipe_nutrition_from_row(self, row: dict) -> dict:
         """Parses the nutrition data from the row"""
-
-        nut_dict: dict = {}
-
-        nut_dict["calories"] = get_value_as_string_or_none(row, "Calories")
-        nut_dict["fatContent"] = get_value_as_string_or_none(row, "Fat")
-        nut_dict["proteinContent"] = get_value_as_string_or_none(row, "Protein")
-        nut_dict["carbohydrateContent"] = get_value_as_string_or_none(row, "Carbohydrate")
-        nut_dict["fiberContent"] = get_value_as_string_or_none(row, "Fiber")
-        nut_dict["sodiumContent"] = get_value_as_string_or_none(row, "Sodium")
-        nut_dict["sugarContent"] = get_value_as_string_or_none(row, "Sugar")
-
-        # FIXME: do we have the other schema.org values here to migrate?
+        nut_dict = {normalized_k: row[k] for k, normalized_k in nutrition_map.items() if k in row}
 
         return cleaner.clean_nutrition(nut_dict)
 

mealie/services/scraper/cleaner.py

@@ -495,7 +495,7 @@ def clean_nutrition(nutrition: dict | None) -> dict[str, str]:
     list of valid keys
 
     Assumptionas:
-        - All units are supplied in grams, expect sodium which maybe be in milligrams
+        - All units are supplied in grams, expect sodium and cholesterol which maybe be in milligrams
 
     Returns:
         dict[str, str]: If the argument is None, or not a dictionary, an empty dictionary is returned
@@ -509,9 +509,10 @@ def clean_nutrition(nutrition: dict | None) -> dict[str, str]:
             if matched_digits := MATCH_DIGITS.search(val):
                 output_nutrition[key] = matched_digits.group(0).replace(",", ".")
 
-    if sodium := nutrition.get("sodiumContent", None):
-        if isinstance(sodium, str) and "m" not in sodium and "g" in sodium:
-            with contextlib.suppress(AttributeError, TypeError):
-                output_nutrition["sodiumContent"] = str(float(output_nutrition["sodiumContent"]) * 1000)
+    for key in ["sodiumContent", "cholesterolContent"]:
+        if val := nutrition.get(key, None):
+            if isinstance(val, str) and "m" not in val and "g" in val:
+                with contextlib.suppress(AttributeError, TypeError):
+                    output_nutrition[key] = str(float(output_nutrition[key]) * 1000)
 
     return output_nutrition

tests/fixtures/fixture_users.py

@@ -173,8 +173,7 @@ def g2_user(session: Session, admin_token, api_client: TestClient):
         pass
 
 
-@fixture(scope="module")
-def unique_user(session: Session, api_client: TestClient):
+def _unique_user(session: Session, api_client: TestClient):
     registration = utils.user_registration_factory()
     response = api_client.post("/api/users/register", json=registration.model_dump(by_alias=True))
     assert response.status_code == 201
@@ -213,6 +212,16 @@ def unique_user(session: Session, api_client: TestClient):
         pass
 
 
+@fixture(scope="function")
+def unique_user_fn_scoped(session: Session, api_client: TestClient):
+    yield from _unique_user(session, api_client)
+
+
+@fixture(scope="module")
+def unique_user(session: Session, api_client: TestClient):
+    yield from _unique_user(session, api_client)
+
+
 @fixture(scope="module")
 def user_tuple(session: Session, admin_token, api_client: TestClient) -> Generator[list[utils.TestUser], None, None]:
     group_name = utils.random_string()

tests/integration_tests/recipe_migration_tests/test_recipe_migrations.py

@@ -1,5 +1,5 @@
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from zipfile import ZipFile
@@ -8,6 +8,7 @@
 from fastapi.testclient import TestClient
 
 from mealie.schema.group.group_migration import SupportedMigrations
+from mealie.schema.recipe.recipe import Recipe
 from mealie.schema.reports.reports import ReportEntryOut
 from tests import data as test_data
 from tests.utils import api_routes
@@ -19,18 +20,94 @@
 class MigrationTestData:
     typ: SupportedMigrations
     archive: Path
+    search_slug: str
+
+    nutrition_filter: set[str] = field(default_factory=set)
+    nutrition_entries: set[str] = field(
+        default_factory=lambda: {
+            "calories",
+            "carbohydrateContent",
+            "cholesterolContent",
+            "fatContent",
+            "fiberContent",
+            "proteinContent",
+            "saturatedFatContent",
+            "sodiumContent",
+            "sugarContent",
+            "transFatContent",
+            "unsaturatedFatContent",
+        }
+    )
 
 
 test_cases = [
-    MigrationTestData(typ=SupportedMigrations.nextcloud, archive=test_data.migrations_nextcloud),
-    MigrationTestData(typ=SupportedMigrations.paprika, archive=test_data.migrations_paprika),
-    MigrationTestData(typ=SupportedMigrations.chowdown, archive=test_data.migrations_chowdown),
-    MigrationTestData(typ=SupportedMigrations.copymethat, archive=test_data.migrations_copymethat),
-    MigrationTestData(typ=SupportedMigrations.mealie_alpha, archive=test_data.migrations_mealie),
-    MigrationTestData(typ=SupportedMigrations.tandoor, archive=test_data.migrations_tandoor),
-    MigrationTestData(typ=SupportedMigrations.plantoeat, archive=test_data.migrations_plantoeat),
-    MigrationTestData(typ=SupportedMigrations.myrecipebox, archive=test_data.migrations_myrecipebox),
-    MigrationTestData(typ=SupportedMigrations.recipekeeper, archive=test_data.migrations_recipekeeper),
+    MigrationTestData(
+        typ=SupportedMigrations.nextcloud,
+        archive=test_data.migrations_nextcloud,
+        search_slug="skillet-shepherd-s-pie",
+        nutrition_filter={
+            "transFatContent",
+            "unsaturatedFatContent",
+        },
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.paprika,
+        archive=test_data.migrations_paprika,
+        search_slug="zucchini-kartoffel-frittata",
+        nutrition_entries=set(),
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.chowdown,
+        archive=test_data.migrations_chowdown,
+        search_slug="roasted-okra",
+        nutrition_entries=set(),
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.copymethat,
+        archive=test_data.migrations_copymethat,
+        search_slug="spam-zoodles",
+        nutrition_entries=set(),
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.mealie_alpha,
+        archive=test_data.migrations_mealie,
+        search_slug="old-fashioned-beef-stew",
+        nutrition_filter={
+            "cholesterolContent",
+            "saturatedFatContent",
+            "transFatContent",
+            "unsaturatedFatContent",
+        },
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.tandoor,
+        archive=test_data.migrations_tandoor,
+        search_slug="texas-red-chili",
+        nutrition_entries=set(),
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.plantoeat,
+        archive=test_data.migrations_plantoeat,
+        search_slug="test-recipe",
+        nutrition_filter={
+            "unsaturatedFatContent",
+            "transFatContent",
+        },
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.myrecipebox,
+        archive=test_data.migrations_myrecipebox,
+        search_slug="beef-cheese-piroshki",
+        nutrition_filter={
+            "cholesterolContent",
+        },
+    ),
+    MigrationTestData(
+        typ=SupportedMigrations.recipekeeper,
+        archive=test_data.migrations_recipekeeper,
+        search_slug="zucchini-bread",
+        nutrition_entries=set(),
+    ),
 ]
 
 test_ids = [
@@ -47,7 +124,8 @@ class MigrationTestData:
 
 
 @pytest.mark.parametrize("mig", test_cases, ids=test_ids)
-def test_recipe_migration(api_client: TestClient, unique_user: TestUser, mig: MigrationTestData) -> None:
+def test_recipe_migration(api_client: TestClient, unique_user_fn_scoped: TestUser, mig: MigrationTestData) -> None:
+    unique_user = unique_user_fn_scoped
     payload = {
         "migration_type": mig.typ.value,
     }
@@ -91,6 +169,18 @@ def test_recipe_migration(api_client: TestClient, unique_user: TestUser, mig: Mi
     events = query_data["items"]
     assert len(events)
 
+    # Validate recipe content
+    response = api_client.get(api_routes.recipes_slug(mig.search_slug), headers=unique_user.token)
+    recipe = Recipe(**assert_deserialize(response))
+
+    if mig.nutrition_entries:
+        assert recipe.nutrition is not None
+        for k in mig.nutrition_entries.difference(mig.nutrition_filter):
+            nutrition = recipe.nutrition.model_dump(by_alias=True)
+            assert k in nutrition and nutrition[k] is not None
+
+    # TODO: validate other types of content
+
 
 def test_bad_mealie_alpha_data_is_ignored(api_client: TestClient, unique_user: TestUser):
     with TemporaryDirectory() as tmpdir:

tests/unit_tests/services_tests/scraper_tests/test_cleaner_parts.py

@@ -481,20 +481,24 @@ def test_cleaner_clean_tags(case: CleanerCase):
         },
     ),
     CleanerCase(
-        test_id="special support for sodiumContent (g -> mg)",
+        test_id="special support for sodiumContent/cholesterolContent (g -> mg)",
         input={
+            "cholesterolContent": "10g",
             "sodiumContent": "10g",
         },
         expected={
+            "cholesterolContent": "10000.0",
             "sodiumContent": "10000.0",
         },
     ),
     CleanerCase(
-        test_id="special support for sodiumContent (mg -> mg)",
+        test_id="special support for sodiumContent/cholesterolContent (mg -> mg)",
         input={
+            "cholesterolContent": "10000mg",
             "sodiumContent": "10000mg",
         },
         expected={
+            "cholesterolContent": "10000",
             "sodiumContent": "10000",
         },
     ),

tests/unit_tests/test_recipe_parser.py

@@ -23,6 +23,12 @@ async def test_recipe_parser(recipe_test_data: RecipeSiteTestCase):
     recipe, _ = await scraper.create_from_html(recipe_test_data.url, translator)
 
     assert recipe.slug == recipe_test_data.expected_slug
+
     assert len(recipe.recipe_instructions or []) == recipe_test_data.num_steps
+
     assert len(recipe.recipe_ingredient) == recipe_test_data.num_ingredients
+
+    actual = recipe.nutrition.model_dump() if recipe.nutrition else {}
+    assert recipe_test_data.num_nutrition_entries == len(actual.items())
+
     assert recipe.org_url == recipe_test_data.url

tests/utils/recipe_data.py

@@ -13,6 +13,7 @@ class RecipeSiteTestCase:
     num_steps: int
     html_file: Path
 
+    num_nutrition_entries: int = 0
     include_tags: bool = False
     expected_tags: set[str] | None = None
 
@@ -26,6 +27,7 @@ def get_recipe_test_cases():
             expected_slug="taiwanese-three-cup-chicken-san-bei-ji-recipe",
             num_ingredients=10,
             num_steps=3,
+            num_nutrition_entries=11,
         ),
         RecipeSiteTestCase(
             url="https://www.rezeptwelt.de/backen-herzhaft-rezepte/schinken-kaese-waffeln-ohne-viel-schnickschnack/4j0bkiig-94d4d-106529-cfcd2-is97x2ml",