Merge branch 'develop'

README.md

@@ -71,7 +71,7 @@ Because of that there are several ways you can support us
 - **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).
 
 ## Contributing
-Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/#contributing-code) **BEFORE** contributing anything!
+Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/guidelines/) **BEFORE** contributing anything!
 
 ## Your Feedback
 

cookbook/connectors/connector_manager.py

@@ -5,6 +5,7 @@
 from asyncio import Task
 from dataclasses import dataclass
 from enum import Enum
+from logging import Logger
 from types import UnionType
 from typing import List, Any, Dict, Optional, Type
 
@@ -39,10 +40,12 @@ class Work:
 # 4. Work is marked as consumed, and next entry of the queue is consumed.
 # Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
 class ConnectorManager:
+    _logger: Logger
     _queue: queue.Queue
     _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig
 
     def __init__(self):
+        self._logger = logging.getLogger("recipes.connector")
         self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
         self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
         self._worker.start()
@@ -65,7 +68,7 @@ def __call__(self, instance: Any, **kwargs) -> None:
         try:
             self._queue.put_nowait(Work(instance, action_type))
         except queue.Full:
-            logging.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+            self._logger.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
             return
 
     def stop(self):
@@ -74,10 +77,12 @@ def stop(self):
 
     @staticmethod
     def worker(worker_id: int, worker_queue: queue.Queue):
+        logger = logging.getLogger("recipes.connector.worker")
+
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
 
-        logging.info(f"started ConnectionManager worker {worker_id}")
+        logger.info(f"started ConnectionManager worker {worker_id}")
 
         # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
         _connectors_cache: Dict[int, List[Connector]] = dict()
@@ -91,6 +96,8 @@ def worker(worker_id: int, worker_queue: queue.Queue):
             if item is None:
                 break
 
+            logger.debug(f"received {item.instance=} with {item.actionType=}")
+
             # If a Connector was changed/updated, refresh connector from the database for said space
             refresh_connector_cache = isinstance(item.instance, ConnectorConfig)
 
@@ -111,7 +118,7 @@ def worker(worker_id: int, worker_queue: queue.Queue):
                         try:
                             connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
                         except BaseException:
-                            logging.exception(f"failed to initialize {config.name}")
+                            logger.exception(f"failed to initialize {config.name}")
                             continue
 
                         if connector is not None:
@@ -123,10 +130,12 @@ def worker(worker_id: int, worker_queue: queue.Queue):
                 worker_queue.task_done()
                 continue
 
+            logger.debug(f"running {len(connectors)} connectors for {item.instance=} with {item.actionType=}")
+
             loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType))
             worker_queue.task_done()
 
-        logging.info(f"terminating ConnectionManager worker {worker_id}")
+        logger.info(f"terminating ConnectionManager worker {worker_id}")
 
         asyncio.set_event_loop(None)
         loop.close()

cookbook/connectors/homeassistant.py

@@ -17,10 +17,11 @@ def __init__(self, config: ConnectorConfig):
         if not config.token or not config.url or not config.todo_entity:
             raise ValueError("config for HomeAssistantConnector in incomplete")
 
+        self._logger = logging.getLogger(f"recipes.connector.homeassistant.{config.name}")
+
         if config.url[-1] != "/":
             config.url += "/"
         self._config = config
-        self._logger = logging.getLogger("connector.HomeAssistant")
 
     async def homeassistant_api_call(self, method: str, path: str, data: Dict) -> str:
         headers = {
@@ -37,7 +38,7 @@ async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry
 
         item, description = _format_shopping_list_entry(shopping_list_entry)
 
-        logging.debug(f"adding {item=} to {self._config.name}")
+        self._logger.debug(f"adding {item=}")
 
         data = {
             "entity_id": self._config.todo_entity,
@@ -48,7 +49,7 @@ async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry
         try:
             await self.homeassistant_api_call("POST", "services/todo/add_item", data)
         except ClientError as err:
-            self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+            self._logger.warning(f"received an exception from the api: {err=}, {type(err)=}")
 
     async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
         if not self._config.on_shopping_list_entry_updated_enabled:
@@ -66,7 +67,7 @@ async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry
 
         item, _ = _format_shopping_list_entry(shopping_list_entry)
 
-        logging.debug(f"removing {item=} from {self._config.name}")
+        self._logger.debug(f"removing {item=}")
 
         data = {
             "entity_id": self._config.todo_entity,
@@ -77,7 +78,7 @@ async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry
             await self.homeassistant_api_call("POST", "services/todo/remove_item", data)
         except ClientError as err:
             # This error will always trigger if the item is not present/found
-            self._logger.debug(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+            self._logger.debug(f"received an exception from the api: {err=}, {type(err)=}")
 
     async def close(self) -> None:
         pass

cookbook/helper/HelperFunctions.py

@@ -1,4 +1,12 @@
+import socket
+from urllib.parse import urlparse
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 from django.db.models import Func
+from ipaddress import ip_address
+
+from recipes import settings
 
 
 class Round(Func):
@@ -11,3 +19,30 @@ def str2bool(v):
         return v
     else:
         return v.lower() in ("yes", "true", "1")
+
+
+"""
+validates an url that is supposed to be imported
+checks that the protocol used is http(s) and that no local address is accessed
+@:param url to test
+@:return true if url is valid, false otherwise
+"""
+
+
+def validate_import_url(url):
+    try:
+        validator = URLValidator(schemes=['http', 'https'])
+        validator(url)
+    except ValidationError:
+        # if schema is not http or https, consider url invalid
+        return False
+
+    # resolve IP address of url
+    try:
+        url_ip_address = ip_address(str(socket.gethostbyname(urlparse(url).hostname)))
+    except (ValueError, AttributeError, TypeError, Exception) as e:
+        # if ip cannot be parsed, consider url invalid
+        return False
+
+    # validate that IP is neither private nor any other special address
+    return not any([url_ip_address.is_private, url_ip_address.is_reserved, url_ip_address.is_loopback,  url_ip_address.is_multicast, url_ip_address.is_link_local, ])

cookbook/helper/unit_conversion_helper.py

@@ -20,6 +20,7 @@
         'gallon': 0.264172,
         'tbsp': 67.628,
         'tsp': 202.884,
+        'us_cup': 4.22675,
         'imperial_fluid_ounce': 35.1951,
         'imperial_pint': 1.75975,
         'imperial_quart': 0.879877,

cookbook/integration/cookbookapp.py

@@ -2,12 +2,12 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
                                                iso_duration_to_minutes)
-from cookbook.helper.scrapers.scrapers import text_scraper
+from recipe_scrapers import scrape_html
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Recipe, Step
 
@@ -20,7 +20,7 @@ def import_file_name_filter(self, zip_info_object):
     def get_recipe_from_file(self, file):
         recipe_html = file.getvalue().decode("utf-8")
 
-        scrape = text_scraper(text=recipe_html)
+        scrape = scrape_html(html=recipe_html, org_url="https://cookbookapp.import", supported_only=False)
         recipe_json = get_from_scraper(scrape, self.request)
         images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
 
@@ -63,7 +63,7 @@ def get_recipe_from_file(self, file):
         if len(images) > 0:
             try:
                 url = images[0]
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/cookmate.py

@@ -1,8 +1,8 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
@@ -69,7 +69,7 @@ def get_recipe_from_file(self, file):
         if recipe_xml.find('imageurl') is not None:
             try:
                 url = recipe_xml.find('imageurl').text.strip()
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                 self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/paprika.py

@@ -6,8 +6,8 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
@@ -87,7 +87,7 @@ def get_recipe_from_file(self, file):
             try:
                 if recipe_json.get("image_url", None):
                     url = recipe_json.get("image_url", None)
-                    if validators.url(url, public=True):
+                    if validate_import_url(url):
                         response = requests.get(url)
                         self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception:

cookbook/integration/plantoeat.py

@@ -1,8 +1,8 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
@@ -75,7 +75,7 @@ def get_recipe_from_file(self, file):
 
         if image_url:
             try:
-                if validators.url(image_url, public=True):
+                if validate_import_url(image_url):
                     response = requests.get(image_url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e: