Merge branch 'main' into fix-translations

Conflicts:
	src/viur/core/i18n.py

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 This file documents any relevant changes done to ViUR-core since version 3.
 
+## [3.7.2]
+
+- feat: Provide `add_or_edit` root-only endpoint for importers (#1380)
+- fix: Provide bone name with assertion message (#1375)
+- fix: `errors` not marked as a reserved word (#1374)
+- doc: Fix `SyntaxWarning: invalid escape sequence '\*'` (#1372)
+- fix: `SkelModule` not able to handle empty index definitions (#1373)
+- fix: Render bones which are `readOnly=True` not as `required=True` (#1371)
+- feat: Provide default `index`-function for `Tree` and `Singleton` (#1365)
+- fix: `FileLeafSkel._inject_serving_url()` is the better choice (#1362)
+
 ## [3.7.1]
 
 - fix: `RelationalBone`: dict size change during iteration (#1359)
@@ -101,6 +112,17 @@ This file documents any relevant changes done to ViUR-core since version 3.
 - refactor: Send emails from `EmailTransport` instances instead of class (#1250)
 - refactor: Sub-class `Session` from `db.Entity` to behave `dict`-compliant (#1153)
 
+## [3.6.32]
+
+- feat: Backport request preflight checks for 3.6 (#1383)
+
+## [3.6.31]
+
+- fix: a `not caseSensitive` bone should lock the lower value (#1378)
+- fix: skip `cached_property` in `Module._update_methods` (#1377)
+- fix: determine a better path for a new `TranslateSkel` (#1367)
+- fix: Ensure derives are generated in `FileBone` inside a `RecordBone` too (#1370)
+
 ## [3.6.30]
 
 - fix: `SelectBone.singleValueFromClient` doesn't accept `Enum` (#1320, #1351)

src/viur/core/bones/base.py

@@ -9,11 +9,12 @@
 import hashlib
 import inspect
 import logging
-from dataclasses import dataclass, field
-from datetime import datetime, timedelta
+import typing as t
 from collections.abc import Iterable
+from dataclasses import dataclass, field
+from datetime import timedelta
 from enum import Enum
-import typing as t
+
 from viur.core import db, utils, current, i18n
 from viur.core.config import conf
 
@@ -1147,24 +1148,28 @@ def buildDBSort(
 
         return query
 
-    def _hashValueForUniquePropertyIndex(self, value: str | int) -> list[str]:
+    def _hashValueForUniquePropertyIndex(
+        self,
+        value: str | int | float | db.Key | list[str | int | float | db.Key],
+    ) -> list[str]:
         """
         Generates a hash of the given value for creating unique property indexes.
 
         This method is called by the framework to create a consistent hash representation of a value
         for constructing unique property indexes. Derived bone classes should overwrite this method to
         implement their own logic for hashing values.
 
-        :param value: The value to be hashed, which can be a string, integer, or a float.
+        :param value: The value(s) to be hashed.
 
         :return: A list containing a string representation of the hashed value. If the bone is multiple,
                 the list may contain more than one hashed value.
         """
-        def hashValue(value: str | int) -> str:
+
+        def hashValue(value: str | int | float | db.Key) -> str:
             h = hashlib.sha256()
             h.update(str(value).encode("UTF-8"))
             res = h.hexdigest()
-            if isinstance(value, int) or isinstance(value, float):
+            if isinstance(value, int | float):
                 return f"I-{res}"
             elif isinstance(value, str):
                 return f"S-{res}"
@@ -1181,9 +1186,9 @@ def keyHash(key):
 
         if not value and not self.unique.lockEmpty:
             return []  # We are zero/empty string and these should not be locked
-        if not self.multiple:
+        if not self.multiple and not isinstance(value, list):
             return [hashValue(value)]
-        # We have an multiple bone here
+        # We have a multiple bone or multiple values here
         if not isinstance(value, list):
             value = [value]
         tmpList = [hashValue(x) for x in value]
@@ -1227,13 +1232,13 @@ def performMagic(self, valuesCache: dict, name: str, isAdd: bool):
         """
         pass  # We do nothing by default
 
-    def postSavedHandler(self, skel: 'viur.core.skeleton.SkeletonInstance', boneName: str, key: str):
+    def postSavedHandler(self, skel: "SkeletonInstance", boneName: str, key: db.Key | None) -> None:
         """
             Can be overridden to perform further actions after the main entity has been written.
 
             :param boneName: Name of this bone
             :param skel: The skeleton this bone belongs to
-            :param key: The (new?) Database Key we've written to
+            :param key: The (new?) Database Key we've written to. In case of a RelSkel the key is None.
         """
         pass
 
@@ -1299,7 +1304,7 @@ def setBoneValue(self,
         the value is valid. If the value is invalid, no modification occurs. The function supports appending values to
         bones with multiple=True and setting or appending language-specific values for bones that support languages.
         """
-        assert not (bool(self.languages) ^ bool(language)), "Language is required or not supported"
+        assert not (bool(self.languages) ^ bool(language)), f"language is required or not supported on {boneName!r}"
         assert not append or self.multiple, "Can't append - bone is not multiple"
 
         if not append and self.multiple:
@@ -1447,14 +1452,14 @@ def structure(self) -> dict:
         ret = {
             "descr": self.descr,
             "type": self.type,
-            "required": self.required,
+            "required": self.required and not self.readOnly,
             "params": self.params,
             "visible": self.visible,
             "readonly": self.readOnly,
             "unique": self.unique.method.value if self.unique else False,
             "languages": self.languages,
             "emptyvalue": self.getEmptyValue(),
-            "indexed": self.indexed
+            "indexed": self.indexed,
         }
 
         # Provide a defaultvalue, if it's not a function.
@@ -1470,6 +1475,8 @@ def structure(self) -> dict:
             }
         else:
             ret["multiple"] = self.multiple
+
+        # Provide compute information
         if self.compute:
             ret["compute"] = {
                 "method": self.compute.interval.method.name

src/viur/core/bones/file.py

@@ -226,12 +226,20 @@ def postSavedHandler(self, skel, boneName, key):
         the derived files directly.
         """
         super().postSavedHandler(skel, boneName, key)
+        from viur.core.skeleton import RelSkel, Skeleton
+
+        if issubclass(skel.skeletonCls, Skeleton):
+            prefix = f"{skel.kindName}_{boneName}"
+        elif issubclass(skel.skeletonCls, RelSkel):  # RelSkel is just a container and has no kindname
+            prefix = f"{skel.skeletonCls.__name__}_{boneName}"
+        else:
+            raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}")
 
         def handleDerives(values):
             if isinstance(values, dict):
                 values = [values]
             for val in (values or ()):  # Ensure derives getting build for each file referenced in this relation
-                ensureDerived(val["dest"]["key"], f"{skel.kindName}_{boneName}", self.derive)
+                ensureDerived(val["dest"]["key"], prefix, self.derive)
 
         values = skel[boneName]
         if self.derive and values:

src/viur/core/bones/record.py

@@ -103,6 +103,22 @@ def singleValueFromClient(self, value, skel, bone_name, client_data):
             )
         return usingSkel, usingSkel.errors
 
+    def postSavedHandler(self, skel, boneName, key) -> None:
+        super().postSavedHandler(skel, boneName, key)
+        for idx, lang, value in self.iter_bone_value(skel, boneName):
+            using = self.using()
+            using.unserialize(value)
+            for bone_name, bone in using.items():
+                bone.postSavedHandler(using, bone_name, None)
+
+    def refresh(self, skel, boneName) -> None:
+        super().refresh(skel, boneName)
+        for idx, lang, value in self.iter_bone_value(skel, boneName):
+            using = self.using()
+            using.unserialize(value)
+            for bone_name, bone in using.items():
+                bone.refresh(using, bone_name)
+
     def getSearchTags(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
         """
         Collects search tags from the 'using' skeleton instance for the given bone.

src/viur/core/bones/relational.py

@@ -475,7 +475,7 @@ def _get_single_destinct_hash(self, value):
 
         return tuple(parts)
 
-    def postSavedHandler(self, skel: "SkeletonInstance", boneName: str, key: db.Key) -> None:
+    def postSavedHandler(self, skel, boneName, key) -> None:
         """
         Handle relational updates after a skeleton is saved.
 
@@ -486,6 +486,8 @@ def postSavedHandler(self, skel: "SkeletonInstance", boneName: str, key: db.Key)
         :param boneName: The name of the relational bone.
         :param key: The key of the saved skeleton instance.
         """
+        if key is None:  # RelSkel container (e.g. RecordBone) has no key, it's covered by it's parent
+            return
         if not skel[boneName]:
             values = []
         elif self.multiple and self.languages:

src/viur/core/bones/string.py

@@ -309,6 +309,13 @@ def getUniquePropertyIndexValues(self, skel: "SkeletonInstance", name: str) -> l
             # Not yet implemented as it's unclear if we should keep each language distinct or not
             raise NotImplementedError()
 
+        if not self.caseSensitive and (value := skel[name]) is not None:
+            if self.multiple:
+                value = [v.lower() for v in value]
+            else:
+                value = value.lower()
+            return self._hashValueForUniquePropertyIndex(value)
+
         return super().getUniquePropertyIndexValues(skel, name)
 
     def refresh(self, skel: "SkeletonInstance", bone_name: str) -> None:

src/viur/core/i18n.py

@@ -77,10 +77,13 @@
 import fnmatch
 import jinja2.ext as jinja2
 import logging
+import sys
 import traceback
 import typing as t
 from pathlib import Path
 
+import jinja2.ext as jinja2
+
 from viur.core import current, db, languages, tasks
 from viur.core.config import conf
 
@@ -158,6 +161,8 @@ class translate:
         "translationCache",
         "force_lang",
         "public",
+        "filename",
+        "lineno",
         "add_missing",
     )
 
@@ -169,6 +174,7 @@ def __init__(
         force_lang: str = None,
         public: bool = False,
         add_missing: bool = False,
+        caller_is_jinja: bool = False,
     ):
         """
         :param key: The unique key defining this text fragment.
@@ -181,24 +187,43 @@ def __init__(
             target language.
         :param force_lang: Use this language instead the one of the request.
         :param public: Flag for public translations, which can be obtained via /json/_translate/get_public.
+        :param caller_is_jinja: Is the call caused by our jinja method?
         """
         super().__init__()
 
         if not isinstance(key, str):
             logging.warning(f"Got non-string (type {type(key)}) as {key=}!", exc_info=True)
 
+        if force_lang is not None and force_lang not in conf.i18n.available_dialects:
+            raise ValueError(f"The language {force_lang=} is not available")
+
         key = str(key)  # ensure key is a str
         self.key = key.lower()
         self.defaultText = defaultText or key
         self.hint = hint
-
         self.translationCache = None
-        if force_lang is not None and force_lang not in conf.i18n.available_dialects:
-            raise ValueError(f"The language {force_lang=} is not available")
-
         self.force_lang = force_lang
         self.public = public
         self.add_missing = add_missing
+        self.filename, self.lineno = None, None
+
+        if self.key not in systemTranslations and conf.i18n.add_missing_translations:
+            # This translation seems to be new and should be added
+            for frame, line in traceback.walk_stack(sys._getframe(0).f_back):
+                if self.filename is None:
+                    # Use the first frame as fallback.
+                    # In case of calling this class directly,
+                    # this is anyway the caller we're looking for.
+                    self.filename = frame.f_code.co_filename
+                    self.lineno = frame.f_lineno
+                    if not caller_is_jinja:
+                        break
+                if caller_is_jinja and not frame.f_code.co_filename.endswith(".py"):
+                    # Look for the latest html, macro (not py) where the
+                    # translate method has been used, that's our caller
+                    self.filename = frame.f_code.co_filename
+                    self.lineno = line
+                    break
 
     def __repr__(self) -> str:
         return f"<translate object for {self.key} with force_lang={self.force_lang}>"
@@ -207,7 +232,6 @@ def __str__(self) -> str:
         if self.translationCache is None:
             global systemTranslations
 
-            from viur.core.render.html.env.viur import translate as jinja_translate
 
             if self.key not in systemTranslations:
                 # either the translate()-object has add_missing set
@@ -225,31 +249,12 @@ def __str__(self) -> str:
 
                 if add_missing:
                     # This translation seems to be new and should be added
-                    filename = lineno = None
-                    is_jinja = False
-                    for frame, line in traceback.walk_stack(None):
-                        if filename is None:
-                            # Use the first frame as fallback.
-                            # In case of calling this class directly,
-                            # this is anyway the caller we're looking for.
-                            filename = frame.f_code.co_filename
-                            lineno = frame.f_lineno
-                        if frame.f_code == jinja_translate.__code__:
-                            # The call was caused by our jinja method
-                            is_jinja = True
-                        if is_jinja and not frame.f_code.co_filename.endswith(".py"):
-                            # Look for the latest html, macro (not py) where the
-                            # translate method has been used, that's our caller
-                            filename = frame.f_code.co_filename
-                            lineno = line
-                            break
-
                     add_missing_translation(
                         key=self.key,
                         hint=self.hint,
                         default_text=self.defaultText,
-                        filename=filename,
-                        lineno=lineno,
+                        filename=self.filename,
+                        lineno=self.lineno,
                         public=self.public,
                     )
 

src/viur/core/module.py

@@ -1,5 +1,6 @@
 import copy
 import enum
+import functools
 import inspect
 import types
 import typing as t
@@ -540,7 +541,7 @@ def _update_methods(self):
         for key in dir(self):
             if key[0] == "_":
                 continue
-            if isinstance(getattr(self.__class__, key, None), property):
+            if isinstance(getattr(self.__class__, key, None), (property, functools.cached_property)):
                 continue
 
             prop = getattr(self, key)