From f502aac8dcc743f221f3c5ae131d8722135c349f Mon Sep 17 00:00:00 2001 From: Matheus Felipe Date: Thu, 1 Dec 2022 01:31:40 -0300 Subject: [PATCH 01/39] Add link to MarkupSafe in FAQ --- docs/faq.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 493dc38c6..a53ae12ff 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -70,6 +70,8 @@ these document types. While automatic escaping means that you are less likely have an XSS problem, it also requires significant extra processing during compiling -and rendering, which can reduce performance. Jinja uses MarkupSafe for +and rendering, which can reduce performance. Jinja uses `MarkupSafe`_ for escaping, which provides optimized C code for speed, but it still introduces overhead to track escaping across methods and formatting. + +.. _MarkupSafe: https://markupsafe.palletsprojects.com/ From 4e7850ce1b26f3108c79e9f59f641f6dcf4d4d6a Mon Sep 17 00:00:00 2001 From: Clay Sweetser Date: Thu, 2 Mar 2023 16:53:14 -0500 Subject: [PATCH 02/39] Clarify what operations the default Undefined supports --- src/jinja2/runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index 53582ae8b..d10fe9d06 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -792,8 +792,8 @@ def __repr__(self) -> str: class Undefined: - """The default undefined type. This undefined type can be printed and - iterated over, but every other access will raise an :exc:`UndefinedError`: + """The default undefined type. This can be printed, iterated, and treated as + a boolean. Any other operation will raise an :exc:`UndefinedError`. >>> foo = Undefined(name='foo') >>> str(foo) From 9c3622c1af8e8e2342c6e7d44de513e2a6c1c441 Mon Sep 17 00:00:00 2001 From: Hugo Vassard Date: Fri, 3 Mar 2023 11:42:07 +0100 Subject: [PATCH 03/39] fix boolean error about whitespace control --- docs/templates.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index 2471cea39..2cb1d7a48 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -202,10 +202,11 @@ option can also be set to strip tabs and spaces from the beginning of a line to the start of a block. (Nothing will be stripped if there are other characters before the start of the block.) -With both `trim_blocks` and `lstrip_blocks` enabled, you can put block tags -on their own lines, and the entire block line will be removed when -rendered, preserving the whitespace of the contents. For example, -without the `trim_blocks` and `lstrip_blocks` options, this template:: +With both ``trim_blocks`` and ``lstrip_blocks`` disabled (the default), block +tags on their own lines will be removed, but a blank line will remain and the +spaces in the content will be preserved. For example, this template: + +.. code-block:: jinja
{% if True %} @@ -213,7 +214,10 @@ without the `trim_blocks` and `lstrip_blocks` options, this template:: {% endif %}
-gets rendered with blank lines inside the div:: +With both ``trim_blocks`` and ``lstrip_blocks`` disabled, the template is +rendered with blank lines inside the div: + +.. code-block:: text
@@ -221,8 +225,10 @@ gets rendered with blank lines inside the div::
-But with both `trim_blocks` and `lstrip_blocks` enabled, the template block -lines are removed and other whitespace is preserved:: +With both ``trim_blocks`` and ``lstrip_blocks`` enabled, the template block +lines are completely removed: + +.. code-block:: text
yay From 8a90b760a8cb3cff9cd9fe8a7b899405b044244b Mon Sep 17 00:00:00 2001 From: Meng Xiangzhuo Date: Tue, 29 Aug 2023 13:19:59 +0800 Subject: [PATCH 04/39] fix a typo in docs/templates.rst --- docs/templates.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templates.rst b/docs/templates.rst index 2cb1d7a48..aff7e172c 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1784,7 +1784,7 @@ It's possible to translate strings in expressions with these functions: - ``_(message)``: Alias for ``gettext``. - ``gettext(message)``: Translate a message. -- ``ngettext(singluar, plural, n)``: Translate a singular or plural +- ``ngettext(singular, plural, n)``: Translate a singular or plural message based on a count variable. - ``pgettext(context, message)``: Like ``gettext()``, but picks the translation based on the context string. From 7d023e5a8600d0db1e6d11f5434737dbcccfcab6 Mon Sep 17 00:00:00 2001 From: Vitor Buxbaum Date: Fri, 17 Nov 2023 09:04:42 -0300 Subject: [PATCH 05/39] Fix typo on filter name --- docs/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index cb62f6c32..c0fa163a0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -666,8 +666,8 @@ Now it can be used in templates: .. sourcecode:: jinja - {{ article.pub_date|datetimeformat }} - {{ article.pub_date|datetimeformat("%B %Y") }} + {{ article.pub_date|datetime_format }} + {{ article.pub_date|datetime_format("%B %Y") }} Some decorators are available to tell Jinja to pass extra information to the filter. The object is passed as the first argument, making the value From 64a6bd1b66fdaa11aa21ac238f40c02c1e0074ca Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Fri, 16 Feb 2024 01:39:44 -0600 Subject: [PATCH 06/39] improve clarity of logical bool ops co-authored-by: David Lord --- docs/templates.rst | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index aff7e172c..2bb28f610 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1412,28 +1412,32 @@ Comparisons Logic ~~~~~ -For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be useful to -combine multiple expressions: +For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be +useful to combine multiple expressions. ``and`` - Return true if the left and the right operand are true. + For ``x and y``, if ``x`` is false, then the value is ``x``, else ``y``. In + a boolean context, this will be treated as ``True`` if both operands are + truthy. ``or`` - Return true if the left or the right operand are true. + For ``x or y``, if ``x`` is true, then the value is ``x``, else ``y``. In a + boolean context, this will be treated as ``True`` if at least one operand is + truthy. ``not`` - negate a statement (see below). + For ``not x``, if ``x`` is false, then the value is ``True``, else + ``False``. -``(expr)`` - Parentheses group an expression. - -.. admonition:: Note - - The ``is`` and ``in`` operators support negation using an infix notation, - too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar`` - and ``not foo in bar``. All other expressions require a prefix notation: + Prefer negating ``is`` and ``in`` using their infix notation: + ``foo is not bar`` instead of ``not foo is bar``; ``foo not in bar`` instead + of ``not foo in bar``. All other expressions require prefix notation: ``not (foo and bar).`` +``(expr)`` + Parentheses group an expression. This is used to change evaluation order, or + to make a long expression easier to read or less ambiguous. + Other Operators ~~~~~~~~~~~~~~~ From 75f0fbf6cb47b531a9c277139a505b9315864330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Lindh=C3=A9?= <7773090+lindhe@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:54:47 +0200 Subject: [PATCH 07/39] fix list comprehension example --- src/jinja2/filters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index 14208770d..4c949cbde 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -1629,8 +1629,8 @@ def sync_do_selectattr( .. code-block:: python - (u for user in users if user.is_active) - (u for user in users if test_none(user.email)) + (user for user in users if user.is_active) + (user for user in users if test_none(user.email)) .. versionadded:: 2.7 """ @@ -1667,8 +1667,8 @@ def sync_do_rejectattr( .. code-block:: python - (u for user in users if not user.is_active) - (u for user in users if not test_none(user.email)) + (user for user in users if not user.is_active) + (user for user in users if not test_none(user.email)) .. versionadded:: 2.7 """ From c667d56de3d0129971175d75f0389d5fef48d0e3 Mon Sep 17 00:00:00 2001 From: David Lord Date: Wed, 18 Dec 2024 09:33:31 -0800 Subject: [PATCH 08/39] change "per default" to "by default" --- CHANGES.rst | 2 +- docs/templates.rst | 2 +- docs/tricks.rst | 2 +- src/jinja2/loaders.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7fb729763..feb1e6c3d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1001,7 +1001,7 @@ Released 2008-07-17, codename Jinjavitus evaluates to ``false``. - Improved error reporting for undefined values by providing a position. -- ``filesizeformat`` filter uses decimal prefixes now per default and +- ``filesizeformat`` filter uses decimal prefixes now by default and can be set to binary mode with the second parameter. - Fixed bug in finalizer diff --git a/docs/templates.rst b/docs/templates.rst index 2bb28f610..0eab8e664 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -528,7 +528,7 @@ However, the name after the `endblock` word must match the block name. Block Nesting and Scope ~~~~~~~~~~~~~~~~~~~~~~~ -Blocks can be nested for more complex layouts. However, per default blocks +Blocks can be nested for more complex layouts. However, by default blocks may not access variables from outer scopes:: {% for item in seq %} diff --git a/docs/tricks.rst b/docs/tricks.rst index b58c5bb09..3a7084a6d 100644 --- a/docs/tricks.rst +++ b/docs/tricks.rst @@ -21,7 +21,7 @@ for a neat trick. Usually child templates extend from one template that adds a basic HTML skeleton. However it's possible to put the `extends` tag into an `if` tag to only extend from the layout template if the `standalone` variable evaluates -to false which it does per default if it's not defined. Additionally a very +to false, which it does by default if it's not defined. Additionally a very basic skeleton is added to the file so that if it's indeed rendered with `standalone` set to `True` a very basic HTML skeleton is added:: diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 8c2c86cd0..b1ad401c0 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -430,7 +430,7 @@ class DictLoader(BaseLoader): >>> loader = DictLoader({'index.html': 'source here'}) - Because auto reloading is rarely useful this is disabled per default. + Because auto reloading is rarely useful this is disabled by default. """ def __init__(self, mapping: t.Mapping[str, str]) -> None: From 786d12b529a2ecf1e2fb586a619a79e2650a6d4d Mon Sep 17 00:00:00 2001 From: David Lord Date: Wed, 18 Dec 2024 09:36:11 -0800 Subject: [PATCH 09/39] clarify block outer scope docs --- docs/templates.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index 0eab8e664..d5f2719e0 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -528,8 +528,8 @@ However, the name after the `endblock` word must match the block name. Block Nesting and Scope ~~~~~~~~~~~~~~~~~~~~~~~ -Blocks can be nested for more complex layouts. However, by default blocks -may not access variables from outer scopes:: +Blocks can be nested for more complex layouts. By default, a block may not +access variables from outside the block (outer scopes):: {% for item in seq %}
  • {% block loop_item %}{{ item }}{% endblock %}
    • From 0c0a3d02d1b103120d41f04e3e8a0974504a244c Mon Sep 17 00:00:00 2001 From: JamesParrott <80779630+JamesParrott@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:03:27 +0000 Subject: [PATCH 10/39] fix Jinja syntax in example --- examples/basic/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/basic/test.py b/examples/basic/test.py index 7a58e1ad4..30f5dd6b3 100644 --- a/examples/basic/test.py +++ b/examples/basic/test.py @@ -6,9 +6,9 @@ { "child.html": """\ {% extends default_layout or 'default.html' %} -{% include helpers = 'helpers.html' %} +{% import 'helpers.html' as helpers %} {% macro get_the_answer() %}42{% endmacro %} -{% title = 'Hello World' %} +{% set title = 'Hello World' %} {% block body %} {{ get_the_answer() }} {{ helpers.conspirate() }} From 955d7daf3d602cba6b3e9afb6fd425fd1577d15f Mon Sep 17 00:00:00 2001 From: Charles-Axel Dein <120501+charlax@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:03:16 +0200 Subject: [PATCH 11/39] Simplify example for ModuleLoader The `ModuleLoader` example seems copy pasted from `ChoiceLoader`. As a result it's not immediately clear how their API differ. --- src/jinja2/loaders.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index b1ad401c0..d2373e5e7 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -613,10 +613,7 @@ class ModuleLoader(BaseLoader): Example usage: - >>> loader = ChoiceLoader([ - ... ModuleLoader('/path/to/compiled/templates'), - ... FileSystemLoader('/path/to/templates') - ... ]) + >>> loader = ModuleLoader('/path/to/compiled/templates') Templates can be precompiled with :meth:`Environment.compile_templates`. """ From d3a0b1a4abac05a071f542aa83edea4551c88fa3 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 9 Aug 2022 10:12:27 +0200 Subject: [PATCH 12/39] use env.concat when calling block reference --- CHANGES.rst | 2 ++ src/jinja2/runtime.py | 6 ++++-- tests/test_nativetypes.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index feb1e6c3d..540d5cccb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,8 @@ Unreleased ``Template.generate_async``. :pr:`1960` - Avoid leaving async generators unclosed in blocks, includes and extends. :pr:`1960` +- The runtime uses the correct ``concat`` function for the current environment + when calling block references. :issue:`1701` Version 3.1.4 diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index d10fe9d06..c2c7c1937 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -367,7 +367,7 @@ def super(self) -> t.Union["BlockReference", "Undefined"]: @internalcode async def _async_call(self) -> str: - rv = concat( + rv = self._context.environment.concat( # type: ignore [x async for x in self._stack[self._depth](self._context)] # type: ignore ) @@ -381,7 +381,9 @@ def __call__(self) -> str: if self._context.environment.is_async: return self._async_call() # type: ignore - rv = concat(self._stack[self._depth](self._context)) + rv = self._context.environment.concat( # type: ignore + self._stack[self._depth](self._context) + ) if self._context.eval_ctx.autoescape: return Markup(rv) diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py index 8c8525251..136908180 100644 --- a/tests/test_nativetypes.py +++ b/tests/test_nativetypes.py @@ -160,3 +160,13 @@ def test_macro(env): result = t.render() assert result == 2 assert isinstance(result, int) + + +def test_block(env): + t = env.from_string( + "{% block b %}{% for i in range(1) %}{{ loop.index }}{% endfor %}" + "{% endblock %}{{ self.b() }}" + ) + result = t.render() + assert result == 11 + assert isinstance(result, int) From 76af7110ead6083e75072f9e76242c9ce79b76ed Mon Sep 17 00:00:00 2001 From: Mehdi ABAAKOUK Date: Fri, 23 Dec 2022 09:46:29 +0100 Subject: [PATCH 13/39] make unique filter async-aware --- CHANGES.rst | 2 ++ src/jinja2/filters.py | 14 +++++++++++++- tests/test_async_filters.py | 7 +++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 540d5cccb..e4bffbfb9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,8 @@ Unreleased :pr:`1960` - The runtime uses the correct ``concat`` function for the current environment when calling block references. :issue:`1701` +- Make ``|unique`` async-aware, allowing it to be used after another + async-aware filter. :issue:`1781` Version 3.1.4 diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index 4c949cbde..af9f6bc0e 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -438,7 +438,7 @@ def do_sort( @pass_environment -def do_unique( +def sync_do_unique( environment: "Environment", value: "t.Iterable[V]", case_sensitive: bool = False, @@ -470,6 +470,18 @@ def do_unique( yield item +@async_variant(sync_do_unique) # type: ignore +async def do_unique( + environment: "Environment", + value: "t.Union[t.AsyncIterable[V], t.Iterable[V]]", + case_sensitive: bool = False, + attribute: t.Optional[t.Union[str, int]] = None, +) -> "t.Iterator[V]": + return sync_do_unique( + environment, await auto_to_list(value), case_sensitive, attribute + ) + + def _min_or_max( environment: "Environment", value: "t.Iterable[V]", diff --git a/tests/test_async_filters.py b/tests/test_async_filters.py index e8cc350d5..e9892f1ed 100644 --- a/tests/test_async_filters.py +++ b/tests/test_async_filters.py @@ -277,6 +277,13 @@ def test_slice(env_async, items): ) +def test_unique_with_async_gen(env_async): + items = ["a", "b", "c", "c", "a", "d", "z"] + tmpl = env_async.from_string("{{ items|reject('==', 'z')|unique|list }}") + out = tmpl.render(items=items) + assert out == "['a', 'b', 'c', 'd']" + + def test_custom_async_filter(env_async, run_async_fn): async def customfilter(val): return str(val) From 2eb4542cbab101910d220229a8f09c94f76c19d1 Mon Sep 17 00:00:00 2001 From: Felipe Moreno Date: Mon, 20 May 2024 11:02:20 -0400 Subject: [PATCH 14/39] int filter handles OverflowError to handle scientific notation --- CHANGES.rst | 2 ++ src/jinja2/filters.py | 2 +- tests/test_filters.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e4bffbfb9..6c998e626 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,8 @@ Unreleased when calling block references. :issue:`1701` - Make ``|unique`` async-aware, allowing it to be used after another async-aware filter. :issue:`1781` +- ``|int`` filter handles ``OverflowError`` from scientific notation. + :issue:`1921` Version 3.1.4 diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index af9f6bc0e..a92832a34 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -999,7 +999,7 @@ def do_int(value: t.Any, default: int = 0, base: int = 10) -> int: # this quirk is necessary so that "42.23"|int gives 42. try: return int(float(value)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): return default diff --git a/tests/test_filters.py b/tests/test_filters.py index d8e9114d0..2cb53ac9d 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -196,6 +196,7 @@ def test_indent_width_string(self, env): ("abc", "0"), ("32.32", "32"), ("12345678901234567890", "12345678901234567890"), + ("1e10000", "0"), ), ) def test_int(self, env, value, expect): From 4936e4d48207c449f7713392e1ae83a1407aec1d Mon Sep 17 00:00:00 2001 From: Anentropic Date: Tue, 3 Sep 2024 22:16:33 +0100 Subject: [PATCH 15/39] make tuple unpacking deterministic in compiler --- CHANGES.rst | 2 ++ src/jinja2/compiler.py | 4 +-- tests/test_compile.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6c998e626..aebb38b58 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,8 @@ Unreleased async-aware filter. :issue:`1781` - ``|int`` filter handles ``OverflowError`` from scientific notation. :issue:`1921` +- Make compiling deterministic for tuple unpacking in a ``{% set ... %}`` + call. :issue:`2021` Version 3.1.4 diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 91720c5f9..074e9b187 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -811,7 +811,7 @@ def pop_assign_tracking(self, frame: Frame) -> None: self.writeline("_block_vars.update({") else: self.writeline("context.vars.update({") - for idx, name in enumerate(vars): + for idx, name in enumerate(sorted(vars)): if idx: self.write(", ") ref = frame.symbols.ref(name) @@ -821,7 +821,7 @@ def pop_assign_tracking(self, frame: Frame) -> None: if len(public_names) == 1: self.writeline(f"context.exported_vars.add({public_names[0]!r})") else: - names_str = ", ".join(map(repr, public_names)) + names_str = ", ".join(map(repr, sorted(public_names))) self.writeline(f"context.exported_vars.update(({names_str}))") # -- Statement Visitors diff --git a/tests/test_compile.py b/tests/test_compile.py index 42a773f21..42efa59c0 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -26,3 +26,64 @@ def test_import_as_with_context_deterministic(tmp_path): expect = [f"'bar{i}': " for i in range(10)] found = re.findall(r"'bar\d': ", content)[:10] assert found == expect + + +def test_top_level_set_vars_unpacking_deterministic(tmp_path): + src = "\n".join(f"{{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10)) + env = Environment(loader=DictLoader({"foo": src})) + env.compile_templates(tmp_path, zip=None) + name = os.listdir(tmp_path)[0] + content = (tmp_path / name).read_text("utf8") + expect = [ + f"context.vars.update({{'a{i}': l_0_a{i}, 'b{i}': l_0_b{i}, 'c{i}': l_0_c{i}}})" + for i in range(10) + ] + found = re.findall( + r"context\.vars\.update\(\{'a\d': l_0_a\d, 'b\d': l_0_b\d, 'c\d': l_0_c\d\}\)", + content, + )[:10] + assert found == expect + expect = [ + f"context.exported_vars.update(('a{i}', 'b{i}', 'c{i}'))" for i in range(10) + ] + found = re.findall( + r"context\.exported_vars\.update\(\('a\d', 'b\d', 'c\d'\)\)", + content, + )[:10] + assert found == expect + + +def test_loop_set_vars_unpacking_deterministic(tmp_path): + src = "\n".join(f" {{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10)) + src = f"{{% for i in seq %}}\n{src}\n{{% endfor %}}" + env = Environment(loader=DictLoader({"foo": src})) + env.compile_templates(tmp_path, zip=None) + name = os.listdir(tmp_path)[0] + content = (tmp_path / name).read_text("utf8") + expect = [ + f"_loop_vars.update({{'a{i}': l_1_a{i}, 'b{i}': l_1_b{i}, 'c{i}': l_1_c{i}}})" + for i in range(10) + ] + found = re.findall( + r"_loop_vars\.update\(\{'a\d': l_1_a\d, 'b\d': l_1_b\d, 'c\d': l_1_c\d\}\)", + content, + )[:10] + assert found == expect + + +def test_block_set_vars_unpacking_deterministic(tmp_path): + src = "\n".join(f" {{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10)) + src = f"{{% block test %}}\n{src}\n{{% endblock test %}}" + env = Environment(loader=DictLoader({"foo": src})) + env.compile_templates(tmp_path, zip=None) + name = os.listdir(tmp_path)[0] + content = (tmp_path / name).read_text("utf8") + expect = [ + f"_block_vars.update({{'a{i}': l_0_a{i}, 'b{i}': l_0_b{i}, 'c{i}': l_0_c{i}}})" + for i in range(10) + ] + found = re.findall( + r"_block_vars\.update\(\{'a\d': l_0_a\d, 'b\d': l_0_b\d, 'c\d': l_0_c\d\}\)", + content, + )[:10] + assert found == expect From d4fb0e8c401a8596be0e67622174b5ad35daeec2 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 1 Oct 2024 12:20:19 -0700 Subject: [PATCH 16/39] preserve `__slots__` on Undefined classes --- CHANGES.rst | 2 ++ src/jinja2/runtime.py | 30 +++++++++++++++----------- tests/test_api.py | 8 ------- tests/test_runtime.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aebb38b58..58dc03214 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,8 @@ Unreleased :issue:`1921` - Make compiling deterministic for tuple unpacking in a ``{% set ... %}`` call. :issue:`2021` +- Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined`` + objects. :issue:`2025` Version 3.1.4 diff --git a/src/jinja2/runtime.py b/src/jinja2/runtime.py index c2c7c1937..09119e2ae 100644 --- a/src/jinja2/runtime.py +++ b/src/jinja2/runtime.py @@ -860,7 +860,11 @@ def _fail_with_undefined_error( @internalcode def __getattr__(self, name: str) -> t.Any: - if name[:2] == "__": + # Raise AttributeError on requests for names that appear to be unimplemented + # dunder methods to keep Python's internal protocol probing behaviors working + # properly in cases where another exception type could cause unexpected or + # difficult-to-diagnose failures. + if name[:2] == "__" and name[-2:] == "__": raise AttributeError(name) return self._fail_with_undefined_error() @@ -984,10 +988,20 @@ class ChainableUndefined(Undefined): def __html__(self) -> str: return str(self) - def __getattr__(self, _: str) -> "ChainableUndefined": + def __getattr__(self, name: str) -> "ChainableUndefined": + # Raise AttributeError on requests for names that appear to be unimplemented + # dunder methods to avoid confusing Python with truthy non-method objects that + # do not implement the protocol being probed for. e.g., copy.copy(Undefined()) + # fails spectacularly if getattr(Undefined(), '__setstate__') returns an + # Undefined object instead of raising AttributeError to signal that it does not + # support that style of object initialization. + if name[:2] == "__" and name[-2:] == "__": + raise AttributeError(name) + return self - __getitem__ = __getattr__ # type: ignore + def __getitem__(self, _name: str) -> "ChainableUndefined": # type: ignore[override] + return self class DebugUndefined(Undefined): @@ -1046,13 +1060,3 @@ class StrictUndefined(Undefined): __iter__ = __str__ = __len__ = Undefined._fail_with_undefined_error __eq__ = __ne__ = __bool__ = __hash__ = Undefined._fail_with_undefined_error __contains__ = Undefined._fail_with_undefined_error - - -# Remove slots attributes, after the metaclass is applied they are -# unneeded and contain wrong data for subclasses. -del ( - Undefined.__slots__, - ChainableUndefined.__slots__, - DebugUndefined.__slots__, - StrictUndefined.__slots__, -) diff --git a/tests/test_api.py b/tests/test_api.py index ff3fcb138..ee11a8d69 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -323,8 +323,6 @@ def test_default_undefined(self): assert und1 == und2 assert und1 != 42 assert hash(und1) == hash(und2) == hash(Undefined()) - with pytest.raises(AttributeError): - getattr(Undefined, "__slots__") # noqa: B009 def test_chainable_undefined(self): env = Environment(undefined=ChainableUndefined) @@ -335,8 +333,6 @@ def test_chainable_undefined(self): assert env.from_string("{{ foo.missing }}").render(foo=42) == "" assert env.from_string("{{ not missing }}").render() == "True" pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render) - with pytest.raises(AttributeError): - getattr(ChainableUndefined, "__slots__") # noqa: B009 # The following tests ensure subclass functionality works as expected assert env.from_string('{{ missing.bar["baz"] }}').render() == "" @@ -368,8 +364,6 @@ def test_debug_undefined(self): str(DebugUndefined(hint=undefined_hint)) == f"{{{{ undefined value printed: {undefined_hint} }}}}" ) - with pytest.raises(AttributeError): - getattr(DebugUndefined, "__slots__") # noqa: B009 def test_strict_undefined(self): env = Environment(undefined=StrictUndefined) @@ -386,8 +380,6 @@ def test_strict_undefined(self): env.from_string('{{ missing|default("default", true) }}').render() == "default" ) - with pytest.raises(AttributeError): - getattr(StrictUndefined, "__slots__") # noqa: B009 assert env.from_string('{{ "foo" if false }}').render() == "" def test_indexing_gives_undefined(self): diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 1978c6410..3cd3be15f 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -1,6 +1,15 @@ +import copy import itertools +import pickle +import pytest + +from jinja2 import ChainableUndefined +from jinja2 import DebugUndefined +from jinja2 import StrictUndefined from jinja2 import Template +from jinja2 import TemplateRuntimeError +from jinja2 import Undefined from jinja2.runtime import LoopContext TEST_IDX_TEMPLATE_STR_1 = ( @@ -73,3 +82,44 @@ def __call__(self, *args, **kwargs): out = t.render(calc=Calc()) # Would be "1" if context argument was passed. assert out == "0" + + +_undefined_types = (Undefined, ChainableUndefined, DebugUndefined, StrictUndefined) + + +@pytest.mark.parametrize("undefined_type", _undefined_types) +def test_undefined_copy(undefined_type): + undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError) + copied = copy.copy(undef) + + assert copied is not undef + assert copied._undefined_hint is undef._undefined_hint + assert copied._undefined_obj is undef._undefined_obj + assert copied._undefined_name is undef._undefined_name + assert copied._undefined_exception is undef._undefined_exception + + +@pytest.mark.parametrize("undefined_type", _undefined_types) +def test_undefined_deepcopy(undefined_type): + undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError) + copied = copy.deepcopy(undef) + + assert copied._undefined_hint is undef._undefined_hint + assert copied._undefined_obj is not undef._undefined_obj + assert copied._undefined_obj == undef._undefined_obj + assert copied._undefined_name is undef._undefined_name + assert copied._undefined_exception is undef._undefined_exception + + +@pytest.mark.parametrize("undefined_type", _undefined_types) +def test_undefined_pickle(undefined_type): + undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError) + copied = pickle.loads(pickle.dumps(undef)) + + assert copied._undefined_hint is not undef._undefined_hint + assert copied._undefined_hint == undef._undefined_hint + assert copied._undefined_obj is not undef._undefined_obj + assert copied._undefined_obj == undef._undefined_obj + assert copied._undefined_name is not undef._undefined_name + assert copied._undefined_name == undef._undefined_name + assert copied._undefined_exception is undef._undefined_exception From 7232b8246200155226adb672db8b3ef305cf29da Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 1 Oct 2024 15:18:27 -0700 Subject: [PATCH 17/39] Fix pickle/copy support for the `missing` singleton --- CHANGES.rst | 2 ++ src/jinja2/utils.py | 13 +++++++++++-- tests/test_utils.py | 12 ++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 58dc03214..1a1a526b5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,8 @@ Unreleased call. :issue:`2021` - Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined`` objects. :issue:`2025` +- Fix `copy`/`pickle` support for the internal ``missing`` object. + :issue:`2027` Version 3.1.4 diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index 5c1ff5d7b..7b52fc03e 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -18,8 +18,17 @@ F = t.TypeVar("F", bound=t.Callable[..., t.Any]) -# special singleton representing missing values for the runtime -missing: t.Any = type("MissingType", (), {"__repr__": lambda x: "missing"})() + +class _MissingType: + def __repr__(self) -> str: + return "missing" + + def __reduce__(self) -> str: + return "missing" + + +missing: t.Any = _MissingType() +"""Special singleton representing missing values for the runtime.""" internal_code: t.MutableSet[CodeType] = set() diff --git a/tests/test_utils.py b/tests/test_utils.py index 7b58af144..86e0f0420 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,4 @@ +import copy import pickle import random from collections import deque @@ -183,3 +184,14 @@ def test_consume(): consume(x) with pytest.raises(StopIteration): next(x) + + +@pytest.mark.parametrize("protocol", range(pickle.HIGHEST_PROTOCOL + 1)) +def test_pickle_missing(protocol: int) -> None: + """Test that missing can be pickled while remaining a singleton.""" + assert pickle.loads(pickle.dumps(missing, protocol)) is missing + + +def test_copy_missing() -> None: + """Test that missing can be copied while remaining a singleton.""" + assert copy.copy(missing) is missing From b5120582706b9e997cfd3277b02a32fd35b1c9d4 Mon Sep 17 00:00:00 2001 From: Dylan Scott Date: Fri, 4 Oct 2024 13:17:07 -0700 Subject: [PATCH 18/39] sandbox disallows `clear` and `pop` on mutable sequence --- CHANGES.rst | 2 ++ src/jinja2/sandbox.py | 4 +++- tests/test_security.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 1a1a526b5..f48eb0399 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,6 +26,8 @@ Unreleased objects. :issue:`2025` - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` +- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence + types. :issue:`2032` Version 3.1.4 diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index ce276156c..8200195db 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -60,7 +60,9 @@ ), ( abc.MutableSequence, - frozenset(["append", "reverse", "insert", "sort", "extend", "remove"]), + frozenset( + ["append", "clear", "pop", "reverse", "insert", "sort", "extend", "remove"] + ), ), ( deque, diff --git a/tests/test_security.py b/tests/test_security.py index 0e8dc5c03..9c7c4427a 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -58,6 +58,8 @@ def test_unsafe(self, env): def test_immutable_environment(self, env): env = ImmutableSandboxedEnvironment() pytest.raises(SecurityError, env.from_string("{{ [].append(23) }}").render) + pytest.raises(SecurityError, env.from_string("{{ [].clear() }}").render) + pytest.raises(SecurityError, env.from_string("{{ [1].pop() }}").render) pytest.raises(SecurityError, env.from_string("{{ {1:2}.clear() }}").render) def test_restricted(self, env): From 0871c71d0166152801798c1e59b3b32d3fb7469e Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 19 Dec 2024 12:07:14 -0800 Subject: [PATCH 19/39] rearrange change entry --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f48eb0399..4d201a5dc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,8 @@ Version 3.1.5 Unreleased +- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence + types. :issue:`2032` - Calling sync ``render`` for an async template uses ``asyncio.run``. :pr:`1952` - Avoid unclosed ``auto_aiter`` warnings. :pr:`1960` @@ -26,8 +28,6 @@ Unreleased objects. :issue:`2025` - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` -- Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence - types. :issue:`2032` Version 3.1.4 From 91a972f5808973cd441f4dc06873b2f8378f30c7 Mon Sep 17 00:00:00 2001 From: Lydxn Date: Mon, 23 Sep 2024 15:09:10 -0700 Subject: [PATCH 20/39] sandbox indirect calls to str.format --- CHANGES.rst | 3 ++ src/jinja2/sandbox.py | 81 ++++++++++++++++++++++-------------------- tests/test_security.py | 17 +++++++++ 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4d201a5dc..0a5694757 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Version 3.1.5 Unreleased +- The sandboxed environment handles indirect calls to ``str.format``, such as + by passing a stored reference to a filter that calls its argument. + :ghsa:`q2x7-8rv6-6q7h` - Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence types. :issue:`2032` - Calling sync ``render`` for an async template uses ``asyncio.run``. diff --git a/src/jinja2/sandbox.py b/src/jinja2/sandbox.py index 8200195db..9c9dae22f 100644 --- a/src/jinja2/sandbox.py +++ b/src/jinja2/sandbox.py @@ -8,6 +8,7 @@ from _string import formatter_field_name_split # type: ignore from collections import abc from collections import deque +from functools import update_wrapper from string import Formatter from markupsafe import EscapeFormatter @@ -83,20 +84,6 @@ ) -def inspect_format_method(callable: t.Callable[..., t.Any]) -> t.Optional[str]: - if not isinstance( - callable, (types.MethodType, types.BuiltinMethodType) - ) or callable.__name__ not in ("format", "format_map"): - return None - - obj = callable.__self__ - - if isinstance(obj, str): - return obj - - return None - - def safe_range(*args: int) -> range: """A range that can't generate ranges with a length of more than MAX_RANGE items. @@ -316,6 +303,9 @@ def getitem( except AttributeError: pass else: + fmt = self.wrap_str_format(value) + if fmt is not None: + return fmt if self.is_safe_attribute(obj, argument, value): return value return self.unsafe_undefined(obj, argument) @@ -333,6 +323,9 @@ def getattr(self, obj: t.Any, attribute: str) -> t.Union[t.Any, Undefined]: except (TypeError, LookupError): pass else: + fmt = self.wrap_str_format(value) + if fmt is not None: + return fmt if self.is_safe_attribute(obj, attribute, value): return value return self.unsafe_undefined(obj, attribute) @@ -348,34 +341,49 @@ def unsafe_undefined(self, obj: t.Any, attribute: str) -> Undefined: exc=SecurityError, ) - def format_string( - self, - s: str, - args: t.Tuple[t.Any, ...], - kwargs: t.Dict[str, t.Any], - format_func: t.Optional[t.Callable[..., t.Any]] = None, - ) -> str: - """If a format call is detected, then this is routed through this - method so that our safety sandbox can be used for it. + def wrap_str_format(self, value: t.Any) -> t.Optional[t.Callable[..., str]]: + """If the given value is a ``str.format`` or ``str.format_map`` method, + return a new function than handles sandboxing. This is done at access + rather than in :meth:`call`, so that calls made without ``call`` are + also sandboxed. """ + if not isinstance( + value, (types.MethodType, types.BuiltinMethodType) + ) or value.__name__ not in ("format", "format_map"): + return None + + f_self: t.Any = value.__self__ + + if not isinstance(f_self, str): + return None + + str_type: t.Type[str] = type(f_self) + is_format_map = value.__name__ == "format_map" formatter: SandboxedFormatter - if isinstance(s, Markup): - formatter = SandboxedEscapeFormatter(self, escape=s.escape) + + if isinstance(f_self, Markup): + formatter = SandboxedEscapeFormatter(self, escape=f_self.escape) else: formatter = SandboxedFormatter(self) - if format_func is not None and format_func.__name__ == "format_map": - if len(args) != 1 or kwargs: - raise TypeError( - "format_map() takes exactly one argument" - f" {len(args) + (kwargs is not None)} given" - ) + vformat = formatter.vformat + + def wrapper(*args: t.Any, **kwargs: t.Any) -> str: + if is_format_map: + if kwargs: + raise TypeError("format_map() takes no keyword arguments") + + if len(args) != 1: + raise TypeError( + f"format_map() takes exactly one argument ({len(args)} given)" + ) + + kwargs = args[0] + args = () - kwargs = args[0] - args = () + return str_type(vformat(f_self, args, kwargs)) - rv = formatter.vformat(s, args, kwargs) - return type(s)(rv) + return update_wrapper(wrapper, value) def call( __self, # noqa: B902 @@ -385,9 +393,6 @@ def call( **kwargs: t.Any, ) -> t.Any: """Call an object from sandboxed code.""" - fmt = inspect_format_method(__obj) - if fmt is not None: - return __self.format_string(fmt, args, kwargs, __obj) # the double prefixes are to avoid double keyword argument # errors when proxying the call. diff --git a/tests/test_security.py b/tests/test_security.py index 9c7c4427a..864d5f7f9 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -173,3 +173,20 @@ def test_safe_format_all_okay(self): '{{ ("a{x.foo}b{y}"|safe).format_map({"x":{"foo": 42}, "y":""}) }}' ) assert t.render() == "a42b<foo>" + + def test_indirect_call(self): + def run(value, arg): + return value.run(arg) + + env = SandboxedEnvironment() + env.filters["run"] = run + t = env.from_string( + """{% set + ns = namespace(run="{0.__call__.__builtins__[__import__]}".format) + %} + {{ ns | run(not_here) }} + """ + ) + + with pytest.raises(SecurityError): + t.render() From 56a724644b1ad9cb03745c10cca732715cdc79e9 Mon Sep 17 00:00:00 2001 From: Sigurd Spieckermann Date: Fri, 26 May 2023 14:32:36 +0200 Subject: [PATCH 21/39] fix f-string syntax error in code generation --- CHANGES.rst | 3 +++ src/jinja2/compiler.py | 7 ++++++- tests/test_compile.py | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0a5694757..5d64b267c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,9 @@ Unreleased - The sandboxed environment handles indirect calls to ``str.format``, such as by passing a stored reference to a filter that calls its argument. :ghsa:`q2x7-8rv6-6q7h` +- Escape template name before formatting it into error messages, to avoid + issues with names that contain f-string syntax. + :issue:`1792`, :ghsa:`gmj6-6f8f-6699` - Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence types. :issue:`2032` - Calling sync ``render`` for an async template uses ``asyncio.run``. diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 074e9b187..23295ec1f 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1141,9 +1141,14 @@ def visit_FromImport(self, node: nodes.FromImport, frame: Frame) -> None: ) self.writeline(f"if {frame.symbols.ref(alias)} is missing:") self.indent() + # The position will contain the template name, and will be formatted + # into a string that will be compiled into an f-string. Curly braces + # in the name must be replaced with escapes so that they will not be + # executed as part of the f-string. + position = self.position(node).replace("{", "{{").replace("}", "}}") message = ( "the template {included_template.__name__!r}" - f" (imported on {self.position(node)})" + f" (imported on {position})" f" does not export the requested name {name!r}" ) self.writeline( diff --git a/tests/test_compile.py b/tests/test_compile.py index 42efa59c0..e1a5391ea 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -1,6 +1,9 @@ import os import re +import pytest + +from jinja2 import UndefinedError from jinja2.environment import Environment from jinja2.loaders import DictLoader @@ -87,3 +90,19 @@ def test_block_set_vars_unpacking_deterministic(tmp_path): content, )[:10] assert found == expect + + +def test_undefined_import_curly_name(): + env = Environment( + loader=DictLoader( + { + "{bad}": "{% from 'macro' import m %}{{ m() }}", + "macro": "", + } + ) + ) + + # Must not raise `NameError: 'bad' is not defined`, as that would indicate + # that `{bad}` is being interpreted as an f-string. It must be escaped. + with pytest.raises(UndefinedError): + env.get_template("{bad}").render() From e45bc745a76b7aa573e68888bf49ecef70001815 Mon Sep 17 00:00:00 2001 From: SamyCookie Date: Thu, 19 Dec 2024 10:59:57 +0100 Subject: [PATCH 22/39] Bugfix: wrong default argument for `Environment.overlay(enable_async)` parameter --- CHANGES.rst | 1 + src/jinja2/environment.py | 7 +++++-- tests/test_api.py | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0a5694757..745b9690a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -31,6 +31,7 @@ Unreleased objects. :issue:`2025` - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` +- ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` Version 3.1.4 diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index f062e4074..821971779 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -406,7 +406,7 @@ def overlay( cache_size: int = missing, auto_reload: bool = missing, bytecode_cache: t.Optional["BytecodeCache"] = missing, - enable_async: bool = False, + enable_async: bool = missing, ) -> "Environment": """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. @@ -419,8 +419,11 @@ def overlay( copied over so modifications on the original environment may not shine through. + .. versionchanged:: 3.1.5 + ``enable_async`` is applied correctly. + .. versionchanged:: 3.1.2 - Added the ``newline_sequence``,, ``keep_trailing_newline``, + Added the ``newline_sequence``, ``keep_trailing_newline``, and ``enable_async`` parameters to match ``__init__``. """ args = dict(locals()) diff --git a/tests/test_api.py b/tests/test_api.py index ee11a8d69..4472b85ac 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -425,3 +425,11 @@ class CustomEnvironment(Environment): env = CustomEnvironment() tmpl = env.from_string("{{ foo }}") assert tmpl.render() == "resolve-foo" + + +def test_overlay_enable_async(env): + assert not env.is_async + assert not env.overlay().is_async + env_async = env.overlay(enable_async=True) + assert env_async.is_async + assert not env_async.overlay(enable_async=False).is_async From ed5f76206aa0a1e86ef087a18bc69643b484f029 Mon Sep 17 00:00:00 2001 From: Yourun-Proger Date: Mon, 2 May 2022 15:42:19 +0300 Subject: [PATCH 23/39] FileSystemLoader includes search paths in error --- CHANGES.rst | 2 ++ src/jinja2/loaders.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 384c6122d..569ae69f7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,6 +35,8 @@ Unreleased - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` - ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` +- Paths where the loader searched for the template were added + to the error message. :issue:`1661` Version 3.1.4 diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index d2373e5e7..65dfbe1a0 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -204,7 +204,10 @@ def get_source( if os.path.isfile(filename): break else: - raise TemplateNotFound(template) + raise TemplateNotFound( + f"{template} not found in the following search path(s):" + f" {self.searchpath}" + ) with open(filename, encoding=self.encoding) as f: contents = f.read() From 227edfd372f174fdae1ff74972de6a532af6c76e Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 19 Dec 2024 19:34:34 -0800 Subject: [PATCH 24/39] clean up message, add test --- CHANGES.rst | 4 ++-- src/jinja2/loaders.py | 6 ++++-- tests/test_loader.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 569ae69f7..cd2a4ef08 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -35,8 +35,8 @@ Unreleased - Fix `copy`/`pickle` support for the internal ``missing`` object. :issue:`2027` - ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` -- Paths where the loader searched for the template were added - to the error message. :issue:`1661` +- The error message from ``FileSystemLoader`` includes the paths that were + searched. :issue:`1661` Version 3.1.4 diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 65dfbe1a0..35799584c 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -204,9 +204,11 @@ def get_source( if os.path.isfile(filename): break else: + plural = "path" if len(self.searchpath) == 1 else "paths" + paths_str = ", ".join(repr(p) for p in self.searchpath) raise TemplateNotFound( - f"{template} not found in the following search path(s):" - f" {self.searchpath}" + template, + f"{template!r} not found in search {plural}: {paths_str}", ) with open(filename, encoding=self.encoding) as f: diff --git a/tests/test_loader.py b/tests/test_loader.py index e0cff6720..5a4e1a9da 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -179,6 +179,24 @@ def test_filename_normpath(self): t = e.get_template("foo/test.html") assert t.filename == str(self.searchpath / "foo" / "test.html") + def test_error_includes_paths(self, env, filesystem_loader): + env.loader = filesystem_loader + + with pytest.raises(TemplateNotFound) as info: + env.get_template("missing") + + e_str = str(info.value) + assert e_str.startswith("'missing' not found in search path: ") + + filesystem_loader.searchpath.append("other") + + with pytest.raises(TemplateNotFound) as info: + env.get_template("missing") + + e_str = str(info.value) + assert e_str.startswith("'missing' not found in search paths: ") + assert ", 'other'" in e_str + class TestModuleLoader: archive = None From f54fa113d38230cd7a8e8de1a8ced0f24e344a4e Mon Sep 17 00:00:00 2001 From: Lily Foote Date: Thu, 11 Aug 2022 15:22:08 +0100 Subject: [PATCH 25/39] Improve the PackageLoader error message This exception is raised when the `package_path` directory (default "templates") is not found, so explain this. --- src/jinja2/loaders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 35799584c..0cdeca14a 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -353,8 +353,8 @@ def __init__( if template_root is None: raise ValueError( - f"The {package_name!r} package was not installed in a" - " way that PackageLoader understands." + f"PackageLoader could not find a '{package_path}' directory for the " + f"{package_name!r} package." ) self._template_root = template_root From aaa083d265307f44f00b010acec61b6eb8e3c3a7 Mon Sep 17 00:00:00 2001 From: David Lord Date: Thu, 19 Dec 2024 20:15:10 -0800 Subject: [PATCH 26/39] separate messages, add test --- CHANGES.rst | 2 ++ src/jinja2/loaders.py | 18 +++++++++++------- tests/test_loader.py | 5 +++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cd2a4ef08..2e83ab3f6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,6 +37,8 @@ Unreleased - ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061` - The error message from ``FileSystemLoader`` includes the paths that were searched. :issue:`1661` +- ``PackageLoader`` shows a clearer error message when the package does not + contain the templates directory. :issue:`1705` Version 3.1.4 diff --git a/src/jinja2/loaders.py b/src/jinja2/loaders.py index 0cdeca14a..3913ee51e 100644 --- a/src/jinja2/loaders.py +++ b/src/jinja2/loaders.py @@ -327,7 +327,6 @@ def __init__( assert loader is not None, "A loader was not found for the package." self._loader = loader self._archive = None - template_root = None if isinstance(loader, zipimport.zipimporter): self._archive = loader.archive @@ -344,18 +343,23 @@ def __init__( elif spec.origin is not None: roots.append(os.path.dirname(spec.origin)) + if not roots: + raise ValueError( + f"The {package_name!r} package was not installed in a" + " way that PackageLoader understands." + ) + for root in roots: root = os.path.join(root, package_path) if os.path.isdir(root): template_root = root break - - if template_root is None: - raise ValueError( - f"PackageLoader could not find a '{package_path}' directory for the " - f"{package_name!r} package." - ) + else: + raise ValueError( + f"PackageLoader could not find a {package_path!r} directory" + f" in the {package_name!r} package." + ) self._template_root = template_root diff --git a/tests/test_loader.py b/tests/test_loader.py index 5a4e1a9da..377290b71 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -429,3 +429,8 @@ def exec_module(self, module): assert "test.html" in package_loader.list_templates() finally: sys.meta_path[:] = before + + +def test_package_loader_no_dir() -> None: + with pytest.raises(ValueError, match="could not find a 'templates' directory"): + PackageLoader("jinja2") From ded9915fc5db23d1a45ad3b210def7b4a96dc287 Mon Sep 17 00:00:00 2001 From: Victor Westerhuis Date: Wed, 23 Aug 2023 09:01:55 +0200 Subject: [PATCH 27/39] improve annotations for methods returning copies --- CHANGES.rst | 1 + src/jinja2/compiler.py | 4 ++-- src/jinja2/environment.py | 4 ++-- src/jinja2/ext.py | 2 +- src/jinja2/idtracking.py | 5 ++++- src/jinja2/utils.py | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2e83ab3f6..3955a32ba 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -39,6 +39,7 @@ Unreleased searched. :issue:`1661` - ``PackageLoader`` shows a clearer error message when the package does not contain the templates directory. :issue:`1705` +- Improve annotations for methods returning copies. :pr:`1880` Version 3.1.4 diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 23295ec1f..ca079070a 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -216,7 +216,7 @@ def __init__( # or compile time. self.soft_frame = False - def copy(self) -> "Frame": + def copy(self) -> "te.Self": """Create a copy of the current one.""" rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) @@ -229,7 +229,7 @@ def inner(self, isolated: bool = False) -> "Frame": return Frame(self.eval_ctx, level=self.symbols.level + 1) return Frame(self.eval_ctx, self) - def soft(self) -> "Frame": + def soft(self) -> "te.Self": """Return a soft frame. A soft frame may not be modified as standalone thing as it shares the resources with the frame it was created of, but it's not a rootlevel frame any longer. diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 821971779..0fc6e5be8 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -123,7 +123,7 @@ def load_extensions( return result -def _environment_config_check(environment: "Environment") -> "Environment": +def _environment_config_check(environment: _env_bound) -> _env_bound: """Perform a sanity check on the environment.""" assert issubclass( environment.undefined, Undefined @@ -407,7 +407,7 @@ def overlay( auto_reload: bool = missing, bytecode_cache: t.Optional["BytecodeCache"] = missing, enable_async: bool = missing, - ) -> "Environment": + ) -> "te.Self": """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. Extensions cannot be removed for an overlayed environment. An overlayed diff --git a/src/jinja2/ext.py b/src/jinja2/ext.py index 8d0810cd4..c7af8d45f 100644 --- a/src/jinja2/ext.py +++ b/src/jinja2/ext.py @@ -89,7 +89,7 @@ def __init_subclass__(cls) -> None: def __init__(self, environment: Environment) -> None: self.environment = environment - def bind(self, environment: Environment) -> "Extension": + def bind(self, environment: Environment) -> "te.Self": """Create a copy of this extension bound to another environment.""" rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) diff --git a/src/jinja2/idtracking.py b/src/jinja2/idtracking.py index d6cb635b2..cb4bccb0e 100644 --- a/src/jinja2/idtracking.py +++ b/src/jinja2/idtracking.py @@ -3,6 +3,9 @@ from . import nodes from .visitor import NodeVisitor +if t.TYPE_CHECKING: + import typing_extensions as te + VAR_LOAD_PARAMETER = "param" VAR_LOAD_RESOLVE = "resolve" VAR_LOAD_ALIAS = "alias" @@ -83,7 +86,7 @@ def ref(self, name: str) -> str: ) return rv - def copy(self) -> "Symbols": + def copy(self) -> "te.Self": rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) rv.refs = self.refs.copy() diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index 7b52fc03e..d7149bc31 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -462,7 +462,7 @@ def __setstate__(self, d: t.Mapping[str, t.Any]) -> None: def __getnewargs__(self) -> t.Tuple[t.Any, ...]: return (self.capacity,) - def copy(self) -> "LRUCache": + def copy(self) -> "te.Self": """Return a shallow copy of the instance.""" rv = self.__class__(self.capacity) rv._mapping.update(self._mapping) From 0cd6948192591b2c31e3dba294ed9300813d1d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 20 Jul 2023 04:20:13 +0200 Subject: [PATCH 28/39] don't apply `urlize` to `@a@b` --- CHANGES.rst | 1 + src/jinja2/utils.py | 2 ++ tests/test_utils.py | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 3955a32ba..e31de857d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -40,6 +40,7 @@ Unreleased - ``PackageLoader`` shows a clearer error message when the package does not contain the templates directory. :issue:`1705` - Improve annotations for methods returning copies. :pr:`1880` +- ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` Version 3.1.4 diff --git a/src/jinja2/utils.py b/src/jinja2/utils.py index d7149bc31..7c922629a 100644 --- a/src/jinja2/utils.py +++ b/src/jinja2/utils.py @@ -333,6 +333,8 @@ def trim_url(x: str) -> str: elif ( "@" in middle and not middle.startswith("www.") + # ignore values like `@a@b` + and not middle.startswith("@") and ":" not in middle and _email_re.match(middle) ): diff --git a/tests/test_utils.py b/tests/test_utils.py index 86e0f0420..b50a6b4c6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,14 @@ def test_escape_urlize_target(self): "http://example.org" ) + def test_urlize_mail_mastodon(self): + fr = "nabijaczleweli@nabijaczleweli.xyz\n@eater@cijber.social\n" + to = ( + '' + "nabijaczleweli@nabijaczleweli.xyz\n@eater@cijber.social\n" + ) + assert urlize(fr) == to + class TestLoremIpsum: def test_lorem_ipsum_markup(self): From d05bd3858c3f4990e91201dae058147caf317462 Mon Sep 17 00:00:00 2001 From: Rens Groothuijsen Date: Wed, 16 Nov 2022 01:25:49 +0100 Subject: [PATCH 29/39] Pass context when using select --- CHANGES.rst | 2 ++ src/jinja2/filters.py | 2 +- tests/test_regression.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e31de857d..521f5a08a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -41,6 +41,8 @@ Unreleased contain the templates directory. :issue:`1705` - Improve annotations for methods returning copies. :pr:`1880` - ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` +- Tests decorated with `@pass_context`` can be used with the ``|select`` + filter. :issue:`1624` Version 3.1.4 diff --git a/src/jinja2/filters.py b/src/jinja2/filters.py index a92832a34..e5b5a00c5 100644 --- a/src/jinja2/filters.py +++ b/src/jinja2/filters.py @@ -1780,7 +1780,7 @@ def transfunc(x: V) -> V: args = args[1 + off :] def func(item: t.Any) -> t.Any: - return context.environment.call_test(name, item, args, kwargs) + return context.environment.call_test(name, item, args, kwargs, context) except LookupError: func = bool # type: ignore diff --git a/tests/test_regression.py b/tests/test_regression.py index 7bd4d1564..10df2d1bd 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -737,6 +737,18 @@ def test_nested_loop_scoping(self, env): ) assert tmpl.render() == "hellohellohello" + def test_pass_context_with_select(self, env): + @pass_context + def is_foo(ctx, s): + assert ctx is not None + return s == "foo" + + env.tests["foo"] = is_foo + tmpl = env.from_string( + "{% for x in ['one', 'foo'] | select('foo') %}{{ x }}{% endfor %}" + ) + assert tmpl.render() == "foo" + @pytest.mark.parametrize("unicode_char", ["\N{FORM FEED}", "\x85"]) def test_unicode_whitespace(env, unicode_char): From ae68c961dc52d48580dc9005c9ebe9117590f690 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 20 Dec 2024 07:57:11 -0800 Subject: [PATCH 30/39] document SandboxedNativeEnvironment pattern --- docs/nativetypes.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/nativetypes.rst b/docs/nativetypes.rst index 1a08700b0..fb2a76718 100644 --- a/docs/nativetypes.rst +++ b/docs/nativetypes.rst @@ -55,6 +55,17 @@ Foo >>> print(result.value) 15 +Sandboxed Native Environment +---------------------------- + +You can combine :class:`.SandboxedEnvironment` and :class:`NativeEnvironment` to +get both behaviors. + +.. code-block:: python + + class SandboxedNativeEnvironment(SandboxedEnvironment, NativeEnvironment): + pass + API --- From d6998ab74e628c9851042248c9e10dc25936630b Mon Sep 17 00:00:00 2001 From: ratchek Date: Tue, 12 Dec 2023 20:43:36 -0500 Subject: [PATCH 31/39] Make ease of use update to template documentation Add the phrases 'multiline comment' and 'triple quotes' to docs in the templates/#block-assignments section. This allows for new users to find this alternative easily. --- docs/templates.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index d5f2719e0..758ba90ce 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1090,9 +1090,10 @@ Block Assignments Starting with Jinja 2.8, it's possible to also use block assignments to capture the contents of a block into a variable name. This can be useful -in some situations as an alternative for macros. In that case, instead of -using an equals sign and a value, you just write the variable name and then -everything until ``{% endset %}`` is captured. +in some situations as an alternative for macros. It can also be used to create +multiline strings instead of triple quotes (''' and """), which Jinja does not +support. In that case, instead of using an equals sign and a value, you just +write the variable name and then everything until ``{% endset %}`` is captured. Example:: From 8a8eafc6b992ba177f1d3dd483f8465f18a11116 Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 20 Dec 2024 08:29:04 -0800 Subject: [PATCH 32/39] edit block assignment section --- docs/templates.rst | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/templates.rst b/docs/templates.rst index 758ba90ce..8db8ccaf9 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1086,35 +1086,34 @@ Assignments use the `set` tag and can have multiple targets:: Block Assignments ~~~~~~~~~~~~~~~~~ -.. versionadded:: 2.8 +It's possible to use `set` as a block to assign the content of the block to a +variable. This can be used to create multi-line strings, since Jinja doesn't +support Python's triple quotes (``"""``, ``'''``). -Starting with Jinja 2.8, it's possible to also use block assignments to -capture the contents of a block into a variable name. This can be useful -in some situations as an alternative for macros. It can also be used to create -multiline strings instead of triple quotes (''' and """), which Jinja does not -support. In that case, instead of using an equals sign and a value, you just -write the variable name and then everything until ``{% endset %}`` is captured. +Instead of using an equals sign and a value, you only write the variable name, +and everything until ``{% endset %}`` is captured. -Example:: +.. code-block:: jinja {% set navigation %}
  • Index
  • Downloads {% endset %} -The `navigation` variable then contains the navigation HTML source. - -.. versionchanged:: 2.10 - -Starting with Jinja 2.10, the block assignment supports filters. +Filters applied to the variable name will be applied to the block's content. -Example:: +.. code-block:: jinja {% set reply | wordwrap %} You wrote: {{ message }} {% endset %} +.. versionadded:: 2.8 + +.. versionchanged:: 2.10 + + Block assignment supports filters. .. _extends: From ee832194cd9f55f75e5a51359b709d535efe957f Mon Sep 17 00:00:00 2001 From: Kevin Brown-Silva Date: Mon, 2 May 2022 12:01:08 -0600 Subject: [PATCH 33/39] Add support for namespaces in tuple assignment This fixes a bug that existed because namespaces within `{% set %}` were treated as a special case. This special case had the side-effect of bypassing the code which allows for tuples to be assigned to. The solution was to make tuple handling (and by extension, primary token handling) aware of namespaces so that namespace tokens can be handled appropriately. This is handled in a backwards-compatible way which ensures that we do not try to parse namespace tokens when we otherwise would be expecting to parse out name tokens with attributes. Namespace instance checks are moved earlier, and deduplicated, so that all checks are done before the assignment. Otherwise, the check could be emitted in the middle of the tuple. --- CHANGES.rst | 2 ++ docs/templates.rst | 3 +++ src/jinja2/compiler.py | 23 ++++++++++++++++------- src/jinja2/parser.py | 30 +++++++++++++++++------------- tests/test_core_tags.py | 8 ++++++++ 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 521f5a08a..2b8179855 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -43,6 +43,8 @@ Unreleased - ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870` - Tests decorated with `@pass_context`` can be used with the ``|select`` filter. :issue:`1624` +- Using ``set`` for multiple assignment (``a, b = 1, 2``) does not fail when the + target is a namespace attribute. :issue:`1413` Version 3.1.4 diff --git a/docs/templates.rst b/docs/templates.rst index 8db8ccaf9..9f376a13c 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -1678,6 +1678,9 @@ The following functions are available in the global scope by default: .. versionadded:: 2.10 + .. versionchanged:: 3.2 + Namespace attributes can be assigned to in multiple assignment. + Extensions ---------- diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index ca079070a..0666cddf7 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1581,6 +1581,22 @@ def visit_Output(self, node: nodes.Output, frame: Frame) -> None: def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: self.push_assign_tracking() + + # NSRef can only ever be used during assignment so we need to check + # to make sure that it is only being used to assign using a Namespace. + # This check is done here because it is used an expression during the + # assignment and therefore cannot have this check done when the NSRef + # node is visited + for nsref in node.find_all(nodes.NSRef): + ref = frame.symbols.ref(nsref.name) + self.writeline(f"if not isinstance({ref}, Namespace):") + self.indent() + self.writeline( + "raise TemplateRuntimeError" + '("cannot assign attribute on non-namespace object")' + ) + self.outdent() + self.newline(node) self.visit(node.target, frame) self.write(" = ") @@ -1641,13 +1657,6 @@ def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: # `foo.bar` notation they will be parsed as a normal attribute access # when used anywhere but in a `set` context ref = frame.symbols.ref(node.name) - self.writeline(f"if not isinstance({ref}, Namespace):") - self.indent() - self.writeline( - "raise TemplateRuntimeError" - '("cannot assign attribute on non-namespace object")' - ) - self.outdent() self.writeline(f"{ref}[{node.attr!r}]") def visit_Const(self, node: nodes.Const, frame: Frame) -> None: diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 22f3f81f7..107232631 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -487,21 +487,18 @@ def parse_assign_target( """ target: nodes.Expr - if with_namespace and self.stream.look().type == "dot": - token = self.stream.expect("name") - next(self.stream) # dot - attr = self.stream.expect("name") - target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) - elif name_only: + if name_only: token = self.stream.expect("name") target = nodes.Name(token.value, "store", lineno=token.lineno) else: if with_tuple: target = self.parse_tuple( - simplified=True, extra_end_rules=extra_end_rules + simplified=True, + extra_end_rules=extra_end_rules, + with_namespace=with_namespace, ) else: - target = self.parse_primary() + target = self.parse_primary(with_namespace=with_namespace) target.set_ctx("store") @@ -643,7 +640,7 @@ def parse_unary(self, with_filter: bool = True) -> nodes.Expr: node = self.parse_filter_expr(node) return node - def parse_primary(self) -> nodes.Expr: + def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: token = self.stream.current node: nodes.Expr if token.type == "name": @@ -651,6 +648,11 @@ def parse_primary(self) -> nodes.Expr: node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) elif token.value in ("none", "None"): node = nodes.Const(None, lineno=token.lineno) + elif with_namespace and self.stream.look().type == "dot": + next(self.stream) # token + next(self.stream) # dot + attr = self.stream.current + node = nodes.NSRef(token.value, attr.value, lineno=token.lineno) else: node = nodes.Name(token.value, "load", lineno=token.lineno) next(self.stream) @@ -683,6 +685,7 @@ def parse_tuple( with_condexpr: bool = True, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, explicit_parentheses: bool = False, + with_namespace: bool = False, ) -> t.Union[nodes.Tuple, nodes.Expr]: """Works like `parse_expression` but if multiple expressions are delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. @@ -704,13 +707,14 @@ def parse_tuple( """ lineno = self.stream.current.lineno if simplified: - parse = self.parse_primary - elif with_condexpr: - parse = self.parse_expression + + def parse() -> nodes.Expr: + return self.parse_primary(with_namespace=with_namespace) + else: def parse() -> nodes.Expr: - return self.parse_expression(with_condexpr=False) + return self.parse_expression(with_condexpr=with_condexpr) args: t.List[nodes.Expr] = [] is_tuple = False diff --git a/tests/test_core_tags.py b/tests/test_core_tags.py index 4bb95e024..2d847a2c9 100644 --- a/tests/test_core_tags.py +++ b/tests/test_core_tags.py @@ -538,6 +538,14 @@ def test_namespace_macro(self, env_trim): ) assert tmpl.render() == "13|37" + def test_namespace_set_tuple(self, env_trim): + tmpl = env_trim.from_string( + "{% set ns = namespace(a=12, b=36) %}" + "{% set ns.a, ns.b = ns.a + 1, ns.b + 1 %}" + "{{ ns.a }}|{{ ns.b }}" + ) + assert tmpl.render() == "13|37" + def test_block_escaping_filtered(self): env = Environment(autoescape=True) tmpl = env.from_string( From b8f4831d41e6a7cb5c40d42f074ffd92d2daccfc Mon Sep 17 00:00:00 2001 From: David Lord Date: Fri, 20 Dec 2024 14:02:31 -0800 Subject: [PATCH 34/39] more comments about nsref assignment only emit nsref instance check once per ref name refactor primary name parsing a bit --- src/jinja2/compiler.py | 24 ++++++++++++++++-------- src/jinja2/parser.py | 18 +++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index 0666cddf7..a4ff6a1b1 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1582,12 +1582,19 @@ def visit_Output(self, node: nodes.Output, frame: Frame) -> None: def visit_Assign(self, node: nodes.Assign, frame: Frame) -> None: self.push_assign_tracking() - # NSRef can only ever be used during assignment so we need to check - # to make sure that it is only being used to assign using a Namespace. - # This check is done here because it is used an expression during the - # assignment and therefore cannot have this check done when the NSRef - # node is visited + # ``a.b`` is allowed for assignment, and is parsed as an NSRef. However, + # it is only valid if it references a Namespace object. Emit a check for + # that for each ref here, before assignment code is emitted. This can't + # be done in visit_NSRef as the ref could be in the middle of a tuple. + seen_refs: t.Set[str] = set() + for nsref in node.find_all(nodes.NSRef): + if nsref.name in seen_refs: + # Only emit the check for each reference once, in case the same + # ref is used multiple times in a tuple, `ns.a, ns.b = c, d`. + continue + + seen_refs.add(nsref.name) ref = frame.symbols.ref(nsref.name) self.writeline(f"if not isinstance({ref}, Namespace):") self.indent() @@ -1653,9 +1660,10 @@ def visit_Name(self, node: nodes.Name, frame: Frame) -> None: self.write(ref) def visit_NSRef(self, node: nodes.NSRef, frame: Frame) -> None: - # NSRefs can only be used to store values; since they use the normal - # `foo.bar` notation they will be parsed as a normal attribute access - # when used anywhere but in a `set` context + # NSRef is a dotted assignment target a.b=c, but uses a[b]=c internally. + # visit_Assign emits code to validate that each ref is to a Namespace + # object only. That can't be emitted here as the ref could be in the + # middle of a tuple assignment. ref = frame.symbols.ref(node.name) self.writeline(f"{ref}[{node.attr!r}]") diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 107232631..f4117754a 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -641,21 +641,24 @@ def parse_unary(self, with_filter: bool = True) -> nodes.Expr: return node def parse_primary(self, with_namespace: bool = False) -> nodes.Expr: + """Parse a name or literal value. If ``with_namespace`` is enabled, also + parse namespace attr refs, for use in assignments.""" token = self.stream.current node: nodes.Expr if token.type == "name": + next(self.stream) if token.value in ("true", "false", "True", "False"): node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) elif token.value in ("none", "None"): node = nodes.Const(None, lineno=token.lineno) - elif with_namespace and self.stream.look().type == "dot": - next(self.stream) # token - next(self.stream) # dot - attr = self.stream.current + elif with_namespace and self.stream.current.type == "dot": + # If namespace attributes are allowed at this point, and the next + # token is a dot, produce a namespace reference. + next(self.stream) + attr = self.stream.expect("name") node = nodes.NSRef(token.value, attr.value, lineno=token.lineno) else: node = nodes.Name(token.value, "load", lineno=token.lineno) - next(self.stream) elif token.type == "string": next(self.stream) buf = [token.value] @@ -693,8 +696,9 @@ def parse_tuple( if no commas where found. The default parsing mode is a full tuple. If `simplified` is `True` - only names and literals are parsed. The `no_condexpr` parameter is - forwarded to :meth:`parse_expression`. + only names and literals are parsed; ``with_namespace`` allows namespace + attr refs as well. The `no_condexpr` parameter is forwarded to + :meth:`parse_expression`. Because tuples do not require delimiters and may end in a bogus comma an extra hint is needed that marks the end of a tuple. For example From 66587ce989e5a478e0bb165371fa2b9d42b7040f Mon Sep 17 00:00:00 2001 From: Kevin Brown-Silva Date: Mon, 2 May 2022 15:33:58 -0600 Subject: [PATCH 35/39] Fix bug where set would sometimes fail within if There was a bug that came as the result of an early optimization done within ID tracking that caused loading parameters to fail in a very specific and rare edge case. That edge case only occurred when the parameter was being set within all 3 standard branches of an if block, since the optimization would assume that the parameter was never being referenced and was only ever being set. This would cause the variable to be set to undefined. The fix for this was to remove the optimization and still continue to load in the parameter even if it is set in all 3 branches. --- CHANGES.rst | 3 +++ src/jinja2/idtracking.py | 17 +++++++---------- tests/test_regression.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2b8179855..b6a5a1af5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -45,6 +45,9 @@ Unreleased filter. :issue:`1624` - Using ``set`` for multiple assignment (``a, b = 1, 2``) does not fail when the target is a namespace attribute. :issue:`1413` +- Using ``set`` in all branches of ``{% if %}{% elif %}{% else %}`` blocks + does not cause the variable to be considered initially undefined. + :issue:`1253` Version 3.1.4 diff --git a/src/jinja2/idtracking.py b/src/jinja2/idtracking.py index cb4bccb0e..e6dd8cd11 100644 --- a/src/jinja2/idtracking.py +++ b/src/jinja2/idtracking.py @@ -121,23 +121,20 @@ def load(self, name: str) -> None: self._define_ref(name, load=(VAR_LOAD_RESOLVE, name)) def branch_update(self, branch_symbols: t.Sequence["Symbols"]) -> None: - stores: t.Dict[str, int] = {} + stores: t.Set[str] = set() + for branch in branch_symbols: - for target in branch.stores: - if target in self.stores: - continue - stores[target] = stores.get(target, 0) + 1 + stores.update(branch.stores) + + stores.difference_update(self.stores) for sym in branch_symbols: self.refs.update(sym.refs) self.loads.update(sym.loads) self.stores.update(sym.stores) - for name, branch_count in stores.items(): - if branch_count == len(branch_symbols): - continue - - target = self.find_ref(name) # type: ignore + for name in stores: + target = self.find_ref(name) assert target is not None, "should not happen" if self.parent is not None: diff --git a/tests/test_regression.py b/tests/test_regression.py index 10df2d1bd..93d72c5e6 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -750,6 +750,16 @@ def is_foo(ctx, s): assert tmpl.render() == "foo" +def test_load_parameter_when_set_in_all_if_branches(env): + tmpl = env.from_string( + "{% if True %}{{ a.b }}{% set a = 1 %}" + "{% elif False %}{% set a = 2 %}" + "{% else %}{% set a = 3 %}{% endif %}" + "{{ a }}" + ) + assert tmpl.render(a={"b": 0}) == "01" + + @pytest.mark.parametrize("unicode_char", ["\N{FORM FEED}", "\x85"]) def test_unicode_whitespace(env, unicode_char): content = "Lorem ipsum\n" + unicode_char + "\nMore text" From eda8fe86fd716dfce24910294e9f1fc81fbc740c Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:14:25 -0800 Subject: [PATCH 36/39] update dev dependencies --- .github/workflows/publish.yaml | 6 +++--- .github/workflows/tests.yaml | 2 +- .pre-commit-config.yaml | 2 +- requirements/build.txt | 2 +- requirements/dev.txt | 32 ++++++++++++++++---------------- requirements/docs.txt | 4 ++-- requirements/tests.txt | 6 +++--- requirements/typing.txt | 2 +- src/jinja2/lexer.py | 6 +++--- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 727518c69..af983de40 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -23,7 +23,7 @@ jobs: - name: generate hash id: hash run: cd dist && echo "hash=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: path: ./dist provenance: @@ -64,10 +64,10 @@ jobs: id-token: write steps: - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3 + - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: repository-url: https://test.pypi.org/legacy/ packages-dir: artifact/ - - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # v1.10.3 + - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: packages-dir: artifact/ diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 515a7a5e4..1062ebe44 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -43,7 +43,7 @@ jobs: cache: pip cache-dependency-path: requirements*/*.txt - name: cache mypy - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ./.mypy_cache key: mypy|${{ hashFiles('pyproject.toml') }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74b54e8f1..a9f102b5e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.8.4 hooks: - id: ruff - id: ruff-format diff --git a/requirements/build.txt b/requirements/build.txt index 1b13b0552..9d6dd1040 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -6,7 +6,7 @@ # build==1.2.2.post1 # via -r build.in -packaging==24.1 +packaging==24.2 # via build pyproject-hooks==1.2.0 # via build diff --git a/requirements/dev.txt b/requirements/dev.txt index ba73d911c..c90a78168 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ # alabaster==1.0.0 # via sphinx -attrs==24.2.0 +attrs==24.3.0 # via # outcome # trio @@ -16,7 +16,7 @@ build==1.2.2.post1 # via pip-tools cachetools==5.5.0 # via tox -certifi==2024.8.30 +certifi==2024.12.14 # via requests cfgv==3.4.0 # via pre-commit @@ -38,7 +38,7 @@ filelock==3.16.1 # via # tox # virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit idna==3.10 # via @@ -52,15 +52,15 @@ jinja2==3.1.4 # via sphinx markupsafe==3.0.2 # via jinja2 -mypy==1.13.0 - # via -r typing.in +mypy==1.14.0 + # via -r /Users/david/Projects/jinja/requirements/typing.in mypy-extensions==1.0.0 # via mypy nodeenv==1.9.1 # via pre-commit outcome==1.3.0.post0 # via trio -packaging==24.1 +packaging==24.2 # via # build # pallets-sphinx-themes @@ -69,8 +69,8 @@ packaging==24.1 # sphinx # tox pallets-sphinx-themes==2.3.0 - # via -r docs.in -pip-compile-multi==2.6.4 + # via -r /Users/david/Projects/jinja/requirements/docs.in +pip-compile-multi==2.7.1 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi @@ -92,8 +92,8 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.3.3 - # via -r tests.in +pytest==8.3.4 + # via -r /Users/david/Projects/jinja/requirements/tests.in pyyaml==6.0.2 # via pre-commit requests==2.32.3 @@ -106,13 +106,13 @@ sortedcontainers==2.4.0 # via trio sphinx==8.1.3 # via - # -r docs.in + # -r /Users/david/Projects/jinja/requirements/docs.in # pallets-sphinx-themes # sphinx-issues # sphinx-notfound-page # sphinxcontrib-log-cabinet sphinx-issues==5.0.0 - # via -r docs.in + # via -r /Users/david/Projects/jinja/requirements/docs.in sphinx-notfound-page==1.0.4 # via pallets-sphinx-themes sphinxcontrib-applehelp==2.0.0 @@ -124,7 +124,7 @@ sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-log-cabinet==1.0.1 - # via -r docs.in + # via -r /Users/david/Projects/jinja/requirements/docs.in sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 @@ -134,16 +134,16 @@ toposort==1.10 tox==4.23.2 # via -r dev.in trio==0.27.0 - # via -r tests.in + # via -r /Users/david/Projects/jinja/requirements/tests.in typing-extensions==4.12.2 # via mypy urllib3==2.2.3 # via requests -virtualenv==20.27.0 +virtualenv==20.28.0 # via # pre-commit # tox -wheel==0.44.0 +wheel==0.45.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docs.txt b/requirements/docs.txt index 453a7cb5d..2283fa9b5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -8,7 +8,7 @@ alabaster==1.0.0 # via sphinx babel==2.16.0 # via sphinx -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -22,7 +22,7 @@ jinja2==3.1.4 # via sphinx markupsafe==3.0.2 # via jinja2 -packaging==24.1 +packaging==24.2 # via # pallets-sphinx-themes # sphinx diff --git a/requirements/tests.txt b/requirements/tests.txt index e019ba988..71dad37da 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -4,7 +4,7 @@ # # pip-compile tests.in # -attrs==24.2.0 +attrs==24.3.0 # via # outcome # trio @@ -14,11 +14,11 @@ iniconfig==2.0.0 # via pytest outcome==1.3.0.post0 # via trio -packaging==24.1 +packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -pytest==8.3.3 +pytest==8.3.4 # via -r tests.in sniffio==1.3.1 # via trio diff --git a/requirements/typing.txt b/requirements/typing.txt index 1cf3727a5..f50d6d667 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -4,7 +4,7 @@ # # pip-compile typing.in # -mypy==1.13.0 +mypy==1.14.0 # via -r typing.in mypy-extensions==1.0.0 # via mypy diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py index 6dc94b67d..9b1c96979 100644 --- a/src/jinja2/lexer.py +++ b/src/jinja2/lexer.py @@ -262,7 +262,7 @@ def __init__( self.message = message self.error_class = cls - def __call__(self, lineno: int, filename: str) -> "te.NoReturn": + def __call__(self, lineno: int, filename: t.Optional[str]) -> "te.NoReturn": raise self.error_class(self.message, lineno, filename) @@ -757,7 +757,7 @@ def tokeniter( for idx, token in enumerate(tokens): # failure group - if token.__class__ is Failure: + if isinstance(token, Failure): raise token(lineno, filename) # bygroup is a bit more complex, in that case we # yield for the current token the first named @@ -778,7 +778,7 @@ def tokeniter( data = groups[idx] if data or token not in ignore_if_empty: - yield lineno, token, data + yield lineno, token, data # type: ignore[misc] lineno += data.count("\n") + newlines_stripped newlines_stripped = 0 From 8d588592653b052f957b720e1fc93196e06f207f Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:14:49 -0800 Subject: [PATCH 37/39] remove test pypi --- .github/workflows/publish.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index af983de40..d609abdb6 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -64,10 +64,6 @@ jobs: id-token: write steps: - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 - with: - repository-url: https://test.pypi.org/legacy/ - packages-dir: artifact/ - uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3 with: packages-dir: artifact/ From 877f6e51be8e1765b06d911cfaa9033775f051d1 Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:16:13 -0800 Subject: [PATCH 38/39] release version 3.1.5 --- CHANGES.rst | 2 +- src/jinja2/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b6a5a1af5..e1b339198 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,7 @@ Version 3.1.5 ------------- -Unreleased +Released 2024-12-21 - The sandboxed environment handles indirect calls to ``str.format``, such as by passing a stored reference to a filter that calls its argument. diff --git a/src/jinja2/__init__.py b/src/jinja2/__init__.py index 578940091..d669f295b 100644 --- a/src/jinja2/__init__.py +++ b/src/jinja2/__init__.py @@ -35,4 +35,4 @@ from .utils import pass_eval_context as pass_eval_context from .utils import select_autoescape as select_autoescape -__version__ = "3.1.5.dev" +__version__ = "3.1.5" From ab8218c7a1b66b62e0ad6b941bd514e3a64a358f Mon Sep 17 00:00:00 2001 From: David Lord Date: Sat, 21 Dec 2024 10:47:08 -0800 Subject: [PATCH 39/39] use project advisory link instead of global --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 02c74a86b..280610e53 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ extlinks = { "issue": ("https://github.com/pallets/jinja/issues/%s", "#%s"), "pr": ("https://github.com/pallets/jinja/pull/%s", "#%s"), - "ghsa": ("https://github.com/advisories/GHSA-%s", "GHSA-%s"), + "ghsa": ("https://github.com/pallets/jinja/security/advisories/GHSA-%s", "GHSA-%s"), } intersphinx_mapping = { "python": ("https://docs.python.org/3/", None),