Merge branch 'master' into validate_host

CHANGES.rst

@@ -14,6 +14,60 @@ Changelog
 
 .. towncrier release notes start
 
+1.11.0
+======
+
+*(2024-09-08)*
+
+
+Features
+--------
+
+- Added :meth:`URL.extend_query() <yarl.URL.extend_query>` method, which can be used to extend parameters without replacing same named keys -- by :user:`bdraco`.
+
+  This method was primarily added to replace the inefficient hand rolled method currently used in ``aiohttp``.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1128`.
+
+
+Miscellaneous internal changes
+------------------------------
+
+- Improved performance of the Cython ``cached_property`` implementation -- by :user:`bdraco`.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1122`.
+
+- Simplified computing ports by removing unnecessary code -- by :user:`bdraco`.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1123`.
+
+- Improved performance of encoding non IPv6 hosts -- by :user:`bdraco`.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1125`.
+
+- Improved performance of :meth:`URL.build() <yarl.URL.build>` when the path, query string, or fragment is an empty string -- by :user:`bdraco`.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1126`.
+
+- Improved performance of the :meth:`URL.update_query() <yarl.URL.update_query>` method -- by :user:`bdraco`.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1130`.
+
+- Improved performance of processing query string changes when arguments are :class:`str` -- by :user:`bdraco`.
+
+  *Related issues and pull requests on GitHub:*
+  :issue:`1131`.
+
+
+----
+
+
 1.10.0
 ======
 

CHANGES/1136.bugfix.rst

@@ -0,0 +1 @@
+Allow empty host for URL schemes other than the special schemes listed in the WHATWG URL spec -- by :user:`bdraco`.

CHANGES/1137.misc.rst

@@ -0,0 +1 @@
+Improved performance of normalizing paths -- by :user:`bdraco`.

docs/api.rst

@@ -659,6 +659,48 @@ section generates a new :class:`URL` instance.
       Support subclasses of :class:`int` (except :class:`bool`) and :class:`float`
       as a query parameter value.
 
+.. method:: URL.extend_query(query)
+            URL.extend_query(**kwargs)
+
+   Returns a new URL with *query* part extended.
+
+   Unlike :meth:`update_query`, this method keeps duplicate keys.
+
+   Returned :class:`URL` object will contain query string which extends
+   parts from passed query parts (or parts of parsed query string).
+
+   Accepts any :class:`~collections.abc.Mapping` (e.g. :class:`dict`,
+   :class:`~multidict.MultiDict` instances) or :class:`str`,
+   auto-encode the argument if needed.
+
+   A sequence of ``(key, value)`` pairs is supported as well.
+
+   Also it can take an arbitrary number of keyword arguments.
+
+   Returns the same :class:`URL` if *query* of ``None`` is passed.
+
+   .. note::
+
+      The library accepts :class:`str`, :class:`float`, :class:`int` and their
+      subclasses except :class:`bool` as query argument values.
+
+      If a mapping such as :class:`dict` is used, the values may also be
+      :class:`list` or :class:`tuple` to represent a key has many values.
+
+      Please see :ref:`yarl-bools-support` for the reason why :class:`bool` is not
+      supported out-of-the-box.
+
+   .. doctest::
+
+      >>> URL('http://example.com/path?a=b&b=1').extend_query(b='2')
+      URL('http://example.com/path?a=b&b=1&b=2')
+      >>> URL('http://example.com/path?a=b&b=1').extend_query([('b', '2')])
+      URL('http://example.com/path?a=b&b=1&b=2')
+      >>> URL('http://example.com/path?a=b&c=e&c=f').extend_query(c='d')
+      URL('http://example.com/path?a=b&c=e&c=f&c=d')
+
+   .. versionadded:: 1.11.0
+
 .. method:: URL.update_query(query)
             URL.update_query(**kwargs)
 

tests/test_normalize_path.py

@@ -8,13 +8,15 @@
     ("/", "/"),
     ("//", "//"),
     ("///", "///"),
+    ("path", "path"),
     # Single-dot
     ("path/to", "path/to"),
     ("././path/to", "path/to"),
     ("path/./to", "path/to"),
     ("path/././to", "path/to"),
     ("path/to/.", "path/to/"),
     ("path/to/./.", "path/to/"),
+    ("/path/to/.", "/path/to/"),
     # Double-dots
     ("../path/to", "path/to"),
     ("path/../to", "to"),

tests/test_update_query.py

@@ -374,3 +374,81 @@ def test_update_query_with_mod_operator():
     assert str(url % {"a": "1"} % {"b": "2"}) == "http://example.com/?a=1&b=2"
     assert str(url % {"a": "1"} % {"a": "3", "b": "2"}) == "http://example.com/?a=3&b=2"
     assert str(url / "foo" % {"a": "1"}) == "http://example.com/foo?a=1"
+
+
+def test_extend_query():
+    url = URL("http://example.com/")
+    assert str(url.extend_query({"a": "1"})) == "http://example.com/?a=1"
+    assert str(URL("test").extend_query(a=1)) == "test?a=1"
+
+    url = URL("http://example.com/?foo=bar")
+    expected_url = URL("http://example.com/?foo=bar&baz=foo")
+
+    assert url.extend_query({"baz": "foo"}) == expected_url
+    assert url.extend_query(baz="foo") == expected_url
+    assert url.extend_query("baz=foo") == expected_url
+
+
+def test_extend_query_with_args_and_kwargs():
+    url = URL("http://example.com/")
+
+    with pytest.raises(ValueError):
+        url.extend_query("a", foo="bar")
+
+
+def test_extend_query_with_multiple_args():
+    url = URL("http://example.com/")
+
+    with pytest.raises(ValueError):
+        url.extend_query("a", "b")
+
+
+def test_extend_query_with_none_arg():
+    url = URL("http://example.com/?foo=bar&baz=foo")
+    assert url.extend_query(None) == url
+
+
+def test_extend_query_with_empty_dict():
+    url = URL("http://example.com/?foo=bar&baz=foo")
+    assert url.extend_query({}) == url
+
+
+def test_extend_query_existing_keys():
+    url = URL("http://example.com/?a=2")
+    assert str(url.extend_query({"a": "1"})) == "http://example.com/?a=2&a=1"
+    assert str(URL("test").extend_query(a=1)) == "test?a=1"
+
+    url = URL("http://example.com/?foo=bar&baz=original")
+    expected_url = URL("http://example.com/?foo=bar&baz=original&baz=foo")
+
+    assert url.extend_query({"baz": "foo"}) == expected_url
+    assert url.extend_query(baz="foo") == expected_url
+    assert url.extend_query("baz=foo") == expected_url
+
+
+def test_extend_query_with_args_and_kwargs_with_existing():
+    url = URL("http://example.com/?a=original")
+
+    with pytest.raises(ValueError):
+        url.extend_query("a", foo="bar")
+
+
+def test_extend_query_with_non_ascii():
+    url = URL("http://example.com/?foo=bar&baz=foo")
+    expected = URL("http://example.com/?foo=bar&baz=foo&%F0%9D%95%A6=%F0%9D%95%A6")
+    assert url.extend_query({"𝕦": "𝕦"}) == expected
+
+
+def test_extend_query_with_non_ascii_as_str():
+    url = URL("http://example.com/?foo=bar&baz=foo&")
+    expected = URL("http://example.com/?foo=bar&baz=foo&%F0%9D%95%A6=%F0%9D%95%A6")
+    assert url.extend_query("𝕦=𝕦") == expected
+
+
+def test_extend_query_with_non_ascii_same_key():
+    url = URL("http://example.com/?foo=bar&baz=foo&%F0%9D%95%A6=%F0%9D%95%A6")
+    expected = URL(
+        "http://example.com/?foo=bar&baz=foo"
+        "&%F0%9D%95%A6=%F0%9D%95%A6&%F0%9D%95%A6=%F0%9D%95%A6"
+    )
+    assert url.extend_query({"𝕦": "𝕦"}) == expected

tests/test_url.py

@@ -993,6 +993,35 @@ def test_joinpath_path_starting_from_slash_is_forbidden():
         assert url.joinpath("/to/others")
 
 
+PATHS = [
+    # No dots
+    ("", ""),
+    ("path", "path"),
+    # Single-dot
+    ("path/to", "path/to"),
+    ("././path/to", "path/to"),
+    ("path/./to", "path/to"),
+    ("path/././to", "path/to"),
+    ("path/to/.", "path/to/"),
+    ("path/to/./.", "path/to/"),
+    # Double-dots
+    ("../path/to", "path/to"),
+    ("path/../to", "to"),
+    ("path/../../to", "to"),
+    # Non-ASCII characters
+    ("μονοπάτι/../../να/ᴜɴɪ/ᴄᴏᴅᴇ", "να/ᴜɴɪ/ᴄᴏᴅᴇ"),
+    ("μονοπάτι/../../να/𝕦𝕟𝕚/𝕔𝕠𝕕𝕖/.", "να/𝕦𝕟𝕚/𝕔𝕠𝕕𝕖/"),
+]
+
+
+@pytest.mark.parametrize("original,expected", PATHS)
+def test_join_path_normalized(original: str, expected: str) -> None:
+    """Test that joinpath normalizes paths."""
+    base_url = URL("http://example.com")
+    new_url = base_url.joinpath(original)
+    assert new_url.path == f"/{expected}"
+
+
 # with_path
 
 

tests/test_url_parsing.py

@@ -210,19 +210,14 @@ def test_no_path(self):
         assert u.query_string == ""
         assert u.fragment == ""
 
-    @pytest.mark.xfail(
-        # FIXME: remove "no cover" pragmas upon xfail marker deletion
-        reason="https://github.com/aio-libs/yarl/issues/821",
-        raises=ValueError,
-    )
     def test_no_host(self):
-        u = URL("//:80")
-        assert u.scheme == ""  # pragma: no cover
-        assert u.host == ""  # pragma: no cover
-        assert u.port == 80  # pragma: no cover
-        assert u.path == "/"  # pragma: no cover
-        assert u.query_string == ""  # pragma: no cover
-        assert u.fragment == ""  # pragma: no cover
+        u = URL("//:77")
+        assert u.scheme == ""
+        assert u.host == ""
+        assert u.port == 77
+        assert u.path == "/"
+        assert u.query_string == ""
+        assert u.fragment == ""
 
     def test_double_port(self):
         with pytest.raises(ValueError):
@@ -457,9 +452,19 @@ def test_complex_frag(self):
 
 
 class TestStripEmptyParts:
-    def test_all_empty(self):
+    def test_all_empty_http(self):
         with pytest.raises(ValueError):
-            URL("//@:?#")
+            URL("http://@:?#")
+
+    def test_all_empty(self):
+        u = URL("//@:?#")
+        assert u.scheme == ""
+        assert u.user is None
+        assert u.password is None
+        assert u.host == ""
+        assert u.path == ""
+        assert u.query_string == ""
+        assert u.fragment == ""
 
     def test_path_only(self):
         u = URL("///path")
@@ -580,3 +585,22 @@ def test_empty_path(self):
         assert u.path == ""
         assert u.query_string == ""
         assert u.fragment == ""
+
+
+@pytest.mark.parametrize(
+    ("scheme"),
+    [
+        ("http"),
+        ("https"),
+        ("ws"),
+        ("wss"),
+        ("ftp"),
+    ],
+)
+def test_schemes_that_require_host(scheme: str) -> None:
+    """Verify that schemes that require a host raise with empty host."""
+    expect = (
+        "Invalid URL: host is required for " f"absolute urls with the {scheme} scheme"
+    )
+    with pytest.raises(ValueError, match=expect):
+        URL(f"{scheme}://:1")

yarl/__init__.py

@@ -8,7 +8,7 @@
     cache_info,
 )
 
-__version__ = "1.11.0.dev0"
+__version__ = "1.11.1.dev0"
 
 __all__ = (
     "URL",