Merge pull request #478 from danieldotnl/httpx_issue

Merge url params with provided params since httpx doesn't do it anymore

custom_components/multiscrape/http.py

@@ -2,6 +2,7 @@
 import asyncio
 import logging
 from collections.abc import Callable
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import httpx
 from homeassistant.const import (CONF_AUTHENTICATION, CONF_HEADERS,
@@ -88,12 +89,15 @@ async def async_request(self, context, resource, method=None, request_data=None,
         headers = self._headers_renderer(variables)
         params = self._params_renderer(variables)
 
+        # Merging params in multiscrape since httpx doesn't do it anymore: https://github.com/encode/httpx/issues/3433
+        merged_resource = merge_url_with_params(resource, params)
+
         _LOGGER.debug(
             "%s # Executing %s-request with a %s to url: %s with headers: %s and cookies: %s.",
             self._config_name,
             context,
             method,
-            resource,
+            merged_resource,
             headers,
             cookies
         )
@@ -108,9 +112,8 @@ async def async_request(self, context, resource, method=None, request_data=None,
         try:
             response = await self._client.request(
                 method,
-                resource,
+                merged_resource,
                 headers=headers,
-                params=params,
                 auth=self._auth,
                 data=data,
                 timeout=self._timeout,
@@ -127,8 +130,10 @@ async def async_request(self, context, resource, method=None, request_data=None,
                 task1 = self._async_file_log(
                     "response_headers", context, response.headers
                 )
-                task2 = self._async_file_log("response_body", context, response.text)
-                task3 = self._async_file_log("response_cookies", context, response.cookies)
+                task2 = self._async_file_log(
+                    "response_body", context, response.text)
+                task3 = self._async_file_log(
+                    "response_cookies", context, response.cookies)
                 await asyncio.gather(task1, task2, task3)
 
             # bit of a hack since httpx also raises an exception for redirects: https://github.com/encode/httpx/blob/c6c8cb1fe2da9380f8046a19cdd5aade586f69c8/CHANGELOG.md#0200-13th-october-2021
@@ -140,7 +145,7 @@ async def async_request(self, context, resource, method=None, request_data=None,
                 "%s # Timeout error while executing %s request to url: %s.\n Error message:\n %s",
                 self._config_name,
                 method,
-                resource,
+                merged_resource,
                 repr(ex),
             )
             await self._handle_request_exception(context, response)
@@ -150,7 +155,7 @@ async def async_request(self, context, resource, method=None, request_data=None,
                 "%s # Request error while executing %s request to url: %s.\n Error message:\n %s",
                 self._config_name,
                 method,
-                resource,
+                merged_resource,
                 repr(ex),
             )
             await self._handle_request_exception(context, response)
@@ -160,7 +165,7 @@ async def async_request(self, context, resource, method=None, request_data=None,
                 "%s # Error executing %s request to url: %s.\n Error message:\n %s",
                 self._config_name,
                 method,
-                resource,
+                merged_resource,
                 repr(ex),
             )
             await self._handle_request_exception(context, response)
@@ -208,3 +213,18 @@ async def _async_file_log(self, content_name, context, content):
                 content_name,
                 filename,
             )
+
+
+def merge_url_with_params(url, params):
+    """Merge URL with parameters."""
+    if not params:
+        return url
+
+    url_parts = list(urlparse(url))
+    query = parse_qs(url_parts[4])
+    query.update(params)
+    url_parts[4] = urlencode(query, doseq=True)
+    try:
+        return urlunparse(url_parts)
+    except Exception as ex:
+        raise ValueError(f"Failed to merge URL with parameters: {ex}") from ex

tests/test_http.py

@@ -0,0 +1,88 @@
+"""Tests for the http module."""
+from custom_components.multiscrape.http import merge_url_with_params
+
+
+def test_merge_url_with_params():
+    """Test merge_url_with_params function."""
+    url = "https://example.com"
+    params = {"param1": "value1", "param2": "value2"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1&param2=value2"
+
+    url = "https://example.com"
+    params = {"param1": "value1", "param2": 2}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1&param2=2"
+
+    url = "https://example.com?param1=value1"
+    params = {"param2": "value2"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1&param2=value2"
+
+    url = "https://example.com?param1=value1"
+    params = {}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1"
+
+    url = "https://example.com?param1=33"
+    params = None
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=33"
+
+
+def test_merge_url_with_params_existing_params():
+    """Test merge_url_with_params with existing URL parameters."""
+    url = "https://example.com?param1=value1"
+    params = {"param2": "value2"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1&param2=value2"
+
+    url = "https://example.com?param1=value1&param2=value2"
+    params = {"param3": "value3"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1&param2=value2&param3=value3"
+
+
+def test_merge_url_with_params_override_existing():
+    """Test merge_url_with_params overriding existing URL parameters."""
+    url = "https://example.com?param1=value1"
+    params = {"param1": "new_value1", "param2": "value2"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=new_value1&param2=value2"
+
+
+def test_merge_url_with_params_array_values():
+    """Test merge_url_with_params with array values."""
+    url = "https://example.com"
+    params = {"param1": ["value1", "value2"]}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1&param1=value2"
+
+    url = "https://example.com?param1=value1"
+    params = {"param1": ["value2", "value3"]}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value2&param1=value3"
+
+
+def test_merge_url_with_params_special_characters():
+    """Test merge_url_with_params with special characters in parameters."""
+    url = "https://example.com"
+    params = {"param1": "value with spaces",
+              "param2": "value&with&special&chars"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value+with+spaces&param2=value%26with%26special%26chars"
+
+
+def test_merge_url_with_params_url_components():
+    """Test merge_url_with_params with various URL components."""
+    # Test URL with port
+    url = "https://example.com:8080"
+    params = {"param1": "value1"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com:8080?param1=value1"
+
+    # Test URL with fragment
+    url = "https://example.com#section1"
+    params = {"param1": "value1"}
+    result = merge_url_with_params(url, params)
+    assert result == "https://example.com?param1=value1#section1"