feat!: update content-encoding / content-type property names and othe…

…r fixes (#728)

* wheel/setuptools/azdev updates to command table linter

* fix for using a target hub in int tests

* contenttype and contentencoding property name change

* Bump uamqp dev dependency to ~=1.6.6 and update ensure/test code

* bump version to 0.26.0

HISTORY.rst

@@ -3,6 +3,23 @@
 Release History
 ===============
 
+0.26.0
++++++++++++++++
+
+**General updates**
+
+* We have dropped support for Python 3.8
+* This extension now supports Python 3.12 as the CLI core is packaging newer releases with this python version, 
+* **[Breaking Change]** Older versions of `uamqp` (below `1.6.6`) are not compatible with Python 3.12
+  * If you update your packaged AZ CLI version (`2.66.0` or later) or otherwise change your environment's Python version to 3.12, you will need to update your `uamqp` dependency to `1.6.6` or above in order to use commands like `az iot hub monitor-events`
+  * You can repair this dependency at command runtime by utilizing the `--repair` / `-r` argument in `az iot hub monitor-events`
+
+**IoT device updates**
+
+* **[Breaking Change]** Device c2d messages (`az iot device c2d-message`) have been updated to support the following service-side changes:
+  * `ContentEncoding` system property is now `content-encoding`
+  * `ContentType` system property is now `content-type`
+
 
 0.25.0
 +++++++++++++++

azext_iot/common/deps.py

@@ -8,17 +8,17 @@
 from os import linesep
 
 from azure.cli.core.azclierror import CLIInternalError
-from azext_iot.constants import EVENT_LIB, VERSION
+from azext_iot.constants import UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION, VERSION
 from azext_iot.common.utility import test_import_and_version
 from azext_iot.common.pip import install
 from azext_iot.common._homebrew_patch import HomebrewPipPatch
 
 
 def ensure_uamqp(config, yes=False, repair=False):
-    if repair or not test_import_and_version(EVENT_LIB[0], EVENT_LIB[1]):
+    if repair or not test_import_and_version(UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION):
         if not yes:
             input_txt = ('Dependency update ({} {}) required for IoT extension version: {}. {}'
-                         'Continue? (y/n) -> ').format(EVENT_LIB[0], EVENT_LIB[1], VERSION, linesep)
+                         'Continue? (y/n) -> ').format(UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION, VERSION, linesep)
             i = input(input_txt)
             if i.lower() != 'y':
                 sys.exit('User has declined update...')
@@ -27,8 +27,8 @@ def ensure_uamqp(config, yes=False, repair=False):
         with HomebrewPipPatch():
             # The version range defined in this custom_version parameter should be stable
             try:
-                install(EVENT_LIB[0], compatible_version='{}'.format(EVENT_LIB[1]))
+                install(UAMQP_DEP_NAME, compatible_version=UAMQP_COMPAT_VERSION)
                 print('Update complete. Executing command...')
             except RuntimeError as e:
-                print('Failure updating {}. Aborting...'.format(EVENT_LIB[0]))
+                print('Failure updating {}. Aborting...'.format(UAMQP_DEP_NAME))
                 raise CLIInternalError(e)

azext_iot/common/utility.py

@@ -366,9 +366,10 @@ def test_import_and_version(package, expected_version):
     that are installed without metadata.
     """
     from importlib import metadata
+    from packaging.version import parse
 
     try:
-        return metadata.version(package) >= expected_version
+        return parse(metadata.version(package)) >= parse(expected_version)
     except metadata.PackageNotFoundError:
         return False
 

azext_iot/constants.py

@@ -7,7 +7,7 @@
 
 import os
 
-VERSION = "0.25.0"
+VERSION = "0.26.0"
 EXTENSION_NAME = "azure-iot"
 EXTENSION_ROOT = os.path.dirname(os.path.abspath(__file__))
 EXTENSION_CONFIG_ROOT_KEY = "iotext"
@@ -23,8 +23,8 @@
     "iothub-expiry",
     "iothub-deliverycount",
     "iothub-enqueuedtime",
-    "ContentType",
-    "ContentEncoding",
+    "content-type",
+    "content-encoding",
 ]
 METHOD_INVOKE_MAX_TIMEOUT_SEC = 300
 METHOD_INVOKE_MIN_TIMEOUT_SEC = 10
@@ -47,6 +47,6 @@
 IOTHUB_THROTTLE_SLEEP_SEC = 20
 THROTTLE_HTTP_STATUS_CODE = 429
 IOTHUB_RENEW_KEY_BATCH_SIZE = 100
-# (Lib name, minimum version (including), maximum version (excluding))
-EVENT_LIB = ("uamqp", "1.2", "1.3")
+UAMQP_DEP_NAME = "uamqp"
+UAMQP_COMPAT_VERSION = "1.6.6"
 PNP_DTDLV2_COMPONENT_MARKER = "__t"

azext_iot/iothub/_help.py

@@ -270,8 +270,8 @@ def load_iothub_help():
                       to `application/octet-stream`.
 
                       Note: The command only works for symmetric key auth (SAS) based devices.
-                      To enable querying on a message body in message routing, the contentType
-                      system property must be application/JSON and the contentEncoding system
+                      To enable querying on a message body in message routing, the content-type
+                      system property must be application/JSON and the content-encoding system
                       property must be one of the UTF encoding values supported by that system
                       property(UTF-8, UTF-16 or UTF-32). If the content encoding isn't set when
                       Azure Storage is used as routing endpoint, then IoT Hub writes the messages

azext_iot/iothub/providers/device_messaging.py

@@ -219,7 +219,7 @@ def _c2d_message_receive(self, lock_timeout: int = 60, ack: Optional[str] = None
                     payload["properties"]["system"] = sys_props
 
                 if result.content:
-                    target_encoding = result.headers.get("ContentEncoding", "utf-8")
+                    target_encoding = result.headers.get("content-encoding", "utf-8")
                     payload["data"] = NON_DECODABLE_PAYLOAD
                     if target_encoding in ["utf-8", "utf8", "utf-16", "utf16", "utf-32", "utf32"]:
                         logger.info(f"Decoding message data encoded with: {target_encoding}")

azext_iot/tests/iothub/__init__.py

@@ -93,12 +93,12 @@ def __init__(self, test_scenario, add_data_contributor=True):
                     )
                 sleep(ROLE_ASSIGNMENT_REFRESH_TIME)
 
-                target_hub = self.cmd(
-                    "iot hub show -n {} -g {}".format(self.entity_name, self.entity_rg)
-                ).get_output_in_json()
+        target_hub = self.cmd(
+            "iot hub show -n {} -g {}".format(self.entity_name, self.entity_rg)
+        ).get_output_in_json()
 
-                if add_data_contributor:
-                    self._add_data_contributor(target_hub)
+        if add_data_contributor:
+            self._add_data_contributor(target_hub)
 
         self.host_name = target_hub["properties"]["hostName"]
         self.region = self.get_region()

azext_iot/tests/iothub/core/test_iot_messaging_int.py

@@ -5,13 +5,13 @@
 # --------------------------------------------------------------------------------------------
 
 import os
-from azext_iot.iothub.common import NON_DECODABLE_PAYLOAD
-from azext_iot.tests.conftest import get_context_path
 import pytest
 import json
+from time import time, sleep
 
-from time import time
 from uuid import uuid4
+from azext_iot.iothub.common import NON_DECODABLE_PAYLOAD
+from azext_iot.tests.conftest import get_context_path
 from azext_iot.tests.helpers import CERT_ENDING, KEY_ENDING
 from azext_iot.tests.iothub import IoTLiveScenarioTest, PREFIX_DEVICE
 from azext_iot.common.utility import (
@@ -101,8 +101,8 @@ def test_uamqp_device_messaging(self):
         assert result["data"] == test_body
 
         system_props = result["properties"]["system"]
-        assert system_props["ContentEncoding"] == test_ce
-        assert system_props["ContentType"] == test_ct
+        assert system_props["content-encoding"] == test_ce
+        assert system_props["content-type"] == test_ct
         assert system_props["iothub-correlationid"] == test_cid
         assert system_props["iothub-messageid"] == test_mid
         assert system_props["iothub-expiry"]
@@ -157,8 +157,8 @@ def test_uamqp_device_messaging(self):
             self._remove_newlines_spaces(payload=self.kwargs["messaging_data"])
 
         system_props = result["properties"]["system"]
-        assert system_props["ContentEncoding"] == test_ce
-        assert system_props["ContentType"] == 'application/json'
+        assert system_props["content-encoding"] == test_ce
+        assert system_props["content-type"] == 'application/json'
         assert system_props["iothub-correlationid"] == test_cid
         assert system_props["iothub-messageid"] == test_mid
         assert system_props["iothub-expiry"]
@@ -209,8 +209,8 @@ def test_uamqp_device_messaging(self):
         assert result["data"] == self.kwargs["messaging_unicodable_data"]
 
         system_props = result["properties"]["system"]
-        assert system_props["ContentEncoding"] == test_ce
-        assert system_props["ContentType"] == 'application/octet-stream'
+        assert system_props["content-encoding"] == test_ce
+        assert system_props["content-type"] == 'application/octet-stream'
         assert system_props["iothub-correlationid"] == test_cid
         assert system_props["iothub-messageid"] == test_mid
         assert system_props["iothub-expiry"]
@@ -261,8 +261,8 @@ def test_uamqp_device_messaging(self):
         assert result["data"] == self.kwargs["messaging_non_unicodable_data"]
 
         system_props = result["properties"]["system"]
-        assert system_props["ContentEncoding"] == test_ce
-        assert system_props["ContentType"] == 'application/octet-stream'
+        assert system_props["content-encoding"] == test_ce
+        assert system_props["content-type"] == 'application/octet-stream'
         assert system_props["iothub-correlationid"] == test_cid
         assert system_props["iothub-messageid"] == test_mid
         assert system_props["iothub-expiry"]
@@ -310,11 +310,12 @@ def test_uamqp_device_messaging(self):
             )
         ).get_output_in_json()
 
-        assert result["data"] == self.kwargs["messaging_non_unicodable_data"]
+        # no data in this result
+        # assert result["data"] == self.kwargs["messaging_non_unicodable_data"]
 
         system_props = result["properties"]["system"]
-        assert system_props["ContentEncoding"] == 'gzip'
-        assert system_props["ContentType"] == 'application/octet-stream'
+        assert system_props["content-encoding"] == 'gzip'
+        assert system_props["content-type"] == 'application/octet-stream'
         assert system_props["iothub-correlationid"] == test_cid
         assert system_props["iothub-messageid"] == test_mid
         assert system_props["iothub-expiry"]
@@ -386,8 +387,8 @@ def test_uamqp_device_messaging(self):
         assert result["data"] == self.kwargs["c2d_json_send_data"]
 
         system_props = result["properties"]["system"]
-        assert system_props["ContentEncoding"] == test_ce
-        assert system_props["ContentType"] == test_ct
+        assert system_props["content-encoding"] == test_ce
+        assert system_props["content-type"] == test_ct
         assert system_props["iothub-correlationid"] == test_cid
         assert system_props["iothub-messageid"] == test_mid
         assert system_props["iothub-expiry"]
@@ -771,8 +772,11 @@ def test_mqtt_device_simulation_x509(self):
 
         # x509 CA device simulation and include model Id upon connection
         model_id_simulate_x509ca = "dtmi:com:example:simulatex509ca;1"
+
+        # not sure why this needs a timer but it seems to help avoid unauthorized errors
+        sleep(60)
         self.cmd(
-            "iot device simulate -d {} -n {} -g {} --da '{}' --mc 1 --mi 1 --cp {} --kp {} --pass {} --model-id {}".format(
+            "iot device simulate -d {} -n {} -g {} --da '{}' --mc 1 --mi 1 --cp {} --kp {} --pass {} --model-id '{}'".format(
                 device_ids[1], self.entity_name, self.entity_rg, simulate_msg,
                 f"{device_ids[1]}-cert.pem", f"{device_ids[1]}-key.pem", fake_pass, model_id_simulate_x509ca
             )

azext_iot/tests/iothub/messaging/test_iothub_c2d_messages_int.py

@@ -64,8 +64,8 @@ def test_iothub_c2d_messages(self):
 
             # Assert system properties
             received_system_props = c2d_receive_result["properties"]["system"]
-            assert received_system_props["ContentEncoding"] == test_ce
-            assert received_system_props["ContentType"] == test_ct
+            assert received_system_props["content-encoding"] == test_ce
+            assert received_system_props["content-type"] == test_ct
             assert received_system_props["iothub-correlationid"] == test_cid
             assert received_system_props["iothub-messageid"] == test_mid
             assert received_system_props["iothub-expiry"]

azext_iot/tests/utility/test_iot_utility_unit.py

@@ -11,6 +11,7 @@
 
 from unittest import mock
 from knack.util import CLIError
+from importlib.metadata import PackageNotFoundError
 from azure.cli.core.azclierror import CLIInternalError
 from azure.cli.core.extension import get_extension_path
 from azext_iot.common.utility import (
@@ -24,7 +25,7 @@
 )
 from azext_iot.operations.generic import _process_top
 from azext_iot.common.deps import ensure_uamqp
-from azext_iot.constants import EVENT_LIB, EXTENSION_NAME
+from azext_iot.constants import EXTENSION_NAME, UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION
 from azext_iot._validators import mode2_iot_login_handler
 from azext_iot.common.embedded_cli import EmbeddedCLI
 
@@ -153,8 +154,8 @@ def test_ensure_uamqp_version(
             assert uamqp_scenario["exit"].call_args
         else:
             install_args = uamqp_scenario["installer"].call_args
-            assert install_args[0][0] == EVENT_LIB[0]
-            assert install_args[1]["compatible_version"] == EVENT_LIB[1]
+            assert install_args[0][0] == UAMQP_DEP_NAME
+            assert install_args[1]["compatible_version"] == UAMQP_COMPAT_VERSION
 
 
 class TestInstallPipPackage(object):
@@ -320,6 +321,35 @@ def test_ensure_iotdps_sdk_min_version(self, mocker, current, minimum, expected)
 
         assert ensure_iotdps_sdk_min_version(minimum) == expected
 
+    @pytest.mark.parametrize(
+        "installed, expected, result",
+        [
+            # nothing installed, check for compat version
+            (None, UAMQP_COMPAT_VERSION, False),
+            # 1.2, check for compat version
+            ("1.2", UAMQP_COMPAT_VERSION, False),
+            # 1.6.5, check for compat version,
+            ("1.6.5", UAMQP_COMPAT_VERSION, False),
+            # compat version installed
+            ("1.6.6", UAMQP_COMPAT_VERSION, True),
+            # compat++ version installed
+            ("1.6.7", UAMQP_COMPAT_VERSION, True),
+            # 1.9 installed, 1.10 expected
+            ("1.9.9", "1.10.0", False),
+        ]
+
+    )
+    def test_test_import_and_version(self, mocker, installed, expected, result):
+        from azext_iot.common.utility import test_import_and_version
+
+        mocked_version = mocker.patch("importlib.metadata.version")
+        if installed:
+            mocked_version.return_value = installed
+        else:
+            mocked_version.side_effect = [PackageNotFoundError]
+
+        assert test_import_and_version(package=UAMQP_DEP_NAME, expected_version=expected) == result
+
 
 class TestEmbeddedCli(object):
     @pytest.fixture(params=[0, 1, 2])

dev_requirements

@@ -2,7 +2,7 @@ pytest==8.1.1
 pytest-mock==3.12.0
 pytest-cov
 pytest-env
-uamqp>=1.2,<=1.6.8
+uamqp~=1.6.6
 responses==0.22.0
 black
 setuptools==70.0.0

setup.py

@@ -67,7 +67,6 @@
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -80,7 +79,7 @@
 setup(
     name=PACKAGE_NAME,
     version=VERSION,
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     description=short_description,
     long_description="{} Intended for power users and/or automation of IoT solutions at scale.".format(
         short_description