Migrate Mbed TLS to v3.5.2 (#2989)

.devcontainer/All/Dockerfile.All

@@ -1 +1 @@
-FROM ghcr.io/nanoframework/dev-container-all:v2.48
+FROM ghcr.io/nanoframework/dev-container-all:v2.49

.devcontainer/All/Dockerfile.All.SRC

@@ -78,7 +78,7 @@ RUN git clone --branch v6.4.0_rel --recursive https://github.com/eclipse-threadx
     && git clone --branch v6.3.0_rel --recursive https://github.com/eclipse-threadx/netxduo.git --depth 1 ./sources/NetxDuo
 
 # Clone dependent repos (mbedtls, fatfs and littlefs)
-RUN git clone --branch mbedtls-2.28.5 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
+RUN git clone --branch mbedtls-3.5.2 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
     && git clone --branch R0.15 https://github.com/abbrev/fatfs.git --depth 1 ./sources/fatfs \
     && git clone --branch v2.9.3 https://github.com/littlefs-project/littlefs --depth 1 ./sources/littlefs
 

.devcontainer/All/scripts/git-pull-repos.sh

@@ -15,7 +15,7 @@ git svn clone http://svn.code.sf.net/p/chibios/code/branches/stable_21.11.x -rHE
 cd /sources/ChibiOs-Contrib
 git pull origin chibios-21.11.x
 cd /sources/mbedtls
-git pull origin mbedtls-2.28.5
+git pull origin mbedtls-3.5.2
 cd /sources/fatfs
 git pull origin R0.15
 cd /sources/FreeRTOS
@@ -32,4 +32,4 @@ cd /sources/SimpleLinkCC13
 git pull origin 4.20.01.04
 cd /sources/TI_SysConfig
 git pull origin 1.5.0
-echo "All repos pulled and up to date"
+echo "All repos pulled and up to date"

.devcontainer/AzureRTOS/Dockerfile.AzureRTOS

@@ -1 +1 @@
-FROM ghcr.io/nanoframework/dev-container-azure-rtos:v1.27
+FROM ghcr.io/nanoframework/dev-container-azure-rtos:v1.28

.devcontainer/AzureRTOS/Dockerfile.AzureRTOS.SRC

@@ -67,7 +67,7 @@ RUN git clone --branch v6.4.0_rel --recursive https://github.com/eclipse-threadx
     && git clone --branch v6.4.0_rel --recursive https://github.com/eclipse-threadx/netxduo.git --depth 1 ./sources/NetxDuo
 
 # Clone dependent repos (mbedtls, fatfs and littlefs)
-RUN git clone --branch mbedtls-2.28.5 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
+RUN git clone --branch mbedtls-3.5.2 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
     && git clone --branch R0.15 https://github.com/abbrev/fatfs.git --depth 1 ./sources/fatfs \
     && git clone --branch v2.9.3 https://github.com/littlefs-project/littlefs --depth 1 ./sources/littlefs
 

.devcontainer/ChibiOS/Dockerfile.ChibiOS

@@ -1 +1 @@
-FROM ghcr.io/nanoframework/dev-container-chibios:v1.28
+FROM ghcr.io/nanoframework/dev-container-chibios:v1.29

.devcontainer/ChibiOS/Dockerfile.ChibiOS.SRC

@@ -65,7 +65,7 @@ RUN git clone --branch nf-build https://github.com/nanoframework/STM32CubeL4.git
     && git clone --branch chibios-21.11.x https://github.com/ChibiOS/ChibiOS-Contrib.git --depth 1 ./sources/ChibiOs-Contrib
 
 # Clone dependent repos (mbedtls, fatfs and littlefs etc.)
-RUN git clone --branch mbedtls-2.28.5 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
+RUN git clone --branch mbedtls-3.5.2 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
     && git clone --branch R0.15 https://github.com/abbrev/fatfs.git --depth 1 ./sources/fatfs \
     && git clone --branch v2.9.3 https://github.com/littlefs-project/littlefs --depth 1 ./sources/littlefs \
     && git clone --branch STABLE-2_1_3_RELEASE https://github.com/lwip-tcpip/lwip.git --depth 1 ./sources/lwip

.devcontainer/FreeRTOS-NXP/Dockerfile.FreeRTOS-NXP

@@ -1 +1 @@
-FROM ghcr.io/nanoframework/dev-container-freertos-nxp:v1.01
+FROM ghcr.io/nanoframework/dev-container-freertos-nxp:v1.02

.devcontainer/FreeRTOS-NXP/Dockerfile.FreeRTOS-NXP.SRC

@@ -55,7 +55,7 @@ RUN apt-get update \
 RUN mkdir -p /usr/local/bin/gcc
 
 # Clone libs mbedtls and fatfs etc.
-RUN git clone --branch mbedtls-2.28.5 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
+RUN git clone --branch mbedtls-3.5.2 https://github.com/ARMmbed/mbedtls.git --depth 1 ./sources/mbedtls \
     && git clone --branch R0.15 https://github.com/abbrev/fatfs.git --depth 1 ./sources/fatfs \
     && git clone --branch STABLE-2_1_3_RELEASE https://github.com/lwip-tcpip/lwip.git --depth 1 ./sources/lwip
 

CMake/Modules/FindNF_Network.cmake

@@ -450,6 +450,23 @@ else()
 
         endforeach()
 
+        # unset this warning as error, which is required for these source files
+        # OK to remove after this issue is fixed upstream https://github.com/Mbed-TLS/mbedtls/issues/9425
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_accept_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_add_cert_auth_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_close_socket_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_connect_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_decode_private_key_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_exit_context_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_generic.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_generic_init_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_initialize_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_parse_certificate_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_available_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_read_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_uninitialize_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+        set_source_files_properties(${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/ssl_write_internal.cpp PROPERTIES COMPILE_FLAGS -Wno-undef)
+
     endif()
 
     if(Use_Networking_Extra_Driver)

CMake/binutils.ChibiOS.cmake

@@ -213,6 +213,7 @@ macro(nf_add_platform_dependencies target)
             # security provider is MbedTLS
             if(USE_SECURITY_MBEDTLS_OPTION)
                 add_dependencies(NF_Network nano::NF_Network)
+                target_compile_definitions(NF_Network PUBLIC -DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\")
             endif()
 
         endif()
@@ -263,7 +264,7 @@ macro(nf_add_platform_include_directories target)
 
             # need to add extra include directories for MbedTLS
             target_include_directories(
-                mbedcrypto PUBLIC
+                mbedcrypto PRIVATE
                 ${CHIBIOS_HAL_INCLUDE_DIRS}
                 ${CHIBIOS_INCLUDE_DIRS}
                 ${ChibiOSnfOverlay_INCLUDE_DIRS}
@@ -331,10 +332,11 @@ macro(nf_add_platform_sources target)
 
             if(USE_SECURITY_MBEDTLS_OPTION)
                 target_link_libraries(${target}.elf
-                mbedtls
+                    mbedtls
                 )
 
                 add_dependencies(NF_Network mbedtls)
+
             endif()
 
         endif()

CMake/binutils.common.cmake

@@ -572,6 +572,11 @@ macro(nf_setup_target_build_common)
     nf_add_platform_packages(TARGET ${NANOCLR_PROJECT_NAME})
     nf_add_platform_dependencies(${NANOCLR_PROJECT_NAME})
 
+    if(API_nanoFramework.System.Security.Cryptography)
+        # need to add MbedTLS configuration file
+        target_sources(NF_NativeAssemblies PRIVATE ${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h)
+    endif()
+
     nf_add_common_sources(TARGET ${NANOCLR_PROJECT_NAME} EXTRA_LIBRARIES ${CLR_EXTRA_LIBRARIES})
     nf_add_platform_sources(${NANOCLR_PROJECT_NAME})
 
@@ -585,11 +590,14 @@ macro(nf_setup_target_build_common)
     if(USE_SECURITY_MBEDTLS_OPTION AND NOT RTOS_ESP32_CHECK)
 
         # MbedTLS requires setting a compiler definition in order to pass a config file
-        target_compile_definitions(mbedcrypto PUBLIC "-DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\"")
-
-        # need to add extra include directories for MbedTLS
-        target_include_directories(
-            mbedcrypto PUBLIC
+        target_compile_definitions(mbedtls PUBLIC -DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\")
+        target_compile_definitions(mbedcrypto PUBLIC -DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\")
+        target_compile_definitions(mbedx509 PUBLIC -DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\")
+        target_compile_definitions(p256m PUBLIC -DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\")
+        target_compile_definitions(everest PUBLIC -DMBEDTLS_CONFIG_FILE=\"${CMAKE_SOURCE_DIR}/src/PAL/COM/sockets/ssl/MbedTLS/nf_mbedtls_config.h\")
+
+        # set include directories for MbedTLS
+        set(MBEDTLS_INCLUDE_DIRECTORIES
             ${CMAKE_SOURCE_DIR}/src/CLR/Include
             ${CMAKE_SOURCE_DIR}/src/HAL/Include
             ${CMAKE_SOURCE_DIR}/src/PAL
@@ -600,7 +608,35 @@ macro(nf_setup_target_build_common)
             ${CMAKE_SOURCE_DIR}/targets/${RTOS}/_include
             ${TARGET_BASE_LOCATION}/nanoCLR
             ${TARGET_BASE_LOCATION}
+            ${CMAKE_BINARY_DIR}/targets/${RTOS}/${TARGET_BOARD}
         )
+
+        # need to add extra include directories for MbedTLS
+        target_include_directories(
+            mbedtls PRIVATE
+                ${MBEDTLS_INCLUDE_DIRECTORIES}
+        )
+
+        target_include_directories(
+            mbedcrypto PRIVATE
+                ${MBEDTLS_INCLUDE_DIRECTORIES}
+        )
+
+        target_include_directories(
+            mbedx509 PRIVATE
+                ${MBEDTLS_INCLUDE_DIRECTORIES}
+        )
+
+        target_include_directories(
+            p256m PRIVATE
+                ${MBEDTLS_INCLUDE_DIRECTORIES}
+        )
+
+        target_include_directories(
+            everest PRIVATE
+                ${MBEDTLS_INCLUDE_DIRECTORIES}
+        )
+
         # platform implementation of hardware random provider
         target_sources(mbedcrypto PRIVATE ${BASE_PATH_FOR_CLASS_LIBRARIES_MODULES}/mbedtls_entropy_hardware_pool.c)
 
@@ -611,6 +647,11 @@ macro(nf_setup_target_build_common)
         nf_set_compile_definitions(TARGET mbedx509 BUILD_TARGET ${NANOCLR_PROJECT_NAME})
         nf_set_compile_definitions(TARGET mbedtls BUILD_TARGET ${NANOCLR_PROJECT_NAME})
 
+        # need to unset several flags for MbedTLS to compile correctly
+        target_compile_options(mbedtls PRIVATE -Wno-undef -Wno-error=unused-function -Wno-error=discarded-qualifiers -Wno-error=unused-parameter)
+        target_compile_options(mbedcrypto PRIVATE -Wno-undef -Wno-error=unused-function -Wno-error=discarded-qualifiers -Wno-error=unused-parameter)
+        target_compile_options(mbedx509 PRIVATE -Wno-undef -Wno-error=unused-function -Wno-error=discarded-qualifiers -Wno-error=unused-parameter)
+
     endif()
 
     # set compile definitions
@@ -712,7 +753,7 @@ function(nf_add_mbedtls_library)
 
     # set tag for currently supported version
     # WHEN CHANGING THIS MAKE SURE TO UPDATE THE DEV CONTAINERS
-    set(MBEDTLS_GIT_TAG "mbedtls-2.28.5")
+    set(MBEDTLS_GIT_TAG "mbedtls-3.5.2")
 
     # set options for Mbed TLS
     option(ENABLE_TESTING "no testing when building Mbed TLS." OFF)
@@ -749,9 +790,15 @@ function(nf_add_mbedtls_library)
     FetchContent_GetProperties(mbedtls)
     if(NOT mbedtls_POPULATED)
         # Fetch the content using previously declared details
-        FetchContent_MakeAvailable(mbedtls)
+        FetchContent_Populate(mbedtls)
     endif()
 
+    set(MBEDTLS_AS_SUBPROJECT TRUE)
+    set(DISABLE_PACKAGE_CONFIG_AND_INSTALL OFF)
+
+    # add the MbedTLS library
+    add_subdirectory(${mbedtls_SOURCE_DIR} MbedTLS_Source)
+
 endfunction()
 
 # PLATFORM_INCLUDES with platform and target include paths to be added to lwIP

azure-pipelines-nightly.yml

@@ -293,7 +293,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('STM32_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('STM32_1_11_0_versioncounter', 0)]
       HelperPackageVersion: $[counter('HelperPackageVersioncounter', 0)]
       TargetPlatform: "stm32"
 
@@ -544,7 +544,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('ESP32_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('ESP32_1_11_0_versioncounter', 0)]
       IDF_PATH: "D:/a/1/s/esp-idf"
       PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
       TargetPlatform: "esp32"
@@ -620,7 +620,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('TI_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('TI_1_11_0_versioncounter', 0)]
       HelperPackageVersion: $[counter('HelperPackageVersioncounter', 0)]
       TargetPlatform: "ti_simplelink"
 
@@ -671,7 +671,7 @@ jobs:
 
     variables:
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('AZURERTOS_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('AZURERTOS_1_11_0_versioncounter', 0)]
       HelperPackageVersion: $[counter('HelperPackageVersioncounter', 0)]
       TargetPlatform: "azure_rtos"
 

azure-pipelines.yml

@@ -412,7 +412,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('STM32_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('STM32_1_11_0_versioncounter', 0)]
       HelperPackageVersion: $[counter('HelperPackageVersioncounter', 0)]
       TargetPlatform: "stm32"
 
@@ -562,7 +562,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('ESP32_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('ESP32_1_11_0_versioncounter', 0)]
       IDF_PATH: "D:/a/1/s/esp-idf"
       PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
       TargetPlatform: "esp32"
@@ -643,7 +643,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('NXP_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('NXP_1_11_0_versioncounter', 0)]
       GIT_LFS_SKIP_SMUDGE: 1
       TargetPlatform: "freertos"
 
@@ -721,7 +721,7 @@ jobs:
     variables:
       DOTNET_NOLOGO: true
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('TI_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('TI_1_11_0_versioncounter', 0)]
       HelperPackageVersion: $[counter('HelperPackageVersioncounter', 0)]
       TargetPlatform: "ti_simplelink"
 
@@ -792,7 +792,7 @@ jobs:
 
     variables:
       # creates a counter and assigns it to the revision variable
-      REVISION: $[counter('AZURERTOS_1_10_0_versioncounter', 0)]
+      REVISION: $[counter('AZURERTOS_1_11_0_versioncounter', 0)]
       HelperPackageVersion: $[counter('HelperPackageVersioncounter', 0)]
       TargetPlatform: "azure_rtos"
 
@@ -843,7 +843,7 @@ jobs:
         value: true
       # creates a counter and assigns it to the revision variable
       - name: REVISION
-        value: $[counter('WIN32_1_10_0_versioncounter', 0)]
+        value: $[counter('WIN32_1_11_0_versioncounter', 0)]
       - name: LITTLEFS_PATH
         value: "D:/a/1/s/littlefs"
 

src/DeviceInterfaces/System.Net/sys_net_native.cpp

@@ -347,8 +347,7 @@ const CLR_RT_NativeAssemblyData g_CLR_AssemblyNative_System_Net =
     "System.Net",
     0xD82C1452,
     method_lookup,
-    { 100, 2, 0, 1 }
+    { 100, 2, 0, 11 }
 };
 
 // clang-format on
-

src/DeviceInterfaces/System.Net/sys_net_native.h

@@ -113,6 +113,16 @@
 //     WirelessAPConfiguration_ConfigurationOptions_HiddenSSID = 8,
 // } WirelessAPConfiguration_ConfigurationOptions;
 
+// MOVED TO src\PAL\COM\sockets\ssl\ssl_functions.h for convinience
+// typedef enum __nfpack SslProtocols
+// {
+//     SslProtocols_None = 0,
+//     SslProtocols_Tls = 192,
+//     SslProtocols_Tls11 = 768,
+//     SslProtocols_Tls12 = 3072,
+//     SslProtocols_Tls13 = 12288,
+// } SslProtocols;
+
 struct Library_sys_net_native_System_Net_NetworkInformation_NetworkInterface
 {
     static const int FIELD___interfaceIndex = 1;

src/PAL/COM/sockets/ssl/MbedTLS/mbedtls.h

@@ -7,31 +7,25 @@
 #define MBEDTLS_H
 
 #include <nanoHAL.h>
+#include <sockets_lwip.h>
 
 #ifdef PLATFORM_ESP32
 #include <esp32_idf.h>
 #include <mbedtls/esp_config.h>
-#else
-#include "nf_mbedtls_config.h"
 #endif
 
-#include <sockets_lwip.h>
-
 #ifdef __cplusplus
 extern "C"
 {
 #endif
 
-#include "mbedtls/version.h"
-#include "mbedtls/platform.h"
-#include "mbedtls/net_sockets.h"
-#include "mbedtls/ssl.h"
-#include "mbedtls/entropy.h"
-#include "mbedtls/ctr_drbg.h"
-#include "mbedtls/error.h"
-#if MBEDTLS_VERSION_MAJOR < 3
-#include "mbedtls/certs.h"
-#endif
+#include <mbedtls/version.h>
+#include <mbedtls/platform.h>
+#include <mbedtls/net_sockets.h>
+#include <mbedtls/ssl.h>
+#include <mbedtls/entropy.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/error.h>
 
     typedef struct mbedTLS_NFContext
     {