Merge branch 'feature/delta_ota' into 'main'

Support delta OTA

Closes TZ-1121 and TZ-748

See merge request espressif/esp-zigbee-sdk!141

examples/common/delta_ota/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRC_DIRS "src"
+                       INCLUDE_DIRS "include"
+                       PRIV_REQUIRES esp_delta_ota app_update
+)

examples/common/delta_ota/include/esp_delta_ota_ops.h

@@ -0,0 +1,103 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: CC0-1.0
+ *
+ * Zigbee delta OTA Example
+ *
+ * This example code is in the Public Domain (or CC0 licensed, at your option.)
+ *
+ * Unless required by applicable law or agreed to in writing, this
+ * software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define DELTA_OTA_UPGRADE_IMAGE_HEADER_SIZE       sizeof(esp_image_header_t)
+#define DELTA_OTA_UPGRADE_PATCH_HEADER_SIZE       64
+#define DELTA_OTA_UPGRADE_DIGEST_SIZE             32
+#define DELTA_OTA_UPGRADE_MAGIC                   0xfccdde10
+
+typedef struct esp_delta_ota_ctx_s {
+    char *header_data;
+    int  header_data_read;
+    bool verify_patch_flag;
+    bool chip_id_verified;
+} esp_delta_ota_ctx_t;
+
+/**
+ * @brief   Commence a Delta OTA update writing to the specified partition.
+
+ * The specified partition is erased to the specified image size.
+ *
+ * If image size is not yet known, pass OTA_SIZE_UNKNOWN which will
+ * cause the entire partition to be erased.
+ *
+ * On success, this function allocates memory that remains in use
+ * until esp_delta_ota_end() is called with the returned handle.
+ *
+ * Note: If the rollback option is enabled and the running application has the ESP_OTA_IMG_PENDING_VERIFY state then
+ * it will lead to the ESP_ERR_OTA_ROLLBACK_INVALID_STATE error. Confirm the running app before to run download a new app,
+ * use esp_ota_mark_app_valid_cancel_rollback() function for it (this should be done as early as possible when you first download a new application).
+ *
+ * @param partition Pointer to info for partition which will receive the OTA update. Required.
+ * @param image_size Size of new OTA app image. Partition will be erased in order to receive this size of image. If 0 or OTA_SIZE_UNKNOWN, the entire partition is erased.
+ * @param out_handle On success, returns a handle which should be used for subsequent esp_ota_write() and esp_delta_ota_end() calls.
+
+ * @return
+ *    - ESP_OK: OTA operation commenced successfully.
+ *    - ESP_ERR_INVALID_ARG: partition or out_handle arguments were NULL, or partition doesn't point to an OTA app partition.
+ *    - ESP_ERR_NO_MEM: Cannot allocate memory for OTA operation.
+ *    - ESP_ERR_OTA_PARTITION_CONFLICT: Partition holds the currently running firmware, cannot update in place.
+ *    - ESP_ERR_NOT_FOUND: Partition argument not found in partition table.
+ *    - ESP_ERR_OTA_SELECT_INFO_INVALID: The OTA data partition contains invalid data.
+ *    - ESP_ERR_INVALID_SIZE: Partition doesn't fit in configured flash size.
+ *    - ESP_ERR_FLASH_OP_TIMEOUT or ESP_ERR_FLASH_OP_FAIL: Flash write failed.
+ *    - ESP_ERR_OTA_ROLLBACK_INVALID_STATE: If the running app has not confirmed state. Before performing an update, the application must be valid.
+ */
+esp_err_t esp_delta_ota_begin(const esp_partition_t *partition, size_t image_size, esp_ota_handle_t *out_handle);
+
+/**
+ * @brief   Write Delta OTA update data to partition
+ *
+ * This function can be called multiple times as
+ * data is received during the OTA operation. Data is written
+ * sequentially to the partition.
+ *
+ * @param handle  Handle obtained from esp_ota_begin
+ * @param data    Data buffer to write
+ * @param size    Size of data buffer in bytes.
+ *
+ * @return
+ *    - ESP_OK: Data was written to flash successfully, or size = 0
+ *    - ESP_ERR_INVALID_ARG: handle is invalid.
+ *    - ESP_ERR_OTA_VALIDATE_FAILED: First byte of image contains invalid app image magic byte.
+ *    - ESP_ERR_FLASH_OP_TIMEOUT or ESP_ERR_FLASH_OP_FAIL: Flash write failed.
+ *    - ESP_ERR_OTA_SELECT_INFO_INVALID: OTA data partition has invalid contents
+ */
+esp_err_t esp_delta_ota_write(esp_ota_handle_t handle, uint8_t *data, int size);
+
+/**
+ * @brief Finish Delta OTA update and validate newly written app image.
+ *
+ * @param handle  Handle obtained from esp_delta_ota_begin().
+ *
+ * @note After calling esp_delta_ota_end(), the handle is no longer valid and any memory associated with it is freed (regardless of result).
+ *
+ * @return
+ *    - ESP_OK: Newly written OTA app image is valid.
+ *    - ESP_ERR_NOT_FOUND: OTA handle was not found.
+ *    - ESP_ERR_INVALID_ARG: Handle was never written to.
+ *    - ESP_ERR_OTA_VALIDATE_FAILED: OTA image is invalid (either not a valid app image, or - if secure boot is enabled - signature failed to verify.)
+ *    - ESP_ERR_INVALID_STATE: If flash encryption is enabled, this result indicates an internal error writing the final encrypted bytes to flash.
+ */
+esp_err_t esp_delta_ota_end(esp_ota_handle_t handle);
+
+#ifdef __cplusplus
+}
+#endif

examples/common/delta_ota/src/esp_delta_ota_ops.c

@@ -0,0 +1,189 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: CC0-1.0
+ *
+ * Zigbee delta OTA Example
+ *
+ * This example code is in the Public Domain (or CC0 licensed, at your option.)
+ *
+ * Unless required by applicable law or agreed to in writing, this
+ * software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "esp_check.h"
+#include "esp_log.h"
+#include "esp_ota_ops.h"
+#include "esp_app_format.h"
+#include "esp_delta_ota.h"
+
+#include "esp_delta_ota_ops.h"
+
+static const char *TAG = "ESP_DELTA_OTA_OPS";
+
+static const esp_partition_t *s_cur_partition = NULL;
+static esp_delta_ota_handle_t s_delta_ota_handle = NULL;
+static esp_delta_ota_ctx_t    *s_delta_ota_ctx = NULL;
+
+static esp_err_t delta_ota_patch_header_verify(void *img_hdr_data)
+{
+    uint8_t sha_256[DELTA_OTA_UPGRADE_DIGEST_SIZE] = { 0 };
+    uint32_t recv_magic = 0;
+    uint8_t *digest = NULL;
+
+    if (!img_hdr_data) {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    recv_magic = *(uint32_t *)img_hdr_data;
+    if (recv_magic != DELTA_OTA_UPGRADE_MAGIC) {
+        ESP_LOGE(TAG, "Invalid magic word in patch");
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    digest = (uint8_t *)(img_hdr_data + sizeof(uint32_t));
+    esp_partition_get_sha256(s_cur_partition, sha_256);
+    if (memcmp(sha_256, digest, DELTA_OTA_UPGRADE_DIGEST_SIZE) != 0) {
+        ESP_LOGE(TAG, "Invalid patch, the SHA256 of the current firmware differs from that in the patch header.");
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    return ESP_OK;
+}
+
+static bool delta_ota_chip_id_verify(void *bin_header_data)
+{
+    esp_image_header_t *header = (esp_image_header_t *)bin_header_data;
+    ESP_RETURN_ON_FALSE(header->chip_id == CONFIG_IDF_FIRMWARE_CHIP_ID, false, TAG,
+                        "Mismatch chip id, expected %d, found %d", CONFIG_IDF_FIRMWARE_CHIP_ID, header->chip_id);
+
+    return true;
+}
+
+static esp_err_t delta_ota_write_cb(const uint8_t *buf_p, size_t size, void *user_data)
+{
+    if (size <= 0) {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    esp_ota_handle_t ota_handle = (esp_ota_handle_t)user_data;
+    int index = 0;
+
+    if (!s_delta_ota_ctx->chip_id_verified) {
+        if (s_delta_ota_ctx->header_data_read + size <= DELTA_OTA_UPGRADE_IMAGE_HEADER_SIZE) {
+            memcpy(s_delta_ota_ctx->header_data + s_delta_ota_ctx->header_data_read, buf_p, size);
+            s_delta_ota_ctx->header_data_read += size;
+            return ESP_OK;
+        } else {
+            index = DELTA_OTA_UPGRADE_IMAGE_HEADER_SIZE - s_delta_ota_ctx->header_data_read;
+            memcpy(s_delta_ota_ctx->header_data + s_delta_ota_ctx->header_data_read, buf_p, index);
+
+            if (!delta_ota_chip_id_verify(s_delta_ota_ctx->header_data)) {
+                return ESP_ERR_INVALID_VERSION;
+            }
+            s_delta_ota_ctx->chip_id_verified = true;
+
+            // Write data in header_data buffer.
+            esp_err_t err = esp_ota_write(ota_handle, s_delta_ota_ctx->header_data, DELTA_OTA_UPGRADE_IMAGE_HEADER_SIZE);
+            if (err != ESP_OK) {
+                return err;
+            }
+        }
+    }
+
+    return esp_ota_write(ota_handle, buf_p + index, size - index);
+}
+
+static esp_err_t delta_ota_read_cb(uint8_t *buf_p, size_t size, int src_offset)
+{
+    if (size <= 0) {
+        return ESP_ERR_INVALID_ARG;
+    }
+
+    return esp_partition_read(s_cur_partition, src_offset, buf_p, size);
+}
+
+esp_err_t esp_delta_ota_begin(const esp_partition_t *partition, size_t image_size, esp_ota_handle_t *out_handle)
+{
+    esp_err_t ret = ESP_OK;
+    esp_ota_handle_t ota_handle = 0;
+    esp_delta_ota_cfg_t cfg = {
+        .read_cb = &delta_ota_read_cb,
+        .write_cb_with_user_data = &delta_ota_write_cb,
+    };
+
+    s_cur_partition = esp_ota_get_running_partition();
+    assert(s_cur_partition);
+
+    ESP_RETURN_ON_FALSE(s_cur_partition->subtype < ESP_PARTITION_SUBTYPE_APP_OTA_MAX &&
+                        partition->subtype < ESP_PARTITION_SUBTYPE_APP_OTA_MAX, ESP_ERR_INVALID_ARG, TAG,
+                        "Failed to get partition info of currently or next running app");
+
+    ret = esp_ota_begin(partition, image_size, &ota_handle);
+    ESP_RETURN_ON_ERROR(ret, TAG, "Failed to begin OTA partition, status: %s", esp_err_to_name(ret));
+
+    cfg.user_data = (void *)ota_handle;
+    s_delta_ota_handle = esp_delta_ota_init(&cfg);
+    assert(s_delta_ota_handle);
+
+    s_delta_ota_ctx = calloc(1, sizeof(esp_delta_ota_ctx_t));
+    assert(s_delta_ota_ctx);
+    s_delta_ota_ctx->header_data = calloc(1, sizeof(DELTA_OTA_UPGRADE_IMAGE_HEADER_SIZE));
+    assert(s_delta_ota_ctx->header_data);
+
+    *out_handle = ota_handle;
+
+    return ret;
+}
+
+esp_err_t esp_delta_ota_write(esp_ota_handle_t handle, uint8_t *data, int size)
+{
+    esp_err_t ret = ESP_OK;
+
+    const uint8_t *patch_data = (const uint8_t *)data;
+    int patch_size = size;
+    if (!s_delta_ota_ctx->verify_patch_flag) {
+        ret = (size <= DELTA_OTA_UPGRADE_PATCH_HEADER_SIZE) ? ESP_ERR_INVALID_ARG : ESP_OK;
+        ESP_RETURN_ON_ERROR(ret, TAG, "Invalid size of received data, status: %s", esp_err_to_name(ret));
+        ret = delta_ota_patch_header_verify(data);
+        ESP_RETURN_ON_ERROR(ret, TAG, "Patch Header verification failed, status: %s", esp_err_to_name(ret));
+
+        s_delta_ota_ctx->verify_patch_flag = true;
+        patch_data = (const uint8_t *)data + DELTA_OTA_UPGRADE_PATCH_HEADER_SIZE;
+        patch_size = size - DELTA_OTA_UPGRADE_PATCH_HEADER_SIZE;
+    }
+
+    if (s_delta_ota_ctx->verify_patch_flag) {
+        ret = esp_delta_ota_feed_patch(s_delta_ota_handle, patch_data, patch_size);
+        ESP_RETURN_ON_ERROR(ret, TAG, "Failed to apply the patch on the source data, status: %s", esp_err_to_name(ret));
+    }
+
+    return ret;
+}
+
+esp_err_t esp_delta_ota_end(esp_ota_handle_t handle)
+{
+    esp_err_t ret = ESP_OK;
+
+    if (s_delta_ota_ctx) {
+        if (s_delta_ota_ctx->header_data) {
+            free(s_delta_ota_ctx->header_data);
+            s_delta_ota_ctx->header_data = NULL;
+        }
+        free(s_delta_ota_ctx);
+        s_delta_ota_ctx = NULL;
+    }
+
+    ret = esp_delta_ota_finalize(s_delta_ota_handle);
+    ESP_RETURN_ON_ERROR(ret, TAG, "Failed to finish the patch applying operation, status: %s", esp_err_to_name(ret));
+    ret = esp_delta_ota_deinit(s_delta_ota_handle);
+    ESP_RETURN_ON_ERROR(ret, TAG, "Failed to clean-up delta ota process, status: %s", esp_err_to_name(ret));
+    ret = esp_ota_end(handle);
+
+    return ret;
+}

examples/esp_zigbee_ota/ota_client/CMakeLists.txt

@@ -4,6 +4,7 @@ cmake_minimum_required(VERSION 3.16)
 
 set(EXTRA_COMPONENT_DIRS
     ${CMAKE_CURRENT_SOURCE_DIR}/../../common/switch_driver
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../common/delta_ota
     )
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(esp_ota_client)

examples/esp_zigbee_ota/ota_client/README.md

@@ -128,6 +128,38 @@ Note: The example also supports the user pressing the `boot` button to send the
  ![Zigbee_ota](../../../docs/_static/zigbee-ota-upgrade-process.png)
  * Server gets the upgrade bin file (ota_file.bin) and transmit it through OTA process. After upgrade finish, the client will restart. Upgrade bin file will be loaded from client side and a log "OTA example 2.0 is running" can be seen on the log indicates OTA file upgraded successfully.
 
+## Delta OTA Upgrade Functions
+
+Compressed Delta OTA Updates aims at enabling Over-the-Air firmware update with compressed delta binaries. Instead of the complete binary to be hosted on the OTA update server, a patch file is hosted which is the difference between the base firmware and the new firmware in compressed form.
+
+### Advantages
+
+* Patch file have smaller size than the complete firmware file. This reduces the time and network usage to download the file from server.
+* No additional storage partition is required for the "patch".
+* Only firmware level changes are required. No bootloader related changes required.
+
+### How to use the function
+
+This example uses `esp_delta_ota` component which available though the [IDF component manager](https://components.espressif.com/component/espressif/esp_delta_ota). Please refer to its documentation for more details.
+
+* Enable `ZB_DELTA_OTA` in menuconfig.
+* Install required packages:
+
+```
+pip install -r managed_components/espressif__esp_delta_ota/examples/https_delta_ota/tools/requirements.txt
+```
+
+* Generate the upgrade file:
+    * An old firmware as the base firmware which will flash into the device.
+    * A new firmware to which we want to upgrade to.
+    * Use the [python tool](./managed_components/espressif__esp_delta_ota/examples/https_delta_ota/tools/esp_delta_ota_patch_gen.py) create the patch file.
+
+        ```
+        python esp_delta_ota_patch_gen.py --chip <target> --base_binary <base_binary> --new_binary <new_binary> --patch_file_name <patch_file_name>
+        ```
+
+> **_NOTE:_** Make sure that the firmware present in the device is used as `base_binary` while creating the patch file. For this purpose, user should keep backup of the firmware running in the device as it is required for creating the patch file.
+
 ## OTA Upgrade Rate Optimization
 
 Here are some ways to optimize the OTA upgrade rate:

examples/esp_zigbee_ota/ota_client/main/Kconfig

@@ -0,0 +1,20 @@
+menu "Example OTA Configuration"
+
+    choice ZB_OTA
+        prompt "Configure the OTA mode"
+        default ZB_NORMAL_OTA
+        help
+            Select which mode does the device OTA, support Normal/Delta.
+
+        config ZB_NORMAL_OTA
+            bool 'Normal OTA Updates'
+            help
+                Select this to enable the OTA with normal binaries.
+
+        config ZB_DELTA_OTA
+            bool 'Delta OTA Updates'
+            help
+                Select this to enable the OTA with compressed delta binaries.
+    endchoice
+
+endmenu