Skip to content

Commit

Permalink
Merge branch 'feature/support_external_module_config' into 'master'
Browse files Browse the repository at this point in the history
feat(ESPAT-1918): Supported to use external module_config

See merge request application/esp-at!1540
  • Loading branch information
xcguang committed Mar 27, 2024
2 parents 05ade88 + ad283fe commit 4134a10
Show file tree
Hide file tree
Showing 33 changed files with 643 additions and 14 deletions.
28 changes: 22 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ string(SUBSTRING "$ENV{ESP_AT_PROJECT_PLATFORM}" 9 31 PLATFORM_NAME) # remove PL
string(STRIP ${PLATFORM_NAME} PLATFORM_NAME)
string(TOLOWER ${PLATFORM_NAME} LOWER_NAME)

set(ESP_AT_MODULE_CONFIG_DIR ${CMAKE_SOURCE_DIR}/module_config/module_${LOWER_NAME}_default )
set(ESP_AT_MODULE_CONFIG_DIR ${CMAKE_SOURCE_DIR}/module_config/module_${LOWER_NAME}_default)
endif()

SET(ENV{ESP_AT_PROJECT_PATH} ${CMAKE_SOURCE_DIR} )
SET(ENV{ESP_AT_PROJECT_PATH} ${CMAKE_SOURCE_DIR})

if (NOT DEFINED SILENCE)
if (DEFINED ENV{SILENCE})
Expand All @@ -28,16 +28,32 @@ set(SILENCE 0)
endif()
endif()

# set the sdkconfig default file
if (${SILENCE} EQUAL 1)
set(SDKCONFIG_DEFAULTS ${ESP_AT_MODULE_CONFIG_DIR}/sdkconfig_silence.defaults)
if(DEFINED ENV{AT_EXT_MODULE_CFG} AND EXISTS "$ENV{AT_EXT_MODULE_CFG}/sdkconfig_silence.defaults")
set(SDKCONFIG_DEFAULTS $ENV{AT_EXT_MODULE_CFG}/sdkconfig_silence.defaults)
else()
set(SDKCONFIG_DEFAULTS ${ESP_AT_MODULE_CONFIG_DIR}/sdkconfig_silence.defaults)
endif()
else()
set(SDKCONFIG_DEFAULTS ${ESP_AT_MODULE_CONFIG_DIR}/sdkconfig.defaults)
if(DEFINED ENV{AT_EXT_MODULE_CFG} AND EXISTS "$ENV{AT_EXT_MODULE_CFG}/sdkconfig.defaults")
set(SDKCONFIG_DEFAULTS $ENV{AT_EXT_MODULE_CFG}/sdkconfig.defaults)
else()
set(SDKCONFIG_DEFAULTS ${ESP_AT_MODULE_CONFIG_DIR}/sdkconfig.defaults)
endif()
endif()
message(STATUS "silence:${SILENCE}, sdkconfig:${SDKCONFIG_DEFAULTS}")

message("silence:${SILENCE} sdkconfig:${SDKCONFIG_DEFAULTS}")
# set the base directory of partition table
if(DEFINED ENV{AT_EXT_MODULE_CFG} AND EXISTS "$ENV{AT_EXT_MODULE_CFG}/partitions_at.csv")
set(CONFIG_PARTITION_TABLE_BASE_DIR "$ENV{AT_EXT_MODULE_CFG}")
else()
set(CONFIG_PARTITION_TABLE_BASE_DIR "${CMAKE_SOURCE_DIR}")
endif()
message(STATUS "partition_table_dir: ${CONFIG_PARTITION_TABLE_BASE_DIR}")

if(NOT DEFINED ENV{IDF_PATH})
SET(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf )
SET(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf)
message(STATUS $ENV{IDF_PATH})
endif()

Expand Down
22 changes: 16 additions & 6 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,11 +158,18 @@ def at_submodules_update(platform, module):
config_dir = os.path.join(os.getcwd(), 'module_config', 'module_{}_default'.format(platform.lower()))

pairs = []
idf_ver_file = os.path.join(config_dir, 'IDF_VERSION')
ext_module_cfg = os.environ.get('AT_EXT_MODULE_CFG')
if ext_module_cfg and os.path.exists(os.path.join(ext_module_cfg, 'IDF_VERSION')):
idf_ver_file = os.path.join(ext_module_cfg, 'IDF_VERSION')
else:
idf_ver_file = os.path.join(config_dir, 'IDF_VERSION')
at_parse_idf_version(idf_ver_file, pairs)

try:
submodules_file = os.path.join(config_dir, 'submodules')
if ext_module_cfg and os.path.exists(os.path.join(ext_module_cfg, 'submodules')):
submodules_file = os.path.join(ext_module_cfg, 'submodules')
else:
submodules_file = os.path.join(config_dir, 'submodules')
at_parse_submodules(submodules_file, pairs)
except Exception as e:
ESP_LOGE('Failed to parse submodules:"{}" ({})'.format(submodules_file, e))
Expand All @@ -174,10 +181,13 @@ def at_submodules_update(platform, module):
ESP_LOGI('submodules check completed for updates.')

def at_patch_if_config(platform, module):
config_dir = os.path.join(os.getcwd(), 'module_config', 'module_{}'.format(module.lower()))
if not os.path.exists(config_dir):
config_dir = os.path.join(os.getcwd(), 'module_config', 'module_{}_default'.format(platform.lower()))

ext_module_cfg = os.environ.get('AT_EXT_MODULE_CFG')
if ext_module_cfg and os.path.exists(os.path.join(ext_module_cfg, 'patch')):
config_dir = ext_module_cfg
else:
config_dir = os.path.join(os.getcwd(), 'module_config', 'module_{}'.format(module.lower()))
if not os.path.exists(config_dir):
config_dir = os.path.join(os.getcwd(), 'module_config', 'module_{}_default'.format(platform.lower()))
patch_tool = os.path.join(os.getcwd(), 'tools', 'patch.py')
if os.path.exists(patch_tool) and os.path.exists(config_dir):
cmd = 'python {} {}'.format(patch_tool, config_dir)
Expand Down
12 changes: 10 additions & 2 deletions components/customized_partitions/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,21 @@ idf_build_get_property(project_dir PROJECT_DIR)
set(flash_customized_partition_args ${build_dir}/flash_customized_partition_args)
set(module_info ${build_dir}/module_info)

# set the path of at_customize.csv
if(DEFINED ENV{AT_EXT_MODULE_CFG} AND EXISTS "$ENV{AT_EXT_MODULE_CFG}/${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE}")
set(AT_CUSTOM_PARTITION_TABLE_PATH "$ENV{AT_EXT_MODULE_CFG}/${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE}")
else()
set(AT_CUSTOM_PARTITION_TABLE_PATH "${project_dir}/${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE}")
endif()
message(STATUS "Set custom partition table: ${AT_CUSTOM_PARTITION_TABLE_PATH}")

if (CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_SUPPORT)
# generate at_customize.bin and add it to flash target
STRING(REGEX REPLACE ".*/(.*)\.csv" "\\1" customize_bin ${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE})
set(customize_bin_full_name ${customize_bin}.bin)
execute_process(
COMMAND ${PYTHON} ${IDF_PATH}/components/partition_table/gen_esp32part.py
${project_dir}/${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE} ${build_dir}/${customize_bin_full_name}
${AT_CUSTOM_PARTITION_TABLE_PATH} ${build_dir}/${customize_bin_full_name}
)
esptool_py_custom_target(${customize_bin} ${customize_bin} "${customize_bin}")
esptool_py_flash_target_image(flash "${customize_bin}" "${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_OFFSET}" "${build_dir}/${customize_bin_full_name}")
Expand All @@ -22,7 +30,7 @@ if (CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_SUPPORT)
COMMAND ${PYTHON} ${COMPONENT_DIR}/at_customized_target_generate.py
--sdkconfig_file ${project_dir}/sdkconfig
--dependency_file ${COMPONENT_DIR}/at_customized_config_dependency.txt
--partition_file ${project_dir}/${CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE}
--partition_file ${AT_CUSTOM_PARTITION_TABLE_PATH}
--output_dir ${build_dir}/${COMPONENT_NAME}
--flash_args_file ${flash_customized_partition_args}
)
Expand Down
3 changes: 3 additions & 0 deletions examples/at_override_module_config/IDF_VERSION
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
branch:release/v5.0
commit:bcca689866db3dfda47f77670bf8df2a7ec94721
repository:https://github.com/espressif/esp-idf.git
97 changes: 97 additions & 0 deletions examples/at_override_module_config/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
### 1. Description
This `at_override_module_config` example works as an external directory, allowing you to override the default module configurations (`esp-at/module_config/<your_module>`) for your module, and you will no longer need to modify the source code of the [esp-at repository](https://github.com/espressif/esp-at). This allows you to just host `at_override_module_config` directory in your git repository in a light-weight way, and it also allows you to freely choose any commit from the esp-at repository snapshot starting from March 2024 as the main project.

### 2. About the Override
Here are five optional configurations that you may want to override, and you can choose to override one or more of them. Below are detailed descriptions of each override.

**Notes:**
- If you don't override a configuration, the native configuration will be used.
- You must use the same filename and directory name to override.

#### 2.1 Override the system configuration
The native system configuration file is `esp-at/module_config/<your_module>/sdkconfig.defaults` (disable silence mode) or `esp-at/module_config/<your_module>/sdkconfig_silence.defaults` (enable silence mode). You may want to enable or disable some features, for example, to enable WebSocket functionality and disable mDNS functionality. You can copy the native system configuration file to `at_override_module_config` directory, and add the following lines to `at_override_module_config/sdkconfig.defaults` file:

```
# Enable WebSocket and disable mDNS
CONFIG_AT_WS_COMMAND_SUPPORT=y
CONFIG_AT_MDNS_COMMAND_SUPPORT=n
```

The build system will use `at_override_module_config/sdkconfig.defaults` as your system configuration.

#### 2.2 Override the patch directory
The native patch directory is `esp-at/module_config/<your_module>/patch`. You may want to add more patches, for example, to output one prompt message after esp-at is ready. You can copy the native patch directory to `at_override_module_config` directory, and add the `at_example.patch` file to `at_override_module_config/patch`, then specify this patch by adding the following lines to `at_override_module_config/patch/patch_list.ini` file:

```
[at_example.patch]
path = .
note = "This is a simple example to demonstrate how to override the patch directory"
```

The build system will use `at_override_module_config/patch` as your patch configuration.

**Notes:**
- You can create the patch using either the `git format-patch` or `git diff` command.

#### 2.3 Override the system partition table
The native system partition table file is `esp-at/module_config/<your_module>/partitions_at.csv`. You may want to modify it, for example, to enlarge app partition (`ota_0`) size to support more features. You can copy the native system partition table to `at_override_module_config/partitions_at.csv`, and update the system partition table by replacing the following lines to `at_override_module_config/partitions_at.csv` file:

```
# Name, Type, SubType, Offset, Size
otadata, data, ota, 0xd000, 0x2000
phy_init, data, phy, 0xf000, 0x1000
nvs, data, nvs, 0x10000, 0xE000
at_customize, 0x40, 0, 0x1E000, 0x42000
ota_0, app, ota_0, 0x60000, 0x3a0000
```

The build system will use `at_override_module_config/partitions_at.csv` as your system partition table configuration.

**Notes:**
- In order to increase the size of the `ota_0` partition, the `ota_1` partition has been removed in this example. This means that you cannot use the OTA commands (`AT+USEROTA` and `AT+CIUPDATE`) provided by the AT firmware. Therefore, your host MCU needs to support firmware updates of esp-at via the [esptool protocol](https://docs.espressif.com/projects/esptool/en/latest/esp32/advanced-topics/serial-protocol.html). Otherwise, if any issues occur, the AT firmware will be unable to be updated.
- You need to override `sdkconfig.defaults` as you need to specify the relative path of the system partition table in `sdkconfig.defaults` using `CONFIG_PARTITION_TABLE_CUSTOM_FILENAME` and `CONFIG_PARTITION_TABLE_FILENAME`. The relative path is relative to `at_override_module_config` for `partitions_at.csv`.

#### 2.4 Override the secondary partition table
The native secondary partition table (also called user partition table) file is `esp-at/module_config/<your_module>/at_customize.csv`. You may want to modify it, for example, to enlarge manufacturing nvs partition (`mfg_nvs`) size to support more parameters. You can copy the native secondary partition table to `at_override_module_config/at_customize.csv`, and update the secondary partition table by replacing the following lines to `at_override_module_config/at_customize.csv` file:

```
# Name, Type, SubType, Offset, Size
mfg_nvs, data, nvs, 0x1f000, 0x41000
```

The build system will use `at_override_module_config/at_customize.csv` as your secondary partition table configuration.

**Notes:**
- In order to increase the size of the `mfg_nvs` partition, the `fatfs` partition has been removed in this example. This means that you cannot use the FileSystem command (`AT+FS`) provided by the AT firmware.
- You need to override `sdkconfig.defaults` as you need to specify the relative path of the secondary partition table in `sdkconfig.defaults` using `CONFIG_AT_CUSTOMIZED_PARTITION_TABLE_FILE`. The relative path is relative to `at_override_module_config` for `at_customize.csv`.

#### 2.5 Override the esp-idf version
It is strongly recommended to use the default `ESP-IDF` version of the `ESP-AT` project, and **NOT** recommended to update it (because of the potential [application binary interface](https://en.wikipedia.org/wiki/Application_binary_interface) incompatibility between the underlying ESP-IDF versions of `libesp_at_core.a` and the esp-at repository, which may cause incorrect operation of the firmware.), but we still keep the option to update the ESP-IDF for testing purposes. The native esp-idf version file is `esp-at/module_config/<your_module>/IDF_VERSION`. You can copy the native esp-idf version to `at_override_module_config/IDF_VERSION`, and keep it as it is.

```
branch:release/v5.0
commit:bcca689866db3dfda47f77670bf8df2a7ec94721
repository:https://github.com/espressif/esp-idf.git
```

The build system will use `at_override_module_config/IDF_VERSION` as your esp-idf version configuration.

### 3. Usage
3.1 Based on the content of `2. About the Override` above and your needs, modify the configurations.

3.2 Add your `at_override_module_config` directory into the build system of esp-at project.

- Linux or macOS
```
export AT_EXT_MODULE_CFG=(path_of_at_override_module_config)
```

- Windows
```
set AT_EXT_MODULE_CFG=(path_of_at_override_module_config)
```

Notes:
- You need to replace (path_of_at_override_module_config) with your real absolute path of your `at_override_module_config` directory.

3.3 Compile your firmware according to the [Compile ESP-AT Project](https://docs.espressif.com/projects/esp-at/en/latest/esp32/Compile_and_Develop/How_to_clone_project_and_compile_it.html#compile-esp-at-project-locally) guide.
2 changes: 2 additions & 0 deletions examples/at_override_module_config/at_customize.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Name, Type, SubType, Offset, Size
mfg_nvs, data, nvs, 0x1f000, 0x41000
6 changes: 6 additions & 0 deletions examples/at_override_module_config/partitions_at.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Name, Type, SubType, Offset, Size
otadata, data, ota, 0xd000, 0x2000
phy_init, data, phy, 0xf000, 0x1000
nvs, data, nvs, 0x10000, 0xE000
at_customize, 0x40, 0, 0x1E000, 0x42000
ota_0, app, ota_0, 0x60000, 0x3a0000
20 changes: 20 additions & 0 deletions examples/at_override_module_config/patch/at_example.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
diff --git a/main/app_main.c b/main/app_main.c
index 06939b6a..de98fbab 100644
--- a/main/app_main.c
+++ b/main/app_main.c
@@ -3,6 +3,7 @@
*
* SPDX-License-Identifier: Apache-2.0
*/
+#include <string.h>
#include "nvs_flash.h"
#include "esp_event.h"
#include "esp_err.h"
@@ -19,4 +20,7 @@ void app_main(void)
ESP_ERROR_CHECK(esp_event_loop_create_default());

esp_at_init();
+
+ char *prompt = "Welcome to the world of ESP-AT!\r\n";
+ esp_at_port_active_write_data((uint8_t *)prompt, strlen(prompt));
}
9 changes: 9 additions & 0 deletions examples/at_override_module_config/patch/patch_list.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
; Add your patches to the section below

[support_ext_partition.patch]
path = esp-idf
note = "[IDF-9560] Support to use partition table outside of the project directory"

[at_example.patch]
path = .
note = "This is a simple example to demonstrate how to override the patch directory"
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
diff --git a/components/partition_table/project_include.cmake b/components/partition_table/project_include.cmake
index d0969ba408..89a0f02cbd 100644
--- a/components/partition_table/project_include.cmake
+++ b/components/partition_table/project_include.cmake
@@ -9,10 +9,14 @@ if(NOT BOOTLOADER_BUILD)
# Set PARTITION_CSV_PATH to the configured partition CSV file
# absolute path
if(CONFIG_PARTITION_TABLE_CUSTOM)
+ if(NOT CONFIG_PARTITION_TABLE_BASE_DIR)
+ set(CONFIG_PARTITION_TABLE_BASE_DIR "${project_dir}")
+ endif()
+
idf_build_get_property(project_dir PROJECT_DIR)
# Custom filename expands any path relative to the project
get_filename_component(PARTITION_CSV_PATH "${CONFIG_PARTITION_TABLE_FILENAME}"
- ABSOLUTE BASE_DIR "${project_dir}")
+ ABSOLUTE BASE_DIR "${CONFIG_PARTITION_TABLE_BASE_DIR}")

if(NOT EXISTS "${PARTITION_CSV_PATH}")
message(WARNING "Partition table CSV file ${PARTITION_CSV_PATH} not found. "
Loading

0 comments on commit 4134a10

Please sign in to comment.