Merge branch 'feature/partition_api_new_component_2' into 'master'

Storage: Partition APIs moved to the new component 'esp_partition'

Closes IDF-1234

See merge request espressif/esp-idf!20747

.gitlab/CODEOWNERS

@@ -92,6 +92,7 @@
 /components/esp_lcd/                  @esp-idf-codeowners/peripherals
 /components/esp_local_ctrl/           @esp-idf-codeowners/app-utilities
 /components/esp_netif/                @esp-idf-codeowners/network
+/components/esp_partition/            @esp-idf-codeowners/storage
 /components/esp_phy/                  @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi @esp-idf-codeowners/ieee802154
 /components/esp_pm/                   @esp-idf-codeowners/power-management @esp-idf-codeowners/bluetooth @esp-idf-codeowners/wifi
 /components/esp_psram/                @esp-idf-codeowners/peripherals @esp-idf-codeowners/system

.gitlab/ci/host-test.yml

@@ -441,7 +441,7 @@ test_system_cxx:
 test_partition_api_host:
   extends: .host_test_template
   script:
-    - cd ${IDF_PATH}/components/spi_flash/host_test/partition_api_test
+    - cd ${IDF_PATH}/components/esp_partition/host_test/partition_api_test
     - idf.py build
     - timeout 5 ./build/partition_api_test.elf | tee test.txt
     - grep " 0 Failures" test.txt

components/app_update/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRCS "esp_ota_ops.c" "esp_ota_app_desc.c"
                     INCLUDE_DIRS "include"
-                    REQUIRES spi_flash partition_table bootloader_support esp_app_format
-                    PRIV_REQUIRES esptool_py efuse)
+                    REQUIRES partition_table bootloader_support esp_app_format esp_partition
+                    PRIV_REQUIRES esptool_py efuse spi_flash)
 
 if(NOT BOOTLOADER_BUILD)
     partition_table_get_partition_info(otadata_offset "--partition-type data --partition-subtype ota" "offset")

components/app_update/esp_ota_ops.c

@@ -14,7 +14,6 @@
 
 #include "esp_err.h"
 #include "esp_partition.h"
-#include "spi_flash_mmap.h"
 #include "esp_image_format.h"
 #include "esp_secure_boot.h"
 #include "esp_flash_encrypt.h"
@@ -84,16 +83,16 @@ static const esp_partition_t *read_otadata(esp_ota_select_entry_t *two_otadata)
         return NULL;
     }
 
-    spi_flash_mmap_handle_t ota_data_map;
+    esp_partition_mmap_handle_t ota_data_map;
     const void *result = NULL;
-    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, SPI_FLASH_MMAP_DATA, &result, &ota_data_map);
+    esp_err_t err = esp_partition_mmap(otadata_partition, 0, otadata_partition->size, ESP_PARTITION_MMAP_DATA, &result, &ota_data_map);
     if (err != ESP_OK) {
         ESP_LOGE(TAG, "mmap otadata filed. Err=0x%8x", err);
         return NULL;
     } else {
         memcpy(&two_otadata[0], result, sizeof(esp_ota_select_entry_t));
         memcpy(&two_otadata[1], result + SPI_FLASH_SEC_SIZE, sizeof(esp_ota_select_entry_t));
-        spi_flash_munmap(ota_data_map);
+        esp_partition_munmap(ota_data_map);
     }
     return otadata_partition;
 }

components/app_update/test_apps/main/CMakeLists.txt

@@ -1,5 +1,6 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "."
-                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver
-                       WHOLE_ARCHIVE)
+                       PRIV_REQUIRES cmock test_utils app_update bootloader_support nvs_flash driver spi_flash
+                      WHOLE_ARCHIVE)
+
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/driver/test/CMakeLists.txt

@@ -1,7 +1,7 @@
 idf_component_register(SRC_DIRS . param_test
                        PRIV_INCLUDE_DIRS include param_test/include
                        PRIV_REQUIRES cmock test_utils driver nvs_flash
-                                     esp_timer esp_adc esp_event esp_wifi)
+                                     esp_timer esp_adc esp_event esp_wifi spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 
 # A local copy of idf-extra-components esp_serial_slave_link, for stabilities of the SDIO test

components/driver/test/test_spi_bus_lock.c

@@ -8,6 +8,7 @@
 #include "driver/spi_master.h"
 #include "driver/gpio.h"
 #include "esp_flash_spi_init.h"
+#include "spi_flash_mmap.h"
 
 #include "test/test_common_spi.h"
 #include "unity.h"

components/esp_hw_support/test/CMakeLists.txt

@@ -1,6 +1,6 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "${include_dirs}"
-                    PRIV_REQUIRES cmock test_utils esp_hw_support driver efuse esp_timer esp_psram)
+                    PRIV_REQUIRES cmock test_utils esp_hw_support driver efuse esp_timer esp_psram spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 
 target_link_libraries(${COMPONENT_LIB} INTERFACE "-u ld_include_test_dport_xt_highint5")

components/esp_partition/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_partition/host_test/partition_api_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/esp_partition/CMakeLists.txt

@@ -0,0 +1,30 @@
+set(srcs "partition.c")
+set(priv_reqs esp_system bootloader_support spi_flash app_update partition_table)
+set(reqs)
+set(include_dirs "include")
+
+idf_build_get_property(target IDF_TARGET)
+if(${target} STREQUAL "linux")
+    list(APPEND srcs "partition_linux.c")
+    set(priv_reqs partition_table)
+
+    # Steal some include directories from bootloader_support and hal components:
+    idf_component_get_property(hal_dir hal COMPONENT_DIR)
+    idf_component_get_property(bootloader_support_dir bootloader_support COMPONENT_DIR)
+    list(APPEND include_dirs include ${hal_dir}/include ${bootloader_support_dir}/include)
+else()
+    list(APPEND srcs "partition_target.c")
+endif()
+
+idf_component_register(SRCS "${srcs}"
+    INCLUDE_DIRS ${include_dirs}
+    REQUIRES ${reqs}
+    PRIV_REQUIRES ${priv_reqs})
+
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+    # These flags are GCC specific
+    set_property(SOURCE ${cache_srcs} APPEND_STRING PROPERTY COMPILE_FLAGS
+        " -fno-inline-small-functions -fno-inline-functions-called-once")
+endif()
+
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/esp_partition/host_test/partition_api_test/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "partition_api_test.c"
+                       REQUIRES esp_partition unity)

components/spi_flash/host_test/partition_api_test/main/partition_api_test.c → components/esp_partition/host_test/partition_api_test/main/partition_api_test.c

@@ -98,7 +98,7 @@ TEST(partition_api, test_partition_ops)
     memset(buffout, 0, sizeof(buffout));
     size_t sector_off = 0; //erase works per whole sector - offset must be aligned to 4kB boundaries
 
-    err = esp_partition_erase_range(partition_data, sector_off, SPI_FLASH_SEC_SIZE);
+    err = esp_partition_erase_range(partition_data, sector_off, partition_data->erase_size);
     assert(esp_partition_read(partition_data, off, (void *)buffout, bufsize) == ESP_OK);
     TEST_ESP_OK(err);
     TEST_ASSERT_EQUAL(0, memcmp(buffout, buferase, bufsize));

components/spi_flash/include/esp_partition.h → components/esp_partition/include/esp_partition.h

@@ -11,8 +11,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include "esp_err.h"
-#include "esp_flash.h"
-#include "spi_flash_mmap.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,6 +21,22 @@ extern "C" {
  * @brief Partition APIs
  */
 
+/** @cond */
+typedef struct esp_flash_t esp_flash_t;
+/** @endcond */
+
+/**
+ * @brief Enumeration which specifies memory space requested in an mmap call
+ */
+typedef enum {
+    ESP_PARTITION_MMAP_DATA,    /**< map to data memory (Vaddr0), allows byte-aligned access, 4 MB total */
+    ESP_PARTITION_MMAP_INST,    /**< map to instruction memory (Vaddr1-3), allows only 4-byte-aligned access, 11 MB total */
+} esp_partition_mmap_memory_t;
+
+/**
+ * @brief Opaque handle for memory region obtained from esp_partition_mmap.
+ */
+typedef uint32_t esp_partition_mmap_handle_t;
 
 /**
  * @brief Partition type
@@ -114,6 +128,7 @@ typedef struct {
     esp_partition_subtype_t subtype;    /*!< partition subtype */
     uint32_t address;                   /*!< starting address of the partition in flash */
     uint32_t size;                      /*!< size of the partition, in bytes */
+    uint32_t erase_size;                /*!< size the erase operation should be aligned to */
     char label[17];                     /*!< partition label, zero-terminated ASCII string */
     bool encrypted;                     /*!< flag is set to true if partition is encrypted */
 } esp_partition_t;
@@ -318,9 +333,9 @@ esp_err_t esp_partition_write_raw(const esp_partition_t* partition,
  *                  esp_partition_find_first or esp_partition_get.
  *                  Must be non-NULL.
  * @param offset Offset from the beginning of partition where erase operation
- *               should start. Must be aligned to 4 kilobytes.
+ *               should start. Must be aligned to partition->erase_size.
  * @param size Size of the range which should be erased, in bytes.
- *                   Must be divisible by 4 kilobytes.
+ *                   Must be divisible by partition->erase_size.
  *
  * @return ESP_OK, if the range was erased successfully;
  *         ESP_ERR_INVALID_ARG, if iterator or dst are NULL;
@@ -342,7 +357,7 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * requested offset (not necessarily to the beginning of mmap-ed region).
  *
  * To release mapped memory, pass handle returned via out_handle argument to
- * spi_flash_munmap function.
+ * esp_partition_munmap function.
  *
  * @param partition Pointer to partition structure obtained using
  *                  esp_partition_find_first or esp_partition_get.
@@ -351,13 +366,25 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * @param size Size of the area to be mapped.
  * @param memory  Memory space where the region should be mapped
  * @param out_ptr  Output, pointer to the mapped memory region
- * @param out_handle  Output, handle which should be used for spi_flash_munmap call
+ * @param out_handle  Output, handle which should be used for esp_partition_munmap call
  *
  * @return ESP_OK, if successful
  */
 esp_err_t esp_partition_mmap(const esp_partition_t* partition, size_t offset, size_t size,
-                             spi_flash_mmap_memory_t memory,
-                             const void** out_ptr, spi_flash_mmap_handle_t* out_handle);
+                             esp_partition_mmap_memory_t memory,
+                             const void** out_ptr, esp_partition_mmap_handle_t* out_handle);
+
+/**
+ * @brief Release region previously obtained using esp_partition_mmap
+ *
+ * @note Calling this function will not necessarily unmap memory region.
+ *       Region will only be unmapped when there are no other handles which
+ *       reference this region. In case of partially overlapping regions
+ *       it is possible that memory will be unmapped partially.
+ *
+ * @param handle  Handle obtained from spi_flash_mmap
+ */
+void esp_partition_munmap(esp_partition_mmap_handle_t handle);
 
 /**
  * @brief Get SHA-256 digest for required partition.

components/spi_flash/include/esp_private/partition_linux.h → components/esp_partition/include/esp_private/partition_linux.h

@@ -19,6 +19,9 @@ extern "C" {
  * @brief Private API functions used for Linux-target emulation of the Partition APIs (host-side testing)
  */
 
+/** @brief emulated sector size for the partition API on Linux */
+#define ESP_PARTITION_EMULATED_SECTOR_SIZE 0x1000
+
 /**
  * @brief Partition type to string conversion routine
  *

components/spi_flash/partition.c → components/esp_partition/partition.c

@@ -12,10 +12,10 @@
 #include "sdkconfig.h"
 #include "esp_flash_partitions.h"
 #include "esp_attr.h"
-#include "esp_flash.h"
 #include "esp_partition.h"
 
 #if !CONFIG_IDF_TARGET_LINUX
+#include "esp_flash.h"
 #include "esp_flash_encrypt.h"
 #endif
 
@@ -90,9 +90,11 @@ static esp_err_t load_partitions(void)
 
 #if CONFIG_IDF_TARGET_LINUX
     esp_err_t err = esp_partition_file_mmap(&p_start);
+    size_t mapped_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
 #else
     esp_err_t err = spi_flash_mmap(partition_align_pg_size,
                                    SPI_FLASH_SEC_SIZE, SPI_FLASH_MMAP_DATA, (const void **)&p_start, &handle);
+    size_t mapped_size = SPI_FLASH_SEC_SIZE;
 #endif
 
     if (err != ESP_OK) {
@@ -101,7 +103,7 @@ static esp_err_t load_partitions(void)
 
     // calculate partition address within mmap-ed region
     p_start += partition_pad;
-    p_end = p_start + SPI_FLASH_SEC_SIZE;
+    p_end = p_start + mapped_size;
 
     for (const uint8_t *p_entry = p_start; p_entry < p_end; p_entry += sizeof(esp_partition_info_t)) {
         esp_partition_info_t entry;
@@ -136,6 +138,11 @@ static esp_err_t load_partitions(void)
 #endif
         item->info.address = entry.pos.offset;
         item->info.size = entry.pos.size;
+#if CONFIG_IDF_TARGET_LINUX
+        item->info.erase_size = ESP_PARTITION_EMULATED_SECTOR_SIZE;
+#else
+        item->info.erase_size = SPI_FLASH_SEC_SIZE;
+#endif
         item->info.type = entry.type;
         item->info.subtype = entry.subtype;
         item->info.encrypted = entry.flags & PART_FLAG_ENCRYPTED;
@@ -363,9 +370,14 @@ esp_err_t esp_partition_register_external(esp_flash_t *flash_chip, size_t offset
         *out_partition = NULL;
     }
 
+#if CONFIG_IDF_TARGET_LINUX
+    return ESP_ERR_NOT_SUPPORTED;
+
+#else
     if (offset + size > flash_chip->size) {
         return ESP_ERR_INVALID_SIZE;
     }
+#endif // CONFIG_IDF_TARGET_LINUX
 
     esp_err_t err = ensure_partitions_loaded();
     if (err != ESP_OK) {

components/spi_flash/partition_linux.c → components/esp_partition/partition_linux.c

@@ -238,10 +238,10 @@ esp_err_t esp_partition_erase_range(const esp_partition_t *partition, size_t off
 {
     assert(partition != NULL);
 
-    if (offset > partition->size || offset % SPI_FLASH_SEC_SIZE != 0) {
+    if (offset > partition->size || offset % partition->erase_size != 0) {
         return ESP_ERR_INVALID_ARG;
     }
-    if (offset + size > partition->size || size % SPI_FLASH_SEC_SIZE != 0) {
+    if (offset + size > partition->size || size % partition->erase_size != 0) {
         return ESP_ERR_INVALID_SIZE;
     }
 

components/spi_flash/partition_target.c → components/esp_partition/partition_target.c

@@ -143,8 +143,8 @@ esp_err_t esp_partition_erase_range(const esp_partition_t *partition,
  * mmaped pointers, and a single handle for all these regions.
  */
 esp_err_t esp_partition_mmap(const esp_partition_t *partition, size_t offset, size_t size,
-                             spi_flash_mmap_memory_t memory,
-                             const void **out_ptr, spi_flash_mmap_handle_t *out_handle)
+                             esp_partition_mmap_memory_t memory,
+                             const void **out_ptr, esp_partition_mmap_handle_t *out_handle)
 {
     assert(partition != NULL);
     if (offset > partition->size) {
@@ -160,7 +160,7 @@ esp_err_t esp_partition_mmap(const esp_partition_t *partition, size_t offset, si
     // offset within mmu page size block
     size_t region_offset = phys_addr & (CONFIG_MMU_PAGE_SIZE - 1);
     size_t mmap_addr = phys_addr & ~(CONFIG_MMU_PAGE_SIZE - 1);
-    esp_err_t rc = spi_flash_mmap(mmap_addr, size + region_offset, memory, out_ptr, out_handle);
+    esp_err_t rc = spi_flash_mmap(mmap_addr, size + region_offset, (spi_flash_mmap_memory_t) memory, out_ptr, (spi_flash_mmap_handle_t*) out_handle);
     // adjust returned pointer to point to the correct offset
     if (rc == ESP_OK) {
         *out_ptr = (void *) (((ptrdiff_t) * out_ptr) + region_offset);
@@ -168,6 +168,11 @@ esp_err_t esp_partition_mmap(const esp_partition_t *partition, size_t offset, si
     return rc;
 }
 
+void esp_partition_munmap(esp_partition_mmap_handle_t handle)
+{
+    spi_flash_munmap((spi_flash_mmap_handle_t) handle);
+}
+
 esp_err_t esp_partition_get_sha256(const esp_partition_t *partition, uint8_t *sha_256)
 {
     return bootloader_common_get_sha256_of_partition(partition->address, partition->size, partition->type, sha_256);

components/esp_partition/test/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(SRC_DIRS "."
+                       PRIV_INCLUDE_DIRS "."
+                       PRIV_REQUIRES test_utils esp_partition esp_system app_update bootloader_support spi_flash)
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/partition_table/test/test_partition.c → components/esp_partition/test/test_partition.c

@@ -49,7 +49,7 @@ TEST_CASE("Can write, read, mmap partition", "[partition][ignore]")
     const esp_partition_t *p = get_test_data_partition();
     printf("Using partition %s at 0x%x, size 0x%x\n", p->label, p->address, p->size);
     TEST_ASSERT_NOT_NULL(p);
-    const size_t max_size = 2 * SPI_FLASH_SEC_SIZE;
+    const size_t max_size = 2 * p->erase_size;
     uint8_t *data = (uint8_t *) malloc(max_size);
     TEST_ASSERT_NOT_NULL(data);
 
@@ -85,10 +85,10 @@ TEST_CASE("Can write, read, mmap partition", "[partition][ignore]")
     free(data);
 
     const uint32_t *mmap_data;
-    spi_flash_mmap_handle_t mmap_handle;
+    esp_partition_mmap_handle_t mmap_handle;
     size_t begin = 3000;
     size_t size = 64000; //chosen so size is smaller than 64K but the mmap straddles 2 MMU blocks
-    TEST_ASSERT_EQUAL(ESP_OK, esp_partition_mmap(p, begin, size, SPI_FLASH_MMAP_DATA,
+    TEST_ASSERT_EQUAL(ESP_OK, esp_partition_mmap(p, begin, size, ESP_PARTITION_MMAP_DATA,
                       (const void **)&mmap_data, &mmap_handle));
     srand(0);
     for (size_t offset = 0; offset < p->size; offset += block_size) {
@@ -107,5 +107,5 @@ TEST_CASE("Can write, read, mmap partition", "[partition][ignore]")
         }
     }
 
-    spi_flash_munmap(mmap_handle);
+    esp_partition_munmap(mmap_handle);
 }

components/spi_flash/test/test_partition_ext.c → components/esp_partition/test/test_partition_ext.c

@@ -1,4 +1,10 @@
+/*
+ * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
 #include "esp_flash.h"
+#include "spi_flash_mmap.h"
 #include "esp_partition.h"
 #include "unity.h"
 

components/spi_flash/test/test_partitions.c → components/esp_partition/test/test_partitions.c

@@ -18,6 +18,8 @@
 #include <esp_log.h>
 #include <esp_partition.h>
 #include <esp_attr.h>
+#include "esp_flash.h"
+#include "spi_flash_mmap.h"
 
 TEST_CASE("Test erase partition", "[spi_flash][esp_flash]")
 {

components/esp_psram/test_apps/psram/main/test_psram.c

@@ -17,6 +17,8 @@
 #include "esp_private/esp_psram_io.h"
 #include "esp_psram.h"
 #include "esp_private/esp_psram_extram.h"
+#include "esp_flash.h"
+#include "esp_partition.h"
 
 __attribute__((unused)) const static char *TAG = "PSRAM";
 

components/esp_system/CMakeLists.txt

@@ -47,7 +47,9 @@ else()
                                     # link-time registration is used.
                                     # [refactor-todo] requires "driver" for headers:
                                     # - spi_common_internal.h
-                                    pthread bootloader_support efuse driver
+                                    # [refactor-todo] esp_partition required for virtual efuse
+                                    # init code. Move to esp_efuse component.
+                                    pthread bootloader_support efuse driver esp_partition
                         LDFRAGMENTS "linker.lf" "app.lf")
     add_subdirectory(port)
 

components/esp_timer/test/CMakeLists.txt

@@ -1,4 +1,4 @@
 idf_component_register(SRC_DIRS "."
                        PRIV_INCLUDE_DIRS "../private_include"
-                       PRIV_REQUIRES cmock test_utils esp_timer)
+                       PRIV_REQUIRES cmock test_utils esp_timer spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/espcoredump/CMakeLists.txt

@@ -24,7 +24,7 @@ idf_component_register(SRCS ${srcs}
                     INCLUDE_DIRS ${includes}
                     PRIV_INCLUDE_DIRS ${priv_includes}
                     LDFRAGMENTS linker.lf
-                    PRIV_REQUIRES spi_flash bootloader_support mbedtls esp_rom soc esp_system driver)
+                    PRIV_REQUIRES esp_partition spi_flash bootloader_support mbedtls esp_rom soc esp_system driver)
 
 if(CONFIG_ESP_COREDUMP_DATA_FORMAT_ELF)
   target_link_libraries(${COMPONENT_LIB} PRIVATE idf::esp_app_format)

components/fatfs/test_fatfs_host/Makefile.files

@@ -40,4 +40,5 @@ INCLUDE_DIRS := \
 		hal/include \
 		spi_flash/include \
 		wear_levelling/include \
+		esp_partition/include \
 	)

components/mbedtls/test_apps/main/CMakeLists.txt

@@ -7,7 +7,7 @@ set(TEST_CRTS "crts/server_cert_chain.pem"
 
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock test_utils mbedtls esp_timer unity
+                    PRIV_REQUIRES cmock test_utils mbedtls esp_timer unity spi_flash
                     EMBED_TXTFILES ${TEST_CRTS}
                     WHOLE_ARCHIVE)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/mbedtls/test_apps/main/test_sha.c

@@ -12,6 +12,7 @@
 #include "esp_heap_caps.h"
 #include "idf_performance.h"
 #include "esp_private/esp_clk.h"
+#include "spi_flash_mmap.h"
 
 #include "soc/soc_caps.h"
 

components/mqtt/test/CMakeLists.txt

@@ -5,5 +5,5 @@ if(CONFIG_MQTT_PROTOCOL_5)
 endif()
 
 idf_component_register(SRCS "${srcs}"
-                    PRIV_REQUIRES cmock test_utils mqtt nvs_flash app_update esp_eth esp_netif)
+                    PRIV_REQUIRES cmock test_utils mqtt nvs_flash app_update esp_eth esp_netif spi_flash)
  target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/mqtt/test/test_mqtt.c

@@ -23,6 +23,7 @@
 #include "test_mqtt_client_broker.h"
 #include "test_mqtt_connection.h"
 #include "esp_mac.h"
+#include "spi_flash_mmap.h"
 
 static void test_leak_setup(const char * file, long line)
 {

components/mqtt/test/test_mqtt5.c

@@ -17,6 +17,7 @@
 #include "test_mqtt5_client_broker.h"
 #include "test_mqtt_connection.h"
 #include "esp_mac.h"
+#include "spi_flash_mmap.h"
 
 static esp_mqtt5_user_property_item_t user_property_arr[3] = {
     {"board", "esp32"},

components/newlib/test/CMakeLists.txt

@@ -1,4 +1,4 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock test_utils driver esp_timer)
+                    PRIV_REQUIRES cmock test_utils driver esp_timer spi_flash)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/nvs_flash/CMakeLists.txt

@@ -14,7 +14,8 @@ set(srcs "src/nvs_api.cpp"
          "src/nvs_types.cpp")
 
 idf_component_register(SRCS "${srcs}"
-                    REQUIRES "spi_flash"
+                    REQUIRES "esp_partition"
+                    PRIV_REQUIRES spi_flash
                     INCLUDE_DIRS "include"
                     PRIV_INCLUDE_DIRS "private_include")
 

components/nvs_flash/host_test/nvs_page_test/CMakeLists.txt

@@ -6,6 +6,7 @@ set(COMPONENTS main)
 list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/driver/")
 list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/freertos/")
 list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/spi_flash/")
+list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/esp_partition/")
 
 idf_build_set_property(COMPILE_DEFINITIONS "NO_DEBUG_STORAGE" APPEND)
 project(test_nvs_page_host)

components/nvs_flash/host_test/nvs_page_test/main/CMakeLists.txt

@@ -5,7 +5,7 @@ idf_component_register(SRCS "nvs_page_test.cpp"
                     "${CMAKE_CURRENT_SOURCE_DIR}/../../../src"
                     PRIV_INCLUDE_DIRS
                     "${CMAKE_CURRENT_SOURCE_DIR}/../../../private_include"
-                    REQUIRES cmock nvs_flash spi_flash partition_table)
+                    REQUIRES cmock nvs_flash spi_flash partition_table esp_partition)
 
 target_compile_options(${COMPONENT_LIB} PUBLIC --coverage)
 target_link_libraries(${COMPONENT_LIB} --coverage)

components/nvs_flash/test/CMakeLists.txt

@@ -1,6 +1,6 @@
 idf_component_register(SRC_DIRS "."
                     PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock test_utils nvs_flash bootloader_support
+                    PRIV_REQUIRES cmock test_utils nvs_flash bootloader_support spi_flash
                     EMBED_TXTFILES encryption_keys.bin partition_encrypted.bin sample.bin)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
 

components/nvs_flash/test_nvs_host/Makefile

@@ -37,7 +37,7 @@ else
 COMPILER := gcc
 endif
 
-CPPFLAGS += -I../private_include -I../include -I../src -I../../esp_rom/include -I../../esp_rom/include/linux -I../../log/include -I./ -I../../esp_common/include -I../../esp32/include -I ../../mbedtls/mbedtls/include -I ../../spi_flash/include -I ../../hal/include -I ../../xtensa/include -I ../../../tools/catch -fprofile-arcs -ftest-coverage -g2 -ggdb
+CPPFLAGS += -I../private_include -I../include -I../src -I../../esp_rom/include -I../../esp_rom/include/linux -I../../log/include -I./ -I../../esp_common/include -I../../esp32/include -I ../../mbedtls/mbedtls/include -I ../../spi_flash/include -I ../../esp_partition/include -I ../../hal/include -I ../../xtensa/include -I ../../../tools/catch -fprofile-arcs -ftest-coverage -g2 -ggdb
 CFLAGS += -fprofile-arcs -ftest-coverage -DLINUX_TARGET -DLINUX_HOST_LEGACY_TEST
 CXXFLAGS += -std=c++11 -Wall -Werror -DLINUX_TARGET -DLINUX_HOST_LEGACY_TEST
 LDFLAGS += -lstdc++ -Wall -fprofile-arcs -ftest-coverage

components/partition_table/test/CMakeLists.txt

@@ -1,4 +0,0 @@
-idf_component_register(SRC_DIRS "."
-                    PRIV_INCLUDE_DIRS "."
-                    PRIV_REQUIRES cmock test_utils)
-target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/spi_flash/.build-test-rules.yml

@@ -1,10 +1,5 @@
 # Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
 
-components/spi_flash/host_test/partition_api_test:
-  enable:
-    - if: IDF_TARGET == "linux"
-      reason: only test on linux
-
 components/spi_flash/test_apps/esp_flash:
   disable:
     - if: IDF_TARGET == "esp32c6"

components/spi_flash/CMakeLists.txt

@@ -1,14 +1,5 @@
 idf_build_get_property(target IDF_TARGET)
 if(${target} STREQUAL "linux")
-    set(srcs "partition.c"
-         "partition_linux.c")
-    idf_component_get_property(hal_dir hal COMPONENT_DIR)
-    idf_component_get_property(bootloader_support_dir bootloader_support COMPONENT_DIR)
-
-    idf_component_register(SRCS "${srcs}"
-        INCLUDE_DIRS include ${hal_dir}/include ${bootloader_support_dir}/include
-        PRIV_INCLUDE_DIRS include/spi_flash
-        PRIV_REQUIRES partition_table)
     return()
 endif()
 
@@ -23,11 +14,7 @@ else()
         "${target}/flash_ops_${target}.c"
     )
 
-    set(srcs
-        "partition.c"
-        "partition_target.c"
-        "flash_brownout_hook.c"
-    )
+    set(srcs "flash_brownout_hook.c")
 
     if(CONFIG_SOC_SPI_MEM_SUPPORT_OPI_MODE)
         list(APPEND srcs "${target}/spi_flash_oct_flash_init.c")

components/spi_flash/host_test/partition_api_test/main/CMakeLists.txt

@@ -1,2 +0,0 @@
-idf_component_register(SRCS "partition_api_test.c"
-                       REQUIRES spi_flash unity)

components/spi_flash/sim/Makefile.files

@@ -3,8 +3,8 @@ SOURCE_FILES := \
     flash_mock.cpp \
     flash_mock_util.c \
     $(addprefix ../, \
-    partition.c \
-    ../spi_flash/partition_target.c \
+    ../esp_partition/partition.c \
+    ../esp_partition/partition_target.c \
     flash_ops.c \
     ../esp_rom/linux/esp_rom_efuse.c \
     ) \
@@ -46,4 +46,5 @@ INCLUDE_DIRS := \
     hal/esp32/include \
     hal/platform_port/include \
     spi_flash/include \
+    esp_partition/include \
     )

components/spi_flash/sim/flash_mock.cpp

@@ -1,11 +1,8 @@
-#include "SpiFlash.h"
-
 #include <string.h>
 #include <stdlib.h>
 
+#include "SpiFlash.h"
 #include "spi_flash_mmap.h"
-#include "esp_partition.h"
-
 #include "esp_err.h"
 #include "esp_rom_spiflash.h"
 #include "esp_flash.h"

components/spi_flash/sim/flash_mock_util.c

@@ -1,6 +1,3 @@
-#include "spi_flash_mmap.h"
-#include "esp_partition.h"
-
 #include "esp_err.h"
 #include "esp_rom_spiflash.h"
 

components/spi_flash/sim/stubs/Makefile.files

@@ -37,5 +37,6 @@ INCLUDE_DIRS := \
 	bootloader_support/bootloader_flash/include \
 	app_update/include \
 	hal/include \
+	esp_partition/include \
 	spi_flash/include \
 	)

components/spi_flash/test/test_esp_flash.c

@@ -7,6 +7,7 @@
 #include <unity.h>
 #include "esp_flash.h"
 #include "esp_private/spi_common_internal.h"
+#include "spi_flash_mmap.h"
 #include "esp_flash_spi_init.h"
 #include "memspi_host_driver.h"
 #include <esp_attr.h>

components/spi_flash/test/test_large_flash_writes.c

@@ -20,6 +20,7 @@
 #include "esp_rom_spiflash.h"
 #include "esp_private/cache_utils.h"
 #include "soc/timer_periph.h"
+#include "esp_flash.h"
 
 static const uint8_t large_const_buffer[16400] = {
     203, // first byte

components/spi_flash/test/test_out_of_bounds_write.c

@@ -3,6 +3,7 @@
 #include "unity.h"
 #include "spi_flash_mmap.h"
 #include "esp_ota_ops.h"
+#include "esp_flash.h"
 
 #if CONFIG_SPI_FLASH_DANGEROUS_WRITE_ABORTS || CONFIG_SPI_FLASH_DANGEROUS_WRITE_FAILS
 

components/spi_flash/test/test_read_write.c

@@ -20,6 +20,8 @@
 #include "esp_attr.h"
 #include "esp_heap_caps.h"
 #include "esp_rom_spiflash.h"
+#include "esp_flash.h"
+
 #if CONFIG_IDF_TARGET_ESP32
 // Used for rom_fix function
 #include "esp32/rom/spi_flash.h"

components/spi_flash/test_apps/esp_flash/main/test_esp_flash_drv.c

@@ -28,6 +28,7 @@
 #include "esp_rom_sys.h"
 #include "esp_timer.h"
 #include "test_esp_flash_def.h"
+#include "spi_flash_mmap.h"
 
 #if CONFIG_IDF_TARGET_ESP32S2
 #include "esp32s2/rom/cache.h"

components/spi_flash/test_apps/flash_mmap/main/test_mmap.c

@@ -16,6 +16,7 @@
 #include <esp_attr.h>
 #include <esp_partition.h>
 #include <esp_flash_encrypt.h>
+#include "esp_flash.h"
 
 #include "test_utils.h"
 

components/spiffs/CMakeLists.txt

@@ -15,8 +15,8 @@ endif()
 idf_component_register(SRCS ${srcs}
                        INCLUDE_DIRS "include"
                        PRIV_INCLUDE_DIRS "." "spiffs/src"
-                       REQUIRES spi_flash
-                       PRIV_REQUIRES ${pr})
+                       REQUIRES esp_partition
+                       PRIV_REQUIRES ${pr} spi_flash)
 
 if(CMAKE_C_COMPILER_ID MATCHES "GNU")
     set_source_files_properties(spiffs/src/spiffs_nucleus.c PROPERTIES COMPILE_FLAGS -Wno-stringop-truncation)

components/wear_levelling/CMakeLists.txt

@@ -7,6 +7,7 @@ idf_component_register(SRCS "Partition.cpp"
                             "wear_levelling.cpp"
                     INCLUDE_DIRS include
                     PRIV_INCLUDE_DIRS private_include
-                    REQUIRES spi_flash)
+                    REQUIRES esp_partition
+                    PRIV_REQUIRES spi_flash)
 
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/wear_levelling/private_include/Partition.h

@@ -11,6 +11,7 @@
 
 #include "Flash_Access.h"
 #include "esp_partition.h"
+#include "spi_flash_mmap.h"  // for SPI_FLASH_SEC_SIZE
 
 /**
 * @brief This class is used to access partition. Class implements Flash_Access interface

components/wear_levelling/test_apps/main/CMakeLists.txt

@@ -1,6 +1,6 @@
 idf_component_register(SRCS test_wl.c
                        PRIV_INCLUDE_DIRS .
-                       PRIV_REQUIRES wear_levelling unity
+                       PRIV_REQUIRES wear_levelling unity spi_flash
                        EMBED_FILES test_partition_v1.bin
                       )
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

components/wear_levelling/test_apps/main/test_wl.c

@@ -13,6 +13,7 @@
 #include "esp_private/esp_clk.h"
 #include "sdkconfig.h"
 #include "esp_cpu.h"
+#include "spi_flash_mmap.h"
 
 
 TEST_GROUP(wear_levelling);

components/wear_levelling/test_wl_host/Makefile.files

@@ -39,4 +39,5 @@ INCLUDE_DIRS := \
 	app_update/include \
 	hal/include \
 	spi_flash/include \
+	esp_partition/include \
 	)

docs/doxygen/Doxyfile

@@ -149,6 +149,7 @@ INPUT = \
     $(PROJECT_PATH)/components/esp_netif/include/esp_netif_types.h \
     $(PROJECT_PATH)/components/esp_netif/include/esp_netif.h \
     $(PROJECT_PATH)/components/esp_netif/include/esp_vfs_l2tap.h \
+    $(PROJECT_PATH)/components/esp_partition/include/esp_partition.h \
     $(PROJECT_PATH)/components/esp_phy/include/esp_phy_init.h \
     $(PROJECT_PATH)/components/esp_pm/include/$(IDF_TARGET)/pm.h \
     $(PROJECT_PATH)/components/esp_pm/include/esp_pm.h \
@@ -237,7 +238,6 @@ INPUT = \
     $(PROJECT_PATH)/components/soc/$(IDF_TARGET)/include/soc/uart_channel.h \
     $(PROJECT_PATH)/components/spi_flash/include/esp_flash_spi_init.h \
     $(PROJECT_PATH)/components/spi_flash/include/esp_flash.h \
-    $(PROJECT_PATH)/components/spi_flash/include/esp_partition.h \
     $(PROJECT_PATH)/components/spi_flash/include/spi_flash_mmap.h \
     $(PROJECT_PATH)/components/spiffs/include/esp_spiffs.h \
     $(PROJECT_PATH)/components/touch_element/include/touch_element/touch_button.h \

docs/en/migration-guides/release-5.x/5.0/storage.rst

@@ -1,6 +1,34 @@
 Storage
 =======
 
+New Component for the Partition APIs
+------------------------------------
+
+Breaking change: all the Partition API code has been moved to a new component :component:`esp_partition`. For the complete list of affected functions and data-types, see header file :component_file:`esp_partition.h <esp_partition/include/esp_partition.h>`.
+
+These API functions and data-types were previously a part of the :component:`spi_flash` component, and thus possible dependencies on the ``spi_flash`` in existing applications may cause the build failure, in case of direct esp_partition_* APIs/data-types use (for instance, ``fatal error: esp_partition.h: No such file or directory`` at lines with ``#include "esp_partition.h"``). If you encounter such an issue, please update your project's CMakeLists.txt file as follows:
+
+Original dependency setup:
+
+.. code-block:: cmake
+
+   idf_component_register(...
+                          REQUIRES spi_flash)
+
+Updated dependency setup:
+
+.. code-block:: cmake
+
+   idf_component_register(...
+                          REQUIRES spi_flash esp_partition)
+
+.. note:: 
+
+   Please update relevant ``REQUIRES`` or ``PRIV_REQUIRES`` section according to your project. The above-presented code snippet is just an example.
+
+If the issue persists, please let us know and we will assist you with your code migration.
+
+
 SDMMC/SDSPI
 -----------
 

docs/zh_CN/migration-guides/release-5.x/5.0/storage.rst

@@ -1,6 +1,34 @@
 存储
 =======
 
+分区 API 的新组件
+------------------------------------
+
+非兼容性更新：所有的分区 API 代码都已迁移到新组件 :component:`esp_partition` 中。如需查看所有受影响的函数及数据类型，请参见头文件 :component_file:`esp_partition.h <esp_partition/include/esp_partition.h>`。
+
+在以前，这些 API 函数和数据类型属于 :component:`spi_flash` 组件。因此，在现有的应用程序中或将依赖 ``spi_flash``，这也意味着在直接使用 esp_partition_* API/数据类型时，可能会导致构建过程失败（比如，在出现 ``#include "esp_partition.h"`` 的行中报错 ``fatal error: esp_partition.h: No such file or directory``）。如果遇到类似问题，请按以下步骤更新项目中的 CMakeLists.txt 文件：
+
+原有的依赖性设置：
+
+.. code-block:: cmake
+
+   idf_component_register(...
+                          REQUIRES spi_flash)
+
+更新后的依赖性设置：
+
+.. code-block:: cmake
+
+   idf_component_register(...
+                          REQUIRES spi_flash esp_partition)
+
+.. note:: 
+
+   请根据项目的实际情况，更新相应的 ``REQUIRES`` 或是 ``PRIV_REQUIRES`` 部分。上述代码片段仅为范例。
+
+如果问题仍未解决，请联系我们，我们将协助您进行代码迁移。
+
+
 SDMMC/SDSPI
 -----------
 

examples/protocols/mqtt/ssl/main/app_main.c

@@ -12,6 +12,8 @@
 #include <stddef.h>
 #include <string.h>
 #include "esp_system.h"
+#include "esp_partition.h"
+#include "spi_flash_mmap.h"
 #include "nvs_flash.h"
 #include "esp_event.h"
 #include "esp_netif.h"

examples/protocols/mqtt/ssl_ds/main/idf_component.yml

@@ -1,3 +1,3 @@
 ## IDF Component Manager Manifest File
 dependencies:
-  espressif/esp_secure_cert_mgr: "^2.0.0"
+  espressif/esp_secure_cert_mgr: "^2.0.2"

examples/storage/partition_api/partition_mmap/main/CMakeLists.txt

@@ -1,2 +1,4 @@
 idf_component_register(SRCS "main.c"
-                    INCLUDE_DIRS ".")
+                    INCLUDE_DIRS "."
+                    REQUIRES esp_partition
+                    PRIV_REQUIRES spi_flash)

examples/storage/partition_api/partition_mmap/main/main.c

@@ -8,6 +8,7 @@
 #include <string.h>
 #include <assert.h>
 #include "esp_partition.h"
+#include "spi_flash_mmap.h"
 #include "esp_log.h"
 
 static const char *TAG = "example";

examples/storage/partition_api/partition_ops/main/CMakeLists.txt

@@ -1,2 +1,4 @@
 idf_component_register(SRCS "main.c"
-                    INCLUDE_DIRS ".")
+                    INCLUDE_DIRS "."
+                    REQUIRES esp_partition
+                    PRIV_REQUIRES spi_flash)

examples/storage/partition_api/partition_ops/main/main.c

@@ -8,6 +8,7 @@
 #include <string.h>
 #include <assert.h>
 #include "esp_partition.h"
+#include "spi_flash_mmap.h"
 #include "esp_log.h"
 
 static const char *TAG = "example";

examples/system/flash_suspend/main/app_main.c

@@ -19,6 +19,7 @@
 #include "esp_cpu.h"
 #include "esp_partition.h"
 #include "driver/gptimer.h"
+#include "esp_flash.h"
 
 
 #define TIMER_RESOLUTION_HZ     (1 * 1000 * 1000) // 1MHz resolution

tools/idf_py_actions/hints.yml

@@ -273,3 +273,8 @@
     re: "fatal error: (tinyusb.h): No such file or directory"
     hint: "{} was removed. Please use esp_tinyusb component from IDF component manager instead.\nYou can install `esp_tinyusb` using 'idf.py add-dependency espressif/esp_tinyusb' command.\nRefer to the migration guide for more details."
     match_to_output: True
+
+-
+    re: "fatal error: esp_partition.h: No such file or directory"
+    hint: "All the Partition APIs have been moved to the new component 'esp_partition' - please, update your project dependencies. See Storage migration guide 5.x for more details."
+    match_to_output: True

tools/mocks/esp_partition/CMakeLists.txt

@@ -0,0 +1,8 @@
+message(STATUS "building ESP_PARTITION MOCKS")
+
+idf_component_get_property(original_esp_partition_dir esp_partition COMPONENT_OVERRIDEN_DIR)
+
+idf_component_mock(INCLUDE_DIRS "${original_esp_partition_dir}/include"
+    REQUIRES spi_flash
+    MOCK_HEADER_FILES
+    ${original_esp_partition_dir}/include/esp_partition.h)

tools/mocks/esp_partition/mock/mock_config.yaml

@@ -0,0 +1,10 @@
+        :cmock:
+          :includes_h_pre_orig_header:
+            - esp_flash.h
+          :plugins:
+            - expect
+            - expect_any_args
+            - return_thru_ptr
+            - array
+            - ignore_arg
+            - callback

tools/mocks/spi_flash/CMakeLists.txt

@@ -2,7 +2,7 @@
 #       On Espressif chips, too many dependencies are missing at the moment.
 #       Furthermore, this component can only mock the interfaces of
 #       spi_master.h and gpio.h.
-message(STATUS "building SPI FLASH MOCKS (only esp_partition* API)")
+message(STATUS "building SPI FLASH MOCKS (only esp_partition support)")
 
 idf_component_get_property(original_spi_flash_dir spi_flash COMPONENT_OVERRIDEN_DIR)
 
@@ -19,6 +19,5 @@ endif()
 idf_component_mock(INCLUDE_DIRS ${include_dirs}
     REQUIRES esp_common
     MOCK_HEADER_FILES
-    ${original_spi_flash_dir}/include/esp_partition.h
     ${original_spi_flash_dir}/include/esp_flash.h
     ${original_spi_flash_dir}/include/spi_flash_mmap.h)

tools/test_apps/system/g1_components/CMakeLists.txt

@@ -59,6 +59,9 @@ set(extra_components_which_shouldnt_be_included
     mbedtls
     # partition_table is pulled in by app_update, esptool_py, bootloader; all to be removed
     partition_table
+    # esp_partition is a new component for separated IDF partition APIs. Added due to its involvement in the spi_flash
+    #   code. To be possibly removed (?)
+    esp_partition
     # pthread is required by esp_system (for initialization only, can be made a weak dependency)
     #   and cxx. See also [refactor-todo] about cxx, can it work without pthread?
     pthread

tools/test_apps/system/panic/main/CMakeLists.txt

@@ -1,3 +1,3 @@
 idf_component_register(SRCS "test_panic_main.c"
                        INCLUDE_DIRS "."
-                       REQUIRES spi_flash esp_system)
+                       REQUIRES spi_flash esp_system esp_partition)

tools/unit-test-app/components/test_utils/CMakeLists.txt

@@ -25,6 +25,6 @@ endif()
 idf_component_register(SRCS ${srcs}
                     INCLUDE_DIRS include
                     PRIV_INCLUDE_DIRS private_include
-                    REQUIRES spi_flash idf_test cmock
+                    REQUIRES esp_partition idf_test cmock
                     PRIV_REQUIRES perfmon driver esp_netif)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")