Merge branch 'feature/ext_flash_partition' into 'master'

Support for partitions in external flash

See merge request idf/esp-idf!5277

.gitlab-ci.yml

@@ -1568,6 +1568,18 @@ UT_001_44:
     - ESP32_IDF
     - UT_T1_1
 
+UT_001_45:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+UT_001_46:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
 UT_002_01:
   <<: *unit_test_template
   tags:

components/nvs_flash/test_nvs_host/Makefile

@@ -21,7 +21,7 @@ SOURCE_FILES = \
 	crc.cpp \
 	main.cpp
 
-CPPFLAGS += -I../include -I../src -I./ -I../../esp_common/include -I../../esp32/include -I ../../mbedtls/mbedtls/include -I ../../spi_flash/include -I ../../../tools/catch -fprofile-arcs -ftest-coverage -DCONFIG_NVS_ENCRYPTION
+CPPFLAGS += -I../include -I../src -I./ -I../../esp_common/include -I../../esp32/include -I ../../mbedtls/mbedtls/include -I ../../spi_flash/include -I ../../soc/include -I ../../../tools/catch -fprofile-arcs -ftest-coverage
 CFLAGS += -fprofile-arcs -ftest-coverage
 CXXFLAGS += -std=c++11 -Wall -Werror
 LDFLAGS += -lstdc++ -Wall -fprofile-arcs -ftest-coverage
@@ -61,10 +61,10 @@ clean:
 	rm -f $(COVERAGE_FILES) *.gcov
 	rm -rf coverage_report/
 	rm -f coverage.info
-	rm ../nvs_partition_generator/partition_single_page.bin
-	rm ../nvs_partition_generator/partition_multipage_blob.bin
-	rm ../nvs_partition_generator/partition_encrypted.bin
-	rm ../nvs_partition_generator/partition_encrypted_using_keygen.bin
-	rm ../nvs_partition_generator/partition_encrypted_using_keyfile.bin
+	rm -f ../nvs_partition_generator/partition_single_page.bin
+	rm -f ../nvs_partition_generator/partition_multipage_blob.bin
+	rm -f ../nvs_partition_generator/partition_encrypted.bin
+	rm -f ../nvs_partition_generator/partition_encrypted_using_keygen.bin
+	rm -f ../nvs_partition_generator/partition_encrypted_using_keyfile.bin
 
 .PHONY: clean all test long-test

components/nvs_flash/test_nvs_host/sdkconfig.h

@@ -0,0 +1,3 @@
+#define CONFIG_NVS_ENCRYPTION 1
+//currently use the legacy implementation, since the stubs for new HAL are not done yet
+#define CONFIG_SPI_FLASH_USE_LEGACY_IMPL 1

components/nvs_flash/test_nvs_host/test_nvs.cpp

@@ -14,6 +14,7 @@
 #include "catch.hpp"
 #include "nvs.hpp"
 #include "nvs_test_api.h"
+#include "sdkconfig.h"
 #ifdef CONFIG_NVS_ENCRYPTION
 #include "nvs_encr.hpp"
 #endif
@@ -2470,10 +2471,10 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit
     if (childpid == 0) {
         exit(execlp("bash", "bash",
                     "-c",
-                    "rm -rf ../../../tools/mass_mfg/host_test | \
-                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata | \
-                    cp -rf ../nvs_partition_generator/testdata . | \
-                    mkdir -p ../../../tools/mass_mfg/host_test",NULL));
+                    "rm -rf ../../../tools/mass_mfg/host_test && \
+                    cp -rf ../../../tools/mass_mfg/testdata mfg_testdata && \
+                    cp -rf ../nvs_partition_generator/testdata . && \
+                    mkdir -p ../../../tools/mass_mfg/host_test", NULL));
     } else {
         CHECK(childpid > 0);
         waitpid(childpid, &status, 0);

components/spi_flash/esp_flash_api.c

@@ -621,6 +621,8 @@ inline static IRAM_ATTR bool regions_overlap(uint32_t a_start, uint32_t a_len,ui
     Adapter layer to original api before IDF v4.0
 ------------------------------------------------------------------------------*/
 
+#ifndef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+
 static esp_err_t spi_flash_translate_rc(esp_err_t err)
 {
     switch (err) {
@@ -644,7 +646,6 @@ static esp_err_t spi_flash_translate_rc(esp_err_t err)
     return ESP_OK;
 }
 
-#ifndef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
 esp_err_t spi_flash_erase_range(uint32_t start_addr, uint32_t size)
 {
     esp_err_t err = esp_flash_erase_region(NULL, start_addr, size);
@@ -655,8 +656,6 @@ esp_err_t spi_flash_write(size_t dst, const void *srcv, size_t size)
 {
     esp_err_t err = esp_flash_write(NULL, srcv, dst, size);
     return spi_flash_translate_rc(err);
-
-    //CHECK_WRITE_ADDRESS(dst, size);
 }
 
 esp_err_t spi_flash_read(size_t src, void *dstv, size_t size)
@@ -671,4 +670,4 @@ esp_err_t spi_flash_unlock()
     return spi_flash_translate_rc(err);
 }
 
-#endif
+#endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL

components/spi_flash/flash_ops.c

@@ -208,7 +208,7 @@ esp_err_t IRAM_ATTR spi_flash_erase_sector(size_t sec)
 
 #ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
 //deprecated, only used in compatible mode
-esp_err_t IRAM_ATTR spi_flash_erase_range(uint32_t start_addr, uint32_t size)
+esp_err_t IRAM_ATTR spi_flash_erase_range(size_t start_addr, size_t size)
 {
     CHECK_WRITE_ADDRESS(start_addr, size);
     if (start_addr % SPI_FLASH_SEC_SIZE != 0) {

components/spi_flash/include/esp_partition.h

@@ -19,8 +19,10 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include "esp_err.h"
+#include "esp_flash.h"
 #include "esp_spi_flash.h"
 
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -98,6 +100,7 @@ typedef struct esp_partition_iterator_opaque_* esp_partition_iterator_t;
  * However, this is the format used by this API.
  */
 typedef struct {
+    esp_flash_t* flash_chip;            /*!< SPI flash chip on which the partition resides */
     esp_partition_type_t type;          /*!< partition type (app/data) */
     esp_partition_subtype_t subtype;    /*!< partition subtype */
     uint32_t address;                   /*!< starting address of the partition in flash */
@@ -257,7 +260,7 @@ esp_err_t esp_partition_write(const esp_partition_t* partition,
  *         or one of error codes from lower-level flash driver.
  */
 esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
-                                    uint32_t start_addr, uint32_t size);
+                                    size_t start_addr, size_t size);
 
 /**
  * @brief Configure MMU to map partition into data memory
@@ -284,7 +287,7 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  *
  * @return ESP_OK, if successful
  */
-esp_err_t esp_partition_mmap(const esp_partition_t* partition, uint32_t offset, uint32_t size,
+esp_err_t esp_partition_mmap(const esp_partition_t* partition, size_t offset, size_t size,
                              spi_flash_mmap_memory_t memory,
                              const void** out_ptr, spi_flash_mmap_handle_t* out_handle);
 
@@ -320,6 +323,43 @@ esp_err_t esp_partition_get_sha256(const esp_partition_t *partition, uint8_t *sh
  */
 bool esp_partition_check_identity(const esp_partition_t *partition_1, const esp_partition_t *partition_2);
 
+/**
+ * @brief Register a partition on an external flash chip
+ *
+ * This API allows designating certain areas of external flash chips (identified by the esp_flash_t structure)
+ * as partitions. This allows using them with components which access SPI flash through the esp_partition API.
+ *
+ * @param flash_chip  Pointer to the structure identifying the flash chip
+ * @param offset  Address in bytes, where the partition starts
+ * @param size  Size of the partition in bytes
+ * @param label  Partition name
+ * @param type  One of the partition types (ESP_PARTITION_TYPE_*). Note that applications can not be booted from external flash
+ *              chips, so using ESP_PARTITION_TYPE_APP is not supported.
+ * @param subtype  One of the partition subtypes (ESP_PARTITION_SUBTYPE_*)
+ * @param[out] out_partition  Output, if non-NULL, receives the pointer to the resulting esp_partition_t structure
+ * @return
+ *      - ESP_OK on success
+ *      - ESP_ERR_NOT_SUPPORTED if CONFIG_CONFIG_SPI_FLASH_USE_LEGACY_IMPL is enabled
+ *      - ESP_ERR_NO_MEM if memory allocation has failed
+ *      - ESP_ERR_INVALID_ARG if the new partition overlaps another partition on the same flash chip
+ *      - ESP_ERR_INVALID_SIZE if the partition doesn't fit into the flash chip size
+ */
+esp_err_t esp_partition_register_external(esp_flash_t* flash_chip, size_t offset, size_t size,
+                                     const char* label, esp_partition_type_t type, esp_partition_subtype_t subtype,
+                                     const esp_partition_t** out_partition);
+
+/**
+ * @brief Deregister the partition previously registered using esp_partition_register_external
+ * @param partition  pointer to the partition structure obtained from esp_partition_register_external,
+ * @return
+ *      - ESP_OK on success
+ *      - ESP_ERR_NOT_FOUND if the partition pointer is not found
+ *      - ESP_ERR_INVALID_ARG if the partition comes from the partition table
+ *      - ESP_ERR_INVALID_ARG if the partition was not registered using
+ *        esp_partition_register_external function.
+ */
+esp_err_t esp_partition_deregister_external(const esp_partition_t* partition);
+
 #ifdef __cplusplus
 }
 #endif

components/spi_flash/partition.c

@@ -19,11 +19,13 @@
 #include <sys/lock.h>
 #include "esp_flash_partitions.h"
 #include "esp_attr.h"
+#include "esp_flash.h"
 #include "esp_spi_flash.h"
 #include "esp_partition.h"
 #include "esp_flash_encrypt.h"
 #include "esp_log.h"
 #include "bootloader_common.h"
+#include "bootloader_util.h"
 #include "esp_ota_ops.h"
 
 #define HASH_LEN 32 /* SHA-256 digest length */
@@ -35,8 +37,10 @@
 #include "sys/queue.h"
 
 
+
 typedef struct partition_list_item_ {
     esp_partition_t info;
+    bool user_registered;
     SLIST_ENTRY(partition_list_item_) next;
 } partition_list_item_t;
 
@@ -163,12 +167,18 @@ static esp_err_t load_partitions()
             break;
         }
         // allocate new linked list item and populate it with data from partition table
-        partition_list_item_t* item = (partition_list_item_t*) malloc(sizeof(partition_list_item_t));
+        partition_list_item_t* item = (partition_list_item_t*) calloc(sizeof(partition_list_item_t), 1);
+        if (item == NULL) {
+            err = ESP_ERR_NO_MEM;
+            break;
+        }
+        item->info.flash_chip = esp_flash_default_chip;
         item->info.address = it->pos.offset;
         item->info.size = it->pos.size;
         item->info.type = it->type;
         item->info.subtype = it->subtype;
         item->info.encrypted = it->flags & PART_FLAG_ENCRYPTED;
+        item->user_registered = false;
 
         if (!esp_flash_encryption_enabled()) {
             /* If flash encryption is not turned on, no partitions should be treated as encrypted */
@@ -193,7 +203,7 @@ static esp_err_t load_partitions()
         last = item;
     }
     spi_flash_munmap(handle);
-    return ESP_OK;
+    return err;
 }
 
 void esp_partition_iterator_release(esp_partition_iterator_t iterator)
@@ -208,6 +218,80 @@ const esp_partition_t* esp_partition_get(esp_partition_iterator_t iterator)
     return iterator->info;
 }
 
+esp_err_t esp_partition_register_external(esp_flash_t* flash_chip, size_t offset, size_t size,
+        const char* label, esp_partition_type_t type, esp_partition_subtype_t subtype,
+        const esp_partition_t** out_partition)
+{
+    if (out_partition != NULL) {
+        *out_partition = NULL;
+    }
+#ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+    return ESP_ERR_NOT_SUPPORTED;
+#endif
+
+    if (offset + size > flash_chip->size) {
+        return ESP_ERR_INVALID_SIZE;
+    }
+
+    partition_list_item_t* item = (partition_list_item_t*) calloc(sizeof(partition_list_item_t), 1);
+    if (item == NULL) {
+        return ESP_ERR_NO_MEM;
+    }
+    item->info.flash_chip = flash_chip;
+    item->info.address = offset;
+    item->info.size = size;
+    item->info.type = type;
+    item->info.subtype = subtype;
+    item->info.encrypted = false;
+    item->user_registered = true;
+    strlcpy(item->info.label, label, sizeof(item->info.label));
+
+    _lock_acquire(&s_partition_list_lock);
+    partition_list_item_t *it, *last = NULL;
+    SLIST_FOREACH(it, &s_partition_list, next) {
+        /* Check if the new partition overlaps an existing one */
+        if (it->info.flash_chip == flash_chip &&
+                bootloader_util_regions_overlap(offset, offset + size,
+                                                it->info.address, it->info.address + it->info.size)) {
+            _lock_release(&s_partition_list_lock);
+            free(item);
+            return ESP_ERR_INVALID_ARG;
+        }
+        last = it;
+    }
+    if (last == NULL) {
+        SLIST_INSERT_HEAD(&s_partition_list, item, next);
+    } else {
+        SLIST_INSERT_AFTER(last, item, next);
+    }
+    _lock_release(&s_partition_list_lock);
+    if (out_partition != NULL) {
+        *out_partition = &item->info;
+    }
+    return ESP_OK;
+}
+
+esp_err_t esp_partition_deregister_external(const esp_partition_t* partition)
+{
+    esp_err_t result = ESP_ERR_NOT_FOUND;
+    _lock_acquire(&s_partition_list_lock);
+    partition_list_item_t *it;
+    SLIST_FOREACH(it, &s_partition_list, next) {
+        if (&it->info == partition) {
+            if (!it->user_registered) {
+                result = ESP_ERR_INVALID_ARG;
+                break;
+            }
+            SLIST_REMOVE(&s_partition_list, it, partition_list_item_, next);
+            free(it);
+            result = ESP_OK;
+            break;
+        }
+    }
+    _lock_release(&s_partition_list_lock);
+    return result;
+}
+
 const esp_partition_t *esp_partition_verify(const esp_partition_t *partition)
 {
     assert(partition != NULL);
@@ -218,7 +302,8 @@ const esp_partition_t *esp_partition_verify(const esp_partition_t *partition)
     while (it != NULL) {
         const esp_partition_t *p = esp_partition_get(it);
         /* Can't memcmp() whole structure here as padding contents may be different */
-        if (p->address == partition->address
+        if (p->flash_chip == partition->flash_chip
+            && p->address == partition->address
             && partition->size == p->size
             && partition->encrypted == p->encrypted) {
             esp_partition_iterator_release(it);
@@ -242,9 +327,17 @@ esp_err_t esp_partition_read(const esp_partition_t* partition,
     }
 
     if (!partition->encrypted) {
+#ifndef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+        return esp_flash_read(partition->flash_chip, dst, partition->address + src_offset, size);
+#else
         return spi_flash_read(partition->address + src_offset, dst, size);
+#endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
     } else {
 #if CONFIG_SECURE_FLASH_ENC_ENABLED
+        if (partition->flash_chip != esp_flash_default_chip) {
+            return ESP_ERR_NOT_SUPPORTED;
+        }
+
         /* Encrypted partitions need to be read via a cache mapping */
         const void *buf;
         spi_flash_mmap_handle_t handle;
@@ -276,9 +369,16 @@ esp_err_t esp_partition_write(const esp_partition_t* partition,
     }
     dst_offset = partition->address + dst_offset;
     if (!partition->encrypted) {
+#ifndef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+        return esp_flash_write(partition->flash_chip, src, dst_offset, size);
+#else
         return spi_flash_write(dst_offset, src, size);
+#endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
     } else {
 #if CONFIG_SECURE_FLASH_ENC_ENABLED
+        if (partition->flash_chip != esp_flash_default_chip) {
+            return ESP_ERR_NOT_SUPPORTED;
+        }
         return spi_flash_write_encrypted(dst_offset, src, size);
 #else
         return ESP_ERR_NOT_SUPPORTED;
@@ -302,7 +402,11 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
     if (start_addr % SPI_FLASH_SEC_SIZE != 0) {
         return ESP_ERR_INVALID_ARG;
     }
+#ifndef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+    return esp_flash_erase_region(partition->flash_chip, partition->address + start_addr, size);
+#else
     return spi_flash_erase_range(partition->address + start_addr, size);
+#endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
 
 }
 
@@ -314,7 +418,7 @@ esp_err_t esp_partition_erase_range(const esp_partition_t* partition,
  * we can add esp_partition_mmapv which will accept an array of offsets and sizes, and return array of
  * mmaped pointers, and a single handle for all these regions.
  */
-esp_err_t esp_partition_mmap(const esp_partition_t* partition, uint32_t offset, uint32_t size,
+esp_err_t esp_partition_mmap(const esp_partition_t* partition, size_t offset, size_t size,
                              spi_flash_mmap_memory_t memory,
                              const void** out_ptr, spi_flash_mmap_handle_t* out_handle)
 {
@@ -325,6 +429,9 @@ esp_err_t esp_partition_mmap(const esp_partition_t* partition, uint32_t offset,
     if (offset + size > partition->size) {
         return ESP_ERR_INVALID_SIZE;
     }
+    if (partition->flash_chip != esp_flash_default_chip) {
+        return ESP_ERR_NOT_SUPPORTED;
+    }
     size_t phys_addr = partition->address + offset;
     // offset within 64kB block
     size_t region_offset = phys_addr & 0xffff;

components/spi_flash/sim/flash_mock.cpp

@@ -109,3 +109,5 @@ void *heap_caps_malloc( size_t size, uint32_t caps )
 {
     return NULL;
 }
+
+esp_flash_t* esp_flash_default_chip = NULL;

components/spi_flash/test/test_partition_ext.c

@@ -0,0 +1,47 @@
+#include "esp_flash.h"
+#include "esp_partition.h"
+#include "unity.h"
+
+
+TEST_CASE("Basic handling of a partition in external flash", "[partition]")
+{
+    esp_flash_t flash = {
+            .size = 1 * 1024 * 1024,
+    };
+
+    const esp_partition_type_t t = ESP_PARTITION_TYPE_DATA;
+    const esp_partition_subtype_t st = ESP_PARTITION_SUBTYPE_DATA_FAT;
+    const char* label = "ext_fat";
+
+    /* can register */
+    const esp_partition_t* ext_partition;
+    TEST_ESP_OK(esp_partition_register_external(&flash, 0, flash.size, label, t, st, &ext_partition));
+
+    /* can find the registered partition */
+    const esp_partition_t* found = esp_partition_find_first(t, st, label);
+    TEST_ASSERT_EQUAL_HEX(ext_partition, found);
+    TEST_ASSERT_EQUAL(found->size, flash.size);
+    TEST_ASSERT_EQUAL(found->address, 0);
+
+    /* can deregister */
+    TEST_ESP_OK(esp_partition_deregister_external(ext_partition));
+
+    /* can not deregister one of the partitions on the main flash chip */
+    const esp_partition_t* nvs_partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_NVS, NULL);
+    TEST_ASSERT_NOT_NULL(nvs_partition);
+    TEST_ESP_ERR(ESP_ERR_INVALID_ARG, esp_partition_deregister_external(nvs_partition));
+
+    /* can not deregister an unknown partition */
+    esp_partition_t dummy_partition = {};
+    TEST_ESP_ERR(ESP_ERR_NOT_FOUND, esp_partition_deregister_external(&dummy_partition));
+
+    /* can not register a partition larger than the flash size */
+    TEST_ESP_ERR(ESP_ERR_INVALID_SIZE,
+            esp_partition_register_external(&flash, 0, 2 * flash.size, "ext_fat", t, st, &ext_partition));
+
+    /* can not register an overlapping partition */
+    TEST_ESP_OK(esp_partition_register_external(&flash, 0, 2 * SPI_FLASH_SEC_SIZE, "p1", t, st, &ext_partition));
+    TEST_ESP_ERR(ESP_ERR_INVALID_ARG, esp_partition_register_external(&flash, SPI_FLASH_SEC_SIZE, 2 * SPI_FLASH_SEC_SIZE,
+            "p2", t, st, NULL));
+    TEST_ESP_OK(esp_partition_deregister_external(ext_partition));
+}

components/spiffs/test_spiffs_host/sdkconfig/sdkconfig.h

@@ -18,4 +18,5 @@
 
 #define CONFIG_ESPTOOLPY_FLASHSIZE "8MB"
 //currently use the legacy implementation, since the stubs for new HAL are not done yet
-#define CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+#define CONFIG_SPI_FLASH_USE_LEGACY_IMPL 1
+

components/wear_levelling/test_wl_host/sdkconfig/sdkconfig.h

@@ -5,5 +5,5 @@
 #define CONFIG_PARTITION_TABLE_OFFSET 0x8000
 #define CONFIG_ESPTOOLPY_FLASHSIZE "8MB"
 //currently use the legacy implementation, since the stubs for new HAL are not done yet
-#define CONFIG_SPI_FLASH_USE_LEGACY_IMPL
+#define CONFIG_SPI_FLASH_USE_LEGACY_IMPL 1