paritition_table: Verify the partition table md5sum when loading the app

Additionally, always enable the partition MD5 check if flash encryption is on in
Release mode. This ensures the partition table ciphertext has not been modified
(CVE-2021-27926).

The exception is pre-V3.1 ESP-IDF bootloaders and partition tables, which
don't have support for the MD5 entry.

components/bootloader/Kconfig.projbuild

@@ -615,6 +615,7 @@ menu "Security features"
         config SECURE_FLASH_ENCRYPTION_MODE_RELEASE
             bool "Release"
             select SECURE_ENABLE_SECURE_ROM_DL_MODE if SECURE_TARGET_HAS_SECURE_ROM_DL_MODE && !SECURE_DISABLE_ROM_DL_MODE # NOERROR
+            select PARTITION_TABLE_MD5 if !ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS
 
     endchoice
 

components/bootloader_support/include/esp_flash_partitions.h

@@ -42,6 +42,9 @@ extern "C" {
 
 #define PART_FLAG_ENCRYPTED (1<<0)
 
+/* The md5sum value is found this many bytes after the ESP_PARTITION_MAGIC_MD5 offset */
+#define ESP_PARTITION_MD5_OFFSET 16
+
 /* Pre-partition table fixed flash offsets */
 #define ESP_BOOTLOADER_DIGEST_OFFSET 0x0
 #define ESP_BOOTLOADER_OFFSET 0x1000 /* Offset of bootloader image. Has matching value in bootloader KConfig.projbuild file. */

components/bootloader_support/src/flash_partitions.c

@@ -52,7 +52,7 @@ esp_err_t esp_partition_table_verify(const esp_partition_info_t *partition_table
             MD5Update(&context, (unsigned char *) partition_table, num_parts * sizeof(esp_partition_info_t));
             MD5Final(digest, &context);
 
-            unsigned char *md5sum = ((unsigned char *) part) + 16; // skip the 2B magic number and the 14B fillup bytes
+            unsigned char *md5sum = ((unsigned char *) part) + ESP_PARTITION_MD5_OFFSET;
 
             if (memcmp(md5sum, digest, sizeof(digest)) != 0) {
                 if (log_errors) {

components/esp32/Kconfig

@@ -695,10 +695,11 @@ menu "ESP32-specific"
             that after enabling this Wi-Fi/Bluetooth will not work.
 
     config ESP32_COMPATIBLE_PRE_V2_1_BOOTLOADERS
-        bool "App compatible with bootloaders before IDF v2.1"
+        bool "App compatible with bootloaders before ESP-IDF v2.1"
+        select ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS
         default n
         help
-            Bootloaders before IDF v2.1 did less initialisation of the
+            Bootloaders before ESP-IDF v2.1 did less initialisation of the
             system clock. This setting needs to be enabled to build an app
             which can be booted by these older bootloaders.
 
@@ -710,6 +711,22 @@ menu "ESP32-specific"
 
             Enabling this setting adds approximately 1KB to the app's IRAM usage.
 
+    config ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS
+        bool "App compatible with bootloader and partition table before ESP-IDF v3.1"
+        default n
+        help
+            Partition tables before ESP-IDF V3.1 do not contain an MD5 checksum
+            field, and the bootloader before ESP-IDF v3.1 cannot read a partition
+            table that contains an MD5 checksum field.
+
+            Enable this option only if your app needs to boot on a bootloader and/or
+            partition table that was generated from a version *before* ESP-IDF v3.1.
+
+            If this option and Flash Encryption are enabled at the same time, and any
+            data partitions in the partition table are marked Encrypted, then the
+            partition encrypted flag should be manually verified in the app before accessing
+            the partition (see CVE-2021-27926).
+
     config ESP32_APP_INIT_CLK
         bool
         default y if ESP32_COMPATIBLE_PRE_V2_1_BOOTLOADERS

components/partition_table/Kconfig.projbuild

@@ -52,6 +52,7 @@ menu "Partition Table"
     config PARTITION_TABLE_MD5
         bool "Generate an MD5 checksum for the partition table"
         default y
+        depends on !ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS
         help
             Generate an MD5 checksum for the partition table for protecting the
             integrity of the table. The generation should be turned off for legacy

components/spi_flash/partition.c

@@ -17,6 +17,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <sys/lock.h>
+#include "esp32/rom/md5_hash.h"
 #include "esp_flash_partitions.h"
 #include "esp_attr.h"
 #include "esp_flash.h"
@@ -30,14 +31,14 @@
 
 #define HASH_LEN 32 /* SHA-256 digest length */
 
+#define MD5_DIGEST_LEN 16
+
 #ifndef NDEBUG
 // Enable built-in checks in queue.h in debug builds
 #define INVARIANTS
 #endif
 #include "sys/queue.h"
 
-
-
 typedef struct partition_list_item_ {
     esp_partition_t info;
     bool user_registered;
@@ -159,24 +160,53 @@ static esp_partition_iterator_opaque_t* iterator_create(esp_partition_type_t typ
 // This function is called only once, with s_partition_list_lock taken.
 static esp_err_t load_partitions(void)
 {
-    const uint32_t* ptr;
+    const uint8_t *p_start;
+    const uint8_t *p_end;
     spi_flash_mmap_handle_t handle;
+
+    // Temporary list of loaded partitions, if valid then we copy this to s_partition_list
+    typeof(s_partition_list) new_partitions_list = SLIST_HEAD_INITIALIZER(s_partition_list);
+    partition_list_item_t* last = NULL;
+
+#if CONFIG_PARTITION_TABLE_MD5
+    const uint8_t *md5_part = NULL;
+    const uint8_t *stored_md5;
+    uint8_t calc_md5[MD5_DIGEST_LEN];
+    struct MD5Context context;
+
+    MD5Init(&context);
+#endif
+
     // map 64kB block where partition table is located
     esp_err_t err = spi_flash_mmap(ESP_PARTITION_TABLE_OFFSET & 0xffff0000,
-            SPI_FLASH_SEC_SIZE, SPI_FLASH_MMAP_DATA, (const void**) &ptr, &handle);
+                                   SPI_FLASH_SEC_SIZE, SPI_FLASH_MMAP_DATA, (const void **)&p_start, &handle);
     if (err != ESP_OK) {
         return err;
     }
     // calculate partition address within mmap-ed region
-    const esp_partition_info_t* it = (const esp_partition_info_t*)
-            (ptr + (ESP_PARTITION_TABLE_OFFSET & 0xffff) / sizeof(*ptr));
-    const esp_partition_info_t* end = it + SPI_FLASH_SEC_SIZE / sizeof(*it);
-    // tail of the linked list of partitions
-    partition_list_item_t* last = NULL;
-    for (; it != end; ++it) {
-        if (it->magic != ESP_PARTITION_MAGIC) {
+    p_start += (ESP_PARTITION_TABLE_OFFSET & 0xffff);
+    p_end = p_start + SPI_FLASH_SEC_SIZE;
+
+    for(const uint8_t *p_entry = p_start; p_entry < p_end; p_entry += sizeof(esp_partition_info_t)) {
+        esp_partition_info_t entry;
+        // copying to RAM instead of using pointer to flash to avoid any chance of TOCTOU due to cache miss
+        // when flash encryption is used
+        memcpy(&entry, p_entry, sizeof(entry));
+
+#if CONFIG_PARTITION_TABLE_MD5
+        if (entry.magic == ESP_PARTITION_MAGIC_MD5) {
+            md5_part = p_entry;
+            break;
+        }
+#endif
+        if (entry.magic != ESP_PARTITION_MAGIC) {
             break;
         }
+
+#if CONFIG_PARTITION_TABLE_MD5
+        MD5Update(&context, (void *)&entry, sizeof(entry));
+#endif
+
         // allocate new linked list item and populate it with data from partition table
         partition_list_item_t* item = (partition_list_item_t*) calloc(sizeof(partition_list_item_t), 1);
         if (item == NULL) {
@@ -184,35 +214,70 @@ static esp_err_t load_partitions(void)
             break;
         }
         item->info.flash_chip = esp_flash_default_chip;
-        item->info.address = it->pos.offset;
-        item->info.size = it->pos.size;
-        item->info.type = it->type;
-        item->info.subtype = it->subtype;
-        item->info.encrypted = it->flags & PART_FLAG_ENCRYPTED;
+        item->info.address = entry.pos.offset;
+        item->info.size = entry.pos.size;
+        item->info.type = entry.type;
+        item->info.subtype = entry.subtype;
+        item->info.encrypted = entry.flags & PART_FLAG_ENCRYPTED;
         item->user_registered = false;
 
         if (!esp_flash_encryption_enabled()) {
             /* If flash encryption is not turned on, no partitions should be treated as encrypted */
             item->info.encrypted = false;
-        } else if (it->type == PART_TYPE_APP
-                || (it->type == PART_TYPE_DATA && it->subtype == PART_SUBTYPE_DATA_OTA)
-                || (it->type == PART_TYPE_DATA && it->subtype == PART_SUBTYPE_DATA_NVS_KEYS)) {
+        } else if (entry.type == PART_TYPE_APP
+                || (entry.type == PART_TYPE_DATA && entry.subtype == PART_SUBTYPE_DATA_OTA)
+                || (entry.type == PART_TYPE_DATA && entry.subtype == PART_SUBTYPE_DATA_NVS_KEYS)) {
             /* If encryption is turned on, all app partitions and OTA data
                are always encrypted */
             item->info.encrypted = true;
         }
 
-        // it->label may not be zero-terminated
-        strncpy(item->info.label, (const char*) it->label, sizeof(item->info.label) - 1);
-        item->info.label[sizeof(it->label)] = 0;
+        // item->info.label is initialized by calloc, so resulting string will be null terminated
+        strncpy(item->info.label, (const char*) entry.label, sizeof(item->info.label) - 1);
+
         // add it to the list
         if (last == NULL) {
-            SLIST_INSERT_HEAD(&s_partition_list, item, next);
+            SLIST_INSERT_HEAD(&new_partitions_list, item, next);
         } else {
             SLIST_INSERT_AFTER(last, item, next);
         }
         last = item;
     }
+
+#if CONFIG_PARTITION_TABLE_MD5
+    if (md5_part == NULL) {
+        ESP_LOGE(TAG, "No MD5 found in partition table");
+        err = ESP_ERR_NOT_FOUND;
+    } else {
+        stored_md5 = md5_part + ESP_PARTITION_MD5_OFFSET;
+
+        MD5Final(calc_md5, &context);
+
+        ESP_LOG_BUFFER_HEXDUMP("calculated md5", calc_md5, MD5_DIGEST_LEN, ESP_LOG_VERBOSE);
+        ESP_LOG_BUFFER_HEXDUMP("stored md5", stored_md5, MD5_DIGEST_LEN, ESP_LOG_VERBOSE);
+
+        if (memcmp(calc_md5, stored_md5, MD5_DIGEST_LEN) != 0) {
+            ESP_LOGE(TAG, "Partition table MD5 mismatch");
+            err = ESP_ERR_INVALID_STATE;
+        } else {
+            ESP_LOGD(TAG, "Partition table MD5 verified");
+        }
+    }
+#endif
+
+    if (err == ESP_OK) {
+        /* Don't copy the list to the static variable unless it's verified */
+        s_partition_list = new_partitions_list;
+    } else {
+        /* Otherwise, free all the memory we just allocated */
+        partition_list_item_t *it = new_partitions_list.slh_first;
+        while (it) {
+            partition_list_item_t *next = it->next.sle_next;
+            free(it);
+            it = next;
+        }
+    }
+
     spi_flash_munmap(handle);
     return err;
 }

docs/en/api-guides/bootloader.rst

@@ -10,6 +10,40 @@ Bootloader performs the following functions:
 
 Bootloader is located at the address `0x1000` in the flash.
 
+Bootloader compatibility
+-------------------------
+
+It is recommended to update to newer :doc:`versions of ESP-IDF </versions>`: when they are released. The OTA (over the air) update process can flash new apps in the field but cannot flash a new bootloader. For this reason, the bootloader supports booting apps built from newer versions of ESP-IDF.
+
+The bootloader does not support booting apps from older versions of ESP-IDF. When updating ESP-IDF manually on an existing product that might need to downgrade the app to an older version, keep using the older ESP-IDF bootloader binary as well.
+
+.. note::
+
+   If testing an OTA update for an existing product in production, always test it using the same ESP-IDF bootloader binary that is deployed in production.
+
+.. only:: esp32
+
+    Before ESP-IDF V2.1
+    ^^^^^^^^^^^^^^^^^^^
+
+    Bootloaders built from very old versions of ESP-IDF (before ESP-IDF V2.1) perform less hardware configuration than newer versions. When using a bootloader from these early ESP-IDF versions and building a new app, enable the config option :ref:`CONFIG_ESP32_COMPATIBLE_PRE_V2_1_BOOTLOADERS`.
+
+    Before ESP-IDF V3.1
+    ^^^^^^^^^^^^^^^^^^^
+
+    Bootloaders built from versions of ESP-IDF before V3.1 do not support MD5 checksums in the partition table binary. When using a bootloader from these ESP-IDF versions and building a new app, enable the config option :ref:`CONFIG_ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS`.
+
+SPI Flash Configuration
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Each ESP-IDF application or bootloader .bin file contains a header with :ref:`CONFIG_ESPTOOLPY_FLASHMODE`, :ref:`CONFIG_ESPTOOLPY_FLASHFREQ`, :ref:`CONFIG_ESPTOOLPY_FLASHSIZE` embedded in it. These are used to configure the SPI flash during boot.
+
+The :first stage bootloader in ROM reads the second stage bootloader header from flash and uses these settings to load it. However, at this time the system clock speed is lower than configured and not all flash modes are supported.  When the second stage bootloader then runs and re-configures the flash, it reads values from the currently selected app binary header not the bootloader header. This allows an OTA update to change the SPI flash settings in use.
+
+.. only:: esp32
+
+   Bootloaders prior to ESP-IDF V4.0 used the bootloader's own header to configure the SPI flash, meaning these values could not be changed in an update. To maintain compatibility with older bootloaders, the app re-initializes the flash settings during app startup using the configuration found in the app header.
+
 FACTORY reset
 ---------------------------
 The user can write a basic working firmware and load it into the factory partition. 

docs/en/api-guides/partition-tables.rst

@@ -171,7 +171,14 @@ MD5 checksum
 
 The binary format of the partition table contains an MD5 checksum computed based on the partition table. This checksum is used for checking the integrity of the partition table during the boot.
 
-The MD5 checksum generation can be disabled by the ``--disable-md5sum`` option of ``gen_esp32part.py`` or by the :ref:`CONFIG_PARTITION_TABLE_MD5` option. This is useful for example when one uses a legacy bootloader which cannot process MD5 checksums and the boot fails with the error message ``invalid magic number 0xebeb``.
+.. only:: esp32
+
+    The MD5 checksum generation can be disabled by the ``--disable-md5sum`` option of ``gen_esp32part.py`` or by the :ref:`CONFIG_PARTITION_TABLE_MD5` option. This is useful for example when one :ref:`uses a bootloader from ESP-IDF before v3.1 <CONFIG_ESP32_COMPATIBLE_PRE_V3_1_BOOTLOADERS>` which cannot process MD5 checksums and the boot fails with the error message ``invalid magic number 0xebeb``.
+
+.. only:: not esp32
+
+    The MD5 checksum generation can be disabled by the ``--disable-md5sum`` option of ``gen_esp32part.py`` or by the :ref:`CONFIG_PARTITION_TABLE_MD5` option.
+
 
 Flashing the partition table
 ----------------------------