Merge branch 'feature/partial_image_download' into 'master'

esp_https_ota: Add support for partial image download

Closes IDF-1669, IDFGH-4573, IDFGH-4574, and IDFGH-4824

See merge request espressif/esp-idf!12141

components/esp_https_ota/include/esp_https_ota.h

@@ -31,6 +31,8 @@ typedef struct {
     const esp_http_client_config_t *http_config;   /*!< ESP HTTP client configuration */
     http_client_init_cb_t http_client_init_cb;     /*!< Callback after ESP HTTP client is initialised */
     bool bulk_flash_erase;                         /*!< Erase entire flash partition during initialization. By default flash partition is erased during write operation and in chunk of 4K sector size */
+    bool partial_http_download;                    /*!< Enable Firmware image to be downloaded over multiple HTTP requests */
+    int max_http_request_size;                     /*!< Maximum request size for partial HTTP download */
 } esp_https_ota_config_t;
 
 #define ESP_ERR_HTTPS_OTA_BASE            (0x9000)

components/esp_https_ota/src/esp_https_ota.c

@@ -23,6 +23,7 @@
 
 #define IMAGE_HEADER_SIZE sizeof(esp_image_header_t) + sizeof(esp_image_segment_header_t) + sizeof(esp_app_desc_t) + 1
 #define DEFAULT_OTA_BUF_SIZE IMAGE_HEADER_SIZE
+#define DEFAULT_REQUEST_SIZE (64 * 1024)
 static const char *TAG = "esp_https_ota";
 
 typedef enum {
@@ -39,8 +40,11 @@ struct esp_https_ota_handle {
     char *ota_upgrade_buf;
     size_t ota_upgrade_buf_size;
     int binary_file_len;
+    int image_length;
+    int max_http_request_size;
     esp_https_ota_state state;
     bool bulk_flash_erase;
+    bool partial_http_download;
 };
 
 typedef struct esp_https_ota_handle esp_https_ota_t;
@@ -100,11 +104,29 @@ static esp_err_t _http_connect(esp_http_client_handle_t http_client)
     esp_err_t err = ESP_FAIL;
     int status_code, header_ret;
     do {
-        err = esp_http_client_open(http_client, 0);
+        char *post_data = NULL;
+        /* Send POST request if body is set.
+         * Note: Sending POST request is not supported if partial_http_download
+         * is enabled
+         */
+        int post_len = esp_http_client_get_post_field(http_client, &post_data);
+        err = esp_http_client_open(http_client, post_len);
         if (err != ESP_OK) {
             ESP_LOGE(TAG, "Failed to open HTTP connection: %s", esp_err_to_name(err));
             return err;
         }
+        if (post_len) {
+            int write_len = 0;
+            while (post_len > 0) {
+                write_len = esp_http_client_write(http_client, post_data, post_len);
+                if (write_len < 0) {
+                    ESP_LOGE(TAG, "Write failed");
+                    return ESP_FAIL;
+                }
+            }
+            post_len -= write_len;
+            post_data += write_len;
+        }
         header_ret = esp_http_client_fetch_headers(http_client);
         if (header_ret < 0) {
             return header_ret;
@@ -167,6 +189,9 @@ esp_err_t esp_https_ota_begin(esp_https_ota_config_t *ota_config, esp_https_ota_
         return ESP_ERR_NO_MEM;
     }
 
+    https_ota_handle->partial_http_download = ota_config->partial_http_download;
+    https_ota_handle->max_http_request_size = (ota_config->max_http_request_size == 0) ? DEFAULT_REQUEST_SIZE : ota_config->max_http_request_size;
+
     /* Initiate HTTP Connection */
     https_ota_handle->http_client = esp_http_client_init(ota_config->http_config);
     if (https_ota_handle->http_client == NULL) {
@@ -175,6 +200,30 @@ esp_err_t esp_https_ota_begin(esp_https_ota_config_t *ota_config, esp_https_ota_
         goto failure;
     }
 
+    if (https_ota_handle->partial_http_download) {
+        esp_http_client_set_method(https_ota_handle->http_client, HTTP_METHOD_HEAD);
+        err = esp_http_client_perform(https_ota_handle->http_client);
+        if (err != ESP_OK || esp_http_client_get_status_code(https_ota_handle->http_client) != HttpStatus_Ok) {
+            ESP_LOGE(TAG, "Failed to get image length");
+            goto http_cleanup;
+        }
+        https_ota_handle->image_length = esp_http_client_get_content_length(https_ota_handle->http_client);
+        esp_http_client_close(https_ota_handle->http_client);
+
+        if (https_ota_handle->image_length > https_ota_handle->max_http_request_size) {
+            char *header_val = NULL;
+            asprintf(&header_val, "bytes=0-%d", https_ota_handle->max_http_request_size - 1);
+            if (header_val == NULL) {
+                ESP_LOGE(TAG, "Failed to allocate memory for HTTP header");
+                err = ESP_ERR_NO_MEM;
+                goto http_cleanup;
+            }
+            esp_http_client_set_header(https_ota_handle->http_client, "Range", header_val);
+            free(header_val);
+        }
+        esp_http_client_set_method(https_ota_handle->http_client, HTTP_METHOD_GET);
+    }
+
     if (ota_config->http_client_init_cb) {
         err = ota_config->http_client_init_cb(https_ota_handle->http_client);
         if (err != ESP_OK) {
@@ -243,7 +292,7 @@ esp_err_t esp_https_ota_get_img_desc(esp_https_ota_handle_t https_ota_handle, es
      * while loop is added to download complete image headers, even if the headers
      * are not sent in a single packet.
      */
-    while (data_read_size > 0 && !esp_https_ota_is_complete_data_received(https_ota_handle)) {
+    while (data_read_size > 0 && !esp_http_client_is_complete_data_received(handle->http_client)) {
         data_read = esp_http_client_read(handle->http_client,
                                           (handle->ota_upgrade_buf + bytes_read),
                                           data_read_size);
@@ -294,7 +343,13 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle)
                then the image data read there should be written to OTA partition
                */
             if (handle->binary_file_len) {
-                return _ota_write(handle, (const void *)handle->ota_upgrade_buf, handle->binary_file_len);
+                /*
+                 * Header length gets added to handle->binary_file_len in _ota_write
+                 * Clear handle->binary_file_len to avoid additional 289 bytes in binary_file_len
+                 */
+                int binary_file_len = handle->binary_file_len;
+                handle->binary_file_len = 0;
+                return _ota_write(handle, (const void *)handle->ota_upgrade_buf, binary_file_len);
             }
             /* falls through */
         case ESP_HTTPS_OTA_IN_PROGRESS:
@@ -303,10 +358,10 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle)
                                              handle->ota_upgrade_buf_size);
             if (data_read == 0) {
                 /*
-                 *  esp_https_ota_is_complete_data_received is added to check whether
+                 *  esp_http_client_is_complete_data_received is added to check whether
                  *  complete image is received.
                  */
-                bool is_recv_complete = esp_https_ota_is_complete_data_received(https_ota_handle);
+                bool is_recv_complete = esp_http_client_is_complete_data_received(handle->http_client);
                 /*
                  * As esp_http_client_read doesn't return negative error code if select fails, we rely on
                  * `errno` to check for underlying transport connectivity closure if any.
@@ -326,20 +381,51 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle)
             } else {
                 return ESP_FAIL;
             }
-            handle->state = ESP_HTTPS_OTA_SUCCESS;
+            if (!handle->partial_http_download || (handle->partial_http_download && handle->image_length == handle->binary_file_len)) {
+                handle->state = ESP_HTTPS_OTA_SUCCESS;
+            }
             break;
          default:
             ESP_LOGE(TAG, "Invalid ESP HTTPS OTA State");
             return ESP_FAIL;
             break;
     }
+    if (handle->partial_http_download) {
+        if (handle->state == ESP_HTTPS_OTA_IN_PROGRESS && handle->image_length > handle->binary_file_len) {
+            esp_http_client_close(handle->http_client);
+            char *header_val = NULL;
+            if ((handle->image_length - handle->binary_file_len) > handle->max_http_request_size) {
+                asprintf(&header_val, "bytes=%d-%d", handle->binary_file_len, (handle->binary_file_len + handle->max_http_request_size - 1));
+            } else {
+                asprintf(&header_val, "bytes=%d-", handle->binary_file_len);
+            }
+            if (header_val == NULL) {
+                ESP_LOGE(TAG, "Failed to allocate memory for HTTP header");
+                return ESP_ERR_NO_MEM;
+            }
+            esp_http_client_set_header(handle->http_client, "Range", header_val);
+            free(header_val);
+            err = _http_connect(handle->http_client);
+            if (err != ESP_OK) {
+                ESP_LOGE(TAG, "Failed to establish HTTP connection");
+                return ESP_FAIL;
+            }
+            return ESP_ERR_HTTPS_OTA_IN_PROGRESS;
+        }
+    }
     return ESP_OK;
 }
 
 bool esp_https_ota_is_complete_data_received(esp_https_ota_handle_t https_ota_handle)
 {
+    bool ret = false;
     esp_https_ota_t *handle = (esp_https_ota_t *)https_ota_handle;
-    return esp_http_client_is_complete_data_received(handle->http_client);
+    if (handle->partial_http_download) {
+        ret = (handle->image_length == handle->binary_file_len);
+    } else {
+        ret = esp_http_client_is_complete_data_received(handle->http_client);
+    }
+    return ret;
 }
 
 esp_err_t esp_https_ota_finish(esp_https_ota_handle_t https_ota_handle)

docs/en/api-reference/system/esp_https_ota.rst

@@ -29,6 +29,19 @@ Application Example
             return ESP_OK;
         }
 
+Partial Image Download over HTTPS
+---------------------------------
+
+To use partial image download feature, enable ``partial_http_download`` configuration in ``esp_https_ota_config_t``.
+When this configuration is enabled, firmware image will be downloaded in multiple HTTP requests of specified size.
+Maximum content length of each request can be specified by setting ``max_http_request_size`` to required value.
+
+This option is useful while fetching image from a service like AWS S3, where mbedTLS Rx buffer size (:ref:`CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN`)
+can be set to lower value which is not possible without enabling this configuration.
+
+Default value of mbedTLS Rx buffer size is set to 16K. By using partial_http_download with max_http_request_size of 4K,
+size of mbedTLS Rx buffer can be reduced to 4K. With this confiuration, memory saving of around 12K is expected.
+
 .. only:: esp32
 
     Signature Verification

examples/system/ota/native_ota_example/main/native_ota_example.c

@@ -132,9 +132,9 @@ static void ota_example_task(void *pvParameter)
     esp_http_client_fetch_headers(client);
 
     update_partition = esp_ota_get_next_update_partition(NULL);
+    assert(update_partition != NULL);
     ESP_LOGI(TAG, "Writing to partition subtype %d at offset 0x%x",
              update_partition->subtype, update_partition->address);
-    assert(update_partition != NULL);
 
     int binary_file_length = 0;
     /*deal with all receive packet*/
@@ -234,8 +234,9 @@ static void ota_example_task(void *pvParameter)
     if (err != ESP_OK) {
         if (err == ESP_ERR_OTA_VALIDATE_FAILED) {
             ESP_LOGE(TAG, "Image validation failed, image is corrupted");
+        } else {
+            ESP_LOGE(TAG, "esp_ota_end failed (%s)!", esp_err_to_name(err));
         }
-        ESP_LOGE(TAG, "esp_ota_end failed (%s)!", esp_err_to_name(err));
         http_cleanup(client);
         task_fatal_error();
     }