OTA: Fixed OTA with chunked servers and added example_test with chunked server

components/esp_https_ota/src/esp_https_ota.c

@@ -296,18 +296,22 @@ esp_err_t esp_https_ota_perform(esp_https_ota_handle_t https_ota_handle)
                                              handle->ota_upgrade_buf,
                                              handle->ota_upgrade_buf_size);
             if (data_read == 0) {
+                /*
+                 *  esp_https_ota_is_complete_data_received is added to check whether
+                 *  complete image is received.
+                 */
+                bool is_recv_complete = esp_https_ota_is_complete_data_received(https_ota_handle);
                 /*
                  * As esp_http_client_read never returns negative error code, we rely on
-                 * `errno` to check for underlying transport connectivity closure if any
+                 * `errno` to check for underlying transport connectivity closure if any.
+                 * Incase the complete data has not been received but the server has sent
+                 * an ENOTCONN or ECONNRESET, failure is returned. We close with success
+                 * if complete data has been received.
                  */
-                if (errno == ENOTCONN || errno == ECONNRESET) {
+                if ((errno == ENOTCONN || errno == ECONNRESET) && !is_recv_complete) {
                     ESP_LOGE(TAG, "Connection closed, errno = %d", errno);
                     return ESP_FAIL;
-                }
-                /*  esp_https_ota_is_complete_data_received is added to check whether
-                    complete image is received.
-                */
-                if (!esp_https_ota_is_complete_data_received(https_ota_handle)) {
+                } else if (!is_recv_complete) {
                     return ESP_ERR_HTTPS_OTA_IN_PROGRESS;
                 }
                 ESP_LOGI(TAG, "Connection closed");

examples/system/ota/advanced_https_ota/example_test.py

@@ -9,6 +9,7 @@ import ssl
 from tiny_test_fw import DUT
 import ttfw_idf
 import random
+import subprocess
 
 server_cert = "-----BEGIN CERTIFICATE-----\n" \
               "MIIDXTCCAkWgAwIBAgIJAP4LF7E72HakMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV\n"\
@@ -92,6 +93,20 @@ def start_https_server(ota_image_dir, server_ip, server_port):
     httpd.serve_forever()
 
 
+def start_chunked_server(ota_image_dir, server_port):
+    os.chdir(ota_image_dir)
+
+    server_file = os.path.join(ota_image_dir, "server_cert.pem")
+    cert_file_handle = open(server_file, "w+")
+    cert_file_handle.write(server_cert)
+
+    key_file = os.path.join(ota_image_dir, "server_key.pem")
+    key_file_handle = open("server_key.pem", "w+")
+    key_file_handle.write(server_key)
+    chunked_server = subprocess.Popen(["openssl", "s_server", "-WWW", "-key", key_file, "-cert", server_file, "-port", str(server_port)])
+    return chunked_server
+
+
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_advanced_https_ota_example(env, extra_data):
     """
@@ -179,6 +194,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_d
     print("writing to device: {}".format("https://" + host_ip + ":8001/" + truncated_bin_name))
     dut1.write("https://" + host_ip + ":8001/" + truncated_bin_name)
     dut1.expect("Image validation failed, image is corrupted", timeout=30)
+    os.remove(binary_file)
 
 
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")
@@ -224,6 +240,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extr
     print("writing to device: {}".format("https://" + host_ip + ":8001/" + truncated_bin_name))
     dut1.write("https://" + host_ip + ":8001/" + truncated_bin_name)
     dut1.expect("advanced_https_ota_example: esp_https_ota_read_img_desc failed", timeout=30)
+    os.remove(binary_file)
 
 
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")
@@ -268,10 +285,51 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
     print("writing to device: {}".format("https://" + host_ip + ":8001/" + random_bin_name))
     dut1.write("https://" + host_ip + ":8001/" + random_bin_name)
     dut1.expect("esp_ota_ops: OTA image has invalid magic byte", timeout=10)
+    os.remove(binary_file)
+
+
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
+def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
+    """
+    This is a positive test case, which downloads complete binary file multiple number of times.
+    Number of iterations can be specified in variable iterations.
+    steps: |
+      1. join AP
+      2. Fetch OTA image over HTTPS
+      3. Reboot with the new OTA image
+    """
+    dut1 = env.get_dut("advanced_https_ota_example", "examples/system/ota/advanced_https_ota", dut_class=ttfw_idf.ESP32DUT)
+    # File to be downloaded. This file is generated after compilation
+    bin_name = "advanced_https_ota.bin"
+    # check and log bin size
+    binary_file = os.path.join(dut1.app.binary_path, bin_name)
+    bin_size = os.path.getsize(binary_file)
+    ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
+    # start test
+    host_ip = get_my_ip()
+    chunked_server = start_chunked_server(dut1.app.binary_path, 8070)
+    dut1.start_app()
+    dut1.expect("Loaded app from partition at offset", timeout=30)
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
+    except DUT.ExpectTimeout:
+        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
+    dut1.expect("Starting Advanced OTA example", timeout=30)
+
+    print("writing to device: {}".format("https://" + host_ip + ":8070/" + bin_name))
+    dut1.write("https://" + host_ip + ":8070/" + bin_name)
+    dut1.expect("Loaded app from partition at offset", timeout=60)
+    dut1.expect("Starting Advanced OTA example", timeout=30)
+    chunked_server.kill()
+    os.remove(os.path.join(dut1.app.binary_path, "server_cert.pem"))
+    os.remove(os.path.join(dut1.app.binary_path, "server_key.pem"))
 
 
 if __name__ == '__main__':
     test_examples_protocol_advanced_https_ota_example()
+    test_examples_protocol_advanced_https_ota_example_chunked()
     test_examples_protocol_advanced_https_ota_example_truncated_bin()
     test_examples_protocol_advanced_https_ota_example_truncated_header()
     test_examples_protocol_advanced_https_ota_example_random()

examples/system/ota/advanced_https_ota/sdkconfig.ci

@@ -1,4 +1,4 @@
 CONFIG_EXAMPLE_FIRMWARE_UPGRADE_URL="FROM_STDIN"
 CONFIG_EXAMPLE_SKIP_COMMON_NAME_CHECK=y
 CONFIG_EXAMPLE_SKIP_VERSION_CHECK=y
-CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=300
+CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=2000

examples/system/ota/native_ota_example/example_test.py

@@ -9,6 +9,7 @@ import ssl
 from tiny_test_fw import DUT
 import ttfw_idf
 import random
+import subprocess
 
 server_cert = "-----BEGIN CERTIFICATE-----\n" \
               "MIIDXTCCAkWgAwIBAgIJAP4LF7E72HakMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV\n"\
@@ -92,6 +93,22 @@ def start_https_server(ota_image_dir, server_ip, server_port):
     httpd.serve_forever()
 
 
+def start_chunked_server(ota_image_dir, server_port):
+    os.chdir(ota_image_dir)
+
+    server_file = os.path.join(ota_image_dir, "server_cert.pem")
+    cert_file_handle = open(server_file, "w+")
+    cert_file_handle.write(server_cert)
+    cert_file_handle.close()
+
+    key_file = os.path.join(ota_image_dir, "server_key.pem")
+    key_file_handle = open("server_key.pem", "w+")
+    key_file_handle.write(server_key)
+    key_file_handle.close()
+    chunked_server = subprocess.Popen(["openssl", "s_server", "-WWW", "-key", key_file, "-cert", server_file, "-port", str(server_port)])
+    return chunked_server
+
+
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_native_ota_example(env, extra_data):
     """
@@ -179,6 +196,7 @@ def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
     print("writing to device: {}".format("https://" + host_ip + ":8002/" + truncated_bin_name))
     dut1.write("https://" + host_ip + ":8002/" + truncated_bin_name)
     dut1.expect("native_ota_example: Image validation failed, image is corrupted", timeout=20)
+    os.remove(binary_file)
 
 
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")
@@ -224,6 +242,7 @@ def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
     print("writing to device: {}".format("https://" + host_ip + ":8002/" + truncated_bin_name))
     dut1.write("https://" + host_ip + ":8002/" + truncated_bin_name)
     dut1.expect("native_ota_example: received package is not fit len", timeout=20)
+    os.remove(binary_file)
 
 
 @ttfw_idf.idf_example_test(env_tag="Example_WIFI")
@@ -268,10 +287,51 @@ def test_examples_protocol_native_ota_example_random(env, extra_data):
     print("writing to device: {}".format("https://" + host_ip + ":8002/" + random_bin_name))
     dut1.write("https://" + host_ip + ":8002/" + random_bin_name)
     dut1.expect("esp_ota_ops: OTA image has invalid magic byte", timeout=20)
+    os.remove(binary_file)
+
+
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
+def test_examples_protocol_native_ota_example_chunked(env, extra_data):
+    """
+    This is a positive test case, which downloads complete binary file multiple number of times.
+    Number of iterations can be specified in variable iterations.
+    steps: |
+      1. join AP
+      2. Fetch OTA image over HTTPS
+      3. Reboot with the new OTA image
+    """
+    dut1 = env.get_dut("native_ota_example", "examples/system/ota/native_ota_example", dut_class=ttfw_idf.ESP32DUT)
+    # File to be downloaded. This file is generated after compilation
+    bin_name = "native_ota.bin"
+    # check and log bin size
+    binary_file = os.path.join(dut1.app.binary_path, bin_name)
+    bin_size = os.path.getsize(binary_file)
+    ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024)
+    # start test
+    host_ip = get_my_ip()
+    chunked_server = start_chunked_server(dut1.app.binary_path, 8070)
+    dut1.start_app()
+    dut1.expect("Loaded app from partition at offset", timeout=30)
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
+    except DUT.ExpectTimeout:
+        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
+
+    dut1.expect("Starting OTA example", timeout=30)
+    print("writing to device: {}".format("https://" + host_ip + ":8070/" + bin_name))
+    dut1.write("https://" + host_ip + ":8070/" + bin_name)
+    dut1.expect("Loaded app from partition at offset", timeout=60)
+    dut1.expect("Starting OTA example", timeout=30)
+    chunked_server.kill()
+    os.remove(os.path.join(dut1.app.binary_path, "server_cert.pem"))
+    os.remove(os.path.join(dut1.app.binary_path, "server_key.pem"))
 
 
 if __name__ == '__main__':
     test_examples_protocol_native_ota_example()
+    test_examples_protocol_native_ota_example_chunked()
     test_examples_protocol_native_ota_example_truncated_bin()
     test_examples_protocol_native_ota_example_truncated_header()
     test_examples_protocol_native_ota_example_random()

examples/system/ota/native_ota_example/sdkconfig.ci

@@ -1,4 +1,4 @@
 CONFIG_EXAMPLE_FIRMWARE_UPG_URL="FROM_STDIN"
 CONFIG_EXAMPLE_SKIP_COMMON_NAME_CHECK=y
 CONFIG_EXAMPLE_SKIP_VERSION_CHECK=y
-CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=300
+CONFIG_EXAMPLE_OTA_RECV_TIMEOUT=2000