ci: mqtt ssl test updated to output larger data in form of binary partition

Testing with predefined buffer sizes for MQTT as well as mbedTLS to
exercise fragmenting outgoing data on both layers. Example test compares
received partition with the actual binary file.

examples/protocols/mqtt/ssl/main/app_main.c

@@ -11,25 +11,16 @@
 #include <stdint.h>
 #include <stddef.h>
 #include <string.h>
-#include "esp_wifi.h"
 #include "esp_system.h"
 #include "nvs_flash.h"
 #include "esp_event.h"
 #include "tcpip_adapter.h"
 #include "protocol_examples_common.h"
 
-#include "freertos/FreeRTOS.h"
-#include "freertos/task.h"
-#include "freertos/semphr.h"
-#include "freertos/queue.h"
-
-#include "lwip/sockets.h"
-#include "lwip/dns.h"
-#include "lwip/netdb.h"
-
 #include "esp_log.h"
 #include "mqtt_client.h"
 #include "esp_tls.h"
+#include "esp_ota_ops.h"
 
 static const char *TAG = "MQTTS_EXAMPLE";
 
@@ -41,6 +32,20 @@ extern const uint8_t mqtt_eclipse_org_pem_start[]   asm("_binary_mqtt_eclipse_or
 #endif
 extern const uint8_t mqtt_eclipse_org_pem_end[]   asm("_binary_mqtt_eclipse_org_pem_end");
 
+//
+// Note: this function is for testing purposes only publishing the entire active partition
+//       (to be checked against the original binary)
+//
+static void send_binary(esp_mqtt_client_handle_t client)
+{
+    spi_flash_mmap_handle_t out_handle;
+    const void *binary_address;
+    const esp_partition_t* partition = esp_ota_get_running_partition();
+    esp_partition_mmap(partition, 0, partition->size, SPI_FLASH_MMAP_DATA, &binary_address, &out_handle);
+    int msg_id = esp_mqtt_client_publish(client, "/topic/binary", binary_address, partition->size, 0, 0);
+    ESP_LOGI(TAG, "binary sent with msg_id=%d", msg_id);
+}
+
 static esp_err_t mqtt_event_handler_cb(esp_mqtt_event_handle_t event)
 {
     esp_mqtt_client_handle_t client = event->client;
@@ -77,6 +82,10 @@ static esp_err_t mqtt_event_handler_cb(esp_mqtt_event_handle_t event)
             ESP_LOGI(TAG, "MQTT_EVENT_DATA");
             printf("TOPIC=%.*s\r\n", event->topic_len, event->topic);
             printf("DATA=%.*s\r\n", event->data_len, event->data);
+            if (strncmp(event->data, "send binary please", event->data_len) == 0) {
+                ESP_LOGI(TAG, "Sending the binary");
+                send_binary(client);
+            }
             break;
         case MQTT_EVENT_ERROR:
             ESP_LOGI(TAG, "MQTT_EVENT_ERROR");
@@ -121,6 +130,7 @@ void app_main(void)
     ESP_LOGI(TAG, "[APP] IDF version: %s", esp_get_idf_version());
 
     esp_log_level_set("*", ESP_LOG_INFO);
+    esp_log_level_set("esp-tls", ESP_LOG_VERBOSE);
     esp_log_level_set("MQTT_CLIENT", ESP_LOG_VERBOSE);
     esp_log_level_set("MQTT_EXAMPLE", ESP_LOG_VERBOSE);
     esp_log_level_set("TRANSPORT_TCP", ESP_LOG_VERBOSE);

examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py

@@ -27,6 +27,7 @@ import DUT
 event_client_connected = Event()
 event_stop_client = Event()
 event_client_received_correct = Event()
+event_client_received_binary = Event()
 message_log = ""
 
 
@@ -45,9 +46,27 @@ def mqtt_client_task(client):
 # The callback for when a PUBLISH message is received from the server.
 def on_message(client, userdata, msg):
     global message_log
+    global event_client_received_correct
+    global event_client_received_binary
+    if msg.topic == "/topic/binary":
+        binary = userdata
+        size = os.path.getsize(binary)
+        print("Receiving binary from esp and comparing with {}, size {}...".format(binary, size))
+        with open(binary, "rb") as f:
+            bin = f.read()
+            if bin == msg.payload[:size]:
+                print("...matches!")
+                event_client_received_binary.set()
+                return
+            else:
+                recv_binary = binary + ".received"
+                with open(recv_binary, "w") as fw:
+                    fw.write(msg.payload)
+                raise ValueError('Received binary (saved as: {}) does not match the original file: {}'.format(recv_binary, binary))
     payload = msg.payload.decode()
     if not event_client_received_correct.is_set() and payload == "data":
-        client.publish("/topic/qos0", "data_to_esp32")
+        client.subscribe("/topic/binary")
+        client.publish("/topic/qos0", "send binary please")
         if msg.topic == "/topic/qos0" and payload == "data":
             event_client_received_correct.set()
     message_log += "Received data:" + msg.topic + " " + payload + "\n"
@@ -63,6 +82,7 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
       2. Test connects a client to the same broker
       3. Test evaluates python client received correct qos0 message
       4. Test ESP32 client received correct qos0 message
+      5. Test python client receives binary data from running partition and compares it with the binary
     """
     dut1 = env.get_dut("mqtt_ssl", "examples/protocols/mqtt/ssl")
     # check and log bin size
@@ -85,6 +105,7 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
         client = mqtt.Client()
         client.on_connect = on_connect
         client.on_message = on_message
+        client.user_data_set(binary_file)
         client.tls_set(None,
                        None,
                        None, cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None)
@@ -112,7 +133,10 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
         if not event_client_received_correct.wait(timeout=30):
             raise ValueError('Wrong data received, msg log: {}'.format(message_log))
         print("Checking esp-client received msg published from py-client...")
-        dut1.expect(re.compile(r"DATA=data_to_esp32"), timeout=30)
+        dut1.expect(re.compile(r"DATA=send binary please"), timeout=30)
+        print("Receiving binary data from running partition...")
+        if not event_client_received_binary.wait(timeout=30):
+            raise ValueError('Binary not received within timeout')
     finally:
         event_stop_client.set()
         thread1.join()

examples/protocols/mqtt/ssl/sdkconfig.ci

@@ -1,2 +1,12 @@
 CONFIG_BROKER_URI="mqtts://${EXAMPLE_MQTT_BROKER_SSL}"
 CONFIG_BROKER_CERTIFICATE_OVERRIDE="${EXAMPLE_MQTT_BROKER_CERTIFICATE}"
+CONFIG_MQTT_USE_CUSTOM_CONFIG=y
+CONFIG_MQTT_TCP_DEFAULT_PORT=1883
+CONFIG_MQTT_SSL_DEFAULT_PORT=8883
+CONFIG_MQTT_WS_DEFAULT_PORT=80
+CONFIG_MQTT_WSS_DEFAULT_PORT=443
+CONFIG_MQTT_BUFFER_SIZE=16384
+CONFIG_MQTT_TASK_STACK_SIZE=6144
+CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y
+CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN=16384
+CONFIG_MBEDTLS_SSL_OUT_CONTENT_LEN=4096