Merge branch 'feature/build_apps_script' into 'master'

tools: add build_apps.py, script to build multiple apps

Closes IDF-641

See merge request espressif/esp-idf!6101

components/lwip/CMakeLists.txt

@@ -148,3 +148,10 @@ set_source_files_properties(
     PROPERTIES COMPILE_FLAGS
     -Wno-implicit-fallthrough
     )
+# "comparison is always false due to limited range of data type" warning
+# when setting CONFIG_LWIP_TCP_WND_DEFAULT to 65535
+set_source_files_properties(
+    lwip/src/core/tcp.c
+    PROPERTIES COMPILE_FLAGS
+    -Wno-type-limits
+)

components/lwip/component.mk

@@ -38,5 +38,6 @@ CFLAGS += -Wno-address  # lots of LWIP source files evaluate macros that check a
 
 lwip/src/netif/ppp/ppp.o: CFLAGS += -Wno-uninitialized
 lwip/src/netif/ppp/pppos.o: CFLAGS += -Wno-implicit-fallthrough
+lwip/src/core/tcp.o: CFLAGS += -Wno-type-limits
 
 COMPONENT_ADD_LDFRAGMENTS += linker.lf

examples/bluetooth/bluedroid/ble/ble_ancs/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/ble_eddystone/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/ble_hid_device_demo/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/ble_ibeacon/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/ble_spp_client/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/ble_spp_server/sdkconfig.defaults

@@ -6,8 +6,8 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 #
 # ESP32-specific config
 #

examples/bluetooth/bluedroid/ble/ble_throughput/throughput_client/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_CTRL_BLE_MAX_CONN=9
 CONFIG_GATTS_NOTIFY_THROUGHPUT=y
 CONFIG_BTDM_MODEM_SLEEP=n

examples/bluetooth/bluedroid/ble/ble_throughput/throughput_server/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_CTRL_BLE_MAX_CONN=9
 CONFIG_EXAMPLE_GATTS_NOTIFY_THROUGHPUT=y
 CONFIG_BTDM_MODEM_SLEEP=n

examples/bluetooth/bluedroid/ble/blufi/sdkconfig.defaults

@@ -6,8 +6,8 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_CTRL_PINNED_TO_CORE_0=y
 CONFIG_BTDM_CTRL_PINNED_TO_CORE_1=n
 CONFIG_BTDM_CTRL_PINNED_TO_CORE=0

examples/bluetooth/bluedroid/ble/gatt_client/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/gatt_security_client/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/gatt_security_server/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/gatt_server/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/ble/gatt_server_service_table/sdkconfig.defaults

@@ -6,8 +6,8 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 
 #
 # ESP32-specific config

examples/bluetooth/bluedroid/ble/gattc_multi_connect/sdkconfig.defaults

@@ -2,6 +2,6 @@
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_CTRL_BLE_MAX_CONN=9

examples/bluetooth/bluedroid/classic_bt/a2dp_sink/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled and
 # Classic BT is enabled and BT_DRAM_RELEASE is disabled
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_BLUEDROID_ENABLED=y
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_BT_A2DP_ENABLE=y

examples/bluetooth/bluedroid/classic_bt/a2dp_source/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled and
 # Classic BT is enabled
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_BLUEDROID_ENABLED=y
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_BT_A2DP_ENABLE=y

examples/bluetooth/bluedroid/classic_bt/bt_discovery/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled and
 # Classic BT is enabled and BT_DRAM_RELEASE is disabled
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_BT_A2DP_ENABLE=n
 CONFIG_BT_BLE_ENABLED=n

examples/bluetooth/bluedroid/classic_bt/bt_spp_acceptor/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_WIFI_ENABLED=n
 CONFIG_BT_SPP_ENABLED=y

examples/bluetooth/bluedroid/classic_bt/bt_spp_initiator/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_WIFI_ENABLED=n
 CONFIG_BT_SPP_ENABLED=y

examples/bluetooth/bluedroid/classic_bt/bt_spp_vfs_acceptor/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_WIFI_ENABLED=n
 CONFIG_BT_SPP_ENABLED=y

examples/bluetooth/bluedroid/classic_bt/bt_spp_vfs_initiator/sdkconfig.defaults

@@ -1,9 +1,9 @@
 # Override some defaults so BT stack is enabled
 # and WiFi disabled by default in this example
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_CLASSIC_ENABLED=y
 CONFIG_WIFI_ENABLED=n
 CONFIG_BT_SPP_ENABLED=y

examples/bluetooth/bluedroid/coex/a2dp_gatts_coex/sdkconfig.defaults

@@ -1,8 +1,8 @@
 # Override some defaults so BT stack is enabled and
 # Classic BT is enabled and BT_DRAM_RELEASE is disabled
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BTDM=y
 CONFIG_BTDM_CTRL_PINNED_TO_CORE_0=y
 CONFIG_BTDM_CTRL_PINNED_TO_CORE_1=n

examples/bluetooth/bluedroid/coex/gattc_gatts_coex/sdkconfig.defaults

@@ -2,5 +2,5 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/bluedroid/hci/controller_hci_uart/sdkconfig.defaults

@@ -5,8 +5,8 @@
 # BT config
 #
 CONFIG_BT_ENABLED=y
-CONFIG_BTDM_CTRL_MODE_BLE_ONLY=
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
+CONFIG_BTDM_CTRL_MODE_BLE_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
 CONFIG_BTDM_CTRL_MODE_BTDM=y
 CONFIG_BTDM_CTRL_BLE_MAX_CONN=9
 CONFIG_BTDM_CTRL_BR_EDR_MAX_ACL_CONN=7

examples/bluetooth/bluedroid/hci/controller_vhci_ble_adv/sdkconfig.defaults

@@ -6,5 +6,5 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n

examples/bluetooth/esp_ble_mesh/ble_mesh_console/ble_mesh_node/sdkconfig.defaults

@@ -5,8 +5,8 @@ CONFIG_ESPTOOLPY_BAUD_921600B=y
 CONFIG_ESPTOOLPY_MONITOR_BAUD_921600B=y
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -42,7 +42,7 @@ CONFIG_BLE_MESH_RX_SEG_MSG_COUNT=1
 CONFIG_BLE_MESH_RX_SDU_MAX=384
 CONFIG_BLE_MESH_TX_SEG_MAX=32
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BLE_MESH_CFG_CLI=y
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512

examples/bluetooth/esp_ble_mesh/ble_mesh_console/ble_mesh_provisioner/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -29,8 +29,8 @@ CONFIG_BLE_MESH_NET_BUF_POOL_USAGE=y
 CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=y
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BLE_MESH_MSG_CACHE_SIZE=10
 CONFIG_BLE_MESH_ADV_BUF_COUNT=60
 CONFIG_BLE_MESH_TX_SEG_MSG_COUNT=6

examples/bluetooth/esp_ble_mesh/ble_mesh_fast_provision/ble_mesh_fast_prov_client/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -30,8 +30,8 @@ CONFIG_BLE_MESH_NET_BUF_POOL_USAGE=y
 CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=n
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512
 CONFIG_BLE_MESH_CFG_CLI=y
 CONFIG_BLE_MESH_GENERIC_ONOFF_CLI=y

examples/bluetooth/esp_ble_mesh/ble_mesh_fast_provision/ble_mesh_fast_prov_server/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -38,8 +38,8 @@ CONFIG_BLE_MESH_MODEL_GROUP_COUNT=3
 CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=y
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512
 CONFIG_BLE_MESH_CFG_CLI=y
 CONFIG_BLE_MESH_CRPL=60

examples/bluetooth/esp_ble_mesh/ble_mesh_node/onoff_client/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -26,7 +26,7 @@ CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=y
 CONFIG_BLE_MESH_NODE_ID_TIMEOUT=60
 CONFIG_BLE_MESH_PROXY_FILTER_SIZE=1
-CONFIG_BLE_MESH_IV_UPDATE_TEST=
+CONFIG_BLE_MESH_IV_UPDATE_TEST=n
 CONFIG_BLE_MESH_SUBNET_COUNT=1
 CONFIG_BLE_MESH_APP_KEY_COUNT=1
 CONFIG_BLE_MESH_MODEL_KEY_COUNT=1
@@ -40,8 +40,8 @@ CONFIG_BLE_MESH_RX_SEG_MSG_COUNT=1
 CONFIG_BLE_MESH_RX_SDU_MAX=384
 CONFIG_BLE_MESH_TX_SEG_MAX=32
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
-CONFIG_BLE_MESH_CFG_CLI=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
+CONFIG_BLE_MESH_CFG_CLI=n
 CONFIG_BLE_MESH_GENERIC_ONOFF_CLI=y
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512

examples/bluetooth/esp_ble_mesh/ble_mesh_node/onoff_server/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -39,7 +39,7 @@ CONFIG_BLE_MESH_RX_SEG_MSG_COUNT=1
 CONFIG_BLE_MESH_RX_SDU_MAX=384
 CONFIG_BLE_MESH_TX_SEG_MAX=32
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BLE_MESH_CFG_CLI=y
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512

examples/bluetooth/esp_ble_mesh/ble_mesh_provisioner/sdkconfig.defaults

@@ -2,8 +2,8 @@
 # by default in this example
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -30,8 +30,8 @@ CONFIG_BLE_MESH_NET_BUF_POOL_USAGE=y
 CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=n
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BLE_MESH_ADV_BUF_COUNT=60
 CONFIG_BLE_MESH_TX_SEG_MSG_COUNT=6
 CONFIG_BLE_MESH_RX_SEG_MSG_COUNT=6

examples/bluetooth/esp_ble_mesh/ble_mesh_vendor_models/fast_prov_vendor_model/sdkconfig.defaults

@@ -18,8 +18,8 @@ CONFIG_BLE_MESH_NET_BUF_POOL_USAGE=y
 CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=y
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512
 CONFIG_BLE_MESH_CFG_CLI=y
 CONFIG_BLE_MESH_CRPL=60

examples/bluetooth/esp_ble_mesh/ble_mesh_wifi_coexist/sdkconfig.defaults

@@ -6,8 +6,8 @@ CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ=240
 CONFIG_MEMMAP_SMP=y
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BTDM_MODEM_SLEEP=n
 CONFIG_BTDM_BLE_SCAN_DUPL=y
 CONFIG_BTDM_SCAN_DUPL_TYPE=2
@@ -38,8 +38,8 @@ CONFIG_BLE_MESH_NET_BUF_POOL_USAGE=y
 CONFIG_BLE_MESH_PB_GATT=y
 CONFIG_BLE_MESH_GATT_PROXY_SERVER=y
 CONFIG_BLE_MESH_RELAY=y
-CONFIG_BLE_MESH_LOW_POWER=
-CONFIG_BLE_MESH_FRIEND=
+CONFIG_BLE_MESH_LOW_POWER=n
+CONFIG_BLE_MESH_FRIEND=n
 CONFIG_BT_BTU_TASK_STACK_SIZE=4512
 CONFIG_BLE_MESH_CFG_CLI=y
 CONFIG_BLE_MESH_GENERIC_ONOFF_CLI=y
@@ -63,18 +63,18 @@ CONFIG_ESP32_WIFI_TX_BA_WIN=32
 CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
 CONFIG_ESP32_WIFI_RX_BA_WIN=32
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_LWIP_TCP_SND_BUF_DEFAULT=65534
 CONFIG_LWIP_TCP_WND_DEFAULT=65534
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 
 CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
 CONFIG_ESPTOOLPY_FLASHFREQ_40M=y

examples/bluetooth/nimble/blecent/sdkconfig.defaults

@@ -6,7 +6,7 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
-CONFIG_BT_BLUEDROID_ENABLED=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
+CONFIG_BT_BLUEDROID_ENABLED=n
 CONFIG_BT_NIMBLE_ENABLED=y

examples/bluetooth/nimble/blehr/sdkconfig.defaults

@@ -6,7 +6,7 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
-CONFIG_BT_BLUEDROID_ENABLED=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
+CONFIG_BT_BLUEDROID_ENABLED=n
 CONFIG_BT_NIMBLE_ENABLED=y

examples/bluetooth/nimble/blemesh/sdkconfig.defaults

@@ -6,8 +6,8 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
-CONFIG_BT_BLUEDROID_ENABLED=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
+CONFIG_BT_BLUEDROID_ENABLED=n
 CONFIG_BT_NIMBLE_ENABLED=y
 CONFIG_BT_NIMBLE_MESH=y

examples/bluetooth/nimble/bleprph/sdkconfig.defaults

@@ -6,7 +6,7 @@
 #
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
-CONFIG_BT_BLUEDROID_ENABLED=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
+CONFIG_BT_BLUEDROID_ENABLED=n
 CONFIG_BT_NIMBLE_ENABLED=y

examples/get-started/hello_world/loadable_elf_example_test.py

@@ -120,7 +120,7 @@ def test_examples_loadable_elf(env, extra_data):
     idf_path = os.environ['IDF_PATH']
     rel_project_path = os.path.join('examples', 'get-started', 'hello_world')
     proj_path = os.path.join(idf_path, rel_project_path)
-    example = IDF.Example(rel_project_path)
+    example = IDF.Example(rel_project_path, target="esp32")
     sdkconfig = example.get_sdkconfig()
     elf_path = os.path.join(example.get_binary_path(rel_project_path), 'hello-world.elf')
     esp_log_path = os.path.join(proj_path, 'esp.log')

examples/get-started/hello_world/sdkconfig.ci

@@ -1,6 +1,6 @@
 CONFIG_APP_BUILD_TYPE_ELF_RAM=y
-CONFIG_VFS_SUPPORT_TERMIOS=
+CONFIG_VFS_SUPPORT_TERMIOS=n
 CONFIG_NEWLIB_NANO_FORMAT=y
 CONFIG_ESP32_PANIC_PRINT_HALT=y
-CONFIG_ESP32_DEBUG_STUBS_ENABLE=
-CONFIG_ESP_ERR_TO_NAME_LOOKUP=
+CONFIG_ESP32_DEBUG_STUBS_ENABLE=n
+CONFIG_ESP_ERR_TO_NAME_LOOKUP=n

examples/provisioning/ble_prov/sdkconfig.defaults

@@ -1,8 +1,8 @@
 # Override some defaults so BT stack is enabled and
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 
 # Binary is larger than default size
 CONFIG_PARTITION_TABLE_CUSTOM=y

examples/provisioning/manager/sdkconfig.defaults

@@ -1,8 +1,8 @@
 # Override some defaults so BT stack is enabled and
 CONFIG_BT_ENABLED=y
 CONFIG_BTDM_CTRL_MODE_BLE_ONLY=y
-CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CTRL_MODE_BTDM=
+CONFIG_BTDM_CTRL_MODE_BR_EDR_ONLY=n
+CONFIG_BTDM_CTRL_MODE_BTDM=n
 CONFIG_BT_NIMBLE_ENABLED=y
 
 ## For Bluedroid as binary is larger than default size

examples/system/console/example_test.py

@@ -0,0 +1,29 @@
+from __future__ import print_function
+import os
+import sys
+
+try:
+    import IDF
+except ImportError:
+    test_fw_path = os.getenv('TEST_FW_PATH')
+    if test_fw_path and test_fw_path not in sys.path:
+        sys.path.insert(0, test_fw_path)
+    import IDF
+
+
+@IDF.idf_example_test(env_tag='Example_WIFI')
+def test_examples_system_console(env, extra_data):
+    dut = env.get_dut('console_example', 'examples/system/console', app_config_name='history')
+    print("Using binary path: {}".format(dut.app.binary_path))
+    dut.start_app()
+    dut.expect("Command history enabled")
+    env.close_dut(dut.name)
+
+    dut = env.get_dut('console_example', 'examples/system/console', app_config_name='nohistory')
+    print("Using binary path: {}".format(dut.app.binary_path))
+    dut.start_app()
+    dut.expect("Command history disabled")
+
+
+if __name__ == '__main__':
+    test_examples_system_console()

examples/system/console/main/console_example_main.c

@@ -59,6 +59,10 @@ static void initialize_nvs(void)
 
 static void initialize_console(void)
 {
+    /* Drain stdout before reconfiguring it */
+    fflush(stdout);
+    fsync(fileno(stdout));
+
     /* Disable buffering on stdin */
     setvbuf(stdin, NULL, _IONBF, 0);
 
@@ -121,6 +125,9 @@ void app_main(void)
 
 #if CONFIG_STORE_HISTORY
     initialize_filesystem();
+    ESP_LOGI(TAG, "Command history enabled");
+#else
+    ESP_LOGI(TAG, "Command history disabled");
 #endif
 
     initialize_console();

examples/system/console/sdkconfig.ci.history

@@ -0,0 +1 @@
+CONFIG_STORE_HISTORY=y

examples/system/console/sdkconfig.ci.nohistory

@@ -0,0 +1 @@
+CONFIG_STORE_HISTORY=n

examples/system/cpp_rtti/example_test.py

@@ -16,7 +16,7 @@ except ImportError:
 
 
 @IDF.idf_example_test(env_tag='Example_WIFI')
-def test_real_time_stats_example(env, extra_data):
+def test_cpp_rtti_example(env, extra_data):
     dut = env.get_dut('cpp_rtti', 'examples/system/cpp_rtti')
     dut.start_app()
 
@@ -34,4 +34,4 @@ def test_real_time_stats_example(env, extra_data):
 
 
 if __name__ == '__main__':
-    test_real_time_stats_example()
+    test_cpp_rtti_example()

examples/system/himem/sdkconfig.defaults

@@ -1,6 +1,6 @@
 CONFIG_ESP32_SPIRAM_SUPPORT=y
 CONFIG_SPIRAM_BOOT_INIT=y
-CONFIG_SPIRAM_IGNORE_NOTFOUND=
+CONFIG_SPIRAM_IGNORE_NOTFOUND=n
 CONFIG_SPIRAM_USE_MALLOC=y
 CONFIG_SPIRAM_TYPE_AUTO=y
 CONFIG_SPIRAM_SIZE=-1

examples/system/sysview_tracing_heap_log/sdkconfig.defaults

@@ -23,6 +23,6 @@ CONFIG_SYSVIEW_EVT_IDLE_ENABLE=y
 CONFIG_SYSVIEW_EVT_TIMER_ENTER_ENABLE=y
 CONFIG_SYSVIEW_EVT_TIMER_EXIT_ENABLE=y
 # Disable color output in logs
-CONFIG_LOG_COLORS=
+CONFIG_LOG_COLORS=n
 # Enable heap tracing to host
 CONFIG_HEAP_TRACING_TOHOST=y

examples/system/unit_test/test/sdkconfig.defaults

@@ -1 +1 @@
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_TASK_WDT=n

examples/wifi/iperf/iperf_test.py

@@ -467,31 +467,6 @@ class IperfTestUtility(object):
         return ret
 
 
-def build_iperf_with_config(config_name):
-    """
-    we need to build iperf example with different configurations.
-
-    :param config_name: sdkconfig we want to build
-    """
-
-    # switch to iperf example path before build when we're running test with Runner
-    example_path = os.path.dirname(__file__)
-    cwd = os.getcwd()
-    if cwd != example_path and example_path:
-        os.chdir(example_path)
-    try:
-        subprocess.check_call("make clean > /dev/null", shell=True)
-        subprocess.check_call(["cp", "sdkconfig.defaults.{}".format(config_name), "sdkconfig.defaults"])
-        subprocess.check_call(["rm", "-f", "sdkconfig"])
-        subprocess.check_call("make defconfig > /dev/null", shell=True)
-        # save sdkconfig to generate config comparision report
-        subprocess.check_call(["cp", "sdkconfig", "sdkconfig.{}".format(config_name)])
-        subprocess.check_call("make -j5 > /dev/null", shell=True)
-        subprocess.check_call("make print_flash_cmd | tail -n 1 > build/download.config", shell=True)
-    finally:
-        os.chdir(cwd)
-
-
 @IDF.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress")
 def test_wifi_throughput_with_different_configs(env, extra_data):
     """
@@ -512,13 +487,12 @@ def test_wifi_throughput_with_different_configs(env, extra_data):
     sdkconfig_files = dict()
 
     for config_name in CONFIG_NAME_PATTERN.findall(config_names_raw):
-        # 1. build config
-        build_iperf_with_config(config_name)
+        # 1. get the config
         sdkconfig_files[config_name] = os.path.join(os.path.dirname(__file__),
-                                                    "sdkconfig.{}".format(config_name))
+                                                    "sdkconfig.ci.{}".format(config_name))
 
         # 2. get DUT and download
-        dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT)
+        dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=config_name)
         dut.start_app()
         dut.expect("esp32>")
 
@@ -572,15 +546,12 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
         "udp_rx": TestResult("udp", "rx", BEST_PERFORMANCE_CONFIG),
     }
 
-    # 1. build config
-    build_iperf_with_config(BEST_PERFORMANCE_CONFIG)
-
-    # 2. get DUT and download
-    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT)
+    # 1. get DUT and download
+    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=BEST_PERFORMANCE_CONFIG)
     dut.start_app()
     dut.expect("esp32>")
 
-    # 3. run test for each required att value
+    # 2. run test for each required att value
     for ap_info in ap_list:
         test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"], ap_info["password"],
                                         pc_nic_ip, pc_iperf_log_file, test_result)
@@ -598,10 +569,10 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
             assert Attenuator.set_att(att_port, atten_val) is True
             test_utility.run_all_cases(atten_val)
 
-    # 4. check test results
+    # 3. check test results
     env.close_dut("iperf")
 
-    # 5. generate report
+    # 4. generate report
     report = ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"),
                                     test_result)
     report.generate_report()
@@ -621,15 +592,12 @@ def test_wifi_throughput_basic(env, extra_data):
         "password": env.get_variable("ap_password"),
     }
 
-    # 1. build iperf with best config
-    build_iperf_with_config(BEST_PERFORMANCE_CONFIG)
-
-    # 2. get DUT
-    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT)
+    # 1. get DUT
+    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=BEST_PERFORMANCE_CONFIG)
     dut.start_app()
     dut.expect("esp32>")
 
-    # 3. preparing
+    # 2. preparing
     test_result = {
         "tcp_tx": TestResult("tcp", "tx", BEST_PERFORMANCE_CONFIG),
         "tcp_rx": TestResult("tcp", "rx", BEST_PERFORMANCE_CONFIG),
@@ -640,11 +608,11 @@ def test_wifi_throughput_basic(env, extra_data):
     test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"],
                                     ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result)
 
-    # 4. run test for TCP Tx, Rx and UDP Tx, Rx
+    # 3. run test for TCP Tx, Rx and UDP Tx, Rx
     for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE):
         test_utility.run_all_cases(0)
 
-    # 5. log performance and compare with pass standard
+    # 4. log performance and compare with pass standard
     performance_items = []
     for throughput_type in test_result:
         IDF.log_performance("{}_throughput".format(throughput_type),
@@ -652,7 +620,7 @@ def test_wifi_throughput_basic(env, extra_data):
         performance_items.append(["{}_throughput".format(throughput_type),
                                   "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput())])
 
-    # save to report
+    # 5. save to report
     TinyFW.JunitReport.update_performance(performance_items)
     # do check after logging, otherwise test will exit immediately if check fail, some performance can't be logged.
     for throughput_type in test_result:

examples/wifi/iperf/sdkconfig.defaults.00 → examples/wifi/iperf/sdkconfig.ci.00

@@ -2,5 +2,5 @@ CONFIG_MEMMAP_SMP=y
 
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n

examples/wifi/iperf/sdkconfig.defaults.01 → examples/wifi/iperf/sdkconfig.ci.01

@@ -1,11 +1,11 @@
 CONFIG_MEMMAP_SMP=y
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
 CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=12
@@ -21,5 +21,5 @@ CONFIG_LWIP_TCP_WND_DEFAULT=11488
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=12
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=12
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=48
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 

examples/wifi/iperf/sdkconfig.defaults.02 → examples/wifi/iperf/sdkconfig.ci.02

@@ -1,11 +1,11 @@
 CONFIG_MEMMAP_SMP=y
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
 CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
@@ -21,5 +21,5 @@ CONFIG_LWIP_TCP_WND_DEFAULT=11488
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=12
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=12
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=48
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 

examples/wifi/iperf/sdkconfig.defaults.03 → examples/wifi/iperf/sdkconfig.ci.03

@@ -1,11 +1,11 @@
 CONFIG_MEMMAP_SMP=y
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
 CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
@@ -21,5 +21,5 @@ CONFIG_LWIP_TCP_WND_DEFAULT=32768
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 

examples/wifi/iperf/sdkconfig.defaults.04 → examples/wifi/iperf/sdkconfig.ci.04

@@ -1,11 +1,11 @@
 CONFIG_MEMMAP_SMP=y
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
 CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
@@ -21,5 +21,5 @@ CONFIG_LWIP_TCP_WND_DEFAULT=65535
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 

examples/wifi/iperf/sdkconfig.defaults.05 → examples/wifi/iperf/sdkconfig.ci.05

@@ -1,11 +1,11 @@
 CONFIG_MEMMAP_SMP=y
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
 CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=16
@@ -21,7 +21,7 @@ CONFIG_LWIP_TCP_WND_DEFAULT=65534
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 
 CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y

examples/wifi/iperf/sdkconfig.defaults.06 → examples/wifi/iperf/sdkconfig.ci.06

@@ -11,15 +11,15 @@ CONFIG_ESP32_WIFI_RX_BA_WIN=32
 CONFIG_FREERTOS_UNICORE=y
 CONFIG_FREERTOS_HZ=1000
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_LWIP_TCP_SND_BUF_DEFAULT=65534
 CONFIG_LWIP_TCP_WND_DEFAULT=65534
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 
 CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y

examples/wifi/iperf/sdkconfig.defaults.07 → examples/wifi/iperf/sdkconfig.ci.07

@@ -12,18 +12,18 @@ CONFIG_ESP32_WIFI_TX_BA_WIN=32
 CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
 CONFIG_ESP32_WIFI_RX_BA_WIN=32
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_LWIP_TCP_SND_BUF_DEFAULT=65534
 CONFIG_LWIP_TCP_WND_DEFAULT=65534
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 
 CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y

examples/wifi/iperf/sdkconfig.defaults.99 → examples/wifi/iperf/sdkconfig.ci.99

@@ -12,18 +12,18 @@ CONFIG_ESP32_WIFI_TX_BA_WIN=32
 CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
 CONFIG_ESP32_WIFI_RX_BA_WIN=32
 
-CONFIG_FREERTOS_UNICORE=
+CONFIG_FREERTOS_UNICORE=n
 CONFIG_FREERTOS_HZ=1000
 
-CONFIG_ESP_INT_WDT=
-CONFIG_ESP_TASK_WDT=
+CONFIG_ESP_INT_WDT=n
+CONFIG_ESP_TASK_WDT=n
 
 CONFIG_LWIP_TCP_SND_BUF_DEFAULT=65534
 CONFIG_LWIP_TCP_WND_DEFAULT=65534
 CONFIG_LWIP_TCP_RECVMBOX_SIZE=64
 CONFIG_LWIP_UDP_RECVMBOX_SIZE=64
 CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=64
-CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
+CONFIG_LWIP_ETHARP_TRUST_IP_MAC=n
 
 CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
 CONFIG_ESPTOOLPY_FLASHFREQ_80M=y

tools/build_apps.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
+#
+
+import argparse
+import sys
+import logging
+from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS
+
+
+def main():
+    parser = argparse.ArgumentParser(description="ESP-IDF app builder")
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        help="Increase the logging level of the script. Can be specified multiple times.",
+    )
+    parser.add_argument(
+        "--build-verbose",
+        action="store_true",
+        help="Enable verbose output from build system.",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=argparse.FileType("w"),
+        help="Write the script log to the specified file, instead of stderr",
+    )
+    parser.add_argument(
+        "--parallel-count",
+        default=1,
+        type=int,
+        help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
+        "it needs to be executed multiple times with same value of --parallel-count and " +
+        "different values of --parallel-index.",
+    )
+    parser.add_argument(
+        "--parallel-index",
+        default=1,
+        type=int,
+        help="Index (1-based) of the job, out of the number specified by --parallel-count.",
+    )
+    parser.add_argument(
+        "--format",
+        default="json",
+        choices=["json"],
+        help="Format to read the list of builds",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Don't actually build, only print the build commands",
+    )
+    parser.add_argument(
+        "--keep-going",
+        action="store_true",
+        help="Don't exit immediately when a build fails.",
+    )
+    parser.add_argument(
+        "--output-build-list",
+        type=argparse.FileType("w"),
+        help="If specified, the list of builds (with all the placeholders expanded) will be written to this file.",
+    )
+    parser.add_argument(
+        "build_list",
+        type=argparse.FileType("r"),
+        nargs="?",
+        default=sys.stdin,
+        help="Name of the file to read the list of builds from. If not specified, read from stdin.",
+    )
+    args = parser.parse_args()
+
+    setup_logging(args)
+
+    build_items = [BuildItem.from_json(line) for line in args.build_list]
+
+    if not build_items:
+        logging.error("Empty build list!")
+        raise SystemExit(1)
+
+    num_builds = len(build_items)
+    num_jobs = args.parallel_count
+    job_index = args.parallel_index - 1  # convert to 0-based index
+    num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
+    min_job_index = num_builds_per_job * job_index
+    if min_job_index >= num_builds:
+        logging.warn("Nothing to do for job {} (build total: {}, per job: {})".format(
+            job_index + 1, num_builds, num_builds_per_job))
+        raise SystemExit(0)
+
+    max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
+    logging.info("Total {} builds, max. {} builds per job, running builds {}-{}".format(
+        num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))
+
+    builds_for_current_job = build_items[min_job_index:max_job_index + 1]
+    for i, build_info in enumerate(builds_for_current_job):
+        index = i + min_job_index + 1
+        build_info.index = index
+        build_info.dry_run = args.dry_run
+        build_info.verbose = args.build_verbose
+        build_info.keep_going = args.keep_going
+        logging.debug("    Build {}: {}".format(index, repr(build_info)))
+        if args.output_build_list:
+            args.output_build_list.write(build_info.to_json_expanded() + "\n")
+
+    failed_builds = []
+    for build_info in builds_for_current_job:
+        logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
+        build_system_class = BUILD_SYSTEMS[build_info.build_system]
+        try:
+            build_system_class.build(build_info)
+        except BuildError as e:
+            logging.error(e.message)
+            if args.keep_going:
+                failed_builds.append(build_info)
+            else:
+                raise SystemExit(1)
+
+    if failed_builds:
+        logging.error("The following build have failed:")
+        for build in failed_builds:
+            logging.error("    {}".format(build))
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/ci/build_examples_cmake.sh

@@ -1,24 +1,10 @@
 #!/bin/bash
 #
-# Build all examples from the examples directory, out of tree to
+# Build all examples from the examples directory, in BUILD_PATH to
 # ensure they can run when copied to a new directory.
 #
 # Runs as part of CI process.
 #
-# Assumes PWD is an out-of-tree build directory, and will copy examples
-# to individual subdirectories, one by one.
-#
-#
-# Without arguments it just builds all examples
-#
-# With one argument <JOB_NAME> it builds part of the examples. This is a useful for
-#   parallel execution in CI.
-#   <JOB_NAME> must look like this:
-#               <some_text_label>_<num>
-#   It scans .gitlab-ci.yaml to count number of jobs which have name "<some_text_label>_<num>"
-#   It scans the filesystem to count all examples
-#   Based on this, it decides to run qa set of examples.
-#
 
 # -----------------------------------------------------------------------------
 # Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
@@ -31,9 +17,6 @@ fi
 set -o errexit # Exit if command failed.
 set -o pipefail # Exit if pipe failed.
 
-# Remove the initial space and instead use '\n'.
-IFS=$'\n\t'
-
 export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
 
 # -----------------------------------------------------------------------------
@@ -45,151 +28,77 @@ die() {
 
 [ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
 [ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
+[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
 [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
 
-set -o nounset # Exit if variable not set.
-
-echo "build_examples running in ${PWD} for target $IDF_TARGET"
-
-# only 0 or 1 arguments
-[ $# -le 1 ] || die "Have to run as $(basename $0) [<START_EXAMPLE_NUMBER>]"
-
-export BATCH_BUILD=1
-export V=0 # only build verbose if there's an error
-
-shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024)
-
-RESULT=0
-FAILED_EXAMPLES=""
-RESULT_ISSUES=22  # magic number result code for issues found
-LOG_SUSPECTED=${LOG_PATH}/common_log.txt
-touch ${LOG_SUSPECTED}
-SDKCONFIG_DEFAULTS_CI=sdkconfig.ci
-
-EXAMPLE_PATHS=$( get_supported_examples.sh $IDF_TARGET | sed "s#^#${IDF_PATH}\/examples\/#g" | awk '{print $0"/CmakeLists.txt"}' )
-NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l )
-# just a plausibility check
-[ ${NUM_OF_EXAMPLES} -lt 50 ] && die "NUM_OF_EXAMPLES is bad"
-
-echo "All examples found for target $IDF_TARGET:"
-echo $EXAMPLE_PATHS
-echo "Number of examples: $NUM_OF_EXAMPLES"
-
-if [ -z "${CI_NODE_TOTAL:-}" ]
-then
-    START_NUM=0
-    if [ "${1:-}" ]; then
-        START_NUM=$1
-    fi
-    END_NUM=${NUM_OF_EXAMPLES}
-else
-    JOB_NUM=${CI_NODE_INDEX}
-    # count number of the jobs
-    NUM_OF_JOBS=${CI_NODE_TOTAL}
-
-    # separate intervals
-    #57 / 5 == 12
-    NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} ))
-    [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad"
-
-    # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60)
-    START_NUM=$(( (${JOB_NUM} - 1) * ${NUM_OF_EX_PER_JOB} ))
-    [ -z ${START_NUM} ] && die "START_NUM is bad"
-
-    END_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} ))
-    [ -z ${END_NUM} ] && die "END_NUM is bad"
+if [ -z ${CI_NODE_TOTAL} ]; then
+    CI_NODE_TOTAL=1
+    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
+fi
+if [ -z ${CI_NODE_INDEX} ]; then
+    # Gitlab uses a 1-based index
+    CI_NODE_INDEX=1
+    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
 fi
 
-build_example () {
-    local ID=$1
-    shift
-    local CMAKELISTS=$1
-    shift
-
-    local EXAMPLE_DIR=$(dirname "${CMAKELISTS}")
-    local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}")
-
-    echo "Building ${EXAMPLE_NAME} for ${IDF_TARGET} as ${ID}..."
-    mkdir -p "example_builds/${IDF_TARGET}/${ID}"
-    cp -r "${EXAMPLE_DIR}" "example_builds/${IDF_TARGET}/${ID}"
-    pushd "example_builds/${IDF_TARGET}/${ID}/${EXAMPLE_NAME}"
-        # be stricter in the CI build than the default IDF settings
-        export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-        export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-
-        # sdkconfig files are normally not checked into git, but may be present when
-        # a developer runs this script locally
-        rm -f sdkconfig
-
-        # If sdkconfig.ci file is present, append it to sdkconfig.defaults,
-        # replacing environment variables
-        if [[ -f "$SDKCONFIG_DEFAULTS_CI" ]]; then
-            cat $SDKCONFIG_DEFAULTS_CI | $IDF_PATH/tools/ci/envsubst.py >> sdkconfig.defaults
-        fi
-
-        # build non-verbose first
-        local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
-        touch ${BUILDLOG}
-
-        if [ "$EXAMPLE_NAME" != "idf_as_lib" ]; then
-            idf.py fullclean >>${BUILDLOG} 2>&1 &&
-            idf.py build >>${BUILDLOG} 2>&1
-        else
-            rm -rf build &&
-            ./build-esp32.sh >>${BUILDLOG} 2>&1
-        fi ||
-        {
-            RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
-        }
-
-        cat ${BUILDLOG}
-    popd
-
-    grep -i "error\|warning" "${BUILDLOG}" 2>&1 >> "${LOG_SUSPECTED}" || :
-}
-
-EXAMPLE_NUM=0
-
-echo "Current job will build example ${START_NUM} - ${END_NUM}"
-
-for EXAMPLE_PATH in ${EXAMPLE_PATHS}
-do
-    if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]]
-    then
-        EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-        continue
-    fi
-    echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH"
-
-    build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}"
-
-    EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 ))
-done
-
-# show warnings
-echo -e "\nFound issues:"
-
-#       Ignore the next messages:
-# "error.o" or "-Werror" in compiler's command line
-# "reassigning to symbol" or "changes choice state" in sdkconfig
-# 'Compiler and toochain versions is not supported' from crosstool_version_check.cmake
-IGNORE_WARNS="\
-library/error\.o\
-\|.*error.*\.c\.obj\
-\|\ -Werror\
-\|error\.d\
-\|reassigning to symbol\
-\|changes choice state\
-\|crosstool_version_check\.cmake\
-"
-
-sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \
-    && RESULT=$RESULT_ISSUES \
-    || echo -e "\tNone"
 
-[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES"
-[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!"
+set -o nounset # Exit if variable not set.
 
-echo -e "\nReturn code = $RESULT"
+export REALPATH=realpath
+if [ "$(uname -s)" = "Darwin" ]; then
+    export REALPATH=grealpath
+fi
 
-exit $RESULT
+# Convert LOG_PATH and BUILD_PATH to relative, to make the json file less verbose.
+LOG_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${LOG_PATH})
+BUILD_PATH=$(${REALPATH} --relative-to ${IDF_PATH} ${BUILD_PATH})
+
+ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
+JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
+mkdir -p "${BUILD_PATH}/example_builds"
+
+echo "build_examples running for target $IDF_TARGET"
+
+cd ${IDF_PATH}
+
+# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage
+# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
+
+# If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
+
+${IDF_PATH}/tools/find_apps.py examples \
+    -vv \
+    --format json \
+    --build-system cmake \
+    --target ${IDF_TARGET} \
+    --recursive \
+    --exclude examples/build_system/idf_as_lib \
+    --work-dir "${BUILD_PATH}/@f/@w/@t" \
+    --build-dir build \
+    --build-log "${LOG_PATH}/@f.txt" \
+    --output ${ALL_BUILD_LIST_JSON} \
+    --config 'sdkconfig.ci=default' \
+    --config 'sdkconfig.ci.*=' \
+    --config '=default' \
+
+# --config rules above explained:
+# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
+# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
+# 3. If none of the above exist, build the default configuration under the name "default"
+
+# The part below is where the actual builds happen
+
+${IDF_PATH}/tools/build_apps.py \
+    -vv \
+    --format json \
+    --keep-going \
+    --parallel-count ${CI_NODE_TOTAL} \
+    --parallel-index ${CI_NODE_INDEX} \
+    --output-build-list ${JOB_BUILD_LIST_JSON} \
+    ${ALL_BUILD_LIST_JSON}\
+
+
+# Check for build warnings
+${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}

tools/ci/check_build_warnings.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# CI script to check build logs for warnings.
+# Reads the list of builds, in the format produced by find_apps.py or build_apps.py, and finds warnings in the
+# log files for every build.
+# Exits with a non-zero exit code if any warning is found.
+
+import os
+import sys
+import argparse
+import logging
+import re
+
+try:
+    from find_build_apps import BuildItem, setup_logging
+except ImportError:
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+    from find_build_apps import BuildItem, setup_logging
+
+WARNING_REGEX = r"error|warning"
+
+IGNORE_WARNS = [
+    re.compile(r_str) for r_str in [
+        r"library/error\.o",
+        r".*error.*\.c\.obj",
+        r"-Werror",
+        r"error\.d",
+        r"reassigning to symbol",
+        r"changes choice state",
+        r"crosstool_version_check\.cmake",
+    ]
+]
+
+
+def line_has_warnings(line):  # type: (str) -> bool
+    if not re.search(WARNING_REGEX, line):
+        return False
+
+    has_warnings = True
+    for ignored in IGNORE_WARNS:
+        if re.search(ignored, line):
+            has_warnings = False
+            break
+
+    return has_warnings
+
+
+def main():
+    parser = argparse.ArgumentParser(description="ESP-IDF app builder")
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        help="Increase the logging level of the script. Can be specified multiple times.",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=argparse.FileType("w"),
+        help="Write the script log to the specified file, instead of stderr",
+    )
+    parser.add_argument(
+        "build_list",
+        type=argparse.FileType("r"),
+        nargs="?",
+        default=sys.stdin,
+        help="Name of the file to read the list of builds from. If not specified, read from stdin.",
+    )
+    args = parser.parse_args()
+    setup_logging(args)
+
+    build_items = [BuildItem.from_json(line) for line in args.build_list]
+
+    if not build_items:
+        logging.error("Empty build list!")
+        raise SystemExit(1)
+
+    found_warnings = 0
+    for build_item in build_items:
+        if not build_item.build_log_path:
+            logging.debug("No log file for {}".format(build_item.work_dir))
+            continue
+        with open(build_item.build_log_path, "r") as log_file:
+            for line_no, line in enumerate(log_file):
+                if line_has_warnings(line):
+                    logging.error("Issue in app {}, config {}:".format(build_item.app_dir, build_item.config_name))
+                    logging.error(line.rstrip("\n"))
+                    logging.error("See {}:{} for details".format(os.path.basename(build_item.build_log_path),
+                                                                 line_no + 1))
+                    found_warnings += 1
+                    break
+
+    if found_warnings:
+        logging.error("Checked {} builds, found {} warnings".format(len(build_items), found_warnings))
+        raise SystemExit(1)
+
+    logging.info("No warnings found")
+
+
+if __name__ == "__main__":
+    main()

tools/ci/config/assign-test.yml

@@ -8,7 +8,6 @@ assign_test:
   # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
   dependencies:
     - build_ssc
-    - build_esp_idf_tests_make
     - build_esp_idf_tests_cmake
   variables:
     TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"

tools/ci/config/build.yml

@@ -197,17 +197,20 @@ build_examples_make:
   artifacts:
     when: always
     paths:
-      - build_examples/*/*/*/*/build/*.bin
-      - build_examples/*/*/*/*/sdkconfig
-      - build_examples/*/*/*/*/build/*.elf
-      - build_examples/*/*/*/*/build/*.map
-      - build_examples/*/*/*/*/build/flasher_args.json
-      - build_examples/*/*/*/*/build/bootloader/*.bin
-      - build_examples/*/*/*/*/build/partition_table/*.bin
+      - build_examples/list.json
+      - build_examples/list_job_*.json
+      - build_examples/*/*/*/sdkconfig
+      - build_examples/*/*/*/build/*.bin
+      - build_examples/*/*/*/build/*.elf
+      - build_examples/*/*/*/build/*.map
+      - build_examples/*/*/*/build/flasher_args.json
+      - build_examples/*/*/*/build/bootloader/*.bin
+      - build_examples/*/*/*/build/partition_table/*.bin
       - $LOG_PATH
     expire_in: 3 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples"
+    BUILD_PATH: "$CI_PROJECT_DIR/build_examples"
   only:
     variables:
       - $BOT_TRIGGER_WITH_LABEL == null
@@ -218,10 +221,7 @@ build_examples_make:
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
     # mechanism work, but this is the next best thing
-    - rm -rf build_examples
-    - mkdir build_examples
-    - cd build_examples
-    # build some of examples
+    - mkdir -p ${BUILD_PATH}
     - mkdir -p ${LOG_PATH}
     - ${IDF_PATH}/tools/ci/build_examples_cmake.sh
     # Check if the tests demand CMake built binaries. If not, delete them

tools/ci/config/target-test.yml

@@ -21,7 +21,6 @@
       - $BOT_LABEL_EXAMPLE_TEST
   dependencies:
     - assign_test
-    - build_examples_make
     - build_examples_cmake_esp32
   artifacts:
     when: always

tools/ci/executable-list.txt

@@ -27,6 +27,7 @@ examples/system/ota/otatool/get_running_partition.py
 examples/system/ota/otatool/otatool_example.py
 examples/system/ota/otatool/otatool_example.sh
 install.sh
+tools/build_apps.py
 tools/check_kconfigs.py
 tools/check_python_dependencies.py
 tools/ci/apply_bot_filter.py
@@ -34,6 +35,7 @@ tools/ci/build_examples.sh
 tools/ci/build_examples_cmake.sh
 tools/ci/check-executable.sh
 tools/ci/check-line-endings.sh
+tools/ci/check_build_warnings.py
 tools/ci/check_deprecated_kconfigs.py
 tools/ci/check_examples_cmake_make.sh
 tools/ci/check_examples_rom_header.sh
@@ -59,6 +61,7 @@ tools/esp_app_trace/logtrace_proc.py
 tools/esp_app_trace/sysviewtrace_proc.py
 tools/esp_app_trace/test/logtrace/test.sh
 tools/esp_app_trace/test/sysview/test.sh
+tools/find_apps.py
 tools/format.sh
 tools/gen_esp_err_to_name.py
 tools/idf.py

tools/find_apps.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps.
+# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
+
+import argparse
+import os
+import sys
+import re
+import glob
+import logging
+import typing
+from find_build_apps import (
+    BUILD_SYSTEMS,
+    BUILD_SYSTEM_CMAKE,
+    BuildSystem,
+    BuildItem,
+    setup_logging,
+    ConfigRule,
+    config_rules_from_str,
+    DEFAULT_TARGET,
+)
+
+# Helper functions
+
+
+def dict_from_sdkconfig(path):
+    """
+    Parse the sdkconfig file at 'path', return name:value pairs as a dict
+    """
+    regex = re.compile(r"^([^#=]+)=(.+)$")
+    result = {}
+    with open(path) as f:
+        for line in f:
+            m = regex.match(line)
+            if m:
+                result[m.group(1)] = m.group(2)
+    return result
+
+
+# Main logic: enumerating apps and builds
+
+
+def find_builds_for_app(
+        app_path, work_dir, build_dir, build_log, target_arg, build_system,
+        config_rules):  # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem]
+    """
+    Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
+    :param app_path: app directory (can be / usually will be a relative path)
+    :param work_dir: directory where the app should be copied before building.
+                     May contain env. variables and placeholders.
+    :param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders.
+    :param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go
+                      into stdout/stderr.
+    :param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with
+                       a different CONFIG_IDF_TARGET value.
+    :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
+    :param config_rules: mapping of sdkconfig file name patterns to configuration names
+    :return: list of BuildItems representing build configuration of the app
+    """
+    build_items = []  # type: typing.List[BuildItem]
+    default_config_name = ""
+
+    for rule in config_rules:
+        if not rule.file_name:
+            default_config_name = rule.config_name
+            continue
+
+        sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name))
+        sdkconfig_paths = sorted(sdkconfig_paths)
+        for sdkconfig_path in sdkconfig_paths:
+
+            # Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument.
+            sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path)
+            target_from_config = sdkconfig_dict.get("CONFIG_IDF_TARGET")
+            if target_from_config is not None and target_from_config != target_arg:
+                logging.debug("Skipping sdkconfig {} which requires target {}".format(
+                    sdkconfig_path, target_from_config))
+                continue
+
+            # Figure out the config name
+            config_name = rule.config_name or ""
+            if "*" in rule.file_name:
+                # convert glob pattern into a regex
+                regex_str = r".*" + rule.file_name.replace(".", r"\.").replace("*", r"(.*)")
+                groups = re.match(regex_str, sdkconfig_path)
+                assert groups
+                config_name = groups.group(1)
+
+            sdkconfig_path = os.path.relpath(sdkconfig_path, app_path)
+            logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format(
+                app_path, sdkconfig_path, config_name))
+            build_items.append(
+                BuildItem(
+                    app_path,
+                    work_dir,
+                    build_dir,
+                    build_log,
+                    target_arg,
+                    sdkconfig_path,
+                    config_name,
+                    build_system,
+                ))
+
+    if not build_items:
+        logging.debug('Adding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name))
+        return [
+            BuildItem(
+                app_path,
+                work_dir,
+                build_dir,
+                build_log,
+                target_arg,
+                None,
+                default_config_name,
+                build_system,
+            )
+        ]
+
+    return build_items
+
+
+def find_apps(build_system_class, path, recursive, exclude_list,
+              target):  # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
+    """
+    Find app directories in path (possibly recursively), which contain apps for the given build system, compatible
+    with the given target.
+    :param build_system_class: class derived from BuildSystem, representing the build system in use
+    :param path: path where to look for apps
+    :param recursive: whether to recursively descend into nested directories if no app is found
+    :param exclude_list: list of paths to be excluded from the recursive search
+    :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped.
+    :return: list of paths of the apps found
+    """
+    build_system_name = build_system_class.NAME
+    logging.debug("Looking for {} apps in {}{}".format(build_system_name, path, " recursively" if recursive else ""))
+    if not recursive:
+        if exclude_list:
+            logging.warn("--exclude option is ignored when used without --recursive")
+        if not build_system_class.is_app(path):
+            logging.warn("Path {} specified without --recursive flag, but no {} app found there".format(
+                path, build_system_name))
+            return []
+        return [path]
+
+    # The remaining part is for recursive == True
+    apps_found = []  # type: typing.List[str]
+    for root, dirs, _ in os.walk(path, topdown=True):
+        logging.debug("Entering {}".format(root))
+        if root in exclude_list:
+            logging.debug("Skipping {} (excluded)".format(root))
+            del dirs[:]
+            continue
+
+        if build_system_class.is_app(root):
+            logging.debug("Found {} app in {}".format(build_system_name, root))
+            # Don't recurse into app subdirectories
+            del dirs[:]
+
+            supported_targets = build_system_class.supported_targets(root)
+            if supported_targets and target not in supported_targets:
+                logging.debug("Skipping, app only supports targets: " + ", ".join(supported_targets))
+                continue
+
+            apps_found.append(root)
+
+    return apps_found
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Tool to generate build steps for IDF apps")
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        help="Increase the logging level of the script. Can be specified multiple times.",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=argparse.FileType("w"),
+        help="Write the script log to the specified file, instead of stderr",
+    )
+    parser.add_argument(
+        "--recursive",
+        action="store_true",
+        help="Look for apps in the specified directories recursively.",
+    )
+    parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE)
+    parser.add_argument(
+        "--work-dir",
+        help="If set, the app is first copied into the specified directory, and then built." +
+        "If not set, the work directory is the directory of the app.",
+    )
+    parser.add_argument(
+        "--config",
+        action="append",
+        help="Adds configurations (sdkconfig file names) to build. This can either be " +
+        "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " +
+        "relative to the project directory, to be used. Optional NAME can be specified, " +
+        "which can be used as a name of this configuration. FILEPATTERN is the name of " +
+        "the sdkconfig file, relative to the project directory, with at most one wildcard. " +
+        "The part captured by the wildcard is used as the name of the configuration.",
+    )
+    parser.add_argument(
+        "--build-dir",
+        help="If set, specifies the build directory name. Can expand placeholders. Can be either a " +
+        "name relative to the work directory, or an absolute path.",
+    )
+    parser.add_argument(
+        "--build-log",
+        help="If specified, the build log will be written to this file. Can expand placeholders.",
+    )
+    parser.add_argument("--target", help="Build apps for given target.")
+    parser.add_argument(
+        "--format",
+        default="json",
+        choices=["json"],
+        help="Format to write the list of builds as",
+    )
+    parser.add_argument(
+        "--exclude",
+        action="append",
+        help="Ignore specified directory (if --recursive is given). Can be used multiple times.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=argparse.FileType("w"),
+        help="Output the list of builds to the specified file",
+    )
+    parser.add_argument("paths", nargs="+", help="One or more app paths.")
+    args = parser.parse_args()
+    setup_logging(args)
+
+    build_system_class = BUILD_SYSTEMS[args.build_system]
+
+    # If the build target is not set explicitly, get it from the environment or use the default one (esp32)
+    if not args.target:
+        env_target = os.environ.get("IDF_TARGET")
+        if env_target:
+            logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target))
+            args.target = env_target
+        else:
+            logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
+            args.target = DEFAULT_TARGET
+
+    # Prepare the list of app paths
+    app_paths = []  # type: typing.List[str]
+    for path in args.paths:
+        app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
+
+    if not app_paths:
+        logging.critical("No {} apps found".format(build_system_class.NAME))
+        raise SystemExit(1)
+    logging.info("Found {} apps".format(len(app_paths)))
+
+    app_paths = sorted(app_paths)
+
+    # Find compatible configurations of each app, collect them as BuildItems
+    build_items = []  # type: typing.List[BuildItem]
+    config_rules = config_rules_from_str(args.config or [])
+    for app_path in app_paths:
+        build_items += find_builds_for_app(
+            app_path,
+            args.work_dir,
+            args.build_dir,
+            args.build_log,
+            args.target,
+            args.build_system,
+            config_rules,
+        )
+    logging.info("Found {} builds".format(len(build_items)))
+
+    # Write out the BuildItems. Only JSON supported now (will add YAML later).
+    if args.format != "json":
+        raise NotImplementedError()
+    out = args.output or sys.stdout
+    out.writelines([item.to_json() + "\n" for item in build_items])
+
+
+if __name__ == "__main__":
+    main()

tools/find_build_apps/__init__.py

@@ -0,0 +1,31 @@
+from .common import (
+    BuildItem,
+    BuildSystem,
+    BuildError,
+    ConfigRule,
+    config_rules_from_str,
+    setup_logging,
+    DEFAULT_TARGET,
+)
+from .cmake import CMakeBuildSystem, BUILD_SYSTEM_CMAKE
+from .make import MakeBuildSystem, BUILD_SYSTEM_MAKE
+
+BUILD_SYSTEMS = {
+    BUILD_SYSTEM_MAKE: MakeBuildSystem,
+    BUILD_SYSTEM_CMAKE: CMakeBuildSystem,
+}
+
+__all__ = [
+    "BuildItem",
+    "BuildSystem",
+    "BuildError",
+    "ConfigRule",
+    "config_rules_from_str",
+    "setup_logging",
+    "DEFAULT_TARGET",
+    "CMakeBuildSystem",
+    "BUILD_SYSTEM_CMAKE",
+    "MakeBuildSystem",
+    "BUILD_SYSTEM_MAKE",
+    "BUILD_SYSTEMS",
+]

tools/find_build_apps/cmake.py

@@ -0,0 +1,158 @@
+import os
+import sys
+import subprocess
+import logging
+import shutil
+import re
+from .common import BuildSystem, BuildItem, BuildError
+
+BUILD_SYSTEM_CMAKE = "cmake"
+IDF_PY = "idf.py"
+
+# While ESP-IDF component CMakeLists files can be identified by the presence of 'idf_component_register' string,
+# there is no equivalent for the project CMakeLists files. This seems to be the best option...
+CMAKE_PROJECT_LINE = r"include($ENV{IDF_PATH}/tools/cmake/project.cmake)"
+
+SUPPORTED_TARGETS_REGEX = re.compile(r"set\(\s*SUPPORTED_TARGETS\s+([a-z_0-9\- ]+)\s*\)")
+
+
+class CMakeBuildSystem(BuildSystem):
+    NAME = BUILD_SYSTEM_CMAKE
+
+    @staticmethod
+    def build(build_item):  # type: (BuildItem) -> None
+        app_path = build_item.app_dir
+        work_path = build_item.work_dir or app_path
+        if not build_item.build_dir:
+            build_path = os.path.join(work_path, "build")
+        elif os.path.isabs(build_item.build_dir):
+            build_path = build_item.build_dir
+        else:
+            build_path = os.path.join(work_path, build_item.build_dir)
+
+        if work_path != app_path:
+            if os.path.exists(work_path):
+                logging.debug("Work directory {} exists, removing".format(work_path))
+                if not build_item.dry_run:
+                    shutil.rmtree(work_path)
+            logging.debug("Copying app from {} to {}".format(app_path, work_path))
+            if not build_item.dry_run:
+                shutil.copytree(app_path, work_path)
+
+        if os.path.exists(build_path):
+            logging.debug("Build directory {} exists, removing".format(build_path))
+            if not build_item.dry_run:
+                shutil.rmtree(build_path)
+
+        if not build_item.dry_run:
+            os.makedirs(build_path)
+
+        # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
+        # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
+        #
+        # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
+        # CMake variable. However here we do this manually to perform environment variable expansion in the
+        # sdkconfig files.
+        sdkconfig_defaults_list = ["sdkconfig.defaults"]
+        if build_item.sdkconfig_path:
+            sdkconfig_defaults_list.append(build_item.sdkconfig_path)
+
+        sdkconfig_file = os.path.join(work_path, "sdkconfig")
+        if os.path.exists(sdkconfig_file):
+            logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
+            if not build_item.dry_run:
+                os.unlink(sdkconfig_file)
+
+        logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
+        if not build_item.dry_run:
+            with open(sdkconfig_file, "w") as f_out:
+                for sdkconfig_name in sdkconfig_defaults_list:
+                    sdkconfig_path = os.path.join(work_path, sdkconfig_name)
+                    if not sdkconfig_path or not os.path.exists(sdkconfig_path):
+                        continue
+                    logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
+                    with open(sdkconfig_path, "r") as f_in:
+                        for line in f_in:
+                            f_out.write(os.path.expandvars(line))
+            # Also save the sdkconfig file in the build directory
+            shutil.copyfile(
+                os.path.join(work_path, "sdkconfig"),
+                os.path.join(build_path, "sdkconfig"),
+            )
+
+        else:
+            for sdkconfig_name in sdkconfig_defaults_list:
+                sdkconfig_path = os.path.join(app_path, sdkconfig_name)
+                if not sdkconfig_path:
+                    continue
+                logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
+                if not os.path.exists(sdkconfig_path):
+                    continue
+                logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
+
+        # Prepare the build arguments
+        args = [
+            # Assume it is the responsibility of the caller to
+            # set up the environment (run . ./export.sh)
+            IDF_PY,
+            "-B",
+            build_path,
+            "-C",
+            work_path,
+            "-DIDF_TARGET=" + build_item.target,
+        ]
+        if build_item.verbose:
+            args.append("-v")
+        args.append("build")
+        cmdline = format(" ".join(args))
+        logging.info("Running {}".format(cmdline))
+
+        if build_item.dry_run:
+            return
+
+        log_file = None
+        build_stdout = sys.stdout
+        build_stderr = sys.stderr
+        if build_item.build_log_path:
+            logging.info("Writing build log to {}".format(build_item.build_log_path))
+            log_file = open(build_item.build_log_path, "w")
+            build_stdout = log_file
+            build_stderr = log_file
+
+        try:
+            subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
+        except subprocess.CalledProcessError as e:
+            raise BuildError("Build failed with exit code {}".format(e.returncode))
+        finally:
+            if log_file:
+                log_file.close()
+
+    @staticmethod
+    def _read_cmakelists(app_path):
+        cmakelists_path = os.path.join(app_path, "CMakeLists.txt")
+        if not os.path.exists(cmakelists_path):
+            return None
+        with open(cmakelists_path, "r") as cmakelists_file:
+            return cmakelists_file.read()
+
+    @staticmethod
+    def is_app(path):
+        cmakelists_file_content = CMakeBuildSystem._read_cmakelists(path)
+        if not cmakelists_file_content:
+            return False
+        if CMAKE_PROJECT_LINE not in cmakelists_file_content:
+            return False
+        return True
+
+    @staticmethod
+    def supported_targets(app_path):
+        cmakelists_file_content = CMakeBuildSystem._read_cmakelists(app_path)
+        if not cmakelists_file_content:
+            return None
+        match = re.findall(SUPPORTED_TARGETS_REGEX, cmakelists_file_content)
+        if not match:
+            return None
+        if len(match) > 1:
+            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
+        targets = match[0].split(" ")
+        return targets

tools/find_build_apps/common.py

@@ -0,0 +1,231 @@
+# coding=utf-8
+
+import sys
+import os
+from collections import namedtuple
+import logging
+import json
+import typing
+
+DEFAULT_TARGET = "esp32"
+
+TARGET_PLACEHOLDER = "@t"
+WILDCARD_PLACEHOLDER = "@w"
+NAME_PLACEHOLDER = "@n"
+FULL_NAME_PLACEHOLDER = "@f"
+INDEX_PLACEHOLDER = "@i"
+
+# ConfigRule represents one --config argument of find_apps.py.
+# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
+# file_name can also be empty to indicate that the default configuration of the app should be used.
+# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used.
+# For example:
+#   filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
+#   filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
+ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"])
+
+
+def config_rules_from_str(rule_strings):  # type: (typing.List[str]) -> typing.List[ConfigRule]
+    """
+    Helper function to convert strings like 'file_name=config_name' into ConfigRule objects
+    :param rule_strings: list of rules as strings
+    :return: list of ConfigRules
+    """
+    rules = []  # type: typing.List[ConfigRule]
+    for rule_str in rule_strings:
+        items = rule_str.split("=", 2)
+        rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
+    return rules
+
+
+class BuildItem(object):
+    """
+    Instance of this class represents one build of an application.
+    The parameters which distinguish the build are passed to the constructor.
+    """
+    def __init__(
+            self,
+            app_path,
+            work_dir,
+            build_path,
+            build_log_path,
+            target,
+            sdkconfig_path,
+            config_name,
+            build_system,
+    ):
+        # These internal variables store the paths with environment variables and placeholders;
+        # Public properties with similar names use the _expand method to get the actual paths.
+        self._app_dir = app_path
+        self._work_dir = work_dir
+        self._build_dir = build_path
+        self._build_log_path = build_log_path
+
+        self.sdkconfig_path = sdkconfig_path
+        self.config_name = config_name
+        self.target = target
+        self.build_system = build_system
+
+        self._app_name = os.path.basename(os.path.normpath(app_path))
+
+        # Some miscellaneous build properties which are set later, at the build stage
+        self.index = None
+        self.verbose = False
+        self.dry_run = False
+        self.keep_going = False
+
+    @property
+    def app_dir(self):
+        """
+        :return: directory of the app
+        """
+        return self._expand(self._app_dir)
+
+    @property
+    def work_dir(self):
+        """
+        :return: directory where the app should be copied to, prior to the build. Can be None, which means that the app
+                 directory should be used.
+        """
+        return self._expand(self._work_dir)
+
+    @property
+    def build_dir(self):
+        """
+        :return: build directory, either relative to the work directory (if relative path is used) or absolute path.
+        """
+        return self._expand(self._build_dir)
+
+    @property
+    def build_log_path(self):
+        """
+        :return: path of the build log file
+        """
+        return self._expand(self._build_log_path)
+
+    def __repr__(self):
+        return "Build app {} for target {}, sdkconfig {} in {}".format(
+            self.app_dir,
+            self.target,
+            self.sdkconfig_path or "(default)",
+            self.build_dir,
+        )
+
+    def to_json(self):  # type: () -> str
+        """
+        :return: JSON string representing this object
+        """
+        return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path)
+
+    def to_json_expanded(self):  # type: () -> str
+        """
+        :return: JSON string representing this object, with all placeholders in paths expanded
+        """
+        return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path)
+
+    def _to_json(self, app_dir, work_dir, build_dir, build_log_path):  # type: (str, str, str, str) -> str
+        """
+        Internal function, called by to_json and to_json_expanded
+        """
+        return json.dumps({
+            "build_system": self.build_system,
+            "app_dir": app_dir,
+            "work_dir": work_dir,
+            "build_dir": build_dir,
+            "build_log_path": build_log_path,
+            "sdkconfig": self.sdkconfig_path,
+            "config": self.config_name,
+            "target": self.target,
+            "verbose": self.verbose,
+        })
+
+    @staticmethod
+    def from_json(json_str):  # type: (typing.Text) -> BuildItem
+        """
+        :return: Get the BuildItem from a JSON string
+        """
+        d = json.loads(str(json_str))
+        result = BuildItem(
+            app_path=d["app_dir"],
+            work_dir=d["work_dir"],
+            build_path=d["build_dir"],
+            build_log_path=d["build_log_path"],
+            sdkconfig_path=d["sdkconfig"],
+            config_name=d["config"],
+            target=d["target"],
+            build_system=d["build_system"],
+        )
+        result.verbose = d["verbose"]
+        return result
+
+    def _expand(self, path):  # type: (str) -> str
+        """
+        Internal method, expands any of the placeholders in {app,work,build} paths.
+        """
+        if not path:
+            return path
+
+        if self.index is not None:
+            path = path.replace(INDEX_PLACEHOLDER, str(self.index))
+        path = path.replace(TARGET_PLACEHOLDER, self.target)
+        path = path.replace(NAME_PLACEHOLDER, self._app_name)
+        if (FULL_NAME_PLACEHOLDER in path):  # to avoid recursion to the call to app_dir in the next line:
+            path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_"))
+        wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
+        if wildcard_pos != -1:
+            if self.config_name:
+                # if config name is defined, put it in place of the placeholder
+                path = path.replace(WILDCARD_PLACEHOLDER, self.config_name)
+            else:
+                # otherwise, remove the placeholder and one character on the left
+                # (which is usually an underscore, dash, or other delimiter)
+                left_of_wildcard = max(0, wildcard_pos - 1)
+                right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER)
+                path = path[0:left_of_wildcard] + path[right_of_wildcard:]
+        path = os.path.expandvars(path)
+        return path
+
+
+class BuildSystem(object):
+    """
+    Class representing a build system.
+    Derived classes implement the methods below.
+    Objects of these classes aren't instantiated, instead the class (type object) is used.
+    """
+
+    NAME = "undefined"
+
+    @staticmethod
+    def build(self):
+        raise NotImplementedError()
+
+    @staticmethod
+    def is_app(path):
+        raise NotImplementedError()
+
+    @staticmethod
+    def supported_targets(app_path):
+        raise NotImplementedError()
+
+
+class BuildError(RuntimeError):
+    pass
+
+
+def setup_logging(args):
+    """
+    Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument.
+    :param args: namespace obtained from argparse
+    """
+    if not args.verbose:
+        log_level = logging.WARNING
+    elif args.verbose == 1:
+        log_level = logging.INFO
+    else:
+        log_level = logging.DEBUG
+
+    logging.basicConfig(
+        format="%(levelname)s: %(message)s",
+        stream=args.log_file or sys.stderr,
+        level=log_level,
+    )

tools/find_build_apps/make.py

@@ -0,0 +1,30 @@
+import os
+from .common import BuildSystem
+
+# Same for the Makefile projects:
+MAKE_PROJECT_LINE = r"include $(IDF_PATH)/make/project.mk"
+
+BUILD_SYSTEM_MAKE = "make"
+
+
+class MakeBuildSystem(BuildSystem):
+    NAME = BUILD_SYSTEM_MAKE
+
+    @staticmethod
+    def build(build_item):
+        raise NotImplementedError()
+
+    @staticmethod
+    def is_app(path):
+        makefile_path = os.path.join(path, "Makefile")
+        if not os.path.exists(makefile_path):
+            return False
+        with open(makefile_path, "r") as makefile:
+            makefile_content = makefile.read()
+        if MAKE_PROJECT_LINE not in makefile_content:
+            return False
+        return True
+
+    @staticmethod
+    def supported_targets(app_path):
+        return ["esp32"]

tools/tiny-test-fw/App.py

@@ -38,9 +38,11 @@ class BaseApp(object):
     Also implements some common methods.
 
     :param app_path: the path for app.
+    :param config_name: app configuration to be tested
+    :param target: build target
     """
 
-    def __init__(self, app_path):
+    def __init__(self, app_path, config_name=None, target=None):
         pass
 
     @classmethod

tools/tiny-test-fw/DUT.py

@@ -275,6 +275,7 @@ class BaseDUT(object):
     DEFAULT_EXPECT_TIMEOUT = 10
     MAX_EXPECT_FAILURES_TO_SAVED = 10
     RECV_THREAD_CLS = RecvThread
+    TARGET = None
     """ DUT subclass can specify RECV_THREAD_CLS to do add some extra stuff when receive data.
     For example, DUT can implement exception detect & analysis logic in receive thread subclass. """
     LOG_THREAD = _LogThread()
@@ -377,15 +378,14 @@ class BaseDUT(object):
 
     # methods that need to be overwritten by Tool
     @classmethod
-    def confirm_dut(cls, port, app, **kwargs):
+    def confirm_dut(cls, port, **kwargs):
         """
         confirm if it's a DUT, usually used by auto detecting DUT in by Env config.
 
         subclass (tool) must overwrite this method.
 
         :param port: comport
-        :param app: app instance
-        :return: True or False
+        :return: tuple of result (bool), and target (str)
         """
         pass
 

tools/tiny-test-fw/Env.py

@@ -62,7 +62,7 @@ class Env(object):
         self.lock = threading.RLock()
 
     @_synced
-    def get_dut(self, dut_name, app_path, dut_class=None, app_class=None, **dut_init_args):
+    def get_dut(self, dut_name, app_path, dut_class=None, app_class=None, app_config_name=None, **dut_init_args):
         """
         get_dut(dut_name, app_path, dut_class=None, app_class=None)
 
@@ -70,6 +70,7 @@ class Env(object):
         :param app_path: application path, app instance will use this path to process application info
         :param dut_class: dut class, if not specified will use default dut class of env
         :param app_class: app class, if not specified will use default app of env
+        :param app_config_name: app build config
         :keyword dut_init_args: extra kwargs used when creating DUT instance
         :return: dut instance
         """
@@ -80,7 +81,7 @@ class Env(object):
                 dut_class = self.default_dut_cls
             if app_class is None:
                 app_class = self.app_cls
-            app_inst = app_class(app_path)
+            detected_target = None
             try:
                 port = self.config.get_variable(dut_name)
             except ValueError:
@@ -89,10 +90,19 @@ class Env(object):
                 available_ports = dut_class.list_available_ports()
                 for port in available_ports:
                     if port not in allocated_ports:
-                        if dut_class.confirm_dut(port, app_inst):
+                        result, detected_target = dut_class.confirm_dut(port)
+                        if result:
                             break
                 else:
                     port = None
+
+            app_target = dut_class.TARGET
+            if not app_target:
+                app_target = detected_target
+            if not app_target:
+                raise ValueError("DUT class doesn't specify the target, and autodetection failed")
+            app_inst = app_class(app_path, app_config_name, app_target)
+
             if port:
                 try:
                     dut_config = self.get_variable(dut_name + "_port_config")

tools/tiny-test-fw/IDF/IDFApp.py

@@ -29,10 +29,12 @@ class IDFApp(App.BaseApp):
     IDF_DOWNLOAD_CONFIG_FILE = "download.config"
     IDF_FLASH_ARGS_FILE = "flasher_args.json"
 
-    def __init__(self, app_path):
+    def __init__(self, app_path, config_name=None, target=None):
         super(IDFApp, self).__init__(app_path)
+        self.config_name = config_name
+        self.target = target
         self.idf_path = self.get_sdk_path()
-        self.binary_path = self.get_binary_path(app_path)
+        self.binary_path = self.get_binary_path(app_path, config_name)
         self.elf_file = self._get_elf_file_path(self.binary_path)
         assert os.path.exists(self.binary_path)
         sdkconfig_dict = self.get_sdkconfig()
@@ -87,13 +89,14 @@ class IDFApp(App.BaseApp):
                     d[configs[0]] = configs[1].rstrip()
         return d
 
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         """
         get binary path according to input app_path.
 
         subclass must overwrite this method.
 
         :param app_path: path of application
+        :param config_name: name of the application build config
         :return: abs app binary path
         """
         pass
@@ -206,59 +209,65 @@ class Example(IDFApp):
         """
         return [os.path.join(self.binary_path, "..", "sdkconfig")]
 
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         # build folder of example path
         path = os.path.join(self.idf_path, app_path, "build")
-        if not os.path.exists(path):
-            # search for CI build folders
-            app = os.path.basename(app_path)
-            example_path = os.path.join(self.idf_path, "build_examples", "example_builds")
-            # example_path has subdirectories named after targets. So we need to look into only the right
-            # subdirectory. Currently, the target is not known at this moment.
-            for dirpath, dirnames, files in os.walk(example_path):
-                if dirnames:
-                    if dirnames[0] == app:
-                        path = os.path.join(example_path, dirpath, dirnames[0], "build")
-                        break
-            else:
-                raise OSError("Failed to find example binary")
-        return path
+        if os.path.exists(path):
+            return path
+
+        if not config_name:
+            config_name = "default"
+
+        # Search for CI build folders.
+        # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
+        # (see tools/ci/build_examples_cmake.sh)
+        # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
+        app_path_underscored = app_path.replace(os.path.sep, "_")
+        example_path = os.path.join(self.idf_path, "build_examples")
+        for dirpath in os.listdir(example_path):
+            if os.path.basename(dirpath) == app_path_underscored:
+                path = os.path.join(example_path, dirpath, config_name, self.target, "build")
+                return path
+
+        raise OSError("Failed to find example binary")
 
 
 class UT(IDFApp):
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         """
-        :param app_path: app path or app config
+        :param app_path: app path
+        :param config_name: config name
         :return: binary path
         """
-        if not app_path:
-            app_path = "default"
+        if not config_name:
+            config_name = "default"
 
         path = os.path.join(self.idf_path, app_path)
-        if not os.path.exists(path):
-            while True:
-                # try to get by config
-                if app_path == "default":
-                    # it's default config, we first try to get form build folder of unit-test-app
-                    path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
-                    if os.path.exists(path):
-                        # found, use bin in build path
-                        break
-                # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
-                path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", app_path)
-                if os.path.exists(path):
-                    break
-                raise OSError("Failed to get unit-test-app binary path")
-        return path
+        default_build_path = os.path.join(path, "build")
+        if os.path.exists(default_build_path):
+            return path
+
+        # first try to get from build folder of unit-test-app
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
+        if os.path.exists(path):
+            # found, use bin in build path
+            return path
+
+        # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", config_name)
+        if os.path.exists(path):
+            return path
+
+        raise OSError("Failed to get unit-test-app binary path")
 
 
 class SSC(IDFApp):
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         # TODO: to implement SSC get binary path
         return app_path
 
 
 class AT(IDFApp):
-    def get_binary_path(self, app_path):
+    def get_binary_path(self, app_path, config_name=None):
         # TODO: to implement AT get binary path
         return app_path

tools/tiny-test-fw/IDF/IDFDUT.py

@@ -152,7 +152,6 @@ class IDFDUT(DUT.SerialDUT):
     # if need to erase NVS partition in start app
     ERASE_NVS = True
     RECV_THREAD_CLS = IDFRecvThread
-    TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
 
     def __init__(self, name, port, log_file, app, allow_dut_exception=False, **kwargs):
         super(IDFDUT, self).__init__(name, port, log_file, app, **kwargs)
@@ -174,6 +173,7 @@ class IDFDUT(DUT.SerialDUT):
         :param port: serial port as string
         :return: MAC address or None
         """
+        esp = None
         try:
             esp = cls._get_rom()(port)
             esp.connect()
@@ -181,12 +181,13 @@ class IDFDUT(DUT.SerialDUT):
         except RuntimeError:
             return None
         finally:
-            # do hard reset after use esptool
-            esp.hard_reset()
-            esp._port.close()
+            if esp:
+                # do hard reset after use esptool
+                esp.hard_reset()
+                esp._port.close()
 
     @classmethod
-    def confirm_dut(cls, port, app, **kwargs):
+    def confirm_dut(cls, port, **kwargs):
         inst = None
         try:
             expected_rom_class = cls._get_rom()
@@ -199,9 +200,9 @@ class IDFDUT(DUT.SerialDUT):
             inst = esptool.ESPLoader.detect_chip(port)
             if expected_rom_class and type(inst) != expected_rom_class:
                 raise RuntimeError("Target not expected")
-            return inst.read_mac() is not None
+            return inst.read_mac() is not None, get_target_by_rom_class(type(inst))
         except(esptool.FatalError, RuntimeError):
-            return False
+            return False, None
         finally:
             if inst is not None:
                 inst._port.close()
@@ -415,18 +416,31 @@ class IDFDUT(DUT.SerialDUT):
 
 
 class ESP32DUT(IDFDUT):
+    TARGET = "esp32"
+    TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
     @classmethod
     def _get_rom(cls):
         return esptool.ESP32ROM
 
 
 class ESP32S2DUT(IDFDUT):
+    TARGET = "esp32s2beta"
+    TOOLCHAIN_PREFIX = "xtensa-esp32s2-elf-"
     @classmethod
     def _get_rom(cls):
         return esptool.ESP32S2ROM
 
 
 class ESP8266DUT(IDFDUT):
+    TARGET = "esp8266"
+    TOOLCHAIN_PREFIX = "xtensa-lx106-elf-"
     @classmethod
     def _get_rom(cls):
         return esptool.ESP8266ROM
+
+
+def get_target_by_rom_class(cls):
+    for c in [ESP32DUT, ESP32S2DUT, ESP8266DUT]:
+        if c._get_rom() == cls:
+            return c.TARGET
+    return None

tools/tiny-test-fw/IDF/__init__.py

@@ -25,7 +25,7 @@ def format_case_id(chip, case_name):
 
 
 def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
-                     level="example", erase_nvs=True, **kwargs):
+                     level="example", erase_nvs=True, config_name=None, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
@@ -36,6 +36,7 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", e
     :param execution_time: execution time in minutes, int
     :param level: test level, could be used to filter test cases, string
     :param erase_nvs: if need to erase_nvs in DUT.start_app()
+    :param config_name: if specified, name of the app configuration
     :param kwargs: other keyword args
     :return: test method
     """

tools/unit-test-app/unit_test.py

@@ -56,6 +56,7 @@ FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
 END_LIST_STR = r'\r?\nEnter test for running'
 TEST_PATTERN = re.compile(r'\((\d+)\)\s+"([^"]+)" ([^\r\n]+)\r?\n(' + END_LIST_STR + r')?')
 TEST_SUBMENU_PATTERN = re.compile(r'\s+\((\d+)\)\s+"[^"]+"\r?\n(?=(?=\()|(' + END_LIST_STR + r'))')
+UT_APP_PATH = "tools/unit-test-app"
 
 SIMPLE_TEST_ID = 0
 MULTI_STAGE_ID = 1
@@ -284,7 +285,7 @@ def run_unit_test_cases(env, extra_data):
 
     for ut_config in case_config:
         Utility.console_log("Running unit test for config: " + ut_config, "O")
-        dut = env.get_dut("unit-test-app", app_path=ut_config, allow_dut_exception=True)
+        dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True)
         if len(case_config[ut_config]) > 0:
             replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin'))
         dut.start_app()
@@ -423,7 +424,7 @@ def get_dut(duts, env, name, ut_config, app_bin=None):
     if name in duts:
         dut = duts[name]
     else:
-        dut = env.get_dut(name, app_path=ut_config, allow_dut_exception=True)
+        dut = env.get_dut(name, app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True)
         duts[name] = dut
         replace_app_bin(dut, "unit-test-app", app_bin)
         dut.start_app()  # download bin to board
@@ -638,7 +639,7 @@ def run_multiple_stage_cases(env, extra_data):
 
     for ut_config in case_config:
         Utility.console_log("Running unit test for config: " + ut_config, "O")
-        dut = env.get_dut("unit-test-app", app_path=ut_config, allow_dut_exception=True)
+        dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config, allow_dut_exception=True)
         if len(case_config[ut_config]) > 0:
             replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin'))
         dut.start_app()
@@ -671,7 +672,7 @@ def detect_update_unit_test_info(env, extra_data, app_bin):
     case_config = format_test_case_config(extra_data)
 
     for ut_config in case_config:
-        dut = env.get_dut("unit-test-app", app_path=ut_config)
+        dut = env.get_dut("unit-test-app", app_path=UT_APP_PATH, app_config_name=ut_config)
         replace_app_bin(dut, "unit-test-app", app_bin)
         dut.start_app()