Merge branch 'feature/ci_optimize_artifact_download' into 'master'

CI: optimize submodule fetch and artifact download

See merge request espressif/esp-idf!6842

.gitlab-ci.yml

@@ -23,9 +23,16 @@ variables:
   # GIT_STRATEGY is not defined here.
   # Use an option from  "CI / CD Settings" - "General pipelines".
 
-  # "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF.
-  # If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules.
-  GIT_SUBMODULE_STRATEGY: normal
+  # we will download archive for each submodule instead of clone.
+  # we don't do "recursive" when fetch submodule as they're not used in CI now.
+  GIT_SUBMODULE_STRATEGY: none
+  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
+  # by default we will fetch all submodules
+  # jobs can overwrite this variable to only fetch submodules they required
+  # set to "none" if don't need to fetch submodules
+  SUBMODULES_TO_FETCH: "all"
+  # tell build system do not check submodule update as we download archive instead of clone
+  IDF_SKIP_CHECK_SUBMODULES: 1
 
   UNIT_TEST_BUILD_SYSTEM: cmake
   EXAMPLE_TEST_BUILD_SYSTEM: cmake
@@ -44,6 +51,7 @@ variables:
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
 
 
+
 # before each job, we need to check if this job is filtered by bot stage/job filter
 .apply_bot_filter: &apply_bot_filter
   python $APPLY_BOT_FILTER_SCRIPT || exit 0
@@ -72,12 +80,10 @@ variables:
   tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
   fi
 
-.show_submodule_urls: &show_submodule_urls |
-  git config --get-regexp '^submodule\..*\.url$' || true
+.fetch_submodules: &fetch_submodules |
+  python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH
 
 before_script:
-  - echo "Running common script"
-  - *show_submodule_urls
   - source tools/ci/setup_python.sh
   # apply bot filter in before script
   - *apply_bot_filter
@@ -93,6 +99,8 @@ before_script:
 
   - *setup_tools_unless_target_test
 
+  - *fetch_submodules
+
   - *setup_custom_toolchain
 
 # used for check scripts which we want to run unconditionally

examples/bluetooth/nimble/blecent/blecent_test.py

@@ -16,43 +16,20 @@
 
 from __future__ import print_function
 import os
-import sys
 import re
 import uuid
 import subprocess
 
-try:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError as e:
-    print(e)
-    print("\nCheck your IDF_PATH\nOR")
-    print("Try `export TEST_FW_PATH=$IDF_PATH/tools/tiny-test-fw` for resolving the issue\nOR")
-    print("Try `pip install -r $IDF_PATH/tools/tiny-test-fw/requirements.txt` for resolving the issue")
-    import IDF
-
-try:
-    import lib_ble_client
-except ImportError:
-    lib_ble_client_path = os.getenv("IDF_PATH") + "/tools/ble"
-    if lib_ble_client_path and lib_ble_client_path not in sys.path:
-        sys.path.insert(0, lib_ble_client_path)
-    import lib_ble_client
-
-
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
+from ble import lib_ble_client
 
 # When running on local machine execute the following before running this script
 # > make app bootloader
 # > make print_flash_cmd | tail -n 1 > build/download.config
-# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI_BT")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT")
 def test_example_app_ble_central(env, extra_data):
     """
         Steps:
@@ -68,12 +45,12 @@ def test_example_app_ble_central(env, extra_data):
     subprocess.check_output(['rm','-rf','/var/lib/bluetooth/*'])
     subprocess.check_output(['hciconfig','hci0','reset'])
     # Acquire DUT
-    dut = env.get_dut("blecent", "examples/bluetooth/nimble/blecent", dut_class=ESP32DUT)
+    dut = env.get_dut("blecent", "examples/bluetooth/nimble/blecent", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut.app.binary_path, "blecent.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("blecent_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.log_performance("blecent_bin_size", "{}KB".format(bin_size // 1024))
 
     # Upload binary and start testing
     Utility.console_log("Starting blecent example test app")

examples/bluetooth/nimble/blehr/blehr_test.py

@@ -16,42 +16,19 @@
 
 from __future__ import print_function
 import os
-import sys
 import re
 import threading
 import traceback
 import Queue
 import subprocess
 
-try:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError as e:
-    print(e)
-    print("\nCheck your IDF_PATH\nOR")
-    print("Try `export TEST_FW_PATH=$IDF_PATH/tools/tiny-test-fw` for resolving the issue\nOR")
-    print("Try `pip install -r $IDF_PATH/tools/tiny-test-fw/requirements.txt` for resolving the issue\n")
-    import IDF
-
-try:
-    import lib_ble_client
-except ImportError:
-    lib_ble_client_path = os.getenv("IDF_PATH") + "/tools/ble"
-    if lib_ble_client_path and lib_ble_client_path not in sys.path:
-        sys.path.insert(0, lib_ble_client_path)
-    import lib_ble_client
-
-
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
+from ble import lib_ble_client
 
 # When running on local machine execute the following before running this script
 # > make app bootloader
 # > make print_flash_cmd | tail -n 1 > build/download.config
-# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw
 
 
 def blehr_client_task(hr_obj, dut_addr):
@@ -115,7 +92,7 @@ class BleHRThread(threading.Thread):
             self.exceptions_queue.put(traceback.format_exc(), block=False)
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI_BT")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT")
 def test_example_app_ble_hr(env, extra_data):
     """
         Steps:
@@ -129,13 +106,13 @@ def test_example_app_ble_hr(env, extra_data):
     subprocess.check_output(['hciconfig','hci0','reset'])
 
     # Acquire DUT
-    dut = env.get_dut("blehr", "examples/bluetooth/nimble/blehr", dut_class=ESP32DUT)
+    dut = env.get_dut("blehr", "examples/bluetooth/nimble/blehr", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut.app.binary_path, "blehr.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("blehr_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("blehr_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("blehr_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("blehr_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     Utility.console_log("Starting blehr simple example test app")

examples/bluetooth/nimble/bleprph/bleprph_test.py

@@ -16,36 +16,15 @@
 
 from __future__ import print_function
 import os
-import sys
 import re
 import Queue
 import traceback
 import threading
 import subprocess
 
-try:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError as e:
-    print(e)
-    print("Try `export TEST_FW_PATH=$IDF_PATH/tools/tiny-test-fw` for resolving the issue")
-    print("Try `pip install -r $IDF_PATH/tools/tiny-test-fw/requirements.txt` for resolving the issue")
-    import IDF
-
-try:
-    import lib_ble_client
-except ImportError:
-    lib_ble_client_path = os.getenv("IDF_PATH") + "/tools/ble"
-    if lib_ble_client_path and lib_ble_client_path not in sys.path:
-        sys.path.insert(0, lib_ble_client_path)
-    import lib_ble_client
-
-
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
+from ble import lib_ble_client
 
 # When running on local machine execute the following before running this script
 # > make app bootloader
@@ -136,7 +115,7 @@ class BlePrphThread(threading.Thread):
             self.exceptions_queue.put(traceback.format_exc(), block=False)
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI_BT")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT")
 def test_example_app_ble_peripheral(env, extra_data):
     """
         Steps:
@@ -150,13 +129,13 @@ def test_example_app_ble_peripheral(env, extra_data):
     subprocess.check_output(['hciconfig','hci0','reset'])
 
     # Acquire DUT
-    dut = env.get_dut("bleprph", "examples/bluetooth/nimble/bleprph", dut_class=ESP32DUT)
+    dut = env.get_dut("bleprph", "examples/bluetooth/nimble/bleprph", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut.app.binary_path, "bleprph.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("bleprph_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("bleprph_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("bleprph_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("bleprph_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     Utility.console_log("Starting bleprph simple example test app")

examples/get-started/blink/example_test.py

@@ -5,21 +5,10 @@ from __future__ import print_function
 from __future__ import unicode_literals
 import re
 import os
-import sys
 import hashlib
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-
-    import IDF
-
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
 
 
 def verify_elf_sha256_embedding(dut):
@@ -39,13 +28,13 @@ def verify_elf_sha256_embedding(dut):
         raise ValueError('ELF file SHA256 mismatch')
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_blink(env, extra_data):
-    dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ESP32DUT)
+    dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ttfw_idf.ESP32DUT)
     binary_file = os.path.join(dut.app.binary_path, "blink.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("blink_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("blink_bin_size", bin_size // 1024)
 
     dut.start_app()
 

examples/get-started/hello_world/loadable_elf_example_test.py

@@ -1,19 +1,12 @@
 import os
-import pexpect
-import serial
-import sys
 import threading
 import time
 
-try:
-    import IDF
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import pexpect
+import serial
 
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
 
 
 class CustomProcess(object):
@@ -125,15 +118,16 @@ class SerialThread(object):
             Utility.console_log('The pyserial thread is still alive', 'O')
 
 
-@IDF.idf_example_test(env_tag="test_jtag_arm")
+@ttfw_idf.idf_example_test(env_tag="test_jtag_arm")
 def test_examples_loadable_elf(env, extra_data):
 
-    idf_path = os.environ['IDF_PATH']
     rel_project_path = os.path.join('examples', 'get-started', 'hello_world')
+    app_files = ['hello-world.elf', 'partition_table/partition-table.bin']
+    example = ttfw_idf.LoadableElfExample(rel_project_path, app_files, target="esp32")
+    idf_path = example.get_sdk_path()
     proj_path = os.path.join(idf_path, rel_project_path)
-    example = IDF.Example(rel_project_path, target="esp32")
     sdkconfig = example.get_sdkconfig()
-    elf_path = os.path.join(example.get_binary_path(rel_project_path), 'hello-world.elf')
+    elf_path = os.path.join(example.binary_path, 'hello-world.elf')
     esp_log_path = os.path.join(proj_path, 'esp.log')
 
     assert(sdkconfig['CONFIG_IDF_TARGET_ESP32'] == 'y'), "Only ESP32 target is supported"

examples/peripherals/can/can_alert_and_recovery/example_test.py

@@ -1,28 +1,16 @@
 # Need Python 3 string formatting functions
 from __future__ import print_function
 
-import os
-import sys
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # The test cause is dependent on the Tiny Test Framework. Ensure the
-    # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw`
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 # CAN Self Test Example constants
 STR_EXPECT = ("CAN Alert and Recovery: Driver installed", "CAN Alert and Recovery: Driver uninstalled")
 EXPECT_TIMEOUT = 20
 
 
-@IDF.idf_example_test(env_tag='Example_CAN1')
+@ttfw_idf.idf_example_test(env_tag='Example_CAN1')
 def test_can_alert_and_recovery_example(env, extra_data):
-    dut = env.get_dut('dut1', 'examples/peripherals/can/can_alert_and_recovery', dut_class=ESP32DUT)
+    dut = env.get_dut('dut1', 'examples/peripherals/can/can_alert_and_recovery', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     for string in STR_EXPECT:

examples/peripherals/can/can_network/example_test.py

@@ -1,19 +1,9 @@
 # Need Python 3 string formatting functions
 from __future__ import print_function
 
-import os
-import sys
 from threading import Thread
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # The test cause is dependent on the Tiny Test Framework. Ensure the
-    # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw`
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+
+import ttfw_idf
 
 # Define tuple of strings to expect for each DUT.
 master_expect = ("CAN Master: Driver installed", "CAN Master: Driver uninstalled")
@@ -37,13 +27,16 @@ def dut_thread_callback(**kwargs):
     result[0] = True
 
 
-@IDF.idf_example_test(env_tag='Example_CAN2')
+@ttfw_idf.idf_example_test(env_tag='Example_CAN2')
 def test_can_network_example(env, extra_data):
 
     # Get device under test. "dut1", "dut2", and "dut3" must be properly defined in EnvConfig
-    dut_master = env.get_dut("dut1", "examples/peripherals/can/can_network/can_network_master", dut_class=ESP32DUT)
-    dut_slave = env.get_dut("dut2", "examples/peripherals/can/can_network/can_network_slave", dut_class=ESP32DUT)
-    dut_listen_only = env.get_dut("dut3", "examples/peripherals/can/can_network/can_network_listen_only", dut_class=ESP32DUT)
+    dut_master = env.get_dut("dut1", "examples/peripherals/can/can_network/can_network_master",
+                             dut_class=ttfw_idf.ESP32DUT)
+    dut_slave = env.get_dut("dut2", "examples/peripherals/can/can_network/can_network_slave",
+                            dut_class=ttfw_idf.ESP32DUT)
+    dut_listen_only = env.get_dut("dut3", "examples/peripherals/can/can_network/can_network_listen_only",
+                                  dut_class=ttfw_idf.ESP32DUT)
 
     # Flash app onto each DUT, each DUT is reset again at the start of each thread
     dut_master.start_app()

examples/peripherals/can/can_self_test/example_test.py

@@ -1,18 +1,7 @@
 # Need Python 3 string formatting functions
 from __future__ import print_function
 
-import os
-import sys
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # The test cause is dependent on the Tiny Test Framework. Ensure the
-    # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw`
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
 # CAN Self Test Example constants
@@ -20,10 +9,10 @@ STR_EXPECT = ("CAN Self Test: Driver installed", "CAN Self Test: Driver uninstal
 EXPECT_TIMEOUT = 20
 
 
-@IDF.idf_example_test(env_tag='Example_CAN1')
+@ttfw_idf.idf_example_test(env_tag='Example_CAN1')
 def test_can_self_test_example(env, extra_data):
     # Get device under test, flash and start example. "dut1" must be defined in EnvConfig
-    dut = env.get_dut('dut1', 'examples/peripherals/can/can_self_test', dut_class=ESP32DUT)
+    dut = env.get_dut('dut1', 'examples/peripherals/can/can_self_test', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     for string in STR_EXPECT:

examples/peripherals/i2c/i2c_tools/example_test.py

@@ -1,23 +1,14 @@
 from __future__ import print_function
-import os
-import sys
 
-EXPECT_TIMEOUT = 20
+import ttfw_idf
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+EXPECT_TIMEOUT = 20
 
 
-@IDF.idf_example_test(env_tag='Example_I2C_CCS811_SENSOR')
+@ttfw_idf.idf_example_test(env_tag='Example_I2C_CCS811_SENSOR')
 def test_i2ctools_example(env, extra_data):
     # Get device under test, flash and start example. "i2ctool" must be defined in EnvConfig
-    dut = env.get_dut('i2ctools', 'examples/peripherals/i2c/i2c_tools', dut_class=ESP32DUT)
+    dut = env.get_dut('i2ctools', 'examples/peripherals/i2c/i2c_tools', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
     dut.expect("esp32>", timeout=EXPECT_TIMEOUT)
     # Get i2c address

examples/peripherals/rmt/ir_protocols/example_test.py

@@ -1,19 +1,11 @@
 from __future__ import print_function
-import os
-import sys
 
-try:
-    import IDF
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 EXPECT_TIMEOUT = 20
 
 
-@IDF.idf_example_test(env_tag='Example_RMT_IR_PROTOCOLS')
+@ttfw_idf.idf_example_test(env_tag='Example_RMT_IR_PROTOCOLS')
 def test_examples_rmt_ir_protocols(env, extra_data):
     dut = env.get_dut('ir_protocols_example', 'examples/peripherals/rmt/ir_protocols', app_config_name='nec')
     print("Using binary path: {}".format(dut.app.binary_path))

examples/peripherals/sdio/sdio_test.py

@@ -12,25 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" example of writing test with TinyTestFW """
-import os
-import sys
+from tiny_test_fw import TinyFW
+import ttfw_idf
 
-try:
-    import TinyFW
-except ImportError:
-    # if we want to run test case outside `tiny-test-fw` folder,
-    # we need to insert tiny-test-fw path into sys path
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import TinyFW
 
-import IDF
-from IDF.IDFDUT import ESP32DUT
-
-
-@IDF.idf_example_test(env_tag="Example_SDIO", ignore=True)
+@ttfw_idf.idf_example_test(env_tag="Example_SDIO", ignore=True)
 def test_example_sdio_communication(env, extra_data):
     """
     Configurations
@@ -50,8 +36,8 @@ def test_example_sdio_communication(env, extra_data):
     or use sdio test board, which has two wrover modules connect to a same FT3232
     Assume that first dut is host and second is slave
     """
-    dut1 = env.get_dut("sdio_host", "examples/peripherals/sdio/host", dut_class=ESP32DUT)
-    dut2 = env.get_dut("sdio_slave", "examples/peripherals/sdio/slave", dut_class=ESP32DUT)
+    dut1 = env.get_dut("sdio_host", "examples/peripherals/sdio/host", dut_class=ttfw_idf.ESP32DUT)
+    dut2 = env.get_dut("sdio_slave", "examples/peripherals/sdio/slave", dut_class=ttfw_idf.ESP32DUT)
     dut1.start_app()
     # wait until the master is ready to setup the slave
     dut1.expect("host ready, start initializing slave...")
@@ -133,5 +119,5 @@ def test_example_sdio_communication(env, extra_data):
 
 
 if __name__ == '__main__':
-    TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=IDF.IDFDUT)
+    TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=ttfw_idf.IDFDUT)
     test_example_sdio_communication()

examples/protocols/asio/chat_client/asio_chat_client_test.py

@@ -1,22 +1,10 @@
 import re
 import os
-import sys
 import socket
 from threading import Thread
 import time
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 global g_client_response
 global g_msg_to_client
@@ -56,7 +44,7 @@ def chat_server_sketch(my_ip):
     print("server closed")
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_asio_chat_client(env, extra_data):
     """
     steps: |
@@ -70,12 +58,12 @@ def test_examples_protocol_asio_chat_client(env, extra_data):
     global g_client_response
     global g_msg_to_client
     test_msg = "ABC"
-    dut1 = env.get_dut("chat_client", "examples/protocols/asio/chat_client", dut_class=ESP32DUT)
+    dut1 = env.get_dut("chat_client", "examples/protocols/asio/chat_client", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_chat_client.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("asio_chat_client_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("asio_chat_client_size", bin_size // 1024)
+    ttfw_idf.log_performance("asio_chat_client_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("asio_chat_client_size", bin_size // 1024)
     # 1. start a tcp server on the host
     host_ip = get_my_ip()
     thread1 = Thread(target=chat_server_sketch, args=(host_ip,))

examples/protocols/asio/chat_server/asio_chat_server_test.py

@@ -1,24 +1,11 @@
 import re
 import os
-import sys
 import socket
 
+import ttfw_idf
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
 
-
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_asio_chat_server(env, extra_data):
     """
     steps: |
@@ -28,12 +15,12 @@ def test_examples_protocol_asio_chat_server(env, extra_data):
       4. Test evaluates received test message from server
     """
     test_msg = b"   4ABC\n"
-    dut1 = env.get_dut("chat_server", "examples/protocols/asio/chat_server", dut_class=ESP32DUT)
+    dut1 = env.get_dut("chat_server", "examples/protocols/asio/chat_server", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_chat_server.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("asio_chat_server_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("asio_chat_server_size", bin_size // 1024)
+    ttfw_idf.log_performance("asio_chat_server_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("asio_chat_server_size", bin_size // 1024)
     # 1. start test
     dut1.start_app()
     # 2. get the server IP address

examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py

@@ -1,25 +1,11 @@
 import re
 import os
-import sys
 import socket
 
+import ttfw_idf
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
 
-    import IDF
-
-
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_asio_tcp_server(env, extra_data):
     """
     steps: |
@@ -30,12 +16,12 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
       5. Test evaluates received test message on server stdout
     """
     test_msg = b"echo message from client to server"
-    dut1 = env.get_dut("tcp_echo_server", "examples/protocols/asio/tcp_echo_server", dut_class=ESP32DUT)
+    dut1 = env.get_dut("tcp_echo_server", "examples/protocols/asio/tcp_echo_server", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_tcp_echo_server.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("asio_tcp_echo_server_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("asio_tcp_echo_server_size", bin_size // 1024)
+    ttfw_idf.log_performance("asio_tcp_echo_server_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("asio_tcp_echo_server_size", bin_size // 1024)
     # 1. start test
     dut1.start_app()
     # 2. get the server IP address

examples/protocols/asio/udp_echo_server/asio_udp_server_test.py

@@ -1,25 +1,11 @@
 import re
 import os
-import sys
 import socket
 
+import ttfw_idf
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
 
-    import IDF
-
-
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_asio_udp_server(env, extra_data):
     """
     steps: |
@@ -30,12 +16,12 @@ def test_examples_protocol_asio_udp_server(env, extra_data):
       5. Test evaluates received test message on server stdout
     """
     test_msg = b"echo message from client to server"
-    dut1 = env.get_dut("udp_echo_server", "examples/protocols/asio/udp_echo_server", dut_class=ESP32DUT)
+    dut1 = env.get_dut("udp_echo_server", "examples/protocols/asio/udp_echo_server", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "asio_udp_echo_server.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("asio_udp_echo_server_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("asio_udp_echo_server_size", bin_size // 1024)
+    ttfw_idf.log_performance("asio_udp_echo_server_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("asio_udp_echo_server_size", bin_size // 1024)
     # 1. start test
     dut1.start_app()
     # 2. get the server IP address

examples/protocols/esp_http_client/esp_http_client_test.py

@@ -1,35 +1,22 @@
 import re
 import os
-import sys
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
+import ttfw_idf
 
-    import IDF
 
-
-@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True)
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI", ignore=True)
 def test_examples_protocol_esp_http_client(env, extra_data):
     """
     steps: |
       1. join AP
       2. Send HTTP request to httpbin.org
     """
-    dut1 = env.get_dut("esp_http_client", "examples/protocols/esp_http_client", dut_class=ESP32DUT)
+    dut1 = env.get_dut("esp_http_client", "examples/protocols/esp_http_client", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "esp-http-client-example.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("esp_http_client_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("esp_http_client_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("esp_http_client_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("esp_http_client_bin_size", bin_size // 1024)
     # start test
     dut1.start_app()
     dut1.expect("Connected to AP, begin http example", timeout=30)

examples/protocols/http_server/advanced_tests/http_server_advanced_test.py

@@ -19,31 +19,14 @@ from __future__ import print_function
 from __future__ import unicode_literals
 import re
 import os
-import sys
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-
-    import IDF
-
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
+from idf_http_server_test import test as client
 
 # When running on local machine execute the following before running this script
 # > make app bootloader
 # > make print_flash_cmd | tail -n 1 > build/download.config
-# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw
-
-# Import client module
-# TODO: replace with import
-expath = os.path.dirname(os.path.realpath(__file__))
-client = Utility.load_source(expath + "/scripts/test.py")
-
 
 # Due to connectivity issues (between runner host and DUT) in the runner environment,
 # some of the `advanced_tests` are ignored. These tests are intended for verifying
@@ -51,16 +34,16 @@ client = Utility.load_source(expath + "/scripts/test.py")
 # of large HTTP packets and malformed requests, running multiple parallel sessions, etc.
 # It is advised that all these tests be run locally, when making changes or adding new
 # features to this component.
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_http_server_advanced(env, extra_data):
     # Acquire DUT
-    dut1 = env.get_dut("http_server", "examples/protocols/http_server/advanced_tests", dut_class=ESP32DUT)
+    dut1 = env.get_dut("http_server", "examples/protocols/http_server/advanced_tests", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut1.app.binary_path, "tests.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("http_server_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     Utility.console_log("Starting http_server advanced test app")

examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py

@@ -21,42 +21,28 @@ from builtins import str
 from builtins import range
 import re
 import os
-import sys
 import random
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import Utility
+from tiny_test_fw import Utility
+import ttfw_idf
+from idf_http_server_test import adder as client
 
 # When running on local machine execute the following before running this script
 # > make app bootloader
 # > make print_flash_cmd | tail -n 1 > build/download.config
-# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw
-
-# Import client module
-# TODO: replace with import
-expath = os.path.dirname(os.path.realpath(__file__))
-client = Utility.load_source(expath + "/scripts/adder.py")
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_http_server_persistence(env, extra_data):
     # Acquire DUT
-    dut1 = env.get_dut("http_server", "examples/protocols/http_server/persistent_sockets", dut_class=ESP32DUT)
+    dut1 = env.get_dut("http_server", "examples/protocols/http_server/persistent_sockets",
+                       dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut1.app.binary_path, "persistent_sockets.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("http_server_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     Utility.console_log("Starting http_server persistance test app")

examples/protocols/http_server/simple/http_server_simple_test.py

@@ -20,43 +20,29 @@ from __future__ import unicode_literals
 from builtins import range
 import re
 import os
-import sys
 import string
 import random
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
+from tiny_test_fw import Utility
+import ttfw_idf
+from idf_http_server_test import client
 
-    import IDF
-
-import Utility
 
 # When running on local machine execute the following before running this script
 # > make app bootloader
 # > make print_flash_cmd | tail -n 1 > build/download.config
-# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw
-
-# Import client module
-expath = os.path.dirname(os.path.realpath(__file__))
-client = Utility.load_source(expath + "/scripts/client.py")
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_http_server_simple(env, extra_data):
     # Acquire DUT
-    dut1 = env.get_dut("http_server", "examples/protocols/http_server/simple", dut_class=ESP32DUT)
+    dut1 = env.get_dut("http_server", "examples/protocols/http_server/simple", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut1.app.binary_path, "simple.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("http_server_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     Utility.console_log("Starting http_server simple test app")

examples/protocols/https_request/example_test.py

@@ -1,22 +1,10 @@
 import re
 import os
-import sys
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True)
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI", ignore=True)
 def test_examples_protocol_https_request(env, extra_data):
     """
     steps: |
@@ -24,12 +12,12 @@ def test_examples_protocol_https_request(env, extra_data):
       2. connect to www.howsmyssl.com:443
       3. send http request
     """
-    dut1 = env.get_dut("https_request", "examples/protocols/https_request", dut_class=ESP32DUT)
+    dut1 = env.get_dut("https_request", "examples/protocols/https_request", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "https_request.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("https_request_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("https_request_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("https_request_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("https_request_bin_size", bin_size // 1024)
     # start test
     dut1.start_app()
     dut1.expect("Connection established...", timeout=30)

examples/protocols/mdns/mdns_example_test.py

@@ -1,6 +1,5 @@
 import re
 import os
-import sys
 import socket
 import time
 import struct
@@ -8,22 +7,8 @@ import dpkt
 import dpkt.dns
 from threading import Thread, Event
 
-
-# this is a test case write with tiny-test-fw.
-# to run test cases outside tiny-test-fw,
-# we need to set environment variable `TEST_FW_PATH`,
-# then get and insert `TEST_FW_PATH` to sys path before import FW module
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
+from tiny_test_fw import DUT
+import ttfw_idf
 
 # g_run_server = True
 # g_done = False
@@ -103,7 +88,7 @@ def mdns_server(esp_host):
             continue
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_mdns(env, extra_data):
     global stop_mdns_server
     """
@@ -113,12 +98,12 @@ def test_examples_protocol_mdns(env, extra_data):
       3. check the mdns name is accessible
       4. check DUT output if mdns advertized host is resolved
     """
-    dut1 = env.get_dut("mdns-test", "examples/protocols/mdns", dut_class=ESP32DUT)
+    dut1 = env.get_dut("mdns-test", "examples/protocols/mdns", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "mdns-test.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mdns-test_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("mdns-test_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("mdns-test_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("mdns-test_bin_size", bin_size // 1024)
     # 1. start mdns application
     dut1.start_app()
     # 2. get the dut host name (and IP address)

examples/protocols/modbus/serial/example_test.py

@@ -2,20 +2,11 @@
 from __future__ import print_function
 
 import os
-import sys
 import re
 import logging
 from threading import Thread
 
-try:
-    import IDF
-except ImportError:
-    # The test cause is dependent on the Tiny Test Framework. Ensure the
-    # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw`
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 LOG_LEVEL = logging.DEBUG
 LOGGER_NAME = "modbus_test"
@@ -174,13 +165,13 @@ def test_check_mode(dut=None, mode_str=None, value=None):
     return False
 
 
-@IDF.idf_example_test(env_tag='Example_T2_RS485')
+@ttfw_idf.idf_example_test(env_tag='Example_T2_RS485')
 def test_modbus_communication(env, comm_mode):
     global logger
 
     # Get device under test. "dut1 - master", "dut2 - slave" must be properly connected through RS485 interface driver
-    dut_master = env.get_dut("modbus_master", "examples/protocols/modbus/serial/mb_master")
-    dut_slave = env.get_dut("modbus_slave", "examples/protocols/modbus/serial/mb_slave")
+    dut_master = env.get_dut("modbus_master", "examples/protocols/modbus/serial/mb_master", dut_class=ttfw_idf.ESP32DUT)
+    dut_slave = env.get_dut("modbus_slave", "examples/protocols/modbus/serial/mb_slave", dut_class=ttfw_idf.ESP32DUT)
 
     try:
         logger.debug("Environment vars: %s\r\n" % os.environ)

examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py

@@ -8,21 +8,8 @@ import ssl
 import paho.mqtt.client as mqtt
 from threading import Thread, Event
 
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
+from tiny_test_fw import DUT
+import ttfw_idf
 
 
 event_client_connected = Event()
@@ -73,7 +60,7 @@ def on_message(client, userdata, msg):
     message_log += "Received data:" + msg.topic + " " + payload + "\n"
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_mqtt_ssl(env, extra_data):
     broker_url = ""
     broker_port = 0
@@ -85,13 +72,13 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
       4. Test ESP32 client received correct qos0 message
       5. Test python client receives binary data from running partition and compares it with the binary
     """
-    dut1 = env.get_dut("mqtt_ssl", "examples/protocols/mqtt/ssl", dut_class=ESP32DUT)
+    dut1 = env.get_dut("mqtt_ssl", "examples/protocols/mqtt/ssl", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "mqtt_ssl.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mqtt_ssl_bin_size", "{}KB"
-                        .format(bin_size // 1024))
-    IDF.check_performance("mqtt_ssl_size", bin_size // 1024)
+    ttfw_idf.log_performance("mqtt_ssl_bin_size", "{}KB"
+                             .format(bin_size // 1024))
+    ttfw_idf.check_performance("mqtt_ssl_size", bin_size // 1024)
     # Look for host:port in sdkconfig
     try:
         value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"])

examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py

@@ -6,21 +6,8 @@ from threading import Thread
 import struct
 import time
 
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
+from tiny_test_fw import DUT
+import ttfw_idf
 
 msgid = -1
 
@@ -66,7 +53,7 @@ def mqqt_server_sketch(my_ip, port):
     print("server closed")
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_mqtt_qos1(env, extra_data):
     global msgid
     """
@@ -76,12 +63,12 @@ def test_examples_protocol_mqtt_qos1(env, extra_data):
       3. Test evaluates that qos1 message is queued and removed from queued after ACK received
       4. Test the broker received the same message id evaluated in step 3
     """
-    dut1 = env.get_dut("mqtt_tcp", "examples/protocols/mqtt/tcp", dut_class=ESP32DUT)
+    dut1 = env.get_dut("mqtt_tcp", "examples/protocols/mqtt/tcp", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "mqtt_tcp.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mqtt_tcp_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("mqtt_tcp_size", bin_size // 1024)
+    ttfw_idf.log_performance("mqtt_tcp_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("mqtt_tcp_size", bin_size // 1024)
     # 1. start mqtt broker sketch
     host_ip = get_my_ip()
     thread1 = Thread(target=mqqt_server_sketch, args=(host_ip,1883))

examples/protocols/mqtt/ws/mqtt_ws_example_test.py

@@ -7,21 +7,8 @@ import sys
 import paho.mqtt.client as mqtt
 from threading import Thread, Event
 
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except Exception:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
+from tiny_test_fw import DUT
+import ttfw_idf
 
 event_client_connected = Event()
 event_stop_client = Event()
@@ -52,7 +39,7 @@ def on_message(client, userdata, msg):
     message_log += "Received data:" + msg.topic + " " + payload + "\n"
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_mqtt_ws(env, extra_data):
     broker_url = ""
     broker_port = 0
@@ -63,12 +50,12 @@ def test_examples_protocol_mqtt_ws(env, extra_data):
       3. Test evaluates it received correct qos0 message
       4. Test ESP32 client received correct qos0 message
     """
-    dut1 = env.get_dut("mqtt_websocket", "examples/protocols/mqtt/ws", dut_class=ESP32DUT)
+    dut1 = env.get_dut("mqtt_websocket", "examples/protocols/mqtt/ws", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "mqtt_websocket.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mqtt_websocket_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("mqtt_websocket_size", bin_size // 1024)
+    ttfw_idf.log_performance("mqtt_websocket_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("mqtt_websocket_size", bin_size // 1024)
     # Look for host:port in sdkconfig
     try:
         value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"])

examples/protocols/mqtt/wss/mqtt_wss_example_test.py

@@ -8,21 +8,9 @@ import ssl
 import paho.mqtt.client as mqtt
 from threading import Thread, Event
 
+from tiny_test_fw import DUT
+import ttfw_idf
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
 
 event_client_connected = Event()
 event_stop_client = Event()
@@ -53,7 +41,7 @@ def on_message(client, userdata, msg):
     message_log += "Received data:" + msg.topic + " " + payload + "\n"
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_mqtt_wss(env, extra_data):
     broker_url = ""
     broker_port = 0
@@ -64,12 +52,12 @@ def test_examples_protocol_mqtt_wss(env, extra_data):
       3. Test evaluates it received correct qos0 message
       4. Test ESP32 client received correct qos0 message
     """
-    dut1 = env.get_dut("mqtt_websocket_secure", "examples/protocols/mqtt/wss", dut_class=ESP32DUT)
+    dut1 = env.get_dut("mqtt_websocket_secure", "examples/protocols/mqtt/wss", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "mqtt_websocket_secure.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mqtt_websocket_secure_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("mqtt_websocket_secure_size", bin_size // 1024)
+    ttfw_idf.log_performance("mqtt_websocket_secure_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("mqtt_websocket_secure_size", bin_size // 1024)
     # Look for host:port in sdkconfig
     try:
         value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"])

examples/protocols/websocket/example_test.py

@@ -1,19 +1,10 @@
 import re
 import os
-import sys
-import IDF
-from IDF.IDFDUT import ESP32DUT
 
-# this is a test case write with tiny-test-fw.
-# to run test cases outside tiny-test-fw,
-# we need to set environment variable `TEST_FW_PATH`,
-# then get and insert `TEST_FW_PATH` to sys path before import FW module
-test_fw_path = os.getenv("TEST_FW_PATH")
-if test_fw_path and test_fw_path not in sys.path:
-    sys.path.insert(0, test_fw_path)
+import ttfw_idf
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True)
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI", ignore=True)
 def test_examples_protocol_websocket(env, extra_data):
     """
     steps: |
@@ -21,12 +12,12 @@ def test_examples_protocol_websocket(env, extra_data):
       2. connect to ws://echo.websocket.org
       3. send and receive data
     """
-    dut1 = env.get_dut("websocket", "examples/protocols/websocket", dut_class=ESP32DUT)
+    dut1 = env.get_dut("websocket", "examples/protocols/websocket", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "websocket-example.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("websocket_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("websocket_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("websocket_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("websocket_bin_size", bin_size // 1024)
     # start test
     dut1.start_app()
     dut1.expect("Waiting for wifi ...")

examples/provisioning/ble_prov/ble_prov_test.py

@@ -17,40 +17,25 @@
 from __future__ import print_function
 import re
 import os
-import sys
 import time
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-try:
-    import esp_prov
-except ImportError:
-    esp_prov_path = os.getenv("IDF_PATH") + "/tools/esp_prov"
-    if esp_prov_path and esp_prov_path not in sys.path:
-        sys.path.insert(0, esp_prov_path)
-    import esp_prov
+import ttfw_idf
+import esp_prov
 
 # Have esp_prov throw exception
 esp_prov.config_throw_except = True
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI_BT")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT")
 def test_examples_provisioning_ble(env, extra_data):
     # Acquire DUT
-    dut1 = env.get_dut("ble_prov", "examples/provisioning/ble_prov", dut_class=ESP32DUT)
+    dut1 = env.get_dut("ble_prov", "examples/provisioning/ble_prov", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut1.app.binary_path, "ble_prov.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("ble_prov_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("ble_prov_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("ble_prov_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("ble_prov_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     dut1.start_app()

examples/provisioning/manager/wifi_prov_mgr_test.py

@@ -17,40 +17,25 @@
 from __future__ import print_function
 import re
 import os
-import sys
 import time
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-try:
-    import esp_prov
-except ImportError:
-    esp_prov_path = os.getenv("IDF_PATH") + "/tools/esp_prov"
-    if esp_prov_path and esp_prov_path not in sys.path:
-        sys.path.insert(0, esp_prov_path)
-    import esp_prov
+import ttfw_idf
+import esp_prov
 
 # Have esp_prov throw exception
 esp_prov.config_throw_except = True
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI_BT")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT")
 def test_examples_wifi_prov_mgr(env, extra_data):
     # Acquire DUT
-    dut1 = env.get_dut("wifi_prov_mgr", "examples/provisioning/manager", dut_class=ESP32DUT)
+    dut1 = env.get_dut("wifi_prov_mgr", "examples/provisioning/manager", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut1.app.binary_path, "wifi_prov_mgr.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("wifi_prov_mgr_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("wifi_prov_mgr_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("wifi_prov_mgr_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("wifi_prov_mgr_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     dut1.start_app()

examples/provisioning/softap_prov/softap_prov_test.py

@@ -17,48 +17,26 @@
 from __future__ import print_function
 import re
 import os
-import sys
 import time
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-try:
-    import esp_prov
-except ImportError:
-    esp_prov_path = os.getenv("IDF_PATH") + "/tools/esp_prov"
-    if esp_prov_path and esp_prov_path not in sys.path:
-        sys.path.insert(0, esp_prov_path)
-    import esp_prov
-
-try:
-    import wifi_tools
-except ImportError:
-    wifi_tools_path = os.getenv("IDF_PATH") + "/examples/provisioning/softap_prov/utils"
-    if wifi_tools_path and wifi_tools_path not in sys.path:
-        sys.path.insert(0, wifi_tools_path)
-    import wifi_tools
+import ttfw_idf
+import esp_prov
+import wifi_tools
 
 # Have esp_prov throw exception
 esp_prov.config_throw_except = True
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI_BT")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT")
 def test_examples_provisioning_softap(env, extra_data):
     # Acquire DUT
-    dut1 = env.get_dut("softap_prov", "examples/provisioning/softap_prov", dut_class=ESP32DUT)
+    dut1 = env.get_dut("softap_prov", "examples/provisioning/softap_prov", dut_class=ttfw_idf.ESP32DUT)
 
     # Get binary file
     binary_file = os.path.join(dut1.app.binary_path, "softap_prov.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("softap_prov_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("softap_prov_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("softap_prov_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("softap_prov_bin_size", bin_size // 1024)
 
     # Upload binary and start testing
     dut1.start_app()

examples/security/flash_encryption/example_test.py

@@ -1,15 +1,5 @@
 from __future__ import print_function
-import os
-import sys
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
 # To prepare a test runner for this example:
@@ -19,9 +9,9 @@ except ImportError:
 #   espefuse.py --do-not-confirm -p $ESPPORT burn_efuse FLASH_CRYPT_CONFIG 0xf
 #   espefuse.py --do-not-confirm -p $ESPPORT burn_efuse FLASH_CRYPT_CNT 0x1
 #   espefuse.py --do-not-confirm -p $ESPPORT burn_key flash_encryption key.bin
-@IDF.idf_example_test(env_tag='Example_Flash_Encryption')
+@ttfw_idf.idf_example_test(env_tag='Example_Flash_Encryption')
 def test_examples_security_flash_encryption(env, extra_data):
-    dut = env.get_dut('flash_encryption', 'examples/security/flash_encryption', dut_class=ESP32DUT)
+    dut = env.get_dut('flash_encryption', 'examples/security/flash_encryption', dut_class=ttfw_idf.ESP32DUT)
     # start test
     dut.start_app()
     lines = [

examples/storage/ext_flash_fatfs/example_test.py

@@ -1,21 +1,11 @@
 from __future__ import print_function
-import os
-import sys
 
-try:
-    import IDF
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
-from IDF.IDFDUT import ESP32DUT
 
-
-@IDF.idf_example_test(env_tag='Example_ExtFlash')
+@ttfw_idf.idf_example_test(env_tag='Example_ExtFlash')
 def test_examples_storage_ext_flash_fatfs(env, extra_data):
-    dut = env.get_dut('ext_flash_fatfs', 'examples/storage/ext_flash_fatfs', dut_class=ESP32DUT)
+    dut = env.get_dut('ext_flash_fatfs', 'examples/storage/ext_flash_fatfs', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     dut.expect('Initialized external Flash')

examples/storage/parttool/example_test.py

@@ -3,19 +3,12 @@ import os
 import sys
 import subprocess
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_examples_parttool(env, extra_data):
-    dut = env.get_dut('parttool', 'examples/storage/parttool', dut_class=ESP32DUT)
+    dut = env.get_dut('parttool', 'examples/storage/parttool', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app(False)
 
     # Verify factory firmware

examples/storage/spiffsgen/example_test.py

@@ -1,22 +1,14 @@
 from __future__ import print_function
 import os
-import sys
 import hashlib
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_examples_spiffsgen(env, extra_data):
     # Test with default build configurations
-    dut = env.get_dut('spiffsgen', 'examples/storage/spiffsgen', dut_class=ESP32DUT)
+    dut = env.get_dut('spiffsgen', 'examples/storage/spiffsgen', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'spiffs_image')

examples/system/console/example_test.py

@@ -1,17 +1,9 @@
 from __future__ import print_function
-import os
-import sys
 
-try:
-    import IDF
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_examples_system_console(env, extra_data):
     dut = env.get_dut('console_example', 'examples/system/console', app_config_name='history')
     print("Using binary path: {}".format(dut.app.binary_path))

examples/system/cpp_exceptions/example_test.py

@@ -1,20 +1,11 @@
 from __future__ import print_function
-import os
-import sys
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_examples_system_cpp_exceptions(env, extra_data):
-    dut = env.get_dut('cpp_exceptions_example', 'examples/system/cpp_exceptions', dut_class=ESP32DUT)
+    dut = env.get_dut('cpp_exceptions_example', 'examples/system/cpp_exceptions', dut_class=ttfw_idf.ESP32DUT)
     # start test
     dut.start_app()
     lines = ['app_main starting',

examples/system/cpp_rtti/example_test.py

@@ -1,23 +1,11 @@
 from __future__ import print_function
-import os
-import sys
-
-try:
-    import IDF
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-
-@IDF.idf_example_test(env_tag='Example_WIFI')
+
+import ttfw_idf
+
+
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_cpp_rtti_example(env, extra_data):
-    dut = env.get_dut('cpp_rtti', 'examples/system/cpp_rtti')
+    dut = env.get_dut('cpp_rtti', 'examples/system/cpp_rtti', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     dut.expect('Type name of std::cout is: std::ostream')

examples/system/esp_event/default_event_loop/example_test.py

@@ -1,18 +1,6 @@
 from __future__ import print_function
-import os
-import sys
-
-try:
-    import IDF
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+
+import ttfw_idf
 
 # Timer events
 TIMER_EVENT_LIMIT = 3
@@ -91,7 +79,7 @@ def _test_iteration_events(dut):
     print("Deleted task event source")
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_default_event_loop_example(env, extra_data):
     dut = env.get_dut('default_event_loop', 'examples/system/esp_event/default_event_loop')
 

examples/system/esp_event/user_event_loops/example_test.py

@@ -1,18 +1,6 @@
 from __future__ import print_function
-import os
-import sys
-
-try:
-    import IDF
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+
+import ttfw_idf
 
 TASK_ITERATION_LIMIT = 10
 
@@ -20,9 +8,9 @@ TASK_ITERATION_POSTING = "posting TASK_EVENTS:TASK_ITERATION_EVENT to {}, iterat
 TASK_ITERATION_HANDLING = "handling TASK_EVENTS:TASK_ITERATION_EVENT from {}, iteration {}"
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_user_event_loops_example(env, extra_data):
-    dut = env.get_dut('user_event_loops', 'examples/system/esp_event/user_event_loops')
+    dut = env.get_dut('user_event_loops', 'examples/system/esp_event/user_event_loops', dut_class=ttfw_idf.ESP32DUT)
 
     dut.start_app()
 

examples/system/esp_timer/example_test.py

@@ -1,20 +1,7 @@
 from __future__ import print_function
 import re
-import os
-import sys
-
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+
+import ttfw_idf
 
 STARTING_TIMERS_REGEX = re.compile(r'Started timers, time since boot: (\d+) us')
 
@@ -38,9 +25,9 @@ LIGHT_SLEEP_TIME = 500000
 ONE_SHOT_TIMER_PERIOD = 5000000
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_examples_system_esp_timer(env, extra_data):
-    dut = env.get_dut('esp_timer_example', 'examples/system/esp_timer', dut_class=ESP32DUT)
+    dut = env.get_dut('esp_timer_example', 'examples/system/esp_timer', dut_class=ttfw_idf.ESP32DUT)
     # start test
     dut.start_app()
     groups = dut.expect(STARTING_TIMERS_REGEX, timeout=30)

examples/system/freertos/real_time_stats/example_test.py

@@ -1,27 +1,14 @@
 from __future__ import print_function
-import os
-import sys
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 STATS_TASK_ITERS = 3
 STATS_TASK_EXPECT = "Real time stats obtained"
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_real_time_stats_example(env, extra_data):
-    dut = env.get_dut('real_time_stats', 'examples/system/freertos/real_time_stats', dut_class=ESP32DUT)
+    dut = env.get_dut('real_time_stats', 'examples/system/freertos/real_time_stats', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     for iteration in range(0, STATS_TASK_ITERS):

examples/system/light_sleep/example_test.py

@@ -1,17 +1,8 @@
 from __future__ import print_function
 import re
-import os
-import sys
 import time
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
+import ttfw_idf
 
 ENTERING_SLEEP_STR = 'Entering light sleep'
 EXIT_SLEEP_REGEX = re.compile(r'Returned from light sleep, reason: (\w+), t=(\d+) ms, slept for (\d+) ms')
@@ -20,9 +11,9 @@ WAITING_FOR_GPIO_STR = 'Waiting for GPIO0 to go high...'
 WAKEUP_INTERVAL_MS = 2000
 
 
-@IDF.idf_example_test(env_tag='Example_WIFI')
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_examples_system_light_sleep(env, extra_data):
-    dut = env.get_dut('light_sleep_example', 'examples/system/light_sleep', dut_class=ESP32DUT)
+    dut = env.get_dut('light_sleep_example', 'examples/system/light_sleep', dut_class=ttfw_idf.ESP32DUT)
     dut.start_app()
 
     # Ensure DTR and RTS are de-asserted for proper control of GPIO0

examples/system/ota/otatool/example_test.py

@@ -3,23 +3,12 @@ import os
 import sys
 import subprocess
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv('TEST_FW_PATH')
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-
-@IDF.idf_example_test(env_tag='Example_WIFI')
+import ttfw_idf
+
+
+@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
 def test_otatool_example(env, extra_data):
-    dut = env.get_dut('otatool', 'examples/system/ota/otatool', dut_class=ESP32DUT)
+    dut = env.get_dut('otatool', 'examples/system/ota/otatool', dut_class=ttfw_idf.ESP32DUT)
 
     # Verify factory firmware
     dut.start_app()

examples/system/ota/simple_ota_example/example_test.py

@@ -1,26 +1,13 @@
 import re
 import os
-import sys
 import socket
 import BaseHTTPServer
 import SimpleHTTPServer
 from threading import Thread
 import ssl
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
+from tiny_test_fw import DUT
+import ttfw_idf
 
 server_cert = "-----BEGIN CERTIFICATE-----\n" \
               "MIIDXTCCAkWgAwIBAgIJAP4LF7E72HakMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV\n"\
@@ -108,7 +95,7 @@ def start_https_server(ota_image_dir, server_ip, server_port):
     httpd.serve_forever()
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_simple_ota_example(env, extra_data):
     """
     steps: |
@@ -116,12 +103,12 @@ def test_examples_protocol_simple_ota_example(env, extra_data):
       2. Fetch OTA image over HTTPS
       3. Reboot with the new OTA image
     """
-    dut1 = env.get_dut("simple_ota_example", "examples/system/ota/simple_ota_example", dut_class=ESP32DUT)
+    dut1 = env.get_dut("simple_ota_example", "examples/system/ota/simple_ota_example", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "simple_ota.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("simple_ota_bin_size", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("simple_ota_bin_size", bin_size // 1024)
+    ttfw_idf.log_performance("simple_ota_bin_size", "{}KB".format(bin_size // 1024))
+    ttfw_idf.check_performance("simple_ota_bin_size", bin_size // 1024)
     # start test
     host_ip = get_my_ip()
     thread1 = Thread(target=start_https_server, args=(dut1.app.binary_path, host_ip, 8000))

examples/wifi/iperf/iperf_test.py

@@ -26,34 +26,13 @@ from builtins import range
 from builtins import object
 import re
 import os
-import sys
 import time
 import subprocess
 
-try:
-    import IDF
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
-import TinyFW
-import Utility
-from Utility import (Attenuator, PowerControl, LineChart)
-
-try:
-    from test_report import (ThroughputForConfigsReport, ThroughputVsRssiReport)
-except ImportError:
-    # add current folder to system path for importing test_report
-    sys.path.append(os.path.dirname(__file__))
-    from test_report import (ThroughputForConfigsReport, ThroughputVsRssiReport)
+from tiny_test_fw import TinyFW, DUT, Utility
+import ttfw_idf
+from idf_iperf_test_util import (Attenuator, PowerControl, LineChart, TestReport)
+
 
 # configurations
 TEST_TIME = TEST_TIMEOUT = 60
@@ -166,8 +145,8 @@ class TestResult(object):
             throughput = 0.0
 
         if throughput == 0 and rssi > self.ZERO_THROUGHPUT_THRESHOLD:
-                self.error_list.append("[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found"
-                                       .format(ap_ssid, att, rssi))
+            self.error_list.append("[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found"
+                                   .format(ap_ssid, att, rssi))
 
         self._save_result(throughput, ap_ssid, att, rssi, heap_size)
 
@@ -467,7 +446,7 @@ class IperfTestUtility(object):
         return ret
 
 
-@IDF.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress")
+@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress")
 def test_wifi_throughput_with_different_configs(env, extra_data):
     """
     steps: |
@@ -492,7 +471,8 @@ def test_wifi_throughput_with_different_configs(env, extra_data):
                                                     "sdkconfig.ci.{}".format(config_name))
 
         # 2. get DUT and download
-        dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=config_name)
+        dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
+                          app_config_name=config_name)
         dut.start_app()
         dut.expect("esp32>")
 
@@ -519,12 +499,12 @@ def test_wifi_throughput_with_different_configs(env, extra_data):
         env.close_dut("iperf")
 
     # 5. generate report
-    report = ThroughputForConfigsReport(os.path.join(env.log_path, "ThroughputForConfigsReport"),
-                                        ap_info["ssid"], test_result, sdkconfig_files)
+    report = TestReport.ThroughputForConfigsReport(os.path.join(env.log_path, "ThroughputForConfigsReport"),
+                                                   ap_info["ssid"], test_result, sdkconfig_files)
     report.generate_report()
 
 
-@IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress")
+@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox", category="stress")
 def test_wifi_throughput_vs_rssi(env, extra_data):
     """
     steps: |
@@ -547,7 +527,8 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
     }
 
     # 1. get DUT and download
-    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=BEST_PERFORMANCE_CONFIG)
+    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
+                      app_config_name=BEST_PERFORMANCE_CONFIG)
     dut.start_app()
     dut.expect("esp32>")
 
@@ -573,12 +554,12 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
     env.close_dut("iperf")
 
     # 4. generate report
-    report = ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"),
-                                    test_result)
+    report = TestReport.ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"),
+                                               test_result)
     report.generate_report()
 
 
-@IDF.idf_example_test(env_tag="Example_ShieldBox_Basic")
+@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox_Basic")
 def test_wifi_throughput_basic(env, extra_data):
     """
     steps: |
@@ -593,7 +574,8 @@ def test_wifi_throughput_basic(env, extra_data):
     }
 
     # 1. get DUT
-    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=BEST_PERFORMANCE_CONFIG)
+    dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT,
+                      app_config_name=BEST_PERFORMANCE_CONFIG)
     dut.start_app()
     dut.expect("esp32>")
 
@@ -615,8 +597,8 @@ def test_wifi_throughput_basic(env, extra_data):
     # 4. log performance and compare with pass standard
     performance_items = []
     for throughput_type in test_result:
-        IDF.log_performance("{}_throughput".format(throughput_type),
-                            "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput()))
+        ttfw_idf.log_performance("{}_throughput".format(throughput_type),
+                                 "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput()))
         performance_items.append(["{}_throughput".format(throughput_type),
                                   "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput())])
 
@@ -624,8 +606,8 @@ def test_wifi_throughput_basic(env, extra_data):
     TinyFW.JunitReport.update_performance(performance_items)
     # do check after logging, otherwise test will exit immediately if check fail, some performance can't be logged.
     for throughput_type in test_result:
-        IDF.check_performance("{}_throughput".format(throughput_type),
-                              test_result[throughput_type].get_best_throughput())
+        ttfw_idf.check_performance("{}_throughput".format(throughput_type),
+                                   test_result[throughput_type].get_best_throughput())
 
     env.close_dut("iperf")
 

make/project.mk

@@ -144,7 +144,7 @@ EXTRA_COMPONENT_DIRS ?=
 COMPONENT_DIRS := $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components $(PROJECT_PATH)/main
 endif
 # Make sure that every directory in the list is an absolute path without trailing slash.
-# This is necessary to split COMPONENT_DIRS into SINGLE_COMPONENT_DIRS and MULTI_COMPONENT_DIRS below. 
+# This is necessary to split COMPONENT_DIRS into SINGLE_COMPONENT_DIRS and MULTI_COMPONENT_DIRS below.
 COMPONENT_DIRS := $(foreach cd,$(COMPONENT_DIRS),$(abspath $(cd)))
 export COMPONENT_DIRS
 
@@ -153,11 +153,11 @@ $(warning SRCDIRS variable is deprecated. These paths can be added to EXTRA_COMP
 COMPONENT_DIRS += $(abspath $(SRCDIRS))
 endif
 
-# List of component directories, i.e. directories which contain a component.mk file 
+# List of component directories, i.e. directories which contain a component.mk file
 SINGLE_COMPONENT_DIRS := $(abspath $(dir $(dir $(foreach cd,$(COMPONENT_DIRS),\
                              $(wildcard $(cd)/component.mk)))))
 
-# List of components directories, i.e. directories which may contain components 
+# List of components directories, i.e. directories which may contain components
 MULTI_COMPONENT_DIRS := $(filter-out $(SINGLE_COMPONENT_DIRS),$(COMPONENT_DIRS))
 
 # The project Makefile can define a list of components, but if it does not do this
@@ -635,6 +635,11 @@ clean: app-clean bootloader-clean config-clean ldgen-clean
 #
 # This only works for components inside IDF_PATH
 check-submodules:
+# for internal use:
+# skip submodule check if running on Gitlab CI and job is configured as not clone submodules
+ifeq ($(IDF_SKIP_CHECK_SUBMODULES),1)
+	@echo "skip submodule check on internal CI"
+else
 # Check if .gitmodules exists, otherwise skip submodule check, assuming flattened structure
 ifneq ("$(wildcard ${IDF_PATH}/.gitmodules)","")
 
@@ -662,7 +667,7 @@ endef
 # so the argument is suitable for use with 'git submodule' commands
 $(foreach submodule,$(subst $(IDF_PATH)/,,$(filter $(IDF_PATH)/%,$(COMPONENT_SUBMODULES))),$(eval $(call GenerateSubmoduleCheckTarget,$(submodule))))
 endif # End check for .gitmodules existence
-
+endif
 
 # PHONY target to list components in the build and their paths
 list-components:

tools/ble/lib_ble_client.py

@@ -29,7 +29,7 @@ try:
     from gi.repository import GLib
 except ImportError as e:
     if 'linux' not in sys.platform:
-        sys.exit("Error: Only supported on Linux platform")
+        raise e
     print(e)
     print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue")
     print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")

tools/ble/lib_gap.py

@@ -25,7 +25,7 @@ try:
     import dbus.service
 except ImportError as e:
     if 'linux' not in sys.platform:
-        sys.exit("Error: Only supported on Linux platform")
+        raise e
     print(e)
     print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue")
     print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")

tools/ble/lib_gatt.py

@@ -25,7 +25,7 @@ try:
     import dbus.service
 except ImportError as e:
     if 'linux' not in sys.platform:
-        sys.exit("Error: Only supported on Linux platform")
+        raise e
     print(e)
     print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue")
     print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue")

tools/ci/ci_fetch_submodule.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+# internal use only for CI
+# download archive of one commit instead of cloning entire submodule repo
+
+import re
+import os
+import subprocess
+import argparse
+import shutil
+import time
+
+import gitlab_api
+
+SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]")
+PATH_PATTERN = re.compile(r"path\s+=\s+(\S+)")
+URL_PATTERN = re.compile(r"url\s+=\s+(\S+)")
+
+SUBMODULE_ARCHIVE_TEMP_FOLDER = "submodule_archive"
+
+
+class SubModule(object):
+    # We don't need to support recursive submodule clone now
+
+    GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r"\d+\s+commit\s+([0-9a-f]+)\s+")
+
+    def __init__(self, gitlab_inst, path, url):
+        self.path = path
+        self.gitlab_inst = gitlab_inst
+        self.project_id = self._get_project_id(url)
+        self.commit_id = self._get_commit_id(path)
+
+    def _get_commit_id(self, path):
+        output = subprocess.check_output(["git", "ls-tree", "HEAD", path])
+        # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0	components/esp_wifi/lib
+        match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
+        return match.group(1)
+
+    def _get_project_id(self, url):
+        base_name = os.path.basename(url)
+        project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0],  # remove .git
+                                                     namespace="espressif")
+        return project_id
+
+    def download_archive(self):
+        print("Update submodule: {}: {}".format(self.path, self.commit_id))
+        path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER,
+                                                      self.project_id)
+        renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
+        os.rename(path_name, renamed_path)
+        shutil.rmtree(self.path, ignore_errors=True)
+        shutil.move(renamed_path, os.path.dirname(self.path))
+
+
+def update_submodule(git_module_file, submodules_to_update):
+    gitlab_inst = gitlab_api.Gitlab()
+    submodules = []
+    with open(git_module_file, "r") as f:
+        data = f.read()
+    match = SUBMODULE_PATTERN.search(data)
+    while True:
+        next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
+        if next_match:
+            end_pos = next_match.start()
+        else:
+            end_pos = len(data)
+        path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
+        url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
+        path = path_match.group(1)
+        url = url_match.group(1)
+
+        filter_result = True
+        if submodules_to_update:
+            if path not in submodules_to_update:
+                filter_result = False
+        if filter_result:
+            submodules.append(SubModule(gitlab_inst, path, url))
+
+        match = next_match
+        if not match:
+            break
+
+    shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
+
+    for submodule in submodules:
+        submodule.download_archive()
+
+
+if __name__ == '__main__':
+    start_time = time.time()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo_path", "-p", default=".", help="repo path")
+    parser.add_argument("--submodule", "-s", default="all",
+                        help="Submodules to update. By default update all submodules. "
+                             "For multiple submodules, separate them with `;`. "
+                             "`all` and `none` are special values that indicates we fetch all / none submodules")
+    args = parser.parse_args()
+    if args.submodule == "none":
+        print("don't need to update submodules")
+        exit(0)
+    if args.submodule == "all":
+        _submodules = []
+    else:
+        _submodules = args.submodule.split(";")
+    update_submodule(os.path.join(args.repo_path, ".gitmodules"), _submodules)
+    print("total time spent on update submodule: {:.02f}s".format(time.time() - start_time))

tools/ci/config/assign-test.yml

@@ -10,7 +10,7 @@ assign_test:
     - build_ssc
     - build_esp_idf_tests_cmake
   variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
   artifacts:
@@ -18,6 +18,7 @@ assign_test:
       - components/idf_test/*/CIConfigs
       - components/idf_test/*/TC.sqlite
       - $EXAMPLE_CONFIG_OUTPUT_PATH
+      - build_examples/artifact_index.json
     expire_in: 1 week
   only:
     variables:
@@ -27,9 +28,9 @@ assign_test:
       - $BOT_LABEL_EXAMPLE_TEST
   script:
     # assign example tests
-    - python $TEST_FW_PATH/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
+    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
     # assign unit test cases
-    - python $TEST_FW_PATH/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
+    - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
     # clone test script to assign tests
     - git clone $TEST_SCRIPT_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
@@ -55,9 +56,9 @@ update_test_cases:
       - ${CI_PROJECT_DIR}/test-management/*.log
     expire_in: 1 week
   variables:
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
     PYTHON_VER: 3
   script:

tools/ci/config/post_check.yml

@@ -10,7 +10,7 @@ check_submodule_sync:
   retry: 2
   variables:
     GIT_STRATEGY: clone
-    GIT_SUBMODULE_STRATEGY: none
+    SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   before_script: []
   after_script: []

tools/ci/config/target-test.yml

@@ -21,7 +21,6 @@
       - $BOT_LABEL_EXAMPLE_TEST
   dependencies:
     - assign_test
-    - build_examples_cmake_esp32
   artifacts:
     when: always
     paths:
@@ -30,12 +29,11 @@
     reports:
         junit: $LOG_PATH/*/XUNIT_RESULT.xml
   variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
-    GIT_SUBMODULE_STRATEGY: none
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
@@ -43,10 +41,7 @@
     # clone test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    # Get esptool
-    - git submodule sync
-    - git submodule update --init components/esptool_py/esptool
-    - cd $TEST_FW_PATH
+    - cd tools/ci/python_packages/tiny_test_fw/bin
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
@@ -65,8 +60,6 @@
       - $BOT_LABEL_EXAMPLE_TEST
   dependencies:
     - assign_test
-    - build_examples_make
-    - build_examples_cmake_esp32
   artifacts:
     when: always
     paths:
@@ -75,7 +68,6 @@
     reports:
         junit: $LOG_PATH/*/XUNIT_RESULT.xml
   variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
@@ -87,7 +79,7 @@
     # clone test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd $TEST_FW_PATH
+    - cd tools/ci/python_packages/tiny_test_fw/bin
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
@@ -139,7 +131,7 @@
       - $LOG_PATH
     expire_in: 1 week
   variables:
-    GIT_SUBMODULE_STRATEGY: none
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
@@ -170,7 +162,6 @@ test_weekend_mqtt:
       - $BOT_LABEL_WEEKEND_TEST
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
     CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
@@ -187,7 +178,6 @@ test_weekend_network:
       - $BOT_LABEL_WEEKEND_TEST
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
     CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
@@ -202,8 +192,6 @@ example_test_001:
 example_test_002:
   extends: .example_test_template
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
-  variables:
-    GIT_SUBMODULE_STRATEGY: normal
   tags:
     - ESP32
     - Example_ShieldBox_Basic
@@ -214,7 +202,7 @@ example_test_002:
     # clone test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd $TEST_FW_PATH
+    - cd tools/ci/python_packages/tiny_test_fw/bin
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
@@ -286,7 +274,6 @@ example_test_010:
 
 example_test_011:
   extends: .example_debug_template
-  parallel: 4
   tags:
     - ESP32
     - Example_T2_RS485
@@ -316,7 +303,7 @@ UT_001:
 
 UT_002:
   extends: .unit_test_template
-  parallel: 11
+  parallel: 10
   tags:
     - ESP32_IDF
     - UT_T1_1
@@ -466,7 +453,7 @@ UT_034:
 
 UT_035:
   extends: .unit_test_template
-  parallel: 20
+  parallel: 19
   tags:
     - ESP32S2BETA_IDF
     - UT_T1_1
@@ -500,7 +487,6 @@ UT_043:
 
 UT_044:
   extends: .unit_test_template
-  parallel: 4
   tags:
     - ESP32_IDF
     - UT_SDIO

tools/ci/executable-list.txt

@@ -6,7 +6,6 @@ components/espcoredump/espcoredump.py
 components/espcoredump/test/test_espcoredump.py
 components/espcoredump/test/test_espcoredump.sh
 components/heap/test_multi_heap_host/test_all_configs.sh
-components/idf_test/unit_test/TestCaseScript/IDFUnitTest/__init__.py
 components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
 components/partition_table/gen_empty_partition.py
 components/partition_table/gen_esp32part.py

tools/ci/python_packages/gitlab_api.py

@@ -0,0 +1,174 @@
+import os
+import re
+import argparse
+import tempfile
+import tarfile
+import zipfile
+
+import gitlab
+
+
+class Gitlab(object):
+    JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?")
+
+    def __init__(self, project_id=None):
+        config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG")
+        if config_data_from_env:
+            # prefer to load config from env variable
+            with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                temp_file.write(config_data_from_env)
+            config_files = [temp_file.name]
+        else:
+            # otherwise try to use config file at local filesystem
+            config_files = None
+        self.gitlab_inst = gitlab.Gitlab.from_config(config_files=config_files)
+        self.gitlab_inst.auth()
+        if project_id:
+            self.project = self.gitlab_inst.projects.get(project_id)
+        else:
+            self.project = None
+
+    def get_project_id(self, name, namespace=None):
+        """
+        search project ID by name
+
+        :param name: project name
+        :param namespace: namespace to match when we have multiple project with same name
+        :return: project ID
+        """
+        projects = self.gitlab_inst.projects.list(search=name)
+        for project in projects:
+            if namespace is None:
+                if len(projects) == 1:
+                    project_id = project.id
+                    break
+            if project.namespace["path"] == namespace:
+                project_id = project.id
+                break
+        else:
+            raise ValueError("Can't find project")
+        return project_id
+
+    def download_artifacts(self, job_id, destination):
+        """
+        download full job artifacts and extract to destination.
+
+        :param job_id: Gitlab CI job ID
+        :param destination: extract artifacts to path.
+        """
+        job = self.project.jobs.get(job_id)
+
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            job.artifacts(streamed=True, action=temp_file.write)
+
+        with zipfile.ZipFile(temp_file.name, "r") as archive_file:
+            archive_file.extractall(destination)
+
+    def download_artifact(self, job_id, artifact_path, destination=None):
+        """
+        download specific path of job artifacts and extract to destination.
+
+        :param job_id: Gitlab CI job ID
+        :param artifact_path: list of path in artifacts (relative path to artifact root path)
+        :param destination: destination of artifact. Do not save to file if destination is None
+        :return: A list of artifact file raw data.
+        """
+        job = self.project.jobs.get(job_id)
+
+        raw_data_list = []
+
+        for a_path in artifact_path:
+            try:
+                data = job.artifact(a_path)
+            except gitlab.GitlabGetError as e:
+                print("Failed to download '{}' form job {}".format(a_path, job_id))
+                raise e
+            raw_data_list.append(data)
+            if destination:
+                file_path = os.path.join(destination, a_path)
+                try:
+                    os.makedirs(os.path.dirname(file_path))
+                except OSError:
+                    # already exists
+                    pass
+                with open(file_path, "wb") as f:
+                    f.write(data)
+
+        return raw_data_list
+
+    def find_job_id(self, job_name, pipeline_id=None):
+        """
+        Get Job ID from job name of specific pipeline
+
+        :param job_name: job name
+        :param pipeline_id: If None, will get pipeline id from CI pre-defined variable.
+        :return: a list of job IDs (parallel job will generate multiple jobs)
+        """
+        job_id_list = []
+        if pipeline_id is None:
+            pipeline_id = os.getenv("CI_PIPELINE_ID")
+        pipeline = self.project.pipelines.get(pipeline_id)
+        jobs = pipeline.jobs.list(all=True)
+        for job in jobs:
+            match = self.JOB_NAME_PATTERN.match(job.name)
+            if match:
+                if match.group(1) == job_name:
+                    job_id_list.append({"id": job.id, "parallel_num": match.group(3)})
+        return job_id_list
+
+    def download_archive(self, ref, destination, project_id=None):
+        """
+        Download archive of certain commit of a repository and extract to destination path
+
+        :param ref: commit or branch name
+        :param destination: destination path of extracted archive file
+        :param project_id: download project of current instance if project_id is None
+        :return: root path name of archive file
+        """
+        if project_id is None:
+            project = self.project
+        else:
+            project = self.gitlab_inst.projects.get(project_id)
+
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            try:
+                project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
+            except gitlab.GitlabGetError as e:
+                print("Failed to archive from project {}".format(project_id))
+                raise e
+
+        print("archive size: {:.03f}MB".format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
+
+        with tarfile.open(temp_file.name, "r") as archive_file:
+            root_name = archive_file.getnames()[0]
+            archive_file.extractall(destination)
+
+        return os.path.join(os.path.realpath(destination), root_name)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("action")
+    parser.add_argument("project_id", type=int)
+    parser.add_argument("--pipeline_id", "-i", type=int, default=None)
+    parser.add_argument("--ref", "-r", default="master")
+    parser.add_argument("--job_id", "-j", type=int, default=None)
+    parser.add_argument("--job_name", "-n", default=None)
+    parser.add_argument("--project_name", "-m", default=None)
+    parser.add_argument("--destination", "-d", default=None)
+    parser.add_argument("--artifact_path", "-a", nargs="*", default=None)
+    args = parser.parse_args()
+
+    gitlab_inst = Gitlab(args.project_id)
+    if args.action == "download_artifacts":
+        gitlab_inst.download_artifacts(args.job_id, args.destination)
+    if args.action == "download_artifact":
+        gitlab_inst.download_artifact(args.job_id, args.artifact_path, args.destination)
+    elif args.action == "find_job_id":
+        job_ids = gitlab_inst.find_job_id(args.job_name, args.pipeline_id)
+        print(";".join([",".join([str(j["id"]), j["parallel_num"]]) for j in job_ids]))
+    elif args.action == "download_archive":
+        gitlab_inst.download_archive(args.ref, args.destination)
+    elif args.action == "get_project_id":
+        ret = gitlab_inst.get_project_id(args.project_name)
+        print("project id: {}".format(ret))

examples/protocols/http_server/persistent_sockets/scripts/adder.py → tools/ci/python_packages/idf_http_server_test/adder.py

@@ -20,7 +20,8 @@ from builtins import str
 from builtins import range
 import http.client
 import argparse
-import Utility
+
+from tiny_test_fw import Utility
 
 
 def start_session(ip, port):

examples/protocols/http_server/simple/scripts/client.py → tools/ci/python_packages/idf_http_server_test/client.py

@@ -20,19 +20,8 @@ from builtins import str
 import http.client
 import argparse
 
-try:
-    import Utility
-except ImportError:
-    import sys
-    import os
-
-    # This environment variable is expected on the host machine
-    # > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-
-    import Utility
+
+from tiny_test_fw import Utility
 
 
 def verbose_print(verbosity, *args):

tools/ci/python_packages/idf_http_server_test/test.py

@@ -0,0 +1,1025 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Utility for testing the web server. Test cases:
+# Assume the device supports 'n' simultaneous open sockets
+#
+# HTTP Server Tests
+#
+# 0. Firmware Settings:
+# - Create a dormant thread whose sole job is to call httpd_stop() when instructed
+# - Measure the following before httpd_start() is called:
+#     - current free memory
+#     - current free sockets
+# - Measure the same whenever httpd_stop is called
+# - Register maximum possible URI handlers: should be successful
+# - Register one more URI handler: should fail
+# - Deregister on URI handler: should be successful
+# - Register on more URI handler: should succeed
+# - Register separate handlers for /hello, /hello/type_html. Also
+#   ensure that /hello/type_html is registered BEFORE  /hello. (tests
+#   that largest matching URI is picked properly)
+# - Create URI handler /adder. Make sure it uses a custom free_ctx
+#   structure to free it up
+
+# 1. Using Standard Python HTTP Client
+# - simple GET on /hello (returns Hello World. Ensures that basic
+#   firmware tests are complete, or returns error)
+# - POST on /hello (should fail)
+# - PUT on /hello (should fail)
+# - simple POST on /echo (returns whatever the POST data)
+# - simple PUT on /echo (returns whatever the PUT data)
+# - GET on /echo (should fail)
+# - simple GET on /hello/type_html (returns Content type as text/html)
+# - simple GET on /hello/status_500 (returns HTTP status 500)
+# - simple GET on /false_uri (returns HTTP status 404)
+# - largest matching URI handler is picked is already verified because
+#   of /hello and /hello/type_html tests
+#
+#
+# 2. Session Tests
+# - Sessions + Pipelining basics:
+#    - Create max supported sessions
+#    - On session i,
+#          - send 3 back-to-back POST requests with data i on /adder
+#          - read back 3 responses. They should be i, 2i and 3i
+#    - Tests that
+#          - pipelining works
+#          - per-session context is maintained for all supported
+#            sessions
+#    - Close all sessions
+#
+# - Cleanup leftover data: Tests that the web server properly cleans
+#   up leftover data
+#    - Create a session
+#    - POST on /leftover_data with 52 bytes of data (data includes
+#      \r\n)(the handler only
+#      reads first 10 bytes and returns them, leaving the rest of the
+#      bytes unread)
+#    - GET on /hello (should return 'Hello World')
+#    - POST on /false_uri with 52  bytes of data (data includes \r\n)
+#      (should return HTTP 404)
+#    - GET on /hello (should return 'Hello World')
+#
+# - Test HTTPd Asynchronous response
+#   - Create a session
+#   - GET on /async_data
+#   - returns 'Hello World!' as a response
+#   - the handler schedules an async response, which generates a second
+#     response 'Hello Double World!'
+#
+# - Spillover test
+#    - Create max supported sessions with the web server
+#    - GET /hello on all the sessions (should return Hello World)
+#    - Create one more session, this should fail
+#    - GET /hello on all the sessions (should return Hello World)
+#
+# - Timeout test
+#    - Create a session and only Send 'GE' on the same (simulates a
+#      client that left the network halfway through a request)
+#    - Wait for recv-wait-timeout
+#    - Server should automatically close the socket
+
+
+# ############ TODO TESTS #############
+
+# 3. Stress Tests
+#
+# - httperf
+#     - Run the following httperf command:
+#     httperf --server=10.31.130.126 --wsess=8,50,0.5 --rate 8 --burst-length 2
+#
+#     - The above implies that the test suite will open
+#           - 8 simultaneous connections with the server
+#           - the rate of opening the sessions will be 8 per sec. So in our
+#             case, a new connection will be opened every 0.2 seconds for 1 second
+#           - The burst length 2 indicates that 2 requests will be sent
+#             simultaneously on the same connection in a single go
+#           - 0.5 seconds is the time between sending out 2 bursts
+#           - 50 is the total number of requests that will be sent out
+#
+#     - So in the above example, the test suite will open 8
+#       connections, each separated by 0.2 seconds. On each connection
+#       it will send 2 requests in a single burst. The bursts on a
+#       single connection will be separated by 0.5 seconds. A total of
+#       25 bursts (25 x 2 = 50) will be sent out
+
+# 4. Leak Tests
+# - Simple Leak test
+#    - Simple GET on /hello/restart (returns success, stop web server, measures leaks, restarts webserver)
+#    - Simple GET on /hello/restart_results (returns the leak results)
+# - Leak test with open sockets
+#    - Open 8 sessions
+#    - Simple GET on /hello/restart (returns success, stop web server,
+#      measures leaks, restarts webserver)
+#    - All sockets should get closed
+#    - Simple GET on /hello/restart_results (returns the leak results)
+
+
+from __future__ import division
+from __future__ import print_function
+from builtins import str
+from builtins import range
+from builtins import object
+import threading
+import socket
+import time
+import argparse
+import http.client
+import sys
+import string
+import random
+
+from tiny_test_fw import Utility
+
+_verbose_ = False
+
+
+class Session(object):
+    def __init__(self, addr, port, timeout=15):
+        self.client = socket.create_connection((addr, int(port)), timeout=timeout)
+        self.target = addr
+        self.status = 0
+        self.encoding = ''
+        self.content_type = ''
+        self.content_len = 0
+
+    def send_err_check(self, request, data=None):
+        rval = True
+        try:
+            self.client.sendall(request.encode())
+            if data:
+                self.client.sendall(data.encode())
+        except socket.error as err:
+            self.client.close()
+            Utility.console_log("Socket Error in send :", err)
+            rval = False
+        return rval
+
+    def send_get(self, path, headers=None):
+        request = "GET " + path + " HTTP/1.1\r\nHost: " + self.target
+        if headers:
+            for field, value in headers.items():
+                request += "\r\n" + field + ": " + value
+        request += "\r\n\r\n"
+        return self.send_err_check(request)
+
+    def send_put(self, path, data, headers=None):
+        request = "PUT " + path + " HTTP/1.1\r\nHost: " + self.target
+        if headers:
+            for field, value in headers.items():
+                request += "\r\n" + field + ": " + value
+        request += "\r\nContent-Length: " + str(len(data)) + "\r\n\r\n"
+        return self.send_err_check(request, data)
+
+    def send_post(self, path, data, headers=None):
+        request = "POST " + path + " HTTP/1.1\r\nHost: " + self.target
+        if headers:
+            for field, value in headers.items():
+                request += "\r\n" + field + ": " + value
+        request += "\r\nContent-Length: " + str(len(data)) + "\r\n\r\n"
+        return self.send_err_check(request, data)
+
+    def read_resp_hdrs(self):
+        try:
+            state = 'nothing'
+            resp_read = ''
+            while True:
+                char = self.client.recv(1).decode()
+                if char == '\r' and state == 'nothing':
+                    state = 'first_cr'
+                elif char == '\n' and state == 'first_cr':
+                    state = 'first_lf'
+                elif char == '\r' and state == 'first_lf':
+                    state = 'second_cr'
+                elif char == '\n' and state == 'second_cr':
+                    state = 'second_lf'
+                else:
+                    state = 'nothing'
+                resp_read += char
+                if state == 'second_lf':
+                    break
+            # Handle first line
+            line_hdrs = resp_read.splitlines()
+            line_comp = line_hdrs[0].split()
+            self.status = line_comp[1]
+            del line_hdrs[0]
+            self.encoding = ''
+            self.content_type = ''
+            headers = dict()
+            # Process other headers
+            for h in range(len(line_hdrs)):
+                line_comp = line_hdrs[h].split(':')
+                if line_comp[0] == 'Content-Length':
+                    self.content_len = int(line_comp[1])
+                if line_comp[0] == 'Content-Type':
+                    self.content_type = line_comp[1].lstrip()
+                if line_comp[0] == 'Transfer-Encoding':
+                    self.encoding = line_comp[1].lstrip()
+                if len(line_comp) == 2:
+                    headers[line_comp[0]] = line_comp[1].lstrip()
+            return headers
+        except socket.error as err:
+            self.client.close()
+            Utility.console_log("Socket Error in recv :", err)
+            return None
+
+    def read_resp_data(self):
+        try:
+            read_data = ''
+            if self.encoding != 'chunked':
+                while len(read_data) != self.content_len:
+                    read_data += self.client.recv(self.content_len).decode()
+            else:
+                chunk_data_buf = ''
+                while (True):
+                    # Read one character into temp  buffer
+                    read_ch = self.client.recv(1)
+                    # Check CRLF
+                    if (read_ch == '\r'):
+                        read_ch = self.client.recv(1).decode()
+                        if (read_ch == '\n'):
+                            # If CRLF decode length of chunk
+                            chunk_len = int(chunk_data_buf, 16)
+                            # Keep adding to contents
+                            self.content_len += chunk_len
+                            rem_len = chunk_len
+                            while (rem_len):
+                                new_data = self.client.recv(rem_len)
+                                read_data += new_data
+                                rem_len -= len(new_data)
+                            chunk_data_buf = ''
+                            # Fetch remaining CRLF
+                            if self.client.recv(2) != "\r\n":
+                                # Error in packet
+                                Utility.console_log("Error in chunked data")
+                                return None
+                            if not chunk_len:
+                                # If last chunk
+                                break
+                            continue
+                        chunk_data_buf += '\r'
+                    # If not CRLF continue appending
+                    # character to chunked data buffer
+                    chunk_data_buf += read_ch
+            return read_data
+        except socket.error as err:
+            self.client.close()
+            Utility.console_log("Socket Error in recv :", err)
+            return None
+
+    def close(self):
+        self.client.close()
+
+
+def test_val(text, expected, received):
+    if expected != received:
+        Utility.console_log(" Fail!")
+        Utility.console_log("  [reason] " + text + ":")
+        Utility.console_log("        expected: " + str(expected))
+        Utility.console_log("        received: " + str(received))
+        return False
+    return True
+
+
+class adder_thread (threading.Thread):
+    def __init__(self, id, dut, port):
+        threading.Thread.__init__(self)
+        self.id = id
+        self.dut = dut
+        self.depth = 3
+        self.session = Session(dut, port)
+
+    def run(self):
+        self.response = []
+
+        # Pipeline 3 requests
+        if (_verbose_):
+            Utility.console_log("   Thread: Using adder start " + str(self.id))
+
+        for _ in range(self.depth):
+            self.session.send_post('/adder', str(self.id))
+            time.sleep(2)
+
+        for _ in range(self.depth):
+            self.session.read_resp_hdrs()
+            self.response.append(self.session.read_resp_data())
+
+    def adder_result(self):
+        if len(self.response) != self.depth:
+            Utility.console_log("Error : missing response packets")
+            return False
+        for i in range(len(self.response)):
+            if not test_val("Thread" + str(self.id) + " response[" + str(i) + "]",
+                            str(self.id * (i + 1)), str(self.response[i])):
+                return False
+        return True
+
+    def close(self):
+        self.session.close()
+
+
+def get_hello(dut, port):
+    # GET /hello should return 'Hello World!'
+    Utility.console_log("[test] GET /hello returns 'Hello World!' =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("GET", "/hello")
+    resp = conn.getresponse()
+    if not test_val("status_code", 200, resp.status):
+        conn.close()
+        return False
+    if not test_val("data", "Hello World!", resp.read().decode()):
+        conn.close()
+        return False
+    if not test_val("data", "text/html", resp.getheader('Content-Type')):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def put_hello(dut, port):
+    # PUT /hello returns 405'
+    Utility.console_log("[test] PUT /hello returns 405 =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("PUT", "/hello", "Hello")
+    resp = conn.getresponse()
+    if not test_val("status_code", 405, resp.status):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def post_hello(dut, port):
+    # POST /hello returns 405'
+    Utility.console_log("[test] POST /hello returns 405 =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("POST", "/hello", "Hello")
+    resp = conn.getresponse()
+    if not test_val("status_code", 405, resp.status):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def post_echo(dut, port):
+    # POST /echo echoes data'
+    Utility.console_log("[test] POST /echo echoes data =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("POST", "/echo", "Hello")
+    resp = conn.getresponse()
+    if not test_val("status_code", 200, resp.status):
+        conn.close()
+        return False
+    if not test_val("data", "Hello", resp.read().decode()):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def put_echo(dut, port):
+    # PUT /echo echoes data'
+    Utility.console_log("[test] PUT /echo echoes data =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("PUT", "/echo", "Hello")
+    resp = conn.getresponse()
+    if not test_val("status_code", 200, resp.status):
+        conn.close()
+        return False
+    if not test_val("data", "Hello", resp.read().decode()):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def get_echo(dut, port):
+    # GET /echo returns 404'
+    Utility.console_log("[test] GET /echo returns 405 =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("GET", "/echo")
+    resp = conn.getresponse()
+    if not test_val("status_code", 405, resp.status):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def get_test_headers(dut, port):
+    # GET /test_header returns data of Header2'
+    Utility.console_log("[test] GET /test_header =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    custom_header = {"Header1": "Value1", "Header3": "Value3"}
+    header2_values = ["", "  ", "Value2", "   Value2", "Value2  ", "  Value2  "]
+    for val in header2_values:
+        custom_header["Header2"] = val
+        conn.request("GET", "/test_header", headers=custom_header)
+        resp = conn.getresponse()
+        if not test_val("status_code", 200, resp.status):
+            conn.close()
+            return False
+        hdr_val_start_idx = val.find("Value2")
+        if hdr_val_start_idx == -1:
+            if not test_val("header: Header2", "", resp.read().decode()):
+                conn.close()
+                return False
+        else:
+            if not test_val("header: Header2", val[hdr_val_start_idx:], resp.read().decode()):
+                conn.close()
+                return False
+        resp.read()
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def get_hello_type(dut, port):
+    # GET /hello/type_html returns text/html as Content-Type'
+    Utility.console_log("[test] GET /hello/type_html has Content-Type of text/html =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("GET", "/hello/type_html")
+    resp = conn.getresponse()
+    if not test_val("status_code", 200, resp.status):
+        conn.close()
+        return False
+    if not test_val("data", "Hello World!", resp.read().decode()):
+        conn.close()
+        return False
+    if not test_val("data", "text/html", resp.getheader('Content-Type')):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def get_hello_status(dut, port):
+    # GET /hello/status_500 returns status 500'
+    Utility.console_log("[test] GET /hello/status_500 returns status 500 =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("GET", "/hello/status_500")
+    resp = conn.getresponse()
+    if not test_val("status_code", 500, resp.status):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def get_false_uri(dut, port):
+    # GET /false_uri returns status 404'
+    Utility.console_log("[test] GET /false_uri returns status 404 =>", end=' ')
+    conn = http.client.HTTPConnection(dut, int(port), timeout=15)
+    conn.request("GET", "/false_uri")
+    resp = conn.getresponse()
+    if not test_val("status_code", 404, resp.status):
+        conn.close()
+        return False
+    Utility.console_log("Success")
+    conn.close()
+    return True
+
+
+def parallel_sessions_adder(dut, port, max_sessions):
+    # POSTs on /adder in parallel sessions
+    Utility.console_log("[test] POST {pipelined} on /adder in " + str(max_sessions) + " sessions =>", end=' ')
+    t = []
+    # Create all sessions
+    for i in range(max_sessions):
+        t.append(adder_thread(i, dut, port))
+
+    for i in range(len(t)):
+        t[i].start()
+
+    for i in range(len(t)):
+        t[i].join()
+
+    res = True
+    for i in range(len(t)):
+        if not test_val("Thread" + str(i) + " Failed", t[i].adder_result(), True):
+            res = False
+        t[i].close()
+    if (res):
+        Utility.console_log("Success")
+    return res
+
+
+def async_response_test(dut, port):
+    # Test that an asynchronous work is executed in the HTTPD's context
+    # This is tested by reading two responses over the same session
+    Utility.console_log("[test] Test HTTPD Work Queue (Async response) =>", end=' ')
+    s = Session(dut, port)
+
+    s.send_get('/async_data')
+    s.read_resp_hdrs()
+    if not test_val("First Response", "Hello World!", s.read_resp_data()):
+        s.close()
+        return False
+    s.read_resp_hdrs()
+    if not test_val("Second Response", "Hello Double World!", s.read_resp_data()):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def leftover_data_test(dut, port):
+    # Leftover data in POST is purged (valid and invalid URIs)
+    Utility.console_log("[test] Leftover data in POST is purged (valid and invalid URIs) =>", end=' ')
+    s = http.client.HTTPConnection(dut + ":" + port, timeout=15)
+
+    s.request("POST", url='/leftover_data', body="abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz")
+    resp = s.getresponse()
+    if not test_val("Partial data", "abcdefghij", resp.read().decode()):
+        s.close()
+        return False
+
+    s.request("GET", url='/hello')
+    resp = s.getresponse()
+    if not test_val("Hello World Data", "Hello World!", resp.read().decode()):
+        s.close()
+        return False
+
+    s.request("POST", url='/false_uri', body="abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz")
+    resp = s.getresponse()
+    if not test_val("False URI Status", str(404), str(resp.status)):
+        s.close()
+        return False
+    # socket would have been closed by server due to error
+    s.close()
+
+    s = http.client.HTTPConnection(dut + ":" + port, timeout=15)
+    s.request("GET", url='/hello')
+    resp = s.getresponse()
+    if not test_val("Hello World Data", "Hello World!", resp.read().decode()):
+        s.close()
+        return False
+
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def spillover_session(dut, port, max_sess):
+    # Session max_sess_sessions + 1 is rejected
+    Utility.console_log("[test] Session max_sess_sessions (" + str(max_sess) + ") + 1 is rejected =>", end=' ')
+    s = []
+    _verbose_ = True
+    for i in range(max_sess + 1):
+        if (_verbose_):
+            Utility.console_log("Executing " + str(i))
+        try:
+            a = http.client.HTTPConnection(dut + ":" + port, timeout=15)
+            a.request("GET", url='/hello')
+            resp = a.getresponse()
+            if not test_val("Connection " + str(i), "Hello World!", resp.read().decode()):
+                a.close()
+                break
+            s.append(a)
+        except Exception:
+            if (_verbose_):
+                Utility.console_log("Connection " + str(i) + " rejected")
+            a.close()
+            break
+
+    # Close open connections
+    for a in s:
+        a.close()
+
+    # Check if number of connections is equal to max_sess
+    Utility.console_log(["Fail","Success"][len(s) == max_sess])
+    return (len(s) == max_sess)
+
+
+def recv_timeout_test(dut, port):
+    Utility.console_log("[test] Timeout occurs if partial packet sent =>", end=' ')
+    s = Session(dut, port)
+    s.client.sendall(b"GE")
+    s.read_resp_hdrs()
+    resp = s.read_resp_data()
+    if not test_val("Request Timeout", "Server closed this connection", resp):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def packet_size_limit_test(dut, port, test_size):
+    Utility.console_log("[test] send size limit test =>", end=' ')
+    retry = 5
+    while (retry):
+        retry -= 1
+        Utility.console_log("data size = ", test_size)
+        s = http.client.HTTPConnection(dut + ":" + port, timeout=15)
+        random_data = ''.join(string.printable[random.randint(0,len(string.printable)) - 1] for _ in list(range(test_size)))
+        path = "/echo"
+        s.request("POST", url=path, body=random_data)
+        resp = s.getresponse()
+        if not test_val("Error", "200", str(resp.status)):
+            if test_val("Error", "500", str(resp.status)):
+                Utility.console_log("Data too large to be allocated")
+                test_size = test_size // 10
+            else:
+                Utility.console_log("Unexpected error")
+            s.close()
+            Utility.console_log("Retry...")
+            continue
+        resp = resp.read().decode()
+        result = (resp == random_data)
+        if not result:
+            test_val("Data size", str(len(random_data)), str(len(resp)))
+            s.close()
+            Utility.console_log("Retry...")
+            continue
+        s.close()
+        Utility.console_log("Success")
+        return True
+    Utility.console_log("Failed")
+    return False
+
+
+def arbitrary_termination_test(dut, port):
+    Utility.console_log("[test] Arbitrary termination test =>", end=' ')
+    cases = [
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\r\n\r\n",
+            "code": "200",
+            "header": "SomeValue"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\nHost: " + dut + "\r\nCustom: SomeValue\r\n\r\n",
+            "code": "200",
+            "header": "SomeValue"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\nCustom: SomeValue\r\n\r\n",
+            "code": "200",
+            "header": "SomeValue"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\n\r\n",
+            "code": "200",
+            "header": "SomeValue"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\r\n\n",
+            "code": "200",
+            "header": "SomeValue"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\nHost: " + dut + "\nCustom: SomeValue\n\n",
+            "code": "200",
+            "header": "SomeValue"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\r\nABCDE",
+            "code": "200",
+            "body": "ABCDE"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\r\n\nABCDE",
+            "code": "200",
+            "body": "ABCDE"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\nABCDE",
+            "code": "200",
+            "body": "ABCDE"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\n\rABCD",
+            "code": "200",
+            "body": "\rABCD"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\r\nCustom: SomeValue\r\r\n\r\r\n",
+            "code": "400"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\r\nHost: " + dut + "\r\n\r\n",
+            "code": "400"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\n\rHost: " + dut + "\r\n\r\n",
+            "code": "400"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\rCustom: SomeValue\r\n",
+            "code": "400"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: Some\rValue\r\n",
+            "code": "400"
+        },
+        {
+            "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom- SomeValue\r\n\r\n",
+            "code": "400"
+        }
+    ]
+    for case in cases:
+        s = Session(dut, port)
+        s.client.sendall((case['request']).encode())
+        resp_hdrs = s.read_resp_hdrs()
+        resp_body = s.read_resp_data()
+        s.close()
+        if not test_val("Response Code", case["code"], s.status):
+            return False
+        if "header" in case.keys():
+            resp_hdr_val = None
+            if "Custom" in resp_hdrs.keys():
+                resp_hdr_val = resp_hdrs["Custom"]
+            if not test_val("Response Header", case["header"], resp_hdr_val):
+                return False
+        if "body" in case.keys():
+            if not test_val("Response Body", case["body"], resp_body):
+                return False
+    Utility.console_log("Success")
+    return True
+
+
+def code_500_server_error_test(dut, port):
+    Utility.console_log("[test] 500 Server Error test =>", end=' ')
+    s = Session(dut, port)
+    # Sending a very large content length will cause malloc to fail
+    content_len = 2**30
+    s.client.sendall(("POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: " + str(content_len) + "\r\n\r\nABCD").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Server Error", "500", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_501_method_not_impl(dut, port):
+    Utility.console_log("[test] 501 Method Not Implemented =>", end=' ')
+    s = Session(dut, port)
+    path = "/hello"
+    s.client.sendall(("ABC " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    # Presently server sends back 400 Bad Request
+    # if not test_val("Server Error", "501", s.status):
+    #     s.close()
+    #     return False
+    if not test_val("Server Error", "400", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_505_version_not_supported(dut, port):
+    Utility.console_log("[test] 505 Version Not Supported =>", end=' ')
+    s = Session(dut, port)
+    path = "/hello"
+    s.client.sendall(("GET " + path + " HTTP/2.0\r\nHost: " + dut + "\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Server Error", "505", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_400_bad_request(dut, port):
+    Utility.console_log("[test] 400 Bad Request =>", end=' ')
+    s = Session(dut, port)
+    path = "/hello"
+    s.client.sendall(("XYZ " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Client Error", "400", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_404_not_found(dut, port):
+    Utility.console_log("[test] 404 Not Found =>", end=' ')
+    s = Session(dut, port)
+    path = "/dummy"
+    s.client.sendall(("GET " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Client Error", "404", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_405_method_not_allowed(dut, port):
+    Utility.console_log("[test] 405 Method Not Allowed =>", end=' ')
+    s = Session(dut, port)
+    path = "/hello"
+    s.client.sendall(("POST " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Client Error", "405", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_408_req_timeout(dut, port):
+    Utility.console_log("[test] 408 Request Timeout =>", end=' ')
+    s = Session(dut, port)
+    s.client.sendall(("POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 10\r\n\r\nABCD").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Client Error", "408", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def code_411_length_required(dut, port):
+    Utility.console_log("[test] 411 Length Required =>", end=' ')
+    s = Session(dut, port)
+    path = "/echo"
+    s.client.sendall(("POST " + path + " HTTP/1.1\r\nHost: " + dut + "\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    # Presently server sends back 400 Bad Request
+    # if not test_val("Client Error", "411", s.status):
+    #    s.close()
+    #    return False
+    if not test_val("Client Error", "400", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+def send_getx_uri_len(dut, port, length):
+    s = Session(dut, port)
+    method = "GET "
+    version = " HTTP/1.1\r\n"
+    path = "/" + "x" * (length - len(method) - len(version) - len("/"))
+    s.client.sendall(method.encode())
+    time.sleep(1)
+    s.client.sendall(path.encode())
+    time.sleep(1)
+    s.client.sendall((version + "Host: " + dut + "\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    s.close()
+    return s.status
+
+
+def code_414_uri_too_long(dut, port, max_uri_len):
+    Utility.console_log("[test] 414 URI Too Long =>", end=' ')
+    status = send_getx_uri_len(dut, port, max_uri_len)
+    if not test_val("Client Error", "404", status):
+        return False
+    status = send_getx_uri_len(dut, port, max_uri_len + 1)
+    if not test_val("Client Error", "414", status):
+        return False
+    Utility.console_log("Success")
+    return True
+
+
+def send_postx_hdr_len(dut, port, length):
+    s = Session(dut, port)
+    path = "/echo"
+    host = "Host: " + dut
+    custom_hdr_field = "\r\nCustom: "
+    custom_hdr_val = "x" * (length - len(host) - len(custom_hdr_field) - len("\r\n\r\n") + len("0"))
+    request = ("POST " + path + " HTTP/1.1\r\n" + host + custom_hdr_field + custom_hdr_val + "\r\n\r\n").encode()
+    s.client.sendall(request[:length // 2])
+    time.sleep(1)
+    s.client.sendall(request[length // 2:])
+    hdr = s.read_resp_hdrs()
+    resp = s.read_resp_data()
+    s.close()
+    if hdr and ("Custom" in hdr):
+        return (hdr["Custom"] == custom_hdr_val), resp
+    return False, s.status
+
+
+def code_431_hdr_too_long(dut, port, max_hdr_len):
+    Utility.console_log("[test] 431 Header Too Long =>", end=' ')
+    res, status = send_postx_hdr_len(dut, port, max_hdr_len)
+    if not res:
+        return False
+    res, status = send_postx_hdr_len(dut, port, max_hdr_len + 1)
+    if not test_val("Client Error", "431", status):
+        return False
+    Utility.console_log("Success")
+    return True
+
+
+def test_upgrade_not_supported(dut, port):
+    Utility.console_log("[test] Upgrade Not Supported =>", end=' ')
+    s = Session(dut, port)
+    # path = "/hello"
+    s.client.sendall(("OPTIONS * HTTP/1.1\r\nHost:" + dut + "\r\nUpgrade: TLS/1.0\r\nConnection: Upgrade\r\n\r\n").encode())
+    s.read_resp_hdrs()
+    s.read_resp_data()
+    if not test_val("Client Error", "400", s.status):
+        s.close()
+        return False
+    s.close()
+    Utility.console_log("Success")
+    return True
+
+
+if __name__ == '__main__':
+    # Execution begins here...
+    # Configuration
+    # Max number of threads/sessions
+    max_sessions = 7
+    max_uri_len = 512
+    max_hdr_len = 512
+
+    parser = argparse.ArgumentParser(description='Run HTTPD Test')
+    parser.add_argument('-4','--ipv4', help='IPv4 address')
+    parser.add_argument('-6','--ipv6', help='IPv6 address')
+    parser.add_argument('-p','--port', help='Port')
+    args = vars(parser.parse_args())
+
+    dut4 = args['ipv4']
+    dut6 = args['ipv6']
+    port = args['port']
+    dut  = dut4
+
+    _verbose_ = True
+
+    Utility.console_log("### Basic HTTP Client Tests")
+    get_hello(dut, port)
+    post_hello(dut, port)
+    put_hello(dut, port)
+    post_echo(dut, port)
+    get_echo(dut, port)
+    put_echo(dut, port)
+    get_hello_type(dut, port)
+    get_hello_status(dut, port)
+    get_false_uri(dut, port)
+    get_test_headers(dut, port)
+
+    Utility.console_log("### Error code tests")
+    code_500_server_error_test(dut, port)
+    code_501_method_not_impl(dut, port)
+    code_505_version_not_supported(dut, port)
+    code_400_bad_request(dut, port)
+    code_404_not_found(dut, port)
+    code_405_method_not_allowed(dut, port)
+    code_408_req_timeout(dut, port)
+    code_414_uri_too_long(dut, port, max_uri_len)
+    code_431_hdr_too_long(dut, port, max_hdr_len)
+    test_upgrade_not_supported(dut, port)
+
+    # Not supported yet (Error on chunked request)
+    # code_411_length_required(dut, port)
+
+    Utility.console_log("### Sessions and Context Tests")
+    parallel_sessions_adder(dut, port, max_sessions)
+    leftover_data_test(dut, port)
+    async_response_test(dut, port)
+    spillover_session(dut, port, max_sessions)
+    recv_timeout_test(dut, port)
+    packet_size_limit_test(dut, port, 50 * 1024)
+    arbitrary_termination_test(dut, port)
+    get_hello(dut, port)
+
+    sys.exit()

tools/tiny-test-fw/Utility/CIAssignTest.py → tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -44,14 +44,13 @@ import re
 import json
 
 import yaml
-
-from Utility import (CaseConfig, SearchCases, GitlabCIJob, console_log)
-
 try:
     from yaml import CLoader as Loader
 except ImportError:
     from yaml import Loader as Loader
 
+from . import (CaseConfig, SearchCases, GitlabCIJob, console_log)
+
 
 class Group(object):
 

tools/tiny-test-fw/Utility/CaseConfig.py → tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py

@@ -20,7 +20,7 @@ Template Config File::
 
     TestConfig:
       app:
-        path: Users/Test/TinyTestFW/IDF/IDFApp.py
+        package: ttfw_idf
         class: Example
       dut:
         path:
@@ -38,22 +38,20 @@ Template Config File::
         extra_data: some extra data passed to case with kwarg extra_data
         overwrite:  # overwrite test configs
           app:
-            path: Users/Test/TinyTestFW/IDF/IDFApp.py
+            package: ttfw_idf
             class: Example
       - name: xxx
 """
+import importlib
 
 import yaml
-
-import TestCase
-
-from Utility import load_source
-
 try:
     from yaml import CLoader as Loader
 except ImportError:
     from yaml import Loader as Loader
 
+from . import TestCase
+
 
 def _convert_to_lower_case_bytes(item):
     """
@@ -172,8 +170,7 @@ class Parser(object):
         """
         output = dict()
         for key in overwrite:
-            path = overwrite[key]["path"]
-            module = load_source(path)
+            module = importlib.import_module(overwrite[key]["package"])
             output[key] = module.__getattribute__(overwrite[key]["class"])
         return output
 

tools/tiny-test-fw/Utility/SearchCases.py → tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -17,7 +17,8 @@ import os
 import fnmatch
 import types
 import copy
-from Utility import load_source
+
+from . import load_source
 
 
 class Search(object):

tools/tiny-test-fw/Runner.py → tools/ci/python_packages/tiny_test_fw/bin/Runner.py

@@ -26,8 +26,8 @@ import sys
 import argparse
 import threading
 
-import TinyFW
-from Utility import SearchCases, CaseConfig
+from tiny_test_fw import TinyFW
+from tiny_test_fw.Utility import SearchCases, CaseConfig
 
 
 class Runner(threading.Thread):

tools/tiny-test-fw/requirements.txt → tools/ci/python_packages/tiny_test_fw/requirements.txt

@@ -2,4 +2,3 @@ pyserial
 pyyaml
 junit_xml
 netifaces
-matplotlib

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -0,0 +1,90 @@
+# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Command line tool to assign example tests to CI test jobs.
+"""
+
+# TODO: Need to handle running examples on different chips
+import os
+import re
+import argparse
+import json
+
+import gitlab_api
+from tiny_test_fw.Utility import CIAssignTest
+
+
+EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
+IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
+if IDF_PATH_FROM_ENV:
+    ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
+                                       "build_examples", "artifact_index.json")
+else:
+    ARTIFACT_INDEX_FILE = "artifact_index.json"
+
+
+class ExampleGroup(CIAssignTest.Group):
+    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
+
+
+class CIExampleAssignTest(CIAssignTest.AssignTest):
+    CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
+
+
+def create_artifact_index_file(project_id=None, pipeline_id=None):
+    if project_id is None:
+        project_id = os.getenv("CI_PROJECT_ID")
+    if pipeline_id is None:
+        pipeline_id = os.getenv("CI_PIPELINE_ID")
+    gitlab_inst = gitlab_api.Gitlab(project_id)
+    artifact_index_list = []
+
+    def format_build_log_path():
+        return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
+
+    for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
+        job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
+        for job_info in job_info_list:
+            raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
+            build_info_list = [json.loads(line) for line in raw_data.splitlines()]
+            for build_info in build_info_list:
+                build_info["ci_job_id"] = job_info["id"]
+                artifact_index_list.append(build_info)
+    try:
+        os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
+    except OSError:
+        # already created
+        pass
+
+    with open(ARTIFACT_INDEX_FILE, "w") as f:
+        json.dump(artifact_index_list, f)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("test_case",
+                        help="test case folder or file")
+    parser.add_argument("ci_config_file",
+                        help="gitlab ci config file")
+    parser.add_argument("output_path",
+                        help="output path of config files")
+    parser.add_argument("--pipeline_id", "-p", type=int, default=None,
+                        help="pipeline_id")
+    args = parser.parse_args()
+
+    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
+    assign_test.assign_cases()
+    assign_test.output_configs(args.output_path)
+    create_artifact_index_file()

tools/tiny-test-fw/CIAssignUnitTest.py → tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py

@@ -3,8 +3,6 @@ Command line tool to assign unit tests to CI test jobs.
 """
 
 import re
-import os
-import sys
 import argparse
 
 import yaml
@@ -14,13 +12,7 @@ try:
 except ImportError:
     from yaml import Loader as Loader
 
-try:
-    from Utility import CIAssignTest
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path:
-        sys.path.insert(0, test_fw_path)
-    from Utility import CIAssignTest
+from tiny_test_fw.Utility import CIAssignTest
 
 
 class Group(CIAssignTest.Group):
@@ -119,7 +111,7 @@ class Group(CIAssignTest.Group):
         if target:
             overwrite = {
                 "dut": {
-                    "path": "IDF/IDFDUT.py",
+                    "package": "ttfw_idf",
                     "class": self.DUT_CLS_NAME[target],
                 }
             }

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -0,0 +1,411 @@
+# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" IDF Test Applications """
+import subprocess
+
+import os
+import json
+
+from tiny_test_fw import App
+from . import CIAssignExampleTest
+
+try:
+    import gitlab_api
+except ImportError:
+    gitlab_api = None
+
+
+def parse_flash_settings(path):
+    file_name = os.path.basename(path)
+    if file_name == "flasher_args.json":
+        # CMake version using build metadata file
+        with open(path, "r") as f:
+            args = json.load(f)
+        flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""]
+        flash_settings = args["flash_settings"]
+        app_name = os.path.splitext(args["app"]["file"])[0]
+    else:
+        # GNU Make version uses download.config arguments file
+        with open(path, "r") as f:
+            args = f.readlines()[-1].split(" ")
+            flash_files = []
+            flash_settings = {}
+            for idx in range(0, len(args), 2):  # process arguments in pairs
+                if args[idx].startswith("--"):
+                    # strip the -- from the command line argument
+                    flash_settings[args[idx][2:]] = args[idx + 1]
+                else:
+                    # offs, filename
+                    flash_files.append((args[idx], args[idx + 1]))
+            # we can only guess app name in download.config.
+            for p in flash_files:
+                if not os.path.dirname(p[1]) and "partition" not in p[1]:
+                    # app bin usually in the same dir with download.config and it's not partition table
+                    app_name = os.path.splitext(p[1])[0]
+                    break
+            else:
+                app_name = None
+    return flash_files, flash_settings, app_name
+
+
+class Artifacts(object):
+    def __init__(self, dest_root_path, artifact_index_file, app_path, config_name, target):
+        assert gitlab_api
+        # at least one of app_path or config_name is not None. otherwise we can't match artifact
+        assert app_path or config_name
+        assert os.path.exists(artifact_index_file)
+        self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID"))
+        self.dest_root_path = dest_root_path
+        with open(artifact_index_file, "r") as f:
+            artifact_index = json.load(f)
+        self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
+
+    @staticmethod
+    def _find_artifact(artifact_index, app_path, config_name, target):
+        for artifact_info in artifact_index:
+            match_result = True
+            if app_path:
+                match_result = app_path in artifact_info["app_dir"]
+            if config_name:
+                match_result = match_result and config_name == artifact_info["config"]
+            if target:
+                match_result = match_result and target == artifact_info["target"]
+            if match_result:
+                ret = artifact_info
+                break
+        else:
+            ret = None
+        return ret
+
+    def download_artifacts(self):
+        if self.artifact_info:
+            base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
+            job_id = self.artifact_info["ci_job_id"]
+
+            # 1. download flash args file
+            if self.artifact_info["build_system"] == "cmake":
+                flash_arg_file = os.path.join(base_path, "flasher_args.json")
+            else:
+                flash_arg_file = os.path.join(base_path, "download.config")
+
+            self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
+
+            # 2. download all binary files
+            flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path,
+                                                                                      flash_arg_file))
+            artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
+            artifact_files.append(os.path.join(base_path, app_name + ".elf"))
+
+            self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
+
+            # 3. download sdkconfig file
+            self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
+                                               self.dest_root_path)
+        else:
+            base_path = None
+        return base_path
+
+    def download_artifact_files(self, file_names):
+        if self.artifact_info:
+            base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
+            job_id = self.artifact_info["ci_job_id"]
+
+            # download all binary files
+            artifact_files = [os.path.join(base_path, fn) for fn in file_names]
+            self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
+
+            # download sdkconfig file
+            self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
+                                               self.dest_root_path)
+        else:
+            base_path = None
+        return base_path
+
+
+class IDFApp(App.BaseApp):
+    """
+    Implements common esp-idf application behavior.
+    idf applications should inherent from this class and overwrite method get_binary_path.
+    """
+
+    IDF_DOWNLOAD_CONFIG_FILE = "download.config"
+    IDF_FLASH_ARGS_FILE = "flasher_args.json"
+
+    def __init__(self, app_path, config_name=None, target=None):
+        super(IDFApp, self).__init__(app_path)
+        self.config_name = config_name
+        self.target = target
+        self.idf_path = self.get_sdk_path()
+        self.binary_path = self.get_binary_path(app_path, config_name, target)
+        self.elf_file = self._get_elf_file_path(self.binary_path)
+        assert os.path.exists(self.binary_path)
+        if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
+            if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path):
+                msg = ("Neither {} nor {} exists. "
+                       "Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' "
+                       "or 'idf.py build' "
+                       "for resolving the issue."
+                       "").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
+                                  self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
+                raise AssertionError(msg)
+
+        self.flash_files, self.flash_settings = self._parse_flash_download_config()
+        self.partition_table = self._parse_partition_table()
+
+    @classmethod
+    def get_sdk_path(cls):
+        # type: () -> str
+        idf_path = os.getenv("IDF_PATH")
+        assert idf_path
+        assert os.path.exists(idf_path)
+        return idf_path
+
+    def _get_sdkconfig_paths(self):
+        """
+        returns list of possible paths where sdkconfig could be found
+
+        Note: could be overwritten by a derived class to provide other locations or order
+        """
+        return [os.path.join(self.binary_path, "sdkconfig"), os.path.join(self.binary_path, "..", "sdkconfig")]
+
+    def get_sdkconfig(self):
+        """
+        reads sdkconfig and returns a dictionary with all configuredvariables
+
+        :raise: AssertionError: if sdkconfig file does not exist in defined paths
+        """
+        d = {}
+        sdkconfig_file = None
+        for i in self._get_sdkconfig_paths():
+            if os.path.exists(i):
+                sdkconfig_file = i
+                break
+        assert sdkconfig_file is not None
+        with open(sdkconfig_file) as f:
+            for line in f:
+                configs = line.split('=')
+                if len(configs) == 2:
+                    d[configs[0]] = configs[1].rstrip()
+        return d
+
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        # type: (str, str, str) -> str
+        """
+        get binary path according to input app_path.
+
+        subclass must overwrite this method.
+
+        :param app_path: path of application
+        :param config_name: name of the application build config. Will match any config if None
+        :param target: target name. Will match for target if None
+        :return: abs app binary path
+        """
+        pass
+
+    @staticmethod
+    def _get_elf_file_path(binary_path):
+        ret = ""
+        file_names = os.listdir(binary_path)
+        for fn in file_names:
+            if os.path.splitext(fn)[1] == ".elf":
+                ret = os.path.join(binary_path, fn)
+        return ret
+
+    def _parse_flash_download_config(self):
+        """
+        Parse flash download config from build metadata files
+
+        Sets self.flash_files, self.flash_settings
+
+        (Called from constructor)
+
+        Returns (flash_files, flash_settings)
+        """
+
+        if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path):
+            # CMake version using build metadata file
+            path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE)
+        else:
+            # GNU Make version uses download.config arguments file
+            path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
+
+        flash_files, flash_settings, app_name = parse_flash_settings(path)
+        # The build metadata file does not currently have details, which files should be encrypted and which not.
+        # Assume that all files should be encrypted if flash encryption is enabled in development mode.
+        sdkconfig_dict = self.get_sdkconfig()
+        flash_settings["encrypt"] = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
+
+        # make file offsets into integers, make paths absolute
+        flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files]
+
+        return flash_files, flash_settings
+
+    def _parse_partition_table(self):
+        """
+        Parse partition table contents based on app binaries
+
+        Returns partition_table data
+
+        (Called from constructor)
+        """
+        partition_tool = os.path.join(self.idf_path,
+                                      "components",
+                                      "partition_table",
+                                      "gen_esp32part.py")
+        assert os.path.exists(partition_tool)
+
+        for (_, path) in self.flash_files:
+            if "partition" in path:
+                partition_file = os.path.join(self.binary_path, path)
+                break
+        else:
+            raise ValueError("No partition table found for IDF binary path: {}".format(self.binary_path))
+
+        process = subprocess.Popen(["python", partition_tool, partition_file],
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        raw_data = process.stdout.read()
+        if isinstance(raw_data, bytes):
+            raw_data = raw_data.decode()
+        partition_table = dict()
+
+        for line in raw_data.splitlines():
+            if line[0] != "#":
+                try:
+                    _name, _type, _subtype, _offset, _size, _flags = line.split(",")
+                    if _size[-1] == "K":
+                        _size = int(_size[:-1]) * 1024
+                    elif _size[-1] == "M":
+                        _size = int(_size[:-1]) * 1024 * 1024
+                    else:
+                        _size = int(_size)
+                except ValueError:
+                    continue
+                partition_table[_name] = {
+                    "type": _type,
+                    "subtype": _subtype,
+                    "offset": _offset,
+                    "size": _size,
+                    "flags": _flags
+                }
+
+        return partition_table
+
+
+class Example(IDFApp):
+    def _get_sdkconfig_paths(self):
+        """
+        overrides the parent method to provide exact path of sdkconfig for example tests
+        """
+        return [os.path.join(self.binary_path, "..", "sdkconfig")]
+
+    def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None):
+        # build folder of example path
+        path = os.path.join(self.idf_path, app_path, "build")
+        if os.path.exists(path):
+            return path
+
+        if not config_name:
+            config_name = "default"
+
+        # Search for CI build folders.
+        # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
+        # (see tools/ci/build_examples_cmake.sh)
+        # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
+        app_path_underscored = app_path.replace(os.path.sep, "_")
+        example_path = os.path.join(self.idf_path, "build_examples")
+        for dirpath in os.listdir(example_path):
+            if os.path.basename(dirpath) == app_path_underscored:
+                path = os.path.join(example_path, dirpath, config_name, target, "build")
+                if os.path.exists(path):
+                    return path
+                else:
+                    return None
+
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        path = self._try_get_binary_from_local_fs(app_path, config_name, target)
+        if path:
+            return path
+        else:
+            artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+                                  app_path, config_name, target)
+            path = artifacts.download_artifacts()
+            if path:
+                return os.path.join(self.idf_path, path)
+            else:
+                raise OSError("Failed to find example binary")
+
+
+class LoadableElfExample(Example):
+    def __init__(self, app_path, app_files, config_name=None, target=None):
+        # add arg `app_files` for loadable elf example.
+        # Such examples only build elf files, so it doesn't generate flasher_args.json.
+        # So we can't get app files from config file. Test case should pass it to application.
+        super(IDFApp, self).__init__(app_path)
+        self.app_files = app_files
+        self.config_name = config_name
+        self.target = target
+        self.idf_path = self.get_sdk_path()
+        self.binary_path = self.get_binary_path(app_path, config_name, target)
+        self.elf_file = self._get_elf_file_path(self.binary_path)
+        assert os.path.exists(self.binary_path)
+
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        path = self._try_get_binary_from_local_fs(app_path, config_name, target)
+        if path:
+            return path
+        else:
+            artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+                                  app_path, config_name, target)
+            path = artifacts.download_artifact_files(self.app_files)
+            if path:
+                return os.path.join(self.idf_path, path)
+            else:
+                raise OSError("Failed to find example binary")
+
+
+class UT(IDFApp):
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        if not config_name:
+            config_name = "default"
+
+        path = os.path.join(self.idf_path, app_path)
+        default_build_path = os.path.join(path, "build")
+        if os.path.exists(default_build_path):
+            return path
+
+        # first try to get from build folder of unit-test-app
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
+        if os.path.exists(path):
+            # found, use bin in build path
+            return path
+
+        # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
+        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", config_name)
+        if os.path.exists(path):
+            return path
+
+        raise OSError("Failed to get unit-test-app binary path")
+
+
+class SSC(IDFApp):
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        # TODO: to implement SSC get binary path
+        return app_path
+
+
+class AT(IDFApp):
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        # TODO: to implement AT get binary path
+        return app_path

tools/tiny-test-fw/IDF/IDFDUT.py → tools/ci/python_packages/ttfw_idf/IDFDUT.py

@@ -30,8 +30,7 @@ except ImportError:
 
 from serial.tools import list_ports
 
-import DUT
-import Utility
+from tiny_test_fw import DUT, Utility
 
 try:
     import esptool

tools/tiny-test-fw/IDF/__init__.py → tools/ci/python_packages/ttfw_idf/__init__.py

@@ -14,10 +14,9 @@
 import os
 import re
 
-import TinyFW
-import Utility
-from IDF.IDFApp import IDFApp, Example, UT
-from IDF.IDFDUT import IDFDUT
+from tiny_test_fw import TinyFW, Utility
+from IDFApp import IDFApp, Example, LoadableElfExample, UT  # noqa: export all Apps for users
+from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT  # noqa: export DUTs for users
 
 
 def format_case_id(chip, case_name):

tools/ci/setup_python.sh

@@ -51,3 +51,6 @@ else
     echo 'No /opt/pyenv/activate exists and no Python interpreter is found!'
     exit 1
 fi
+
+# add esp-idf local package path to PYTHONPATH so it can be imported directly
+export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"

tools/cmake/git_submodules.cmake

@@ -11,6 +11,14 @@ if(NOT GIT_FOUND)
 else()
 
     function(git_submodule_check root_path)
+        # for internal use:
+        # skip submodule check if running on Gitlab CI and job is configured as not clone submodules
+        if($ENV{IDF_SKIP_CHECK_SUBMODULES})
+            if($ENV{IDF_SKIP_CHECK_SUBMODULES} EQUAL 1)
+                message("skip submodule check on internal CI")
+                return()
+            endif()
+        endif()
 
         execute_process(
             COMMAND ${GIT_EXECUTABLE} submodule status

tools/esp_prov/__init__.py

@@ -0,0 +1 @@
+from esp_prov import *  # noqa: export esp_prov module to users

tools/tiny-test-fw/CIAssignExampleTest.py

@@ -1,56 +0,0 @@
-# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Command line tool to assign example tests to CI test jobs.
-"""
-
-# TODO: Need to handle running examples on different chips
-import os
-import sys
-import re
-import argparse
-
-try:
-    from Utility.CIAssignTest import AssignTest
-except ImportError:
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path:
-        sys.path.insert(0, test_fw_path)
-    from Utility.CIAssignTest import AssignTest
-
-from Utility.CIAssignTest import Group
-
-
-class ExampleGroup(Group):
-    SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
-
-
-class CIExampleAssignTest(AssignTest):
-    CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("test_case",
-                        help="test case folder or file")
-    parser.add_argument("ci_config_file",
-                        help="gitlab ci config file")
-    parser.add_argument("output_path",
-                        help="output path of config files")
-    args = parser.parse_args()
-
-    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
-    assign_test.assign_cases()
-    assign_test.output_configs(args.output_path)

tools/tiny-test-fw/IDF/IDFApp.py

@@ -1,273 +0,0 @@
-# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-""" IDF Test Applications """
-import subprocess
-
-import os
-import json
-import App
-
-
-class IDFApp(App.BaseApp):
-    """
-    Implements common esp-idf application behavior.
-    idf applications should inherent from this class and overwrite method get_binary_path.
-    """
-
-    IDF_DOWNLOAD_CONFIG_FILE = "download.config"
-    IDF_FLASH_ARGS_FILE = "flasher_args.json"
-
-    def __init__(self, app_path, config_name=None, target=None):
-        super(IDFApp, self).__init__(app_path)
-        self.config_name = config_name
-        self.target = target
-        self.idf_path = self.get_sdk_path()
-        self.binary_path = self.get_binary_path(app_path, config_name)
-        self.elf_file = self._get_elf_file_path(self.binary_path)
-        assert os.path.exists(self.binary_path)
-        sdkconfig_dict = self.get_sdkconfig()
-        if "CONFIG_APP_BUILD_GENERATE_BINARIES" in sdkconfig_dict:
-            # There are no flashing targets available when no binaries where generated.
-            if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
-                if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path):
-                    msg = ("Neither {} nor {} exists. "
-                           "Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' "
-                           "or 'idf.py build' "
-                           "for resolving the issue."
-                           "").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
-                                      self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
-                    raise AssertionError(msg)
-
-            self.flash_files, self.flash_settings = self._parse_flash_download_config()
-            self.partition_table = self._parse_partition_table()
-
-    @classmethod
-    def get_sdk_path(cls):
-        idf_path = os.getenv("IDF_PATH")
-        assert idf_path
-        assert os.path.exists(idf_path)
-        return idf_path
-
-    def _get_sdkconfig_paths(self):
-        """
-        returns list of possible paths where sdkconfig could be found
-
-        Note: could be overwritten by a derived class to provide other locations or order
-        """
-        return [os.path.join(self.binary_path, "sdkconfig"), os.path.join(self.binary_path, "..", "sdkconfig")]
-
-    def get_sdkconfig(self):
-        """
-        reads sdkconfig and returns a dictionary with all configuredvariables
-
-        :param sdkconfig_file: location of sdkconfig
-        :raise: AssertionError: if sdkconfig file does not exist in defined paths
-        """
-        d = {}
-        sdkconfig_file = None
-        for i in self._get_sdkconfig_paths():
-            if os.path.exists(i):
-                sdkconfig_file = i
-                break
-        assert sdkconfig_file is not None
-        with open(sdkconfig_file) as f:
-            for line in f:
-                configs = line.split('=')
-                if len(configs) == 2:
-                    d[configs[0]] = configs[1].rstrip()
-        return d
-
-    def get_binary_path(self, app_path, config_name=None):
-        """
-        get binary path according to input app_path.
-
-        subclass must overwrite this method.
-
-        :param app_path: path of application
-        :param config_name: name of the application build config
-        :return: abs app binary path
-        """
-        pass
-
-    @staticmethod
-    def _get_elf_file_path(binary_path):
-        ret = ""
-        file_names = os.listdir(binary_path)
-        for fn in file_names:
-            if os.path.splitext(fn)[1] == ".elf":
-                ret = os.path.join(binary_path, fn)
-        return ret
-
-    def _parse_flash_download_config(self):
-        """
-        Parse flash download config from build metadata files
-
-        Sets self.flash_files, self.flash_settings
-
-        (Called from constructor)
-
-        Returns (flash_files, flash_settings)
-        """
-
-        if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path):
-            # CMake version using build metadata file
-            with open(os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE), "r") as f:
-                args = json.load(f)
-                flash_files = [(offs,file) for (offs,file) in args["flash_files"].items() if offs != ""]
-                flash_settings = args["flash_settings"]
-        else:
-            # GNU Make version uses download.config arguments file
-            with open(os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE), "r") as f:
-                args = f.readlines()[-1].split(" ")
-                flash_files = []
-                flash_settings = {}
-                for idx in range(0, len(args), 2):  # process arguments in pairs
-                    if args[idx].startswith("--"):
-                        # strip the -- from the command line argument
-                        flash_settings[args[idx][2:]] = args[idx + 1]
-                    else:
-                        # offs, filename
-                        flash_files.append((args[idx], args[idx + 1]))
-
-        # The build metadata file does not currently have details, which files should be encrypted and which not.
-        # Assume that all files should be encrypted if flash encryption is enabled in development mode.
-        sdkconfig_dict = self.get_sdkconfig()
-        flash_settings["encrypt"] = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
-
-        # make file offsets into integers, make paths absolute
-        flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files]
-
-        return (flash_files, flash_settings)
-
-    def _parse_partition_table(self):
-        """
-        Parse partition table contents based on app binaries
-
-        Returns partition_table data
-
-        (Called from constructor)
-        """
-        partition_tool = os.path.join(self.idf_path,
-                                      "components",
-                                      "partition_table",
-                                      "gen_esp32part.py")
-        assert os.path.exists(partition_tool)
-
-        for (_, path) in self.flash_files:
-            if "partition" in path:
-                partition_file = os.path.join(self.binary_path, path)
-                break
-        else:
-            raise ValueError("No partition table found for IDF binary path: {}".format(self.binary_path))
-
-        process = subprocess.Popen(["python", partition_tool, partition_file],
-                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        raw_data = process.stdout.read()
-        if isinstance(raw_data, bytes):
-            raw_data = raw_data.decode()
-        partition_table = dict()
-
-        for line in raw_data.splitlines():
-            if line[0] != "#":
-                try:
-                    _name, _type, _subtype, _offset, _size, _flags = line.split(",")
-                    if _size[-1] == "K":
-                        _size = int(_size[:-1]) * 1024
-                    elif _size[-1] == "M":
-                        _size = int(_size[:-1]) * 1024 * 1024
-                    else:
-                        _size = int(_size)
-                except ValueError:
-                    continue
-                partition_table[_name] = {
-                    "type": _type,
-                    "subtype": _subtype,
-                    "offset": _offset,
-                    "size": _size,
-                    "flags": _flags
-                }
-
-        return partition_table
-
-
-class Example(IDFApp):
-    def _get_sdkconfig_paths(self):
-        """
-        overrides the parent method to provide exact path of sdkconfig for example tests
-        """
-        return [os.path.join(self.binary_path, "..", "sdkconfig")]
-
-    def get_binary_path(self, app_path, config_name=None):
-        # build folder of example path
-        path = os.path.join(self.idf_path, app_path, "build")
-        if os.path.exists(path):
-            return path
-
-        if not config_name:
-            config_name = "default"
-
-        # Search for CI build folders.
-        # Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
-        # (see tools/ci/build_examples_cmake.sh)
-        # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
-        app_path_underscored = app_path.replace(os.path.sep, "_")
-        example_path = os.path.join(self.idf_path, "build_examples")
-        for dirpath in os.listdir(example_path):
-            if os.path.basename(dirpath) == app_path_underscored:
-                path = os.path.join(example_path, dirpath, config_name, self.target, "build")
-                return path
-
-        raise OSError("Failed to find example binary")
-
-
-class UT(IDFApp):
-    def get_binary_path(self, app_path, config_name=None):
-        """
-        :param app_path: app path
-        :param config_name: config name
-        :return: binary path
-        """
-        if not config_name:
-            config_name = "default"
-
-        path = os.path.join(self.idf_path, app_path)
-        default_build_path = os.path.join(path, "build")
-        if os.path.exists(default_build_path):
-            return path
-
-        # first try to get from build folder of unit-test-app
-        path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
-        if os.path.exists(path):
-            # found, use bin in build path
-            return path
-
-        # ``make ut-build-all-configs`` or ``make ut-build-CONFIG`` will copy binary to output folder
-        path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", config_name)
-        if os.path.exists(path):
-            return path
-
-        raise OSError("Failed to get unit-test-app binary path")
-
-
-class SSC(IDFApp):
-    def get_binary_path(self, app_path, config_name=None):
-        # TODO: to implement SSC get binary path
-        return app_path
-
-
-class AT(IDFApp):
-    def get_binary_path(self, app_path, config_name=None):
-        # TODO: to implement AT get binary path
-        return app_path

tools/unit-test-app/unit_test.py

@@ -19,33 +19,12 @@ Test script for unit test case.
 """
 
 import re
-import os
-import sys
 import time
 import argparse
 import threading
 
-try:
-    import TinyFW
-except ImportError:
-    # if we want to run test case outside `tiny-test-fw` folder,
-    # we need to insert tiny-test-fw path into sys path
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    else:
-        # or try the copy in IDF
-        idf_path = os.getenv("IDF_PATH")
-        tiny_test_path = idf_path + "/tools/tiny-test-fw"
-        if os.path.exists(tiny_test_path):
-            sys.path.insert(0, tiny_test_path)
-    import TinyFW
-
-import IDF
-import Utility
-import Env
-from DUT import ExpectTimeout
-from IDF.IDFApp import UT
+from tiny_test_fw import TinyFW, Utility, Env, DUT
+import ttfw_idf
 
 
 UT_APP_BOOT_UP_DONE = "Press ENTER to see the list of tests."
@@ -180,7 +159,7 @@ def reset_dut(dut):
         try:
             dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
             break
-        except ExpectTimeout:
+        except DUT.ExpectTimeout:
             pass
     else:
         raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
@@ -255,14 +234,14 @@ def run_one_normal_case(dut, one_case, junit_test_case):
                            (FINISH_PATTERN, handle_test_finish),
                            (UT_APP_BOOT_UP_DONE, handle_reset_finish),
                            timeout=one_case["timeout"])
-        except ExpectTimeout:
+        except DUT.ExpectTimeout:
             Utility.console_log("Timeout in expect", color="orange")
             junit_test_case.add_failure_info("timeout")
             one_case_finish(False)
             break
 
 
-@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True)
+@ttfw_idf.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True)
 def run_unit_test_cases(env, extra_data):
     """
     extra_data can be three types of value
@@ -401,7 +380,7 @@ class Handler(threading.Thread):
             time.sleep(1)
             self.dut.write("\"{}\"".format(self.parent_case_name))
             self.dut.expect("Running " + self.parent_case_name + "...")
-        except ExpectTimeout:
+        except DUT.ExpectTimeout:
             Utility.console_log("No case detected!", color="orange")
         while not self.finish and not self.force_stop.isSet():
             try:
@@ -411,7 +390,7 @@ class Handler(threading.Thread):
                                     (self.SEND_SIGNAL_PATTERN, device_send_action),  # send signal pattern
                                     (self.FINISH_PATTERN, handle_device_test_finish),  # test finish pattern
                                     timeout=self.timeout)
-            except ExpectTimeout:
+            except DUT.ExpectTimeout:
                 Utility.console_log("Timeout in expect", color="orange")
                 one_device_case_finish(False)
                 break
@@ -471,7 +450,7 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, app_bin, junit
     return result
 
 
-@IDF.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True)
+@ttfw_idf.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True)
 def run_multiple_devices_cases(env, extra_data):
     """
      extra_data can be two types of value
@@ -618,7 +597,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case):
                                (FINISH_PATTERN, handle_test_finish),
                                (UT_APP_BOOT_UP_DONE, handle_next_stage),
                                timeout=one_case["timeout"])
-            except ExpectTimeout:
+            except DUT.ExpectTimeout:
                 Utility.console_log("Timeout in expect", color="orange")
                 one_case_finish(False)
                 break
@@ -627,7 +606,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case):
             break
 
 
-@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True)
+@ttfw_idf.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True)
 def run_multiple_stage_cases(env, extra_data):
     """
     extra_data can be 2 types of value
@@ -734,7 +713,7 @@ def detect_update_unit_test_info(env, extra_data, app_bin):
             for _dic in extra_data:
                 if 'type' not in _dic:
                     raise ValueError("Unit test \"{}\" doesn't exist in the flashed device!".format(_dic.get('name')))
-        except ExpectTimeout:
+        except DUT.ExpectTimeout:
             Utility.console_log("Timeout during getting the test list", color="red")
         finally:
             dut.close()
@@ -789,8 +768,8 @@ if __name__ == '__main__':
     TinyFW.set_default_config(env_config_file=args.env_config_file)
 
     env_config = TinyFW.get_default_config()
-    env_config['app'] = UT
-    env_config['dut'] = IDF.IDFDUT
+    env_config['app'] = ttfw_idf.UT
+    env_config['dut'] = ttfw_idf.IDFDUT
     env_config['test_suite_name'] = 'unit_test_parsing'
     test_env = Env.Env(**env_config)
     detect_update_unit_test_info(test_env, extra_data=list_of_dicts, app_bin=args.app_bin)