ci: migrate system test apps to pytest

.gitlab/ci/target-test.yml

@@ -356,6 +356,30 @@ test_app_test_pytest_esp32s2_usb_host:
     - build_pytest_test_apps_esp32s2
   tags: [ esp32s2, usb_host ]
 
+test_app_test_pytest_esp32s3_mspi_f8r8:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s3
+  needs:
+    - build_pytest_test_apps_esp32s3
+  tags: [ esp32s3, MSPI_F8R8 ]
+
+test_app_test_pytest_esp32s3_mspi_f4r8:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s3
+  needs:
+    - build_pytest_test_apps_esp32s3
+  tags: [ esp32s3, MSPI_F4R8 ]
+
+test_app_test_pytest_esp32s3_mspi_f4r4:
+  extends:
+    - .pytest_test_apps_dir_template
+    - .rules:test:custom_test-esp32s3
+  needs:
+    - build_pytest_test_apps_esp32s3
+  tags: [ esp32s3, MSPI_F4R4 ]
+
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
 .define_config_file_name: &define_config_file_name |
@@ -671,24 +695,6 @@ test_app_test_esp32_generic:
     - ESP32
     - Example_GENERIC
 
-test_app_test_flash_psram_f4r4:
-  extends: .test_app_esp32s3_template
-  tags:
-    - ESP32S3
-    - MSPI_F4R4
-
-test_app_test_flash_psram_f4r8:
-  extends: .test_app_esp32s3_template
-  tags:
-    - ESP32S3
-    - MSPI_F4R8
-
-test_app_test_flash_psram_f8r8:
-  extends: .test_app_esp32s3_template
-  tags:
-    - ESP32S3
-    - MSPI_F8R8
-
 .unit_test_template:
   extends: .target_test_job_template
   needs: # the assign already needs all the build jobs

pytest.ini

@@ -54,6 +54,9 @@ markers =
   external_flash: external flash memory connected via VSPI (FSPI)
   sdcard_sdmode: sdcard running in SD mode
   sdcard_spimode: sdcard running in SPI mode
+  MSPI_F8R8: runner with Octal Flash and Octal PSRAM
+  MSPI_F4R8: runner with Quad Flash and Octal PSRAM
+  MSPI_F4R4: runner with Quad Flash and Quad PSRAM
 
   # multi-dut markers
   multi_dut_generic: tests should be run on generic runners, at least have two duts connected.

tools/test_apps/system/flash_psram/app_test.py

@@ -1,54 +0,0 @@
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-
-
-import glob
-import os
-from typing import Any
-
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-def test_loop(env, config_names):  # type: (Any, Any) -> None
-
-    for name in config_names:
-        Utility.console_log("Checking config \"{}\"... ".format(name), end='')
-        dut = env.get_dut('flash_psram', 'tools/test_apps/system/flash_psram', app_config_name=name)
-        dut.start_app()
-        dut.expect('flash psram test success')
-        env.close_dut(dut.name)
-        Utility.console_log('done')
-
-
-# For F8R8 board (Octal Flash and Octal PSRAM)
-@ttfw_idf.idf_custom_test(env_tag='MSPI_F8R8', target=['esp32s3'])
-def test_flash8_psram8(env, _):  # type: (Any, Any) -> None
-
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.f8r8*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-    test_loop(env, config_names)
-
-
-# For F4R8 board (Quad Flash and Octal PSRAM)
-@ttfw_idf.idf_custom_test(env_tag='MSPI_F4R8', target=['esp32s3'])
-def test_flash4_psram8(env, _):  # type: (Any, Any) -> None
-
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.f4r8*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-    test_loop(env, config_names)
-
-
-# For F4R4 board (Quad Flash and Quad PSRAM)
-@ttfw_idf.idf_custom_test(env_tag='MSPI_F4R4', target=['esp32s3'])
-def test_flash4_psram4(env, _):  # type: (Any, Any) -> None
-
-    config_files = glob.glob(os.path.join(os.path.dirname(__file__), 'sdkconfig.ci.f4r4*'))
-    config_names = [os.path.basename(s).replace('sdkconfig.ci.', '') for s in config_files]
-    test_loop(env, config_names)
-
-
-if __name__ == '__main__':
-    test_flash8_psram8()
-    test_flash4_psram8()
-    test_flash4_psram4()

tools/test_apps/system/flash_psram/pytest_flash_psram.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import os
+import pathlib
+
+import pytest
+from pytest_embedded import Dut
+
+MSPI_F8R8_configs = [p.name.replace('sdkconfig.ci.', '') for p in pathlib.Path(os.path.dirname(__file__)).glob('sdkconfig.ci.f8r8*')]
+
+
+@pytest.mark.esp32s3
+@pytest.mark.MSPI_F8R8
+@pytest.mark.parametrize('config', MSPI_F8R8_configs, indirect=True)
+def test_flash8_psram8(dut: Dut) -> None:
+    dut.expect_exact('flash psram test success')
+
+
+# For F4R8 board (Quad Flash and Octal PSRAM)
+MSPI_F4R8_configs = [p.name.replace('sdkconfig.ci.', '') for p in pathlib.Path(os.path.dirname(__file__)).glob('sdkconfig.ci.f4r8*')]
+
+
+@pytest.mark.esp32s3
+@pytest.mark.MSPI_F4R8
+@pytest.mark.parametrize('config', MSPI_F4R8_configs, indirect=True)
+def test_flash4_psram8(dut: Dut) -> None:
+    dut.expect_exact('flash psram test success')
+
+
+# For F4R4 board (Quad Flash and Quad PSRAM)
+MSPI_F4R4_configs = [p.name.replace('sdkconfig.ci.', '') for p in pathlib.Path(os.path.dirname(__file__)).glob('sdkconfig.ci.f4r4*')]
+
+
+@pytest.mark.esp32s3
+@pytest.mark.MSPI_F4R4
+@pytest.mark.parametrize('config', MSPI_F4R4_configs, indirect=True)
+def test_flash4_psram4(dut: Dut) -> None:
+    dut.expect_exact('flash psram test success')

tools/test_apps/system/longjmp_test/app_test.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-
-import ttfw_idf
-from tiny_test_fw import Utility
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC', target=['esp32', 'esp32s2'], group='test-apps')
-def test_longjmp(env, _):
-
-    dut = env.get_dut('longjmp_test', 'tools/test_apps/system/longjmp_test')
-    dut.start_app()
-    dut.expect('Test successful', 15)
-
-    Utility.console_log('longjmp test done.')
-
-
-if __name__ == '__main__':
-    test_longjmp()

tools/test_apps/system/longjmp_test/pytest_longjmp.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s2
+@pytest.mark.generic
+def test_sys_memprot(dut: Dut) -> None:
+    dut.expect_exact('Test successful')

tools/test_apps/system/memprot/app_test.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-
-import ttfw_idf
-from tiny_test_fw import Utility
-
-MEM_TEST_S2 = [
-    ['IRAM0_SRAM', 'WRX'],
-    ['IRAM0_RTCFAST', 'WRX'],
-    ['DRAM0_SRAM', 'WR'],
-    ['DRAM0_RTCFAST', 'WR'],
-    ['PERI1_RTCSLOW', 'WR'],
-    ['PERI2_RTCSLOW_0', 'WRX'],
-    ['PERI2_RTCSLOW_1', 'WRX']
-]
-
-MEM_TEST_C3 = [
-    ['IRAM0_SRAM', 'WRX'],
-    ['DRAM0_SRAM', 'WR'],
-    ['IRAM0_RTCFAST', 'WRX']
-]
-
-MEM_TEST_S3_MULTI = [
-    # instruction execute test temporarily disabled
-    # ['IRAM0_SRAM (core 0)', 'WRX'],
-    ['IRAM0_SRAM (core 0)', 'WR'],
-    ['DRAM0_SRAM (core 0)', 'WR'],
-    # instruction execute test temporarily disabled
-    # ['IRAM0_SRAM (core 1)', 'WRX'],
-    ['IRAM0_SRAM (core 1)', 'WR'],
-    ['DRAM0_SRAM (core 1)', 'WR']
-    # temporarily disabled unless IDF-5208 gets merged
-    # ['IRAM0_RTCFAST', 'WR'],
-]
-
-MEM_TEST_S3_UNI = [
-    ['IRAM0_SRAM (core 0)', 'WRX'],
-    ['DRAM0_SRAM (core 0)', 'WR']
-    # temporarily disabled unless IDF-5208 gets merged
-    # ['IRAM0_RTCFAST', 'WR'],
-]
-
-
-@ttfw_idf.idf_custom_test(env_tag='Example_GENERIC', target=['esp32c3', 'esp32s2', 'esp32s3'], group='test-apps')
-def test_memprot(env, extra_data):
-
-    dut = env.get_dut('memprot', 'tools/test_apps/system/memprot')
-    dut.start_app()
-
-    mem_test_cfg = []
-    current_target = dut.app.get_sdkconfig()['CONFIG_IDF_TARGET'].replace('"','').lower()
-
-    try:
-        unicore = dut.app.get_sdkconfig()['CONFIG_FREERTOS_UNICORE']
-    except KeyError:
-        unicore = 'n'
-
-    if current_target == 'esp32c3':
-        mem_test_cfg = MEM_TEST_C3
-    elif current_target == 'esp32s2':
-        mem_test_cfg = MEM_TEST_S2
-    elif current_target == 'esp32s3':
-        mem_test_cfg = MEM_TEST_S3_UNI if unicore == 'y' else MEM_TEST_S3_MULTI
-
-    Utility.console_log('Test cfg: ' + current_target)
-
-    for i in mem_test_cfg:
-        if 'R' in i[1]:
-            dut.expect(i[0] + ' read low: OK')
-            dut.expect(i[0] + ' read high: OK')
-        if 'W' in i[1]:
-            dut.expect(i[0] + ' write low: OK')
-            dut.expect(i[0] + ' write high: OK')
-        if 'X' in i[1]:
-            dut.expect(i[0] + ' exec low: OK')
-            dut.expect(i[0] + ' exec high: OK')
-
-    Utility.console_log('Memprot test done')
-
-
-if __name__ == '__main__':
-    test_memprot()

tools/test_apps/system/memprot/pytest_memprot.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import logging
+
+import pytest
+from pytest_embedded import Dut
+
+MEM_TEST_UNICORE = {
+    'esp32s2': [
+        ['IRAM0_SRAM', 'WRX'],
+        ['IRAM0_RTCFAST', 'WRX'],
+        ['DRAM0_SRAM', 'WR'],
+        ['DRAM0_RTCFAST', 'WR'],
+        ['PERI1_RTCSLOW', 'WR'],
+        ['PERI2_RTCSLOW_0', 'WRX'],
+        ['PERI2_RTCSLOW_1', 'WRX']
+    ],
+    'esp32c3': [
+        ['IRAM0_SRAM', 'WRX'],
+        ['DRAM0_SRAM', 'WR'],
+        ['IRAM0_RTCFAST', 'WRX']
+    ],
+    'esp32s3': [
+        ['IRAM0_SRAM (core 0)', 'WRX'],
+        ['DRAM0_SRAM (core 0)', 'WR']
+        # temporarily disabled unless IDF-5208 gets merged
+        # ['IRAM0_RTCFAST', 'WR'],
+    ],
+}
+
+MEM_TEST_MULTICORE = {
+    'esp32s3': [
+        # instruction execute test temporarily disabled
+        # ['IRAM0_SRAM (core 0)', 'WRX'],
+        ['IRAM0_SRAM (core 0)', 'WR'],
+        ['DRAM0_SRAM (core 0)', 'WR'],
+        # instruction execute test temporarily disabled
+        # ['IRAM0_SRAM (core 1)', 'WRX'],
+        ['IRAM0_SRAM (core 1)', 'WR'],
+        ['DRAM0_SRAM (core 1)', 'WR']
+        # temporarily disabled unless IDF-5208 gets merged
+        # ['IRAM0_RTCFAST', 'WR'],
+    ]
+}
+
+
+@pytest.mark.esp32s2
+@pytest.mark.esp32s3
+@pytest.mark.esp32c3
+@pytest.mark.generic
+def test_sys_memprot(dut: Dut) -> None:
+
+    current_target = dut.target
+
+    unicore = dut.app.sdkconfig.get('FREERTOS_UNICORE')
+
+    mem_test_cfg = MEM_TEST_UNICORE if unicore else MEM_TEST_MULTICORE
+
+    logging.info(f'Test cfg: {current_target}')
+
+    for i in mem_test_cfg[current_target]:
+        if 'R' in i[1]:
+            dut.expect_exact(i[0] + ' read low: OK')
+            dut.expect_exact(i[0] + ' read high: OK')
+        if 'W' in i[1]:
+            dut.expect_exact(i[0] + ' write low: OK')
+            dut.expect_exact(i[0] + ' write high: OK')
+        if 'X' in i[1]:
+            dut.expect_exact(i[0] + ' exec low: OK')
+            dut.expect_exact(i[0] + ' exec high: OK')