Merge branch 'feat/reorg_examples' into 'master'

feat: new find_build_apps rules with centralized manifest file

Closes IDFCI-1061, IDF-3553, IDFCI-941, IDFCI-1359, and IDFCI-1361

See merge request espressif/esp-idf!18299

.gitignore

@@ -22,14 +22,16 @@ GPATH
 .DS_Store
 
 # Components Unit Test Apps files
-components/**/build
+components/**/build/
+components/**/build_*_*/
 components/**/sdkconfig
 components/**/sdkconfig.old
 
 # Example project files
+examples/**/build/
+examples/**/build_esp*_*/
 examples/**/sdkconfig
 examples/**/sdkconfig.old
-examples/**/build
 
 # Doc build artifacts
 docs/_build/
@@ -43,7 +45,7 @@ docs/_static/NotoSansSC-Regular.otf
 tools/unit-test-app/sdkconfig
 tools/unit-test-app/sdkconfig.old
 tools/unit-test-app/build
-tools/unit-test-app/builds
+tools/unit-test-app/build_*_*/
 tools/unit-test-app/output
 tools/unit-test-app/test_configs
 
@@ -51,7 +53,8 @@ tools/unit-test-app/test_configs
 log_ut_cmake
 
 # test application build files
-tools/test_apps/**/build
+tools/test_apps/**/build/
+tools/test_apps/**/build_*_*/
 tools/test_apps/**/sdkconfig
 tools/test_apps/**/sdkconfig.old
 

.gitlab-ci.yml

@@ -54,6 +54,10 @@ variables:
   BATCH_BUILD: "1"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
+  PYTHON_VER: 3.7.10
+
+  CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
+  IDF_BUILD_APPS_PROJ: 2818  # fuhanxi/idf-build-apps
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
@@ -212,9 +216,27 @@ before_script:
     - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
     # TODO: remove this, IDFCI-1207
     - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
+
+.before_script_build_jobs:
+  before_script:
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - source tools/ci/setup_python.sh
+    - add_gitlab_ssh_keys
+    - source tools/ci/configure_ci_environment.sh
+    - *setup_tools_unless_target_test
+    - fetch_submodules
+    - *download_test_python_contraint_file
+    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest
+    # TODO: remove this, IDFCI-1207
+    - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
+    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
+    # not only need pytest related packages, but also needs ttfw requirements
+    - internal_pip_install $IDF_BUILD_APPS_PROJ idf_build_apps
+    - pip install -r tools/ci/python_packages/ttfw_idf/requirements.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
 
 default:
   retry:

.gitlab/CODEOWNERS

@@ -177,13 +177,11 @@
 /examples/zigbee/             @esp-idf-codeowners/ieee802154
 
 /tools/                       @esp-idf-codeowners/tools
-/tools/*_apps.py              @esp-idf-codeowners/ci
 /tools/ble/                   @esp-idf-codeowners/app-utilities
 /tools/catch/                 @esp-idf-codeowners/ci
 /tools/ci/                    @esp-idf-codeowners/ci
 /tools/cmake/                 @esp-idf-codeowners/build-config
 /tools/esp_prov/              @esp-idf-codeowners/app-utilities
-/tools/find_build_apps/       @esp-idf-codeowners/ci
 /tools/idf_size_yaml/         @esp-idf-codeowners/peripherals
 /tools/kconfig*/              @esp-idf-codeowners/build-config
 /tools/ldgen/                 @esp-idf-codeowners/build-config

.gitlab/ci/README.md

@@ -19,6 +19,13 @@
     - [Functions](#functions)
       - [CI Job Related](#ci-job-related)
       - [Shell Script Related](#shell-script-related)
+  - [Manifest File to Control the Build/Test apps](#manifest-file-to-control-the-buildtest-apps)
+    - [Grammar](#grammar)
+      - [Operands](#operands)
+      - [Operators](#operators)
+      - [Limitation:](#limitation)
+    - [How does it work?](#how-does-it-work)
+    - [Example](#example)
 
 ## General Workflow
 
@@ -224,3 +231,99 @@ To run these commands in shell script locally, place `source tools/ci/utils.sh`
 - `info`: log in green color
 - `run_cmd`: run the command with duration seconds info
 - `retry_failed`: run the command with duration seconds info, retry when failed
+
+## Manifest File to Control the Build/Test apps
+
+`.build-test-rules.yml` file is a manifest file to control if the CI is running the build and test job or not. The Supported Targets table in `README.md` for apps would be auto-generated by `pre-commit` from the app's `.build-test-rules.yml`.
+
+### Grammar
+
+#### Operands
+
+- variables starts with `SOC_`. The value would be parsed from components/soc/[TARGET]/include/soc/soc_caps.h
+- `IDF_TARGET`
+- `INCLUDE_DEFAULT` (The default value of officially supported targets is 1, otherwise is 0)
+- String, must be double-quoted. e.g. `"esp32"`, `"12345"`
+- Integer, support decimal and hex. e.g. `1`, `0xAB`
+- List with String and Integer inside, the type could be mixed. e.g. `["esp32", 1]`
+
+#### Operators
+
+- `==`, `!=`, `>`, `>=`, `<`, `<=`
+- `and`, `or`
+- `in`, `not in` with list
+- parentheses
+
+#### Limitation:
+
+- all operators are binary operator. For more than two operands, you may use nested parentheses trick. For example,
+  - `A == 1 or (B == 2 and C in [1,2,3])`
+  - `(A == 1 and B == 2) or (C not in ["3", "4", 5])`
+
+### How does it work?
+
+By default, we enable build and test jobs for supported targets.
+
+three rules (disable rules are calculated after the `enable` rule):
+- enable: run CI build/test jobs for targets that match any of the specified conditions only
+- disable: will not run CI build/test jobs for targets that match any of the specified conditions
+- disable_test: will not run CI test jobs for targets that match any of the specified conditions
+
+Each key is a test folder. Will apply to all folders inside.
+
+If one sub folder is in a special case, you can overwrite the rules for this folder by adding another entry for this folder itself. Each folder's rules are standalone, and will not inherit its parent's rules. (YAML inheritance is too complicated for reading)
+
+For example in the following codeblock, only `disable` rule exists in `examples/foo/bar`. It's unaware of its parent's `enable` rule. 
+
+```yaml
+examples/foo:
+  enable:
+    - if: IDF_TARGET == "esp32"
+
+examples/foo/bar:
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+```
+
+### Example
+
+```yaml
+examples/foo:
+  enable:
+    - if IDF_TARGET in ["esp32", 1, 2, 3]
+    - if IDF_TARGET not in ["4", "5", 6]
+  # should be run under all targets!
+
+examples/bluetooth:
+  disable:  # disable both build and tests jobs
+    - if: SOC_BT_SUPPORTED != 1
+    # reason is optional if there's no `temporary: true`
+  disable_test:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: lack of ci runners  # required when `temporary: true`
+
+examples/foo:
+  enable:
+    - if IDF_TARGET in ["esp32", 1, 2, 3]
+    - if IDF_TARGET not in ["4", "5", 6]
+  # should be run under all targets!
+
+examples/bluetooth/test_foo:
+  # each folder's settings are standalone
+  disable:
+    - if: IDF_TARGET == "esp32s2"
+      temporary: true
+      reason: no idea
+  # unlike examples/bluetooth, the apps under this folder would not be build nor test for "no idea" under target esp32s2
+
+examples/get-started/hello_world:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: this one only supports linux!
+
+examples/get-started/blink:
+  enable:
+    - if: INCLUDE_DEFAULT == 1 or IDF_TARGET == "linux"
+      reason: This one supports all supported targets and linux
+```

.gitlab/ci/assign-test.yml

@@ -7,12 +7,12 @@
     SUBMODULES_TO_FETCH: "none"
   artifacts:
     paths:
-      - ${TEST_DIR}/test_configs
-      - ${BUILD_DIR}/artifact_index.json
+      - ${TEST_DIR}/test_configs/
+      - artifact_index.json
     when: always
     expire_in: 1 week
   script:
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+    - run_cmd python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
 
 assign_example_test:
   extends:
@@ -36,8 +36,7 @@ assign_example_test:
       optional: true
   variables:
     TEST_TYPE: example_test
-    TEST_DIR: ${CI_PROJECT_DIR}/examples
-    BUILD_DIR: ${CI_PROJECT_DIR}/build_examples
+    TEST_DIR: examples
 
 assign_custom_test:
   extends:
@@ -61,8 +60,7 @@ assign_custom_test:
       optional: true
   variables:
     TEST_TYPE: custom_test
-    TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
-    BUILD_DIR: ${CI_PROJECT_DIR}/build_test_apps
+    TEST_DIR: tools/test_apps
 
 assign_unit_test:
   extends:
@@ -81,10 +79,7 @@ assign_unit_test:
       optional: true
   variables:
     TEST_TYPE: unit_test
-    TEST_DIR: ${CI_PROJECT_DIR}/components/idf_test/unit_test
-    BUILD_DIR: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
-  script:
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py $TEST_TYPE $TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_DIR/test_configs
+    TEST_DIR: components/idf_test/unit_test
 
 assign_integration_test:
   extends:

.gitlab/ci/build.yml

@@ -15,149 +15,178 @@
 .build_pytest_template:
   extends:
     - .build_template
-    - .before_script_pytest
+    - .before_script_build_jobs
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-  variables:
-    PYTHON_VER: 3.7.10
   artifacts:
     paths:
       - "**/build*/size.json"
-      - "**/build*/build.log"
+      - "**/build*/build_log.txt"
       - "**/build*/*.bin"
       - "**/build*/*.elf"
       - "**/build*/*.map"
       - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
       - "**/build*/config/sdkconfig.json"
       - "**/build*/bootloader/*.bin"
       - "**/build*/partition_table/*.bin"
       - $SIZE_INFO_LOCATION
     when: always
     expire_in: 3 days
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --pytest-apps
+      --collect-size-info $SIZE_INFO_LOCATION
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
 
 build_pytest_examples_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: examples
 
 build_pytest_examples_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32s2
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: examples
 
 build_pytest_examples_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:example_test-esp32s3
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: examples
 
-build_pytest_examples_esp32c2:
+build_pytest_examples_esp32c3:
   extends:
     - .build_pytest_template
-    - .rules:build:example_test-esp32c2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+    - .rules:build:example_test-esp32c3
+  parallel: 3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: examples
 
-build_pytest_examples_esp32c3:
+build_pytest_examples_esp32c2:
   extends:
     - .build_pytest_template
-    - .rules:build:example_test-esp32c3
-  parallel: 2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py examples --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv --parallel-count $CI_NODE_TOTAL --parallel-index $CI_NODE_INDEX
+    - .rules:build:example_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: examples
 
 build_pytest_components_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+  parallel: 2
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: components
 
 build_pytest_components_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: components
 
 build_pytest_components_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32s3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: components
 
 build_pytest_components_esp32c3:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: components
 
 build_pytest_components_esp32c2:
   extends:
     - .build_pytest_template
     - .rules:build:component_ut-esp32c2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py components --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: components
 
 build_non_test_components_apps:
   extends:
-    - .build_template
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:component_ut
-  variables:
-    IDF_TARGET: all
-    TEST_PREFIX: component_ut
-    TEST_RELATIVE_DIR: component_ut
+  script:
+    - set_component_ut_vars
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
+      -t all
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
 
 build_pytest_test_apps_esp32:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
 
 build_pytest_test_apps_esp32s2:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32s2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s2 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
 
 build_pytest_test_apps_esp32s3:
   extends:
     - .build_pytest_template
     - .rules:build:custom_test-esp32s3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32s3 --size-info $SIZE_INFO_LOCATION -vv
+  variables:
+    IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
 
-build_pytest_test_apps_esp32c2:
+build_pytest_test_apps_esp32c3:
   extends:
     - .build_pytest_template
-    - .rules:build:custom_test-esp32c2
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c2 --size-info $SIZE_INFO_LOCATION -vv
+    - .rules:build:custom_test-esp32c3
+  variables:
+    IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
 
-build_pytest_test_apps_esp32c3:
+build_pytest_test_apps_esp32c2:
   extends:
     - .build_pytest_template
-    - .rules:build:custom_test-esp32c3
-  script:
-    - run_cmd python tools/ci/build_pytest_apps.py tools/test_apps --target esp32c3 --size-info $SIZE_INFO_LOCATION -vv
+    - .rules:build:custom_test-esp32c2
+  variables:
+    IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
 
 .build_template_app_template:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .before_script_build_jobs
   variables:
     LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
     BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
@@ -179,8 +208,6 @@ build_pytest_test_apps_esp32c3:
     # using on esp-idf. If it doesn't exist then just stick to the default branch
     - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
     - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
     # Only do the default cmake build for each target, remaining part are done in the build_template_app job
     - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
 
@@ -240,34 +267,45 @@ build_ssc_esp32s3:
     TARGET_NAME: "ESP32S3"
 
 .build_esp_idf_tests_cmake_template:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .before_script_build_jobs
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-    - scan_tests
   artifacts:
     paths:
-      - tools/unit-test-app/output/${IDF_TARGET}
-      - tools/unit-test-app/builds/*.json
-      - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json
-      - components/idf_test/unit_test/*.yml
-      - $LOG_PATH
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
       - $SIZE_INFO_LOCATION
+      - components/idf_test/unit_test/*.yml
     when: always
     expire_in: 4 days
   variables:
-    LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
-    BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
-    OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
-    BUILD_SYSTEM: "cmake"
-    TEST_TYPE: "unit_test"
-    PYTHON_VER: 3.7.10
     LDGEN_CHECK_MAPPING: 1
   script:
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
-    - cd $CI_PROJECT_DIR/tools/unit-test-app
-    - python tools/UnitTestParser.py ${BUILD_PATH} ${CI_NODE_INDEX:-1}
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
+      -t $IDF_TARGET
+      --config "configs/*="
+      --copy-sdkconfig
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --preserve-all
+    - run_cmd python tools/unit-test-app/tools/UnitTestParser.py tools/unit-test-app ${CI_NODE_INDEX:-1}
 
 build_esp_idf_tests_cmake_esp32:
   extends:
@@ -306,142 +344,143 @@ build_esp_idf_tests_cmake_esp32c3:
   variables:
     IDF_TARGET: esp32c3
 
-.build_examples_template:
-  extends: .build_template
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
+.build_cmake_template:
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
       artifacts: false
-    - scan_tests
-  variables:
-    TEST_PREFIX: examples
-    TEST_RELATIVE_DIR: examples
-    SCAN_TEST_JSON: ${CI_PROJECT_DIR}/${TEST_RELATIVE_DIR}/test_configs/scan_${IDF_TARGET}_${BUILD_SYSTEM}.json
-    TEST_TYPE: example_test
-    LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
-    BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
-    PYTHON_VER: 3.7.10
+  variables:
     LDGEN_CHECK_MAPPING: 1
-  script:
-    # it's not possible to build 100% out-of-tree and have the "artifacts"
-    # mechanism work, but this is the next best thing
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
-
-.build_examples_cmake_template:
-  extends: .build_examples_template
   artifacts:
     paths:
-      - build_${TEST_PREFIX}/list.json
-      - build_${TEST_PREFIX}/list_job_*.json
-      - build_${TEST_PREFIX}/*/*/*/sdkconfig
-      - build_${TEST_PREFIX}/*/*/*/build/size.json
-      - build_${TEST_PREFIX}/*/*/*/build/*.bin
-      - build_${TEST_PREFIX}/*/*/*/build/*.elf
-      - build_${TEST_PREFIX}/*/*/*/build/*.map
-      - build_${TEST_PREFIX}/*/*/*/build/flasher_args.json
-      - build_${TEST_PREFIX}/*/*/*/build/bootloader/*.bin
-      - build_${TEST_PREFIX}/*/*/*/build/partition_table/*.bin
-      - $LOG_PATH
+      - "**/build*/size.json"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/sdkconfig"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - list_job_*.json
       - $SIZE_INFO_LOCATION
     when: always
     expire_in: 4 days
-  variables:
-    BUILD_SYSTEM: cmake
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
+      -t $IDF_TARGET
+      --copy-sdkconfig
+      --collect-size-info $SIZE_INFO_LOCATION
+      --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+      --extra-preserve-dirs
+        examples/bluetooth/esp_ble_mesh/ble_mesh_console
+        examples/bluetooth/hci/controller_hci_uart_esp32
+        examples/wifi/iperf
 
 build_examples_cmake_esp32:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32
   parallel: 12
   variables:
     IDF_TARGET: esp32
+    TEST_DIR: examples
 
 build_examples_cmake_esp32s2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32s2
   parallel: 8
   variables:
     IDF_TARGET: esp32s2
+    TEST_DIR: examples
 
 build_examples_cmake_esp32s3:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32s3
   parallel: 8
   variables:
     IDF_TARGET: esp32s3
+    TEST_DIR: examples
 
 build_examples_cmake_esp32c2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32c2
   parallel: 8
   variables:
     IDF_TARGET: esp32c2
+    TEST_DIR: examples
 
 build_examples_cmake_esp32c3:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32c3
   parallel: 8
   variables:
     IDF_TARGET: esp32c3
+    TEST_DIR: examples
 
 build_examples_cmake_esp32h2:
   extends:
-    - .build_examples_cmake_template
+    - .build_cmake_template
     - .rules:build:example_test-esp32h2
   variables:
     IDF_TARGET: esp32h2
-
-.build_test_apps_template:
-  extends: .build_examples_cmake_template
-  variables:
-    TEST_PREFIX: test_apps
-    TEST_RELATIVE_DIR: tools/test_apps
-    TEST_TYPE: custom_test
-  script:
-    - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
+    TEST_DIR: examples
 
 build_test_apps_esp32:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32
   parallel: 2
   variables:
     IDF_TARGET: esp32
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32s2:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32s2
   parallel: 2
   variables:
     IDF_TARGET: esp32s2
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32s3:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32s3
   parallel: 2
   variables:
     IDF_TARGET: esp32s3
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32c3:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32c3
   parallel: 2
   variables:
     IDF_TARGET: esp32c3
+    TEST_DIR: tools/test_apps
 
 build_test_apps_esp32c2:
   extends:
-    - .build_test_apps_template
+    - .build_cmake_template
     - .rules:build:custom_test-esp32c2
   variables:
     IDF_TARGET: esp32c2
+    TEST_DIR: tools/test_apps
 
 .test_build_system_template:
   stage: host_test

.gitlab/ci/docs.yml

@@ -57,8 +57,6 @@ check_docs_lang_sync:
 
 .build_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
-  variables:
-    PYTHON_VER: 3.7.10
   tags:
     - build_docs
   dependencies: []
@@ -73,8 +71,6 @@ check_docs_lang_sync:
 
 check_docs_gh_links:
   image: $ESP_IDF_DOC_ENV_IMAGE
-  variables:
-    PYTHON_VER: 3.7.10
   extends:
     - .pre_check_job_template
     - .doc-rules:build:docs
@@ -156,7 +152,6 @@ build_docs_pdf:
     - .before_script_no_sync_submodule
   image: $ESP_IDF_DOC_ENV_IMAGE
   variables:
-    PYTHON_VER: 3.7.10
     DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
     PYTHONUNBUFFERED: 1
   stage: test_deploy

.gitlab/ci/host-test.yml

@@ -2,8 +2,6 @@
   extends: .rules:test:host_test
   stage: host_test
   image: $ESP_ENV_IMAGE
-  variables:
-    PYTHON_VER: 3.7.10
   tags:
     - host_test
   dependencies: []
@@ -438,8 +436,11 @@ test_gen_soc_caps_kconfig:
 test_pytest_qemu:
   extends:
     - .host_test_template
-    - .before_script_pytest
+    - .before_script_build_jobs
   image: $QEMU_IMAGE
   script:
-    - run_cmd python tools/ci/build_pytest_apps.py . --target esp32 -m qemu -vv
+    - run_cmd python tools/ci/ci_build_apps.py . -vv
+      --target esp32
+      --pytest-apps
+      -m qemu
     - pytest --target esp32 -m qemu --embedded-services idf,qemu

.gitlab/ci/pre_check.yml

@@ -51,8 +51,6 @@ check_python_style:
   extends:
     - .pre_check_base_template
     - .rules:patterns:python-files
-  variables:
-    PYTHON_VER: 3.7.10
   artifacts:
     when: on_failure
     paths:
@@ -63,8 +61,6 @@ check_python_style:
 
 test_check_kconfigs:
   extends: .pre_check_job_template
-  variables:
-    PYTHON_VER: 3.7.10
   artifacts:
     when: on_failure
     paths:
@@ -157,38 +153,6 @@ check_esp_system:
   script:
     - python components/esp_system/check_system_init_priorities.py
 
-scan_tests:
-  extends:
-    - .pre_check_base_template
-    - .before_script_pytest
-    - .rules:build:target_test
-  image: $TARGET_TEST_ENV_IMAGE
-  tags:
-    - scan_test  # since this job is used for uploading the cache, the runner tags should be unique
-  artifacts:
-    paths:
-      - $EXAMPLE_TEST_OUTPUT_DIR
-      - $TEST_APPS_OUTPUT_DIR
-      - $COMPONENT_UT_OUTPUT_DIR
-    expire_in: 1 week
-  variables:
-    EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples
-    EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs
-    TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps
-    TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs
-    COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
-    CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py
-    EXTRA_TEST_DIRS: >-
-      examples/bluetooth/esp_ble_mesh/ble_mesh_console
-      examples/bluetooth/hci/controller_hci_uart_esp32
-      examples/wifi/iperf
-    EXTRA_EVALUATE_ARGS: '--evaluate-parallel-count --config "sdkconfig.ci=default" --config "sdkconfig.ci.*=" --config "=default"'
-  script:
-    - run_cmd python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR --extra_test_dirs $EXTRA_TEST_DIRS $EXTRA_EVALUATE_ARGS
-    - run_cmd python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR $EXTRA_EVALUATE_ARGS
-    - set_component_ut_vars
-    - run_cmd python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR --combine-all-targets --except-targets linux $EXTRA_EVALUATE_ARGS
-
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
 check_version_tag:
@@ -211,3 +175,10 @@ check_commit_msg:
     - git log -n10 --oneline ${PIPELINE_COMMIT_SHA}
     # commit start with "WIP: " need to be squashed before merge
     - 'git log --pretty=%s origin/master..${PIPELINE_COMMIT_SHA} -- | grep -i "^WIP:" && exit 1 || exit 0'
+
+check_test_scripts_build_test_rules:
+  extends:
+    - .pre_check_job_template
+    - .before_script_build_jobs
+  script:
+    - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components

.gitlab/ci/rules.yml

@@ -41,14 +41,6 @@
   - "components/**/*"
   - "examples/cxx/experimental/experimental_cpp_component/*"
 
-.patterns-build_target_test: &patterns-build_target_test
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
-  - "tools/ci/build_pytest_apps.py"
-
 .patterns-build_system: &patterns-build_system
   - "tools/cmake/**/*"
   - "tools/kconfig_new/**/*"
@@ -56,6 +48,7 @@
   - "tools/requirements.json"
   - "tools/ci/test_build_system*.sh"
   - "tools/ci/test_build_system*.py"
+  - "tools/ci/ci_build_apps.py"
 
 .patterns-custom_test: &patterns-custom_test
   - "components/espcoredump/**/*"
@@ -64,11 +57,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "tools/test_apps/**/*"
   - "tools/ldgen/**/*"
 
@@ -77,11 +65,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "tools/unit-test-app/**/*"
 
   - "components/**/*"
@@ -91,11 +74,6 @@
   - "tools/ci/python_packages/tiny_test_fw/**/*"
   - "tools/ci/python_packages/ttfw_idf/**/*"
 
-  - "tools/ci/find_apps_build_apps.sh"
-  - "tools/build_apps.py"
-  - "tools/find_apps.py"
-  - "tools/find_build_apps/**/*"
-
   - "components/**/*"
 
 .patterns-integration_test: &patterns-integration_test
@@ -485,8 +463,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -508,8 +484,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -525,8 +499,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
 
 .rules:build:component_ut-esp32c3:
   rules:
@@ -543,8 +515,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -563,8 +533,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -583,8 +551,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
 
@@ -603,8 +569,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -629,8 +593,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -648,8 +610,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -664,8 +624,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
 
 .rules:build:custom_test-esp32c3:
   rules:
@@ -680,8 +638,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -698,8 +654,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -716,8 +670,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -734,8 +686,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-custom_test
 
@@ -772,8 +722,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -801,8 +749,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -824,8 +770,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test-related_changes-bt
     - <<: *if-dev-push
@@ -850,8 +794,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -876,8 +818,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -902,8 +842,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -928,8 +866,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-example_test
     - <<: *if-dev-push
@@ -1012,8 +948,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-component_ut
     - <<: *if-dev-push
@@ -1050,8 +984,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1069,8 +1001,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1085,8 +1015,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
 
 .rules:build:unit_test-esp32c3:
   rules:
@@ -1101,8 +1029,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1119,8 +1045,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1137,8 +1061,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 
@@ -1155,8 +1077,6 @@
       changes: *patterns-build_components
     - <<: *if-dev-push
       changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-build_target_test
     - <<: *if-dev-push
       changes: *patterns-unit_test
 

.gitlab/ci/static-code-analysis.yml

@@ -10,7 +10,6 @@ clang_tidy_check:
     when: always
     expire_in: 1 day
   variables:
-    CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
     CLANG_TIDY_DIRS_TXT: ${CI_PROJECT_DIR}/tools/ci/clang_tidy_dirs.txt
     RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
     OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports

.pre-commit-config.yaml

@@ -149,6 +149,20 @@ repos:
         pass_filenames: false
         additional_dependencies:
           - pyparsing
+      - id: check-all-apps-readmes
+        name: Check if all apps readme files match given .build-test-rules.yml files. Modify the supported target tables
+        entry: tools/ci/check_build_test_rules.py check-readmes
+        language: python
+        files: 'tools/test_apps/.+|examples/.+|components/.+'
+        additional_dependencies:
+          - idf_build_apps
+      - id: sort-build-test-rules-ymls
+        name: sort .build-test-rules.yml files
+        entry: tools/ci/check_build_test_rules.py sort-yaml
+        language: python
+        files: '\.build-test-rules\.yml'
+        additional_dependencies:
+          - ruamel.yaml
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:

.pylintrc

@@ -152,6 +152,7 @@ disable=print-statement,
         too-many-statements,
         ungrouped-imports,  # since we have isort in pre-commit
         no-name-in-module,  # since we have flake8 to check this
+        too-many-instance-attributes,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

components/cxx/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/cxx/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet

components/driver/.build-test-rules.yml

@@ -0,0 +1,33 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/driver/test_apps/i2s_test_apps:
+  disable:
+    - if: SOC_I2S_SUPPORTED != 1
+
+components/driver/test_apps/i2s_test_apps/legacy_i2s_adc_dac:
+  disable:
+    - if: SOC_I2S_SUPPORTS_ADC_DAC != 1
+
+components/driver/test_apps/legacy_pcnt_driver:
+  disable:
+    - if: SOC_PCNT_SUPPORTED != 1
+
+components/driver/test_apps/legacy_rmt_driver:
+  disable:
+    - if: SOC_RMT_SUPPORTED != 1
+
+components/driver/test_apps/legacy_rtc_temp_driver:
+  disable:
+    - if: SOC_TEMP_SENSOR_SUPPORTED != 1
+
+components/driver/test_apps/pulse_cnt:
+  disable:
+    - if: SOC_PCNT_SUPPORTED != 1
+
+components/driver/test_apps/rmt:
+  disable:
+    - if: SOC_RMT_SUPPORTED != 1
+
+components/driver/test_apps/temperature_sensor:
+  disable:
+    - if: SOC_TEMP_SENSOR_SUPPORTED != 1

components/driver/test_apps/gpio/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |

components/driver/test_apps/gpio/pytest_gpio.py

@@ -18,4 +18,4 @@ from pytest_embedded import Dut
 def test_gpio(dut: Dut) -> None:
     dut.expect_exact('Press ENTER to see the list of tests')
     dut.write('*')
-    dut.expect_unity_test_output()
+    dut.expect_unity_test_output(timeout=300)

components/driver/test_apps/gptimer/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |

components/driver/test_apps/i2s_test_apps/i2s/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |

components/driver/test_apps/i2s_test_apps/legacy_i2s_driver/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |

components/driver/test_apps/legacy_rmt_driver/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |

components/driver/test_apps/legacy_rtc_temp_driver/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- | -------- |

components/driver/test_apps/legacy_timer_driver/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |

components/driver/test_apps/rmt/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |

components/driver/test_apps/temperature_sensor/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- | -------- |

components/esp_eth/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_eth/test_apps:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      reason: only test on esp32

components/esp_event/host_test/esp_event_unit_test/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | Linux |
+| ----------------- | ----- |

components/esp_lcd/.build-test-rules.yml

@@ -0,0 +1,9 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_lcd/test_apps/i80_lcd:
+  disable:
+    - if: SOC_LCD_I80_SUPPORTED != 1
+
+components/esp_lcd/test_apps/rgb_lcd:
+  disable:
+    - if: SOC_LCD_RGB_SUPPORTED != 1

components/esp_lcd/test_apps/i2c_lcd/README.md

@@ -1 +1,4 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 This test app is used to test LCDs with I2C interface.

components/esp_lcd/test_apps/spi_lcd/README.md

@@ -1 +1,4 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 This test app is used to test LCDs with SPI interface.

components/esp_netif/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_netif/test_apps:
+  disable_test:
+    - if: IDF_TARGET != "esp32s2"
+      temporary: true
+      reason: lack of runners

components/esp_netif/test_apps/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |

components/esp_psram/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_psram/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2", "esp32s3"]
+      reason: only test on esp32, esp32s2, and esp32s3

components/esp_rom/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_rom/host_test/rom_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/esp_system/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/esp_system/test_apps/rtc_power_modes:
+  enable:
+    - if: IDF_TARGET == "esp32s3"
+      temporary: true
+      reason: the other targets are not tested yet

components/esp_system/test_apps/rtc_8md256/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |

components/espcoredump/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/espcoredump/test_apps:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet

components/espcoredump/test_apps/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-C3 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # ESP Core Dump Tests

components/log/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/log/host_test/log_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/lwip/test_afl_host/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 ## Introduction
 This test uses [american fuzzy lop](http://lcamtuf.coredump.cx/afl/) to mangle real dns, dhcp client, dhcp server packets and look for exceptions caused by the parser.
 

components/mqtt/host_test/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | Linux |
+| ----------------- | ----- |
+
 # Description
 
 This directory contains test code for the mqtt client that runs on host.

components/newlib/test_apps/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |

components/nvs_flash/host_test/nvs_page_test/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | Linux |
+| ----------------- | ----- |
+
 # NVS Page Unit Test
 
 This unit test for the `nvs::Page` C++ class uses CMock to abstract its dependency to the `spi_flash` component. Its `CMakeLists.txt` builds mocks instead of the actual `spi_flash` component by setting the component property `USE_MOCK` for `spi_flash`. It also needs some stubs for CRC32 since the ROM component can not be mocked yet. These are retrieved from the `mock` directory inside `nvs_flash`.

components/spi_flash/.build-test-rules.yml

@@ -0,0 +1,6 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/spi_flash/host_test/partition_api_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux

components/ulp/cmake/CMakeLists.txt

@@ -68,6 +68,8 @@ add_custom_target(${ULP_APP_NAME}_ld_script
                   DEPENDS ${ULP_LD_SCRIPT}
                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
+# To avoid warning "Manually-specified variables were not used by the project"
+set(bypassWarning "${IDF_TARGET}")
 if(ULP_COCPU_IS_RISCV)
     #risc-v ulp uses extra files for building:
     list(APPEND ULP_S_SOURCES

components/wear_levelling/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+components/wear_levelling/test_apps:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet

docs/en/contribute/esp-idf-tests-with-pytest.rst

@@ -390,11 +390,11 @@ Build Job Names
 Build Job Command
 ^^^^^^^^^^^^^^^^^
 
-The command used by CI to build all the relevant tests is: ``python $IDF_PATH/tools/ci/build_pytest_apps.py <parent_dir> --target <target> -vv``
+The command used by CI to build all the relevant tests is: ``python $IDF_PATH/tools/ci/ci_build_apps.py <parent_dir> --target <target> -vv --pytest-apps``
 
 All apps which supported the specified target would be built with all supported sdkconfig files under ``build_<target>_<config>``.
 
-For example, If you run ``python $IDF_PATH/tools/ci/build_pytest_apps.py $IDF_PATH/examples/system/console/basic --target esp32``, the folder structure would be like this:
+For example, If you run ``python $IDF_PATH/tools/ci/ci_build_apps.py $IDF_PATH/examples/system/console/basic --target esp32 --pytest-apps``, the folder structure would be like this:
 
 .. code:: text
 
@@ -442,7 +442,7 @@ For example, if you want to run all the esp32 tests under the ``$IDF_PATH/exampl
    $ cd $IDF_PATH
    $ . ./export.sh
    $ cd examples/system/console/basic
-   $ python $IDF_PATH/tools/ci/build_pytest_apps.py . --target esp32 -vv
+   $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps
    $ pytest --target esp32
 
 Tips and Tricks

examples/bluetooth/.build-test-rules.yml

@@ -0,0 +1,105 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/bluetooth/bluedroid/ble:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c2", "esp32c3", "esp32h2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/ble_50:
+  enable:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32h2", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/ble_50/multi-adv:
+  enable:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/classic_bt:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/coex/a2dp_gatts_coex:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/bluedroid/coex/gattc_gatts_coex:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/blufi:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_ble_mesh:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_ble_mesh/ble_mesh_console:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_hid_device:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/esp_hid_host:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/hci/ble_adv_scan_combined:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/hci/controller_hci_uart_esp32:
+  enable:
+    - if: IDF_TARGET == "esp32"
+
+examples/bluetooth/hci/controller_hci_uart_esp32c3_and_esp32s3:
+  enable:
+    - if: IDF_TARGET in ["esp32c3", "esp32s3"]
+
+examples/bluetooth/hci/controller_vhci_ble_adv:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/nimble:
+  enable:
+    - if: IDF_TARGET == "esp32"
+      temporary: true
+
+      reason: not tested yet
+examples/bluetooth/nimble/ble_spp:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3", "esp32s3"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/bluetooth/nimble/hci:
+  enable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: the other targets are not tested yet

examples/bluetooth/bluedroid/ble/ble_ancs/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF BLE ANCS Example

examples/bluetooth/bluedroid/ble/ble_compatibility_test/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF BLE Compatibility Test Example

examples/bluetooth/bluedroid/ble/ble_eddystone/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Eddystone Example

examples/bluetooth/bluedroid/ble/ble_hid_device_demo/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF BLE HID Example

examples/bluetooth/bluedroid/ble/ble_ibeacon/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-H2 | ESP32-C2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF iBeacon demo

examples/bluetooth/bluedroid/ble/ble_spp_client/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-H2 | ESP32-C2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF SPP GATT CLIENT demo

examples/bluetooth/bluedroid/ble/ble_spp_server/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 ## ESP-IDF GATT SERVER SPP Example

examples/bluetooth/bluedroid/ble/ble_throughput/throughput_client/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-H2 | ESP32-C2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF BLE throughput GATT CLIENT Test

examples/bluetooth/bluedroid/ble/ble_throughput/throughput_server/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-H2 | ESP32-C2 | ESP32-S3 | 
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF BLE throughput GATT SERVER Test

examples/bluetooth/bluedroid/ble/gatt_client/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-H2 | ESP32-C2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Client Example

examples/bluetooth/bluedroid/ble/gatt_security_client/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Security Client Example

examples/bluetooth/bluedroid/ble/gatt_security_server/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Security Server Example

examples/bluetooth/bluedroid/ble/gatt_server/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Server Example

examples/bluetooth/bluedroid/ble/gatt_server_service_table/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 | 
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Server Service Table Example

examples/bluetooth/bluedroid/ble/gattc_multi_connect/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-C3 | ESP32-C2 | ESP32-H2 | ESP32-S3 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Client Multi Connection Example

examples/bluetooth/bluedroid/ble_50/ble50_security_client/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32-C3 | ESP32-S3 | ESP32-C2 | ESP32-H2 |
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Gatt Security Client Example

examples/bluetooth/bluedroid/ble_50/ble50_security_server/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32-C3 | ESP32-S3 | ESP32-C2 | ESP32-H2 |
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF BLE50 Security Server Example

examples/bluetooth/bluedroid/ble_50/multi-adv/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32-C3 | ESP32-S3 | ESP32-C2 | 
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- |
 
 #ESP-IDF Multi Adv Example

examples/bluetooth/bluedroid/ble_50/peroidic_adv/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32-C3 | ESP32-S3 | ESP32-C2 | ESP32-H2 |
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- | -------- |
 
 # ESP_IDF Periodic Adv Example

examples/bluetooth/bluedroid/ble_50/peroidic_sync/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32-C3 | ESP32-S3 | ESP32-C2 | ESP32-H2 |
+| Supported Targets | ESP32-C2 | ESP32-C3 | ESP32-H2 | ESP32-S3 |
 | ----------------- | -------- | -------- | -------- | -------- |
 
 # ESP-IDF Periodic Sync Example

examples/build_system/.build-test-rules.yml

@@ -0,0 +1,18 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/build_system/cmake/import_lib:
+  disable_test:
+    - if: IDF_TARGET in ["esp32c2", "esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners
+
+examples/build_system/cmake/linux_host_app:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux
+
+examples/build_system/cmake/plugins:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners

examples/build_system/cmake/component_manager/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Using the component manager for downloading dependencies
 
 This example demonstrates how to use [IDF Component Manager](https://pypi.org/project/idf-component-manager/) for downloading dependencies from [the component registry](https://components.espressif.com). More details and use cases of IDF Component Manager can be found in the programming guide under `API Guides` -> `Tools` -> `IDF Component Manager`.

examples/build_system/cmake/import_lib/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Import Third-Party CMake Library Example
 
 This example demonstrates how to import third-party CMake libraries.

examples/build_system/cmake/import_prebuilt/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Import Prebuilt Library Example
 
 This example illustrates how to import a prebuilt static library in the ESP-IDF build system.

examples/build_system/cmake/multi_config/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Multiple Build Configurations Example
 
 (See the README.md file in the upper level 'examples' directory for more information about examples.)

examples/build_system/cmake/plugins/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Link Time Plugins Registration
 
 (See the README.md file in the upper level 'examples' directory for more information about examples.)

examples/custom_bootloader/bootloader_hooks/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Bootloader hooks
 
 (See the README.md file in the upper level for more information about bootloader examples.)

examples/custom_bootloader/bootloader_override/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Bootloader override
 
 (See the README.md file in the upper level for more information about bootloader examples.)

examples/cxx/.build-test-rules.yml

@@ -0,0 +1,41 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/cxx/exceptions:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners
+
+examples/cxx/experimental/esp_modem_cxx:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32s2"]
+      temporary: true
+      reason: the other targets are not tested yet
+
+examples/cxx/experimental/experimental_cpp_component/host_test:
+  enable:
+    - if: IDF_TARGET == "linux"
+      reason: only test on linux
+
+examples/cxx/experimental/simple_i2c_rw_example:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/cxx/experimental/simple_spi_rw_example:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/cxx/pthread:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners
+examples/cxx/rtti:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners

examples/cxx/exceptions/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # Example: C++ exception handling

examples/cxx/experimental/blink_cxx/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Example: Blink C++ example
 
 (See the README.md file in the upper level 'examples' directory for more information about examples.)

examples/cxx/experimental/esp_event_async_cxx/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP-Event asynchronous example

examples/cxx/experimental/esp_event_cxx/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # ESP Event synchronous example

examples/cxx/experimental/esp_mqtt_cxx/ssl/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |

examples/cxx/experimental/esp_mqtt_cxx/tcp/README.md

@@ -1,2 +1,2 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |

examples/cxx/experimental/esp_timer_cxx/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # Example: ESPTimer C++ class

examples/cxx/experimental/simple_i2c_rw_example/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # Example: C++ I2C sensor read for MPU9250

examples/cxx/experimental/simple_spi_rw_example/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # Example: C++ SPI sensor read for MCU9250 inertial/giroscope sensor

examples/cxx/pthread/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # C++ pthread Example

examples/cxx/rtti/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 | ESP32-C2 |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- | -------- |
 
 # Example: C++ run-time type info (RTTI)

examples/ethernet/.build-test-rules.yml

@@ -0,0 +1,17 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/ethernet:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+
+examples/ethernet/iperf:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet
+  disable_test:
+    - if: IDF_TARGET in ["esp32c3", "esp32s2", "esp32s3"]
+      temporary: true
+      reason: lack of runners

examples/ethernet/basic/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # Ethernet Example

examples/ethernet/enc28j60/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # ENC28J60 Example

examples/ethernet/eth2ap/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # eth2ap Example

examples/ethernet/iperf/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # Ethernet iperf Example

examples/get-started/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/get-started/blink:
+  disable_test:
+    - if: IDF_TARGET not in ["esp32", "esp32c3"]
+      temporary: true
+      reason: lack of runners

examples/get-started/blink/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Blink Example
 
 (See the README.md file in the upper level 'examples' directory for more information about examples.)

examples/get-started/hello_world/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # Hello World Example
 
 Starts a FreeRTOS task to print "Hello World".

examples/get-started/sample_project/README.md

@@ -1,3 +1,6 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- |
+
 # _Sample project_
 
 (See the README.md file in the upper level 'examples' directory for more information about examples.)

examples/mesh/.build-test-rules.yml

@@ -0,0 +1,7 @@
+# Documentation: .gitlab/ci/README.md#manifest-file-to-control-the-buildtest-apps
+
+examples/mesh:
+  disable:
+    - if: IDF_TARGET == "esp32c2"
+      temporary: true
+      reason: target esp32c2 is not supported yet

examples/mesh/internal_communication/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # Mesh Internal Communication Example

examples/mesh/ip_internal_network/README.md

@@ -1,4 +1,4 @@
-| Supported Targets | ESP32 | ESP32-S2 | ESP32-S3 | ESP32-C3 |
+| Supported Targets | ESP32 | ESP32-C3 | ESP32-S2 | ESP32-S3 |
 | ----------------- | ----- | -------- | -------- | -------- |
 
 # Mesh IP Internal Networking example