Merge branch 'ci/refactor_ci_yaml' into 'master'

CI: Refactor CI rules into separate yaml file: rules.yml

See merge request espressif/esp-idf!10634

.editorconfig

@@ -37,3 +37,7 @@ max_line_length = 119
 indent_style = space
 indent_size = 4
 max_line_length = 120
+
+[{*.sh,*.yml}]
+indent_style = space
+indent_size = 2

.gitlab-ci.yml

@@ -49,13 +49,14 @@ variables:
   IDF_PATH: "$CI_PROJECT_DIR"
   BATCH_BUILD: "1"
   V: "0"
-  APPLY_BOT_FILTER_SCRIPT: "$CI_PROJECT_DIR/tools/ci/apply_bot_filter.py"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
+
   # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
+
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "${GITLAB_SSH_SERVER}/qa/ci-test-runner-configs.git"
   CI_AUTO_TEST_SCRIPT_REPO_URL: "${GITLAB_SSH_SERVER}/qa/auto_test_script.git"
@@ -64,11 +65,6 @@ variables:
   # Versioned esp-idf-doc env image to use for all document building jobs
   ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7"
 
-
-# before each job, we need to check if this job is filtered by bot stage/job filter
-.apply_bot_filter: &apply_bot_filter
-  python $APPLY_BOT_FILTER_SCRIPT || exit 0
-
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
   export IDF_MIRROR_PREFIX_MAP=
@@ -77,92 +73,41 @@ variables:
   tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
   fi
 
-.fetch_submodules: &fetch_submodules |
-  python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH
-
-.add_ssh_keys: &add_ssh_keys |
-  mkdir -p ~/.ssh
-  chmod 700 ~/.ssh
-  echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-  chmod 600 ~/.ssh/id_rsa
-  echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-
 before_script:
+  - source tools/ci/utils.sh
   - source tools/ci/setup_python.sh
-  # apply bot filter in before script
-  - *apply_bot_filter
-  # add gitlab ssh key
-  - *add_ssh_keys
-  # Set some options and environment for CI
+  - apply_bot_filter
+  - add_gitlab_ssh_keys
   - source tools/ci/configure_ci_environment.sh
   - *setup_tools_unless_target_test
-  - *fetch_submodules
-
-# used for component-based unit test apps
-.before_script_for_component_ut:
-  variables:
-    COMPONENT_UT_EXCLUDE_LIST_FP: ${CI_PROJECT_DIR}/tools/ci/component_ut_excludes.txt
-  before_script:
-    - source tools/ci/setup_python.sh
-    - *apply_bot_filter
-    - *add_ssh_keys
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_tools_unless_target_test
-    - *fetch_submodules
-    - export COMPONENT_UT_DIRS=`find components/ -name test_apps -type d`
-    - export COMPONENT_UT_EXCLUDES=`[ -r $COMPONENT_UT_EXCLUDE_LIST_FP ] && cat $COMPONENT_UT_EXCLUDE_LIST_FP | xargs`
+  - fetch_submodules
 
 # used for check scripts which we want to run unconditionally
 .before_script_lesser_nofilter:
-  variables:
-    GIT_SUBMODULE_STRATEGY: none
   before_script:
     - echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
+    - source tools/ci/utils.sh
     - source tools/ci/setup_python.sh
     - source tools/ci/configure_ci_environment.sh
 
 # used for everything else where we want to do no prep, except for bot filter
 .before_script_lesser:
-  variables:
-    GIT_SUBMODULE_STRATEGY: none
   before_script:
     - echo "Not setting up GitLab key, not fetching submodules"
+    - source tools/ci/utils.sh
     - source tools/ci/setup_python.sh
-    # apply bot filter in before script
-    - *apply_bot_filter
+    - apply_bot_filter
     - source tools/ci/configure_ci_environment.sh
 
-.check_job_template:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-  extends: .before_script_lesser_nofilter
-
-.check_job_template_with_filter:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-  extends: .before_script_lesser
-
-.python_lint_template:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - host_test
-  dependencies: []
-
-.macos_build_template:
-  stage: build
-  tags:
-    - macos_shell
-  dependencies: []
+.before_script_slim:
+  before_script:
+    - echo "Only load utils.sh inside"
+    - source tools/ci/utils.sh
+
+.before_script_macos:
   before_script:
-    - *apply_bot_filter
+    - source tools/ci/utils.sh
+    - apply_bot_filter
     - $IDF_PATH/tools/idf_tools.py install-python-env
     # On macOS, these tools need to be installed
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
@@ -173,51 +118,10 @@ before_script:
     - source tools/ci/configure_ci_environment.sh
     # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
     - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-    - *fetch_submodules
-
-.build_template_app_template:
-  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - build
-  variables:
-    BATCH_BUILD: "1"
-    LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
-    BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
-    SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
-    BUILD_DIR: "@t/@w"
-    BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
-    BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
-    BUILD_COMMAND_ARGS: ""
-  artifacts:
-    when: always
-    paths:
-      - log_template_app/*
-      - size_info.txt
-      - build_template_app/**/size.json
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-  script:
-    # Set the variable for 'esp-idf-template' testing
-    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
-    - ./tools/ci/retry_failed.sh git clone ${ESP_IDF_TEMPLATE_GIT}
-    # Try to use the same branch name for esp-idf-template that we're
-    # using on esp-idf. If it doesn't exist then just stick to the default branch
-    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
-    - export PATH="$IDF_PATH/tools:$PATH"
-    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
-    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-    # Only do the default cmake build for each target, remaining part are done in the build_template_app job
-    - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
-    # Check if there are any stray printf/ets_printf references in WiFi libs
-    - cd components/esp_wifi/lib
-    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
-    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
+    - fetch_submodules
 
 include:
+  - '/tools/ci/config/rules.yml'
   - '/tools/ci/config/pre_check.yml'
   - '/tools/ci/config/build.yml'
   - '/tools/ci/config/assign-test.yml'

tools/ci/config/README.md

@@ -0,0 +1,135 @@
+# Rules for `rules.yml`
+
+- [Rules for `rules.yml`](#rules-for-rulesyml)
+  - [How to Develop With `rules.yml`?](#how-to-develop-with-rulesyml)
+    - [How to Add a New `Job`?](#how-to-add-a-new-job)
+    - [How to Add a New `Rules` Template?](#how-to-add-a-new-rules-template)
+    - [How to Add a New `if` Anchor?](#how-to-add-a-new-if-anchor)
+  - [Naming Rules](#naming-rules)
+    - [Common Naming Rules](#common-naming-rules)
+    - [`if` Anchors Naming Rules](#if-anchors-naming-rules)
+      - [Common Phrases/Abbreviations](#common-phrasesabbreviations)
+    - [`rules` Template Naming Rules](#rules-template-naming-rules)
+  - [Reusable Shell Script `tools/ci/utils.sh`](#reusable-shell-script-toolsciutilssh)
+    - [Functions](#functions)
+      - [CI Job Related](#ci-job-related)
+      - [Shell Script Related](#shell-script-related)
+
+## How to Develop With `rules.yml`?
+
+### How to Add a New `Job`?
+
+check if there's a suitable `.rules:<rules-you-need>` template
+
+1. if there is, put this in the job `extends`. All done, now you can close this window. (`extends` could be array or string)
+2. if there isn't
+   1. check [How to Add a New `Rules` Template?](#how-to-add-a-new-rules-template), create a suitable one
+   2. follow step 1
+
+### How to Add a New `Rules` Template?
+
+check if there's a suitable `.if-<if-anchor-you-need>` anchor
+
+ 1. if there is, create a rule following [`rules` Template Naming Rules](#rules-template-naming-rules).For detail information, please refer to [GitLab Documentation `rules-if`](https://docs.gitlab.com/ee/ci/yaml/README.html#rulesif). Here's an example.
+
+```yaml
+.rules:dev:
+rules:
+  - <<: *if-trigger
+  - <<: *if-dev-push
+```
+
+2. if there isn't
+
+   1. check [How to Add a New `if` Anchor?](#how-to-add-a-new-if-anchor), create a suitable one
+   2. follow step 1
+
+### How to Add a New `if` Anchor?
+
+Create an `if` anchor following [`if` Anchors Naming Rules](#if-anchors-naming-rules). For detail information about how to write the condition clause, please refer to [GitLab Documentation `only/except (advanced)](https://docs.gitlab.com/ee/ci/yaml/README.html#onlyexcept-advanced). Here's an example.
+
+```yaml
+.if-schedule: &if-schedule:
+  if: '$CI_PIPELINE_SOURCE == "schedule"'
+```
+
+## Naming Rules
+
+### Common Naming Rules
+
+if a phrase has multi words, use `_` to concat them.
+
+> e.g. `regular_test`
+
+if a name have multi phrase, use `-` to concat them.
+
+> e.g. `regular_test-example_test`
+
+### `if` Anchors Naming Rules
+
+- if it's a label: `.if-label-<label_name>`
+- if it's a ref: `.if-ref-<ref_name>`
+- if it's a branch: `.if-branch-<branch_name>`
+- if it's a tag: `.if-tag-<tag_name>`
+- if it's a operating system: `.if-os-mac`
+- if it's multi-type combination: `.if-ref-<release_name>-branch-<branch_name>`
+
+#### Common Phrases/Abbreviations
+
+- `no_label`
+
+   `$BOT_TRIGGER_WITH_LABEL == null`
+
+- `protected`
+
+  `($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)`
+
+-  `target_test`
+
+   `example_test` or `custom_test` or `unit_test-all_targets`
+
+### `rules` Template Naming Rules
+
+- if it's os related: `.rules:os:<os_name>`
+- if it's tag related: `.rules:tag:<tag_1>-<tag_2>`
+- if it's label related: `.rules:labels:<label_1>-<label_2>`
+
+    By default, all `.rules:labels` should include both `if-label-regular_test` and `if-protected-no-label` implicitly, unless they have special postfixes:
+
+    - slim: `regular_test` not included
+    - only: only have the phrases listed
+
+- if it's target test related: `.rules:tests:<test_type_1>-<test_type_2>`
+
+    By default, all `.rules:tests` should include `if-protected-no_label` implicitly, unless they have special postfixes (same as above)
+
+- if it needs to build at first, then do some target test: `.rules:build_tests:<test_type_1>-<test_type_2>`
+
+    By default, all `.rules:build_tests` should include `if-protected-no-label`, `if-label-build`, `if-label-regular-test` implictly, unless they have special postfixes (same as above)
+
+- if a job supports all targets, use `-all_targets` as postfix
+
+## Reusable Shell Script `tools/ci/utils.sh`
+
+It is used to put all the reusable shell script as small functions. If you want to set `before_script: []` for you job, now you can set `extends: .before_script_slim` instead. it will only run `source tools/ci/utils.sh`
+
+If you're developing CI shell scripts, you can use these functions without source. They're already included in all `before_script`
+
+To run these commands in shell script locally, place `source tools/ci/utils.sh` at the very beginning.
+
+### Functions
+
+#### CI Job Related
+- `apply_bot_filter`
+- `add_gitlab_ssh_keys`
+- `add_github_ssh_keys`
+- `add_doc_server_ssh_keys`
+- `fetch_submodules`
+- `get_all_submodules`
+
+#### Shell Script Related
+- `error`: log in red color
+- `warning`: log in orange color
+- `info`: log in green color
+- `run_cmd`: run the command with duration seconds info
+- `retry_failed`: run the command with duration seconds info, retry when failed

tools/ci/config/assign-test.yml

@@ -1,21 +1,23 @@
 assign_test:
-  extends: .before_script_for_component_ut
+  extends:
+    - .rules:tests:target_test-integration_test-weekend_test
   tags:
     - assign_test
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   stage: assign_test
   # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
   # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
-  dependencies:
+  dependencies: # Here is not a hard dependency relationship, could be skipped. so we do not use "needs" here.
     - build_ssc_esp32
     - build_esp_idf_tests_cmake_esp32
     - build_esp_idf_tests_cmake_esp32s2
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    EXAMPLE_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
-    TEST_APP_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
-    COMPONENT_UT_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
-    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    EXAMPLE_TEST_DIR: "${CI_PROJECT_DIR}/examples"
+    CUSTOM_TEST_DIR: "${CI_PROJECT_DIR}/tools/test_apps"
+    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    # COMPONENT_UT_DIRS is set by `set_component_ut_vars` in `utils.sh`
+    COMPONENT_UT_OUTPUT_DIR: "${CI_PROJECT_DIR}/component_ut"
     INTEGRATION_CONFIG_OUTPUT_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
@@ -32,42 +34,27 @@ assign_test:
       - build_component_ut/artifact_index.json
       - tools/unit-test-app/builds/artifact_index.json
     expire_in: 1 week
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
-      - $BOT_LABEL_INTEGRATION_TEST
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_CUSTOM_TEST
-      - $BOT_LABEL_WEEKEND_TEST
   script:
-    # assign example tests
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $IDF_PATH/examples -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_CONFIG_OUTPUT_PATH
-    # assign test apps
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $IDF_PATH/tools/test_apps -c $CI_TARGET_TEST_CONFIG_FILE -o $TEST_APP_CONFIG_OUTPUT_PATH
-    # assign component ut
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_CONFIG_OUTPUT_PATH
-    # assign unit test cases
-    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_CASE_FILE -c $CI_TARGET_TEST_CONFIG_FILE -o $IDF_PATH/components/idf_test/unit_test/CIConfigs
+    - set_component_ut_vars
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py example_test $EXAMPLE_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $EXAMPLE_TEST_DIR/test_configs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py custom_test $CUSTOM_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $CUSTOM_TEST_DIR/test_configs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py component_ut $COMPONENT_UT_DIRS -c $CI_TARGET_TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR/test_configs
+    - python tools/ci/python_packages/ttfw_idf/IDFAssignTest.py unit_test $UNIT_TEST_DIR -c $CI_TARGET_TEST_CONFIG_FILE -o $UNIT_TEST_DIR/CIConfigs
     # clone test script to assign tests
     # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     # assign integration test cases
     - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin -o $INTEGRATION_CONFIG_OUTPUT_PATH
 
 update_test_cases:
+  extends: .rules:ref:master-schedule
   stage: assign_test
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env
   tags:
     - deploy_test
-  only:
-    refs:
-      - master
-      - schedules
-  dependencies:
+  needs:
     - build_esp_idf_tests_cmake_esp32
     - build_esp_idf_tests_cmake_esp32s2
   artifacts:
@@ -77,18 +64,18 @@ update_test_cases:
     expire_in: 1 week
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    UNIT_TEST_CASE_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
+    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
     AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
     PYTHON_VER: 3.7.7
   script:
     - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
     - python $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
     # update unit test cases
-    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_CASE_DIR -maxdepth 1 -name "*.yml" | xargs)
+    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
     - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
     # update example test cases
     - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA

tools/ci/config/build.yml

@@ -4,33 +4,78 @@
   tags:
     - build
   variables:
-    BATCH_BUILD: "1"
-    V: "0"
     SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
 
+.build_template_app_template:
+  extends:
+    - .build_template
+    - .rules:labels:build
+  variables:
+    LOG_PATH: "${CI_PROJECT_DIR}/log_template_app"
+    BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app"
+    BUILD_DIR: "@t/@w"
+    BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt"
+    BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt"
+    BUILD_COMMAND_ARGS: ""
+  artifacts:
+    when: always
+    paths:
+      - log_template_app/*
+      - size_info.txt
+      - build_template_app/**/size.json
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT}
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default branch
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - export PATH="$IDF_PATH/tools:$PATH"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
+    # Only do the default cmake build for each target, remaining part are done in the build_template_app job
+    - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS}
+    # Check if there are any stray printf/ets_printf references in WiFi libs
+    - cd components/esp_wifi/lib
+    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done;
+    - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done;
+
+# build-related-pre-check-jobs ------------------------------------------------
+# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
+fast_template_app:
+  extends: .build_template_app_template
+  stage: pre_check
+  variables:
+    BUILD_COMMAND_ARGS: "-p"
+
+check_docs_gh_links:
+  extends: .build_docs_template
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
+#------------------------------------------------------------------------------
+
 .build_ssc_template:
-  extends: .build_template
-  parallel: 3
+  extends:
+    - .build_template
+    - .rules:build_tests:integration_test
   artifacts:
     paths:
       - SSC/ssc_bin
     expire_in: 1 week
-  variables:
-    TARGET_NAME: "ESP32"
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_INTEGRATION_TEST
-      - $BOT_LABEL_REGULAR_TEST
   script:
-    - ./tools/ci/retry_failed.sh git clone $SSC_REPOSITORY
+    - retry_failed git clone $SSC_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT SSC SSC
     - cd SSC
     - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME
 
 build_ssc_esp32:
   extends: .build_ssc_template
+  parallel: 3
   variables:
     TARGET_NAME: "ESP32"
 
@@ -41,7 +86,9 @@ build_ssc_esp32s2:
     TARGET_NAME: "ESP32S2"
 
 .build_esp_idf_tests_cmake:
-  extends: .build_template
+  extends:
+    - .build_template
+    - .rules:build_tests:unit_test
   artifacts:
     paths:
       - tools/unit-test-app/output/${IDF_TARGET}
@@ -52,13 +99,6 @@ build_ssc_esp32s2:
       - $SIZE_INFO_LOCATION
     when: always
     expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
-      - $BOT_LABEL_REGULAR_TEST
   variables:
     LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake"
     BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds
@@ -89,17 +129,9 @@ build_esp_idf_tests_cmake_esp32s3:
     IDF_TARGET: esp32s3
 
 .build_examples_template:
-  extends: .build_template
-  artifacts:
-    when: always
-    expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
+  extends:
+    - .build_template
+    - .rules:build_tests:example_test-weekend_test
   variables:
     TEST_PREFIX: examples
     TEST_RELATIVE_DIR: examples
@@ -116,7 +148,8 @@ build_esp_idf_tests_cmake_esp32s3:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
 
 build_examples_make:
-  extends: .build_examples_template
+  extends:
+    - .build_examples_template
   # This is a workaround for a rarely encountered issue with building examples in CI.
   # Probably related to building of Kconfig in 'make clean' stage
   retry: 1
@@ -126,23 +159,16 @@ build_examples_make:
       - $LOG_PATH
       - build_${TEST_PREFIX}/*/*/*/build/size.json
       - $SIZE_INFO_LOCATION
+    when: always
+    expire_in: 4 days
   variables:
     BUILD_SYSTEM: make
     IDF_TARGET: esp32  # currently we only support esp32
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-      - pipelines
-      - web
 
 # same as above, but for CMake
-.build_examples_cmake: &build_examples_cmake
+.build_examples_cmake:
   extends: .build_examples_template
-  dependencies:
+  needs:
     - scan_tests
   artifacts:
     paths:
@@ -174,18 +200,13 @@ build_examples_cmake_esp32s2:
     IDF_TARGET: esp32s2
 
 .build_test_apps:
-  extends: .build_examples_cmake
+  extends:
+    - .build_examples_cmake
+    - .rules:build_tests:custom_test-weekend_test
   variables:
     TEST_PREFIX: test_apps
     TEST_RELATIVE_DIR: tools/test_apps
     TEST_TYPE: custom_test
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_CUSTOM_TEST
-      - $BOT_LABEL_WEEKEND_TEST
   script:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
 
@@ -208,17 +229,12 @@ build_test_apps_esp32s3:
     IDF_TARGET: esp32s3
 
 .build_component_ut:
-  extends: .build_test_apps
+  extends:
+    - .build_test_apps
+    - .rules:build_tests:unit_test
   variables:
     TEST_PREFIX: component_ut
     TEST_RELATIVE_DIR: component_ut
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
 
 build_component_ut_esp32:
   extends: .build_component_ut
@@ -230,128 +246,84 @@ build_component_ut_esp32s2:
   variables:
     IDF_TARGET: esp32s2
 
-# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
-
-.build_docs_template: &build_docs_template
+.build_docs_template:
   stage: build
   image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - build_docs
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build
+  parallel:
+    matrix:
+      - DOCLANG: [ "en", "zh_CN" ]
+        DOCTGT: [ "esp32", "esp32s2" ]
+
+build_docs_html:
+  extends:
+    - .build_docs_template
+    - .rules:labels:build_docs
   artifacts:
     when: always
     paths:
       - docs/_build/*/*/*.txt
       - docs/_build/*/*/html/*
     expire_in: 4 days
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_BUILD_DOCS
-      - $BOT_LABEL_REGULAR_TEST
-  dependencies: []
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs html -l $DOCLANG -t $DOCTGT build
-
-build_docs_html:
-  extends: .build_docs_template
-  parallel:
-    matrix:
-      - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2"]
+  variables:
+    DOC_BUILDERS: "html"
 
 build_docs_pdf:
-  stage: build
-  image: $ESP_IDF_DOC_ENV_IMAGE
-  tags:
-    - build_docs
+  extends:
+    - .build_docs_template
+    - .rules:labels:build_docs-slim
   artifacts:
     when: always
     paths:
       - docs/_build/*/*/latex/*
     expire_in: 4 days
-  rules:
-    - if: '$CI_COMMIT_REF_NAME == "master"'
-    - if: '$CI_COMMIT_REF_NAME =~ "/^release\/v/"'
-    - if: '$CI_COMMIT_REF_NAME =~ "/^v\d+\.\d+(\.\d+)?($|-)/"'
-    - if: $BOT_LABEL_BUILD_DOCS
-  dependencies: []
-  parallel:
-    matrix:
-      - DOCLANG: ["en", "zh_CN"]
-        DOCTGT: ["esp32", "esp32s2"]
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs latex -l $DOCLANG -t $DOCTGT build
+  variables:
+    DOC_BUILDERS: "latex"
 
-test_build_system:
-  extends: .build_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
+.test_build_system_template:
+  extends:
+    - .build_template
+    - .rules:build_tests:weekend_test
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - rm -rf test_build_system
     - mkdir test_build_system
     - cd test_build_system
-    - ${IDF_PATH}/tools/ci/test_build_system.sh
+    - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT}
+
+test_build_system:
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system.sh
 
 test_build_system_cmake:
-  extends: .build_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_WEEKEND_TEST
-  script:
-    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - rm -rf test_build_system
-    - mkdir test_build_system
-    - cd test_build_system
-    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh
+  extends: .test_build_system_template
+  variables:
+    SHELL_TEST_SCRIPT: test_build_system_cmake.sh
 
 test_build_system_cmake_macos:
-  extends: .macos_build_template
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-      - triggers
-      - pipelines
-      - web
-    variables:
-      - $CI_PIPELINE_SOURCE != "push" && $BOT_LABEL_MACOS_TEST != null
-      - $CI_PIPELINE_SOURCE == "push"
-  script:
-    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
-    - rm -rf test_build_system
-    - mkdir test_build_system
-    - cd test_build_system
-    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh
+  extends:
+    - .test_build_system_template
+    - .before_script_macos
+    - .rules:os:mac_os
+  tags:
+    - macos_shell
 
 build_docker:
+  extends:
+    - .before_script_slim
+    - .rules:protected-schedule
   stage: build
   image: espressif/docker-builder:1
   tags:
     - build_docker_amd64_brno
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
   variables:
     DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
-  before_script: []
   script:
     - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL
     - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi
@@ -364,67 +336,50 @@ build_docker:
     # Therefore, build a copy of the example located inside the container.
     - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
 
-build_idf_exe:
+.test-on-windows:
+  extends:
+    - .before_script_slim
+    - .rules:protected-schedule
   stage: build
   image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
   tags:
     - build
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-  before_script: []
-  artifacts:
-    paths:
-      - tools/windows/idf_exe/build/idf-exe-v*.zip
-    expire_in: 4 days
   script:
-    - cd tools/windows/idf_exe/
+    - cd $TEST_DIR
     - mkdir build
     - cd build
     - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
     - cmake --build .
 
+build_idf_exe:
+  extends: .test-on-windows
+  artifacts:
+    paths:
+      - tools/windows/idf_exe/build/idf-exe-v*.zip
+    expire_in: 4 days
+  variables:
+    TEST_DIR: tools/windows/idf_exe
+
 build_cmdlinerunner:
-  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross
-  tags:
-    - build
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-  before_script: []
+  extends: .test-on-windows
   artifacts:
     paths:
       - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll
     expire_in: 4 days
-  script:
-    - cd tools/windows/tool_setup/cmdlinerunner
-    - mkdir build
-    - cd build
-    - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release ..
-    - cmake --build .
+  variables:
+    TEST_DIR: tools/windows/tool_setup/cmdlinerunner
 
 build_installer:
+  extends:
+    - .before_script_slim
+    - .rules:protected-schedule
   # using a different stage here to be able to use artifacts from build_cmdlinerunner job
   stage: host_test
   image: $CI_DOCKER_REGISTRY/wine-innosetup:1
   tags:
     - build
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - schedules
-  dependencies:
+  needs:
     - build_cmdlinerunner
-  before_script: []
   script:
     - cd tools/windows/tool_setup/
     - ./build_installer.sh
@@ -450,14 +405,15 @@ build_template_app:
   image:
     name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2
   before_script:
+    - source tools/ci/utils.sh
     - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
-    - python $SUBMODULE_FETCH_TOOL
+    - fetch_submodules
     # Exclude the submodules, all paths ends with /**
-    # get all submodules configs | get all paths | add /** as suffix | xargs | replace all <space> to <comma>
-    - export SUBMODULES=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g')
+    - export SUBMODULES=$(get_all_submodules)
     # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
     - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-    - export EXCLUSIONS="$SUBMODULES,$CUSTOM_EXCLUDES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
+    # Exclude the report dir
+    - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg"
     - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR
   variables:
     GIT_DEPTH: 0
@@ -465,13 +421,13 @@ build_template_app:
     REPORT_DIR: examples/get-started/hello_world/tidybuild/report
   tags:
     - host_test
+  needs:
+    - clang_tidy_check_regular
 
 code_quality_check:
-  extends: .sonar_scan_template
-  dependencies:
-    - clang_tidy_check_regular
-  only:
-    - triggers
+  extends:
+    - .sonar_scan_template
+    - .rules:trigger
   allow_failure: true
   script:
     - export CI_MERGE_REQUEST_IID=`python ${CI_PROJECT_DIR}/tools/ci/ci_get_latest_mr_iid.py ${CI_COMMIT_BRANCH} | xargs`
@@ -497,14 +453,9 @@ code_quality_check:
       -Dsonar.branch.name=$CI_COMMIT_REF_NAME
 
 code_quality_report:
-  extends: .sonar_scan_template
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-    - schedules
-  dependencies:
-    - clang_tidy_check_regular
+  extends:
+    - .sonar_scan_template
+    - .rules:protected-schedule
   script:
     - sonar-scanner
       -Dsonar.host.url=$SONAR_HOST_URL

tools/ci/config/deploy.yml

@@ -1,16 +1,16 @@
-.clang_tidy_deploy_template:
+.deploy_job_template:
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - deploy
+
+.clang_tidy_deploy_template:
+  extends: .deploy_job_template
   tags:
     - deploy
     - shiny
   script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_SERVER $DOCS_SERVER_USER
     - export GIT_VER=$(git describe --always)
     - cd $IDF_PATH/examples/get-started/hello_world/tidybuild
     - mv report $GIT_VER
@@ -27,89 +27,56 @@ clang_tidy_deploy:
   extends: .clang_tidy_deploy_template
   # Override default stage to happen before the post_check
   stage: test_deploy
-  dependencies:
+  needs:
     - clang_tidy_check
     - clang_tidy_check_all
   variables:
     BOT_NEEDS_TRIGGER_BY_NAME: 1
 
 clang_tidy_deploy_regular:
-  extends: .clang_tidy_deploy_template
-  dependencies:
+  extends:
+    - .clang_tidy_deploy_template
+    - .rules:labels:static_analysis-only
+  needs:
     - clang_tidy_check_regular
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_LABEL_STATIC_ANALYSIS
-      - $BOT_LABEL_STATIC_ANALYSIS_ALL
 
 push_to_github:
-  stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - deploy
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-  when: on_success
-  dependencies: []
-  extends: .before_script_lesser
+  extends:
+    - .deploy_job_template
+    - .before_script_lesser
+    - .rules:protected-no_label
   script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+    - add_github_ssh_keys
     - git remote remove github &>/dev/null || true
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
 .deploy_docs_template:
-  extends: .before_script_lesser
-  stage: deploy
+  extends:
+    - .deploy_job_template
+    - .before_script_lesser
   image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - deploy
     - shiny
-  dependencies:
+  needs:
     - build_docs_html
     - build_docs_pdf
   variables:
     DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
     PYTHONUNBUFFERED: 1
   script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
     - export GIT_VER=$(git describe --always)
-
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
 
-
 # deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
 deploy_docs_preview:
-  extends: .deploy_docs_template
+  extends:
+    - .deploy_docs_template
+    - .rules:labels:build_docs-slim
   # Override default stage to happen before the post_check
   stage: test_deploy
-  only:
-    refs:
-      - triggers
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD_DOCS
   variables:
     TYPE: "preview"
     # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
@@ -121,15 +88,10 @@ deploy_docs_preview:
 
 # deploy docs to production webserver
 deploy_docs_production:
-  extends: .deploy_docs_template
-  only:
-    refs:
-      # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
+  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
+  extends:
+    - .deploy_docs_template
+    - .rules:protected-no_label
   variables:
     TYPE: "preview"
     DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
@@ -139,15 +101,13 @@ deploy_docs_production:
     DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
 
 deploy_test_result:
-  stage: deploy
+  extends:
+    - .deploy_job_template
+    - .before_script_slim
+    - .rules:ref:master-schedule-always
   image: $CI_DOCKER_REGISTRY/bot-env
   tags:
     - deploy_test
-  when: always
-  only:
-    refs:
-      - master
-      - schedules
   artifacts:
     when: always
     paths:
@@ -161,21 +121,15 @@ deploy_test_result:
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
     TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
-  before_script:
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
   script:
+    - add_gitlab_ssh_keys
     - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
     - export REV_COUNT=$(git rev-list --count HEAD)
     - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
     # artifacts of job update_test_cases creates test-management folder
     # we need to remove it so we can clone test-management folder again
     - rm -rf test-management
-    - ./tools/ci/retry_failed.sh git clone $TEST_MANAGEMENT_REPO
+    - retry_failed git clone $TEST_MANAGEMENT_REPO
     - python3 $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}

tools/ci/config/host-test.yml

@@ -1,31 +1,22 @@
 .host_test_template:
+  extends: .rules:labels:host_test
   stage: host_test
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - host_test
   dependencies: []
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_HOST_TEST
-      - $BOT_LABEL_REGULAR_TEST
 
 .host_fuzzer_test_template:
-  stage: host_test
+  extends:
+    - .host_test_template
+    - .rules:labels:fuzzer_test-weekend_test-only
   image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
-  tags:
-    - host_test
-  dependencies: []
   artifacts:
     when: always
     paths:
       - ${FUZZER_TEST_DIR}/out/crashes
       - ${FUZZER_TEST_DIR}/fuzz_output.log
     expire_in: 1 week
-  only:
-    variables:
-      - $BOT_LABEL_FUZZER_TEST
-      - $BOT_LABEL_WEEKEND_TEST
   script:
     - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
     - cd ${FUZZER_TEST_DIR}
@@ -42,16 +33,13 @@ test_nvs_on_host:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh make test
 
 test_nvs_coverage:
-  extends: .host_test_template
+  extends:
+    - .host_test_template
+    - .rules:labels:nvs_coverage-only
   artifacts:
     paths:
       - components/nvs_flash/test_nvs_host/coverage_report
     expire_in: 1 week
-  only:
-    refs:
-      - triggers
-    variables:
-      - $BOT_LABEL_NVS_COVERAGE
   script:
     - cd components/nvs_flash/test_nvs_host
     - make coverage_report
@@ -279,24 +267,16 @@ test_mkdfu:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py
 
 test_docs:
-  stage: host_test
+  extends: .host_test_template
   image: $ESP_IDF_DOC_ENV_IMAGE
-  tags:
-    - host_test
-  dependencies: []
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_HOST_TEST
-      - $BOT_LABEL_REGULAR_TEST
   artifacts:
     when: on_failure
     paths:
-        - docs/test/_build/*/*/*/html/*
+      - docs/test/_build/*/*/*/html/*
     expire_in: 1 week
   script:
-  - cd ${IDF_PATH}/docs/test
-  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
-  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
-  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
+    - cd ${IDF_PATH}/docs/test
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
 

tools/ci/config/post_check.yml

@@ -2,9 +2,27 @@
 .show_submodule_urls: &show_submodule_urls |
   git config --get-regexp '^submodule\..*\.url$' || true
 
-check_submodule_sync:
-  extends: .check_job_template
+.post_check_base_template:
   stage: post_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - host_test
+  dependencies: []
+
+.post_check_job_template:
+  extends:
+    - .post_check_base_template
+    - .before_script_lesser_nofilter
+
+.post_check_job_template_with_filter:
+  extends:
+    - .post_check_base_template
+    - .before_script_lesser
+
+check_submodule_sync:
+  extends:
+    - .before_script_slim
+    - .post_check_job_template
   tags:
     - github_sync
   retry: 2
@@ -12,8 +30,6 @@ check_submodule_sync:
     GIT_STRATEGY: clone
     SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
-  before_script: []
-  after_script: []
   script:
     - git submodule deinit --force .
     # setting the default remote URL to the public one, to resolve relative location URLs
@@ -26,40 +42,31 @@ check_submodule_sync:
   allow_failure: true # remove this line when esp32s3 support in esptool got merged into master
 
 check_ut_cmake_make:
-  extends: .check_job_template_with_filter
-  stage: post_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  extends:
+    - .post_check_job_template_with_filter
+    - .rules:dev
   tags:
     - build
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  dependencies: []
   script:
     - tools/ci/check_ut_cmake_make.sh
 
 check_artifacts_expire_time:
-  extends: .check_job_template
-  stage: post_check
+  extends: .post_check_job_template
   script:
     # check if we have set expire time for all artifacts
     - python tools/ci/check_artifacts_expire_time.py
 
 check_pipeline_triggered_by_label:
-  extends: .check_job_template
-  stage: post_check
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL
+  extends:
+    - .post_check_job_template
+    - .rules:dev
   script:
     # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
     # We want to make sure some jobs are always executed to detect regression.
     - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }
 
 check_commit_msg:
-  extends: .check_job_template
-  stage: post_check
+  extends: .post_check_job_template
   script:
     - git status
     - git log -n10 --oneline

tools/ci/config/post_deploy.yml

@@ -1,11 +1,8 @@
-.check_doc_links_template: &check_doc_links_template
+.check_doc_links_template:
+  extends: .rules:protected
   stage: post_deploy
   image: $ESP_IDF_DOC_ENV_IMAGE
   tags: [ "build", "amd64", "internet" ]
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
   artifacts:
     when: always
     paths:

tools/ci/config/pre_check.yml

@@ -1,61 +1,63 @@
+.pre_check_base_template:
+  stage: pre_check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - host_test
+  dependencies: []
+
+.pre_check_job_template:
+  extends:
+    - .pre_check_base_template
+    - .before_script_lesser_nofilter
+
+.pre_check_job_template_with_filter:
+  extends:
+    - .pre_check_base_template
+    - .before_script_lesser
+
 check_line_endings:
-  extends: .check_job_template
+  extends: .pre_check_job_template
   script:
     - tools/ci/check-line-endings.sh ${IDF_PATH}
 
 check_permissions:
-  extends: .check_job_template
+  extends: .pre_check_job_template
   script:
     - tools/ci/check-executable.sh
 
 check_docs_lang_sync:
-  extends: .check_job_template
-  stage: pre_check
+  extends: .pre_check_job_template
   variables:
     SUBMODULES_TO_FETCH: "none"
   script:
     - cd docs
     - ./check_lang_folder_sync.sh
 
-check_docs_gh_links:
-  extends: .build_docs_template
-  stage: pre_check
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
-
 check_version:
-  extends: .check_job_template
   # Don't run this for feature/bugfix branches, so that it is possible to modify
   # esp_idf_version.h in a branch before tagging the next version.
-  only:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
+  extends:
+    - .pre_check_job_template
+    - .rules:protected
   script:
     - export IDF_PATH=$PWD
     - tools/ci/check_idf_version.sh
 
 check_examples_cmake_make:
-  extends: .check_job_template_with_filter
-  except:
-    - master
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
+  extends:
+    - .pre_check_job_template_with_filter
+    - .rules:dev
   script:
     - python ${IDF_PATH}/tools/ci/check_examples_cmake_make.py
 
 check_rom_api_header:
-  extends: .check_job_template_with_filter
+  extends: .pre_check_job_template_with_filter
   script:
     - tools/ci/check_examples_rom_header.sh
     - tools/ci/check_rom_apis.sh
 
 check_python_style:
-  extends: .python_lint_template
+  extends: .pre_check_base_template
   artifacts:
     when: on_failure
     paths:
@@ -65,7 +67,7 @@ check_python_style:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
 check_kconfigs:
-  extends: .check_job_template_with_filter
+  extends: .pre_check_job_template_with_filter
   artifacts:
     when: on_failure
     paths:
@@ -81,51 +83,31 @@ check_kconfigs:
     - ${IDF_PATH}/tools/check_kconfigs.py
 
 check_deprecated_kconfig_options:
-  extends: .check_job_template_with_filter
+  extends: .pre_check_job_template_with_filter
   script:
     - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py
 
 check_cmake_style:
-  extends: .check_job_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
+  extends: .pre_check_job_template
   script:
     tools/cmake/run_cmake_lint.sh
 
 check_wifi_lib_md5:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  extends: .pre_check_base_template
   tags:
     - build
   variables:
     SUBMODULES_TO_FETCH: "components/esp_wifi/lib"
-  dependencies: []
   script:
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
 
-# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits
-fast_template_app:
-  extends: .build_template_app_template
-  stage: pre_check
-  variables:
-    BUILD_COMMAND_ARGS: "-p"
-
 check_public_headers:
-  stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  extends:
+    - .pre_check_base_template
+    - .rules:labels:build
   tags:
     - build
-  variables:
-    BATCH_BUILD: "1"
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD
-      - $BOT_LABEL_REGULAR_TEST
   script:
     - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf-
 
@@ -140,16 +122,8 @@ check_public_headers:
 
 scan_tests:
   extends:
-    - .before_script_for_component_ut
     - .scan_build_tests
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_REGULAR_TEST
-      - $BOT_LABEL_EXAMPLE_TEST
-      - $BOT_LABEL_CUSTOM_TEST
-      - $BOT_LABEL_UNIT_TEST
-      - $BOT_LABEL_UNIT_TEST_S2
+    - .rules:build_tests:target_test-weekend_test
   artifacts:
     paths:
       - $EXAMPLE_TEST_OUTPUT_DIR
@@ -163,13 +137,14 @@ scan_tests:
     COMPONENT_UT_OUTPUT_DIR: ${CI_PROJECT_DIR}/component_ut/test_configs
     PYTHON_VER: 3
   script:
+    - set_component_ut_vars
     - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b make --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
     - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -b cmake --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR
     - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
     - python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
 
 check_readme_links:
-  extends: .check_job_template
+  extends: .pre_check_job_template
   tags: [ "amd64", "deploy", "internet" ]
   allow_failure: true
   variables:
@@ -178,11 +153,8 @@ check_readme_links:
     - python ${IDF_PATH}/tools/ci/check_readme_links.py
 
 .clang_tidy_check_template:
-  stage: pre_check
+  extends: .pre_check_base_template
   image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
-  tags:
-    - host_test
-  dependencies: []
   artifacts:
     reports:
       junit: $IDF_PATH/output.xml
@@ -191,7 +163,7 @@ check_readme_links:
       - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
     expire_in: 1 day
   script:
-    - ./tools/ci/retry_failed.sh git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
+    - retry_failed git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
     # Setup parameters of triggered/regular job
     - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
     - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
@@ -212,18 +184,15 @@ clang_tidy_check_all:
     BOT_LABEL_STATIC_ANALYSIS_ALL: 1
 
 check_codeowners:
-  extends: .check_job_template
+  extends: .pre_check_job_template
   script:
     - tools/codeowners.py ci-check
 
 # For release tag pipelines only, make sure the tag was created with 'git tag -a' so it will update
 # the version returned by 'git describe'
 check_version_tag:
-  extends: .check_job_template
-  only:
-    refs:
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
+  extends:
+    - .pre_check_job_template
+    - .rules:tag:release-no_label
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || echo "ESP-IDF versions must be annotated tags." && exit 1

tools/ci/config/rules.yml

@@ -0,0 +1,249 @@
+# if anchors
+.if-ref-master: &if-ref-master
+  if: '$CI_COMMIT_REF_NAME == "master"'
+
+.if-tag-release-no_label: &if-tag-release-no_label
+  if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/ && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.if-schedule: &if-schedule
+  if: '$CI_PIPELINE_SOURCE == "schedule"'
+
+.if-trigger: &if-trigger
+  if: '$CI_PIPELINE_SOURCE == "trigger"'
+
+.if-label-regular_test: &if-label-regular_test
+  if: '$BOT_LABEL_REGULAR_TEST'
+
+.if-label-build: &if-label-build
+  if: '$BOT_LABEL_BUILD'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS'
+
+.if-label-integration_test: &if-label-integration_test
+  if: '$BOT_LABEL_INTEGRATION_TEST'
+
+.if-label-unit_test: &if-label-unit_test
+  if: '$BOT_LABEL_UNIT_TEST'
+
+.if-label-unit_test_s2: &if-label-unit_test_s2
+  if: '$BOT_LABEL_UNIT_TEST_S2'
+
+.if-label-unit_test-all_targets: &if-label-unit_test-all_targets
+  if: '$BOT_LABEL_UNIT_TEST || $BOT_LABEL_UNIT_TEST_S2'
+
+.if-label-weekend_test: &if-label-weekend_test
+  if: '$BOT_LABEL_WEEKEND_TEST'
+
+.if-label-example_test: &if-label-example_test
+  if: '$BOT_LABEL_EXAMPLE_TEST'
+
+.if-label-custom_test: &if-label-custom_test
+  if: '$BOT_LABEL_CUSTOM_TEST'
+
+.if-label-host_test: &if-label-host_test
+  if: '$BOT_LABEL_HOST_TEST'
+
+.if-label-fuzzer_test: &if-label-fuzzer_test
+  if: '$BOT_LABEL_FUZZER_TEST'
+
+.if-label-nvs_coverage: &if-label-nvs_coverage
+  if: '$BOT_LABEL_NVS_COVERAGE'
+
+.if-label-static_analysis: &if-label-static_analysis
+  if: '$BOT_LABEL_STATIC_ANALYSIS || $BOT_LABEL_STATIC_ANALYSIS_ALL'
+
+.if-label-iperf_stress_test: &if-label-iperf_stress_test
+  if: '$BOT_LABEL_IPERF_STRESS_TEST'
+
+.if-os-mac: &if-os-mac
+  if: '$BOT_LABEL_MACOS_TEST'
+
+# Rules templates
+.rules:protected:
+  rules:
+    - <<: *if-protected
+
+.rules:protected-no_label:
+  rules:
+    - <<: *if-protected-no_label
+
+.rules:protected-schedule:
+  rules:
+    - <<: *if-protected
+    - <<: *if-schedule
+
+.rules:trigger:
+  rules:
+    - <<: *if-trigger
+
+.rules:dev:
+  rules:
+    - <<: *if-trigger
+    - <<: *if-dev-push
+
+.rules:os:mac_os:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-os-mac
+
+.rules:tag:release-no_label:
+  rules:
+    - <<: *if-tag-release-no_label
+
+.rules:ref:master-schedule:
+  rules:
+    - <<: *if-ref-master
+    - <<: *if-schedule
+
+.rules:ref:master-schedule-always:
+  rules:
+    - <<: *if-ref-master
+      when: always
+    - <<: *if-schedule
+      when: always
+
+.rules:labels:static_analysis-only:
+  rules:
+    - <<: *if-label-static_analysis
+
+.rules:labels:build:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-regular_test
+    - <<: *if-label-build
+
+.rules:labels:build_docs:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-build_docs
+
+.rules:labels:build_docs-slim:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build_docs
+
+.rules:labels:weekend_test-only:
+  rules:
+    - <<: *if-label-weekend_test
+
+.rules:labels:iperf_stress_test-only:
+  rules:
+    - <<: *if-label-iperf_stress_test
+
+.rules:labels:fuzzer_test-weekend_test-only:
+  rules:
+    - <<: *if-label-fuzzer_test
+    - <<: *if-label-weekend_test
+
+.rules:labels:nvs_coverage-only:
+  rules:
+    - <<: *if-label-nvs_coverage
+
+.rules:labels:host_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-regular_test
+    - <<: *if-label-host_test
+
+.rules:tests:example_test-schedule:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-example_test
+    - <<: *if-schedule
+
+.rules:tests:custom_test-schedule:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-custom_test
+    - <<: *if-schedule
+
+.rules:tests:unit_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-unit_test
+
+.rules:tests:unit_test_s2-only:
+  rules:
+    - <<: *if-label-unit_test_s2
+
+.rules:tests:integration_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-integration_test
+
+.rules:tests:target_test-integration_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-example_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-unit_test-all_targets
+    - <<: *if-label-integration_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:integration_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-integration_test
+
+.rules:build_tests:weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:unit_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-unit_test-all_targets
+
+.rules:build_tests:example_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-example_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:custom_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-weekend_test
+
+.rules:build_tests:target_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-example_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-unit_test-all_targets
+
+.rules:build_tests:target_test-weekend_test:
+  rules:
+    - <<: *if-protected-no_label
+    - <<: *if-label-build
+    - <<: *if-label-regular_test
+    - <<: *if-label-example_test
+    - <<: *if-label-custom_test
+    - <<: *if-label-unit_test-all_targets
+    - <<: *if-label-weekend_test

tools/ci/config/target-test.yml

@@ -1,24 +1,13 @@
 # for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
 # we need to convert to pattern "job_name_index.yml"
 .define_config_file_name: &define_config_file_name |
-    JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
-    JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
-    CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"
+  JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
+  JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
+  CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"
 
-.example_test_template:
+.target_test_job_template:
   stage: target_test
-  when: on_success
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_EXAMPLE_TEST
-  dependencies:
+  needs:
     - assign_test
   artifacts:
     when: always
@@ -26,10 +15,9 @@
       - $LOG_PATH
     expire_in: 1 week
     reports:
-        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+      junit: $LOG_PATH/*/XUNIT_RESULT.xml
   variables:
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
@@ -38,133 +26,70 @@
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
     # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     - cd tools/ci/python_packages/tiny_test_fw/bin
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
-.example_debug_template:
-  stage: target_test
-  when: on_success
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_EXAMPLE_TEST
-  dependencies:
-    - assign_test
-  artifacts:
-    when: always
-    paths:
-      - $LOG_PATH
-    expire_in: 1 week
-    reports:
-        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+.example_test_template:
+  extends:
+    - .target_test_job_template
+    - .rules:tests:example_test-schedule
   variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd tools/ci/python_packages/tiny_test_fw/bin
-    # run test
-    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
-.test_app_template:
+.example_debug_template:
   extends: .example_test_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_CUSTOM_TEST
   variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    SUBMODULES_TO_FETCH: "all"
+
+.test_app_template:
+  extends:
+    - .target_test_job_template
+    - .rules:tests:custom_test-schedule
+  variables:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
 
 .component_ut_template:
   extends:
-    - .before_script_for_component_ut
-    - .example_test_template
-  only:
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST
+    - .target_test_job_template
+    - .rules:tests:unit_test
   variables:
+    TEST_CASE_PATH: "$COMPONENT_UT_DIRS"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/component_ut/test_configs"
     PYTHON_VER: 3
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
+    - set_component_ut_vars
     # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     - cd tools/ci/python_packages/tiny_test_fw/bin
     # run test
-    - python Runner.py $COMPONENT_UT_DIRS -c $CONFIG_FILE -e $ENV_FILE
+    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
 .unit_test_template:
-  extends: .example_test_template
-  stage: target_test
-  dependencies:
-    - assign_test
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST
+  extends:
+    - .target_test_job_template
+    - .rules:tests:unit_test
   variables:
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
     PYTHON_VER: 3
 
-.test_template:
-  stage: target_test
-  when: on_success
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_INTEGRATION_TEST
-  dependencies:
+.integration_test_template:
+  extends:
+    - .target_test_job_template
+    - .rules:tests:integration_test
+  needs:
     - assign_test
     - build_ssc_esp32
-  artifacts:
-    when: always
-    reports:
-        junit: $LOG_PATH/*/XUNIT_RESULT.xml
-    paths:
-      - $LOG_PATH
-    expire_in: 1 week
   variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
@@ -179,56 +104,48 @@
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench
     # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
     # run test
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
 
 test_weekend_mqtt:
-  extends: .example_test_template
-  stage: target_test
+  extends:
+    - .example_test_template
+    - .rules:labels:weekend_test-only
   tags:
     - ESP32
     - Example_WIFI
-  only:
-    variables:
-      - $BOT_LABEL_WEEKEND_TEST
   variables:
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
     CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
 
 test_weekend_network:
-  extends: .example_test_template
-  stage: target_test
+  extends:
+    - .example_test_template
+    - .rules:labels:weekend_test-only
   image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
   tags:
     - ESP32
     - Example_WIFI
-  only:
-    variables:
-      - $BOT_LABEL_WEEKEND_TEST
   variables:
-    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
-    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
     CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
 
 example_test_001A:
   extends: .example_test_template
   parallel: 4
   artifacts:
-      when: always
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-      expire_in: 1 week
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
   tags:
     - ESP32
     - Example_WIFI
@@ -252,17 +169,6 @@ example_test_002:
   tags:
     - ESP32
     - Example_ShieldBox_Basic
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    - cd tools/ci/python_packages/tiny_test_fw/bin
-    # run test
-    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
-
 
 .example_test_003:
   extends: .example_test_template
@@ -289,11 +195,10 @@ example_test_005:
     - Example_WIFI_BT
 
 example_test_006:
-  extends: .example_test_template
+  extends:
+    - .example_test_template
+    - .rules:labels:iperf_stress_test-only
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
-  only:
-    variables:
-      - $BOT_LABEL_IPERF_STRESS_TEST
   tags:
     - ESP32
     - Example_ShieldBox
@@ -322,10 +227,8 @@ example_test_009:
     - ESP32
     - test_jtag_arm
   artifacts:
-      when: always
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-      expire_in: 1 week
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
   variables:
     SETUP_TOOLS: "1"
     PYTHON_VER: 3
@@ -342,11 +245,9 @@ example_test_011:
     - ESP32
     - Example_T2_RS485
   artifacts:
-      when: always
-      expire_in: 1 week
-      paths:
-        - $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
-        - $LOG_PATH
+    paths:
+      - $CI_PROJECT_DIR/examples/protocols/modbus/serial/*.log
+      - $LOG_PATH
   variables:
     SETUP_TOOLS: "1"
 
@@ -374,11 +275,9 @@ example_test_015:
     - ESP32
     - Example_PPP
   artifacts:
-      when: always
-      expire_in: 1 week
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-        - $LOG_PATH
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
+      - $LOG_PATH
 
 example_test_016:
   extends: .example_test_template
@@ -386,11 +285,9 @@ example_test_016:
     - ESP32
     - Example_Modbus_TCP
   artifacts:
-      when: always
-      expire_in: 1 week
-      paths:
-        - $CI_PROJECT_DIR/examples/*/*/*.log
-        - $LOG_PATH
+    paths:
+      - $CI_PROJECT_DIR/examples/*/*/*.log
+      - $LOG_PATH
 
 test_app_test_001:
   extends: .test_app_template
@@ -399,10 +296,8 @@ test_app_test_001:
     - ESP32
     - test_jtag_arm
   artifacts:
-      when: always
-      paths:
-        - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
-      expire_in: 1 week
+    paths:
+      - $CI_PROJECT_DIR/tools/test_apps/system/*/*.log
   variables:
     SETUP_TOOLS: "1"
     PYTHON_VER: 3
@@ -578,21 +473,9 @@ UT_034:
     - UT_T1_ESP_FLASH
 
 .unit_test_s2_template:
-  extends: .unit_test_template
-  dependencies:
-    - assign_test
-  only:
-    refs:
-      # Due to lack of runners, the tests are only done by manual trigger
-      # - master
-      # - /^release\/v/
-      # - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-      - schedules
-    variables:
-      # Due to lack of runners, S2 tests will only be triggered with label (@bot test will not trigger)
-      # - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_UNIT_TEST_S2
+  extends:
+    - .unit_test_template
+    - .rules:tests:unit_test_s2-only # due to the lack of runners, s2 tests will only be triggered by label
 
 UT_035:
   extends: .unit_test_s2_template
@@ -610,7 +493,7 @@ UT_036:
 
 # ToDo: re-enable this job when ESP32-S2 LEDC runner installed
 # UT_037:
-#   extends: .unit_test_template
+#   extends: .unit_test_s2_template
 #   tags:
 #     - ESP32S2_IDF
 #     - UT_T1_LEDC
@@ -648,7 +531,7 @@ UT_045:
     - ESP32_IDF
     - UT_SDIO
     - psram
-    
+
 UT_046:
   extends: .unit_test_template
   tags:
@@ -656,7 +539,7 @@ UT_046:
     - UT_T1_GPIO
 
 nvs_compatible_test:
-  extends: .test_template
+  extends: .integration_test_template
   artifacts:
     when: always
     paths:
@@ -671,12 +554,12 @@ nvs_compatible_test:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
-    - ./tools/ci/retry_failed.sh git clone $TEST_ENV_CONFIG_REPO
+    - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench
     # can not retry if downing git lfs files failed, so using empty_branch first.
-    - ./tools/ci/retry_failed.sh git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
-    - ./tools/ci/retry_failed.sh git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
+    - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
     # prepare nvs bins
     - cd auto_test_script
@@ -685,129 +568,129 @@ nvs_compatible_test:
     - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
 
 IT_001:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 3
   tags:
     - ESP32_IDF
     - SSC_T1_4
 
 IT_002:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_2
 
 IT_003:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 14
   tags:
     - ESP32_IDF
     - SSC_T2_5
 
 IT_004:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_APC
 
 IT_005:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 2
   tags:
     - ESP32_IDF
     - SSC_T1_5
 
 IT_006:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 12
   tags:
     - ESP32_IDF
     - SSC_T1_6
 
 IT_007:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 3
   tags:
     - ESP32_IDF
     - SSC_T1_7
 
 IT_008:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_8
 
 IT_009:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_3
 
 IT_011:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_MESH1
 
 IT_012:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 2
   tags:
     - ESP32_IDF
     - SSC_T2_MESH1
 
 IT_013:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T3_MESH1
 
 IT_014:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T6_MESH1
 
 IT_015:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T12_MESH1
 
 IT_016:
-  extends: .test_template
+  extends: .integration_test_template
   allow_failure: true
   tags:
     - ESP32_IDF
     - SSC_T50_MESH1
 
 IT_017:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_MESH2
 
 IT_018:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T1_9
 
 IT_019:
-  extends: .test_template
+  extends: .integration_test_template
   parallel: 2
   tags:
     - ESP32_IDF
     - SSC_T2_2
 
 IT_020:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T2_3
 
 IT_021:
-  extends: .test_template
+  extends: .integration_test_template
   tags:
     - ESP32_IDF
     - SSC_T2_4

tools/ci/executable-list.txt

@@ -58,10 +58,10 @@ tools/ci/mirror-submodule-update.sh
 tools/ci/multirun_with_pyenv.sh
 tools/ci/normalize_clangtidy_path.py
 tools/ci/push_to_github.sh
-tools/ci/retry_failed.sh
 tools/ci/test_build_system.sh
 tools/ci/test_build_system_cmake.sh
 tools/ci/test_configure_ci_environment.sh
+tools/ci/utils.sh
 tools/cmake/convert_to_cmake.py
 tools/cmake/run_cmake_lint.sh
 tools/codeowners.py

tools/ci/python_packages/ttfw_idf/CIScanTests.py

@@ -20,6 +20,7 @@ BUILD_ALL_LABELS = [
     'BOT_LABEL_BUILD',
     'BOT_LABEL_BUILD_ALL_APPS',
     'BOT_LABEL_REGULAR_TEST',
+    'BOT_LABEL_WEEKEND_TEST',
 ]
 
 

tools/ci/retry_failed.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-#
-# Retries a command RETRY_ATTEMPTS times in case of failure
-#
-# Inspired by https://stackoverflow.com/a/8351489
-#
-
-max_attempts=${RETRY_ATTEMPTS-3}
-RETRY_TIMEWAIT=${RETRY_TIMEWAIT-1}
-attempt=1
-exitCode=0
-whole_start=$(date +%s)
-attempt_start=whole_start
-
-while true; do
-  if "$@" ; then
-    exitCode=0
-    break
-  else
-    exitCode=$?
-  fi
-
-  if (( $attempt >= $max_attempts )) ; then
-    break
-  fi
-
-  echo "Failed! ("$@") Spent time $(( $(date '+%s') - ${attempt_start} )) sec. Retrying in ${RETRY_TIMEWAIT}..." 1>&2
-  sleep $RETRY_TIMEWAIT
-  attempt=$(( attempt + 1 ))
-  RETRY_TIMEWAIT=$(( RETRY_TIMEWAIT * 2 ))
-  attempt_start=$(date +%s)
-done
-
-if [[ $exitCode != 0 ]] ; then
-  echo -n "Totally failed! ("$@")" 1>&2
-else
-  echo -n "Done ("$@")" 1>&2
-fi
-
-echo " Spent time $(( $(date '+%s') - ${whole_start} )) sec in total" 1>&2
-
-exit $exitCode

tools/ci/utils.sh

@@ -0,0 +1,119 @@
+# Modified from https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/utils.sh
+
+# before each job, we need to check if this job is filtered by bot stage/job filter
+function apply_bot_filter() {
+  python "${IDF_PATH}"/tools/ci/apply_bot_filter.py || exit 0
+}
+
+function add_ssh_keys() {
+  local key_string="${1}"
+  mkdir -p ~/.ssh
+  chmod 700 ~/.ssh
+  echo -n "${key_string}" >~/.ssh/id_rsa_base64
+  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  chmod 600 ~/.ssh/id_rsa
+}
+
+function add_gitlab_ssh_keys() {
+  add_ssh_keys "${GITLAB_KEY}"
+  echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
+
+  # For gitlab geo nodes
+  if [ "${LOCAL_GITLAB_SSH_SERVER:-}" ]; then
+    SRV=${LOCAL_GITLAB_SSH_SERVER##*@} # remove the chars before @, which is the account
+    SRV=${SRV%%:*}                     # remove the chars after :, which is the port
+    printf "Host %s\n\tStrictHostKeyChecking no\n" "${SRV}" >>~/.ssh/config
+  fi
+}
+
+function add_github_ssh_keys() {
+  add_ssh_keys "${GH_PUSH_KEY}"
+  echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >>~/.ssh/config
+}
+
+function add_doc_server_ssh_keys() {
+  local key_string="${1}"
+  local server_url="${2}"
+  local server_user="${3}"
+  add_ssh_keys "${key_string}"
+  echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
+}
+
+function fetch_submodules() {
+  python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+}
+
+function get_all_submodules() {
+  git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | sed -e 's|$|/**|' | xargs | sed -e 's/ /,/g'
+}
+
+function set_component_ut_vars() {
+  local exclude_list_fp="${IDF_PATH}/tools/ci/component_ut_excludes.txt"
+  export COMPONENT_UT_DIRS=$(find components/ -name test_apps -type d)
+  export COMPONENT_UT_EXCLUDES=$([ -r $exclude_list_fp ] && cat $exclude_list_fp | xargs)
+  echo "COMPONENT_UT_DIRS, COMPONENT_UT_EXCLUDES written into export"
+}
+
+function error() {
+  printf "\033[0;31m%s\n\033[0m" "${1}" >&2
+}
+
+function info() {
+  printf "\033[0;32m%s\n\033[0m" "${1}" >&2
+}
+
+function warning() {
+  printf "\033[0;33m%s\n\033[0m" "${1}" >&2
+}
+
+function run_cmd() {
+  local start=$(date +%s)
+  eval "$@"
+  local ret=$?
+  local end=$(date +%s)
+  local duration=$((end - start))
+
+  if [[ $ret -eq 0 ]]; then
+    info "(\$ $*) succeeded in ${duration} seconds."
+    return 0
+  else
+    error "(\$ $*) failed in ${duration} seconds."
+    return $ret
+  fi
+}
+
+# Retries a command RETRY_ATTEMPTS times in case of failure
+# Inspired by https://stackoverflow.com/a/8351489
+function retry_failed() {
+  local max_attempts=${RETRY_ATTEMPTS-3}
+  local timeout=${RETRY_TIMEWAIT-1}
+  local attempt=1
+  local exitCode=0
+
+  whole_start=$(date +%s)
+  while true; do
+    if run_cmd "$@"; then
+      exitCode=0
+      break
+    else
+      exitCode=$?
+    fi
+
+    if ((attempt >= max_attempts)); then
+      break
+    fi
+
+    error "Retrying in ${timeout} seconds..."
+    sleep $timeout
+    attempt=$((attempt + 1))
+    timeout=$((timeout * 2))
+  done
+
+  local duration=$(($(date '+%s') - whole_start))
+  if [[ $exitCode != 0 ]]; then
+    error "Totally failed! Spent $duration sec in total"
+  else
+    info "Done! Spent $duration sec in total"
+  fi
+  return $exitCode
+}