Merge branch 'feat/dynamic_integration_pipeline' into 'master'

ci: dynamic integration child pipeline

Closes IDFCI-1659 and MTWJ-306

See merge request espressif/esp-idf!25898

.gitlab-ci.yml

@@ -1,7 +1,7 @@
 workflow:
   rules:
     # Disable those non-protected push triggered pipelines
-    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
       when: never
     # when running merged result pipelines, it would create a temp commit id. use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA instead of $CI_COMMIT_SHA.
     # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha
@@ -25,7 +25,7 @@ include:
   - '.gitlab/ci/static-code-analysis.yml'
   - '.gitlab/ci/pre_check.yml'
   - '.gitlab/ci/build.yml'
-  - '.gitlab/ci/assign-test.yml'
+  - '.gitlab/ci/integration_test.yml'
   - '.gitlab/ci/host-test.yml'
   - '.gitlab/ci/target-test.yml'
   - '.gitlab/ci/deploy.yml'

.gitlab/ci/README.md

@@ -23,10 +23,12 @@
     - [Grammar](#grammar)
     - [Special Rules](#special-rules)
   - [Upload/Download Artifacts to Internal Minio Server](#uploaddownload-artifacts-to-internal-minio-server)
-    - [Env Vars](#env-vars)
-    - [Artifacts Types and File Patterns](#artifacts-types-and-file-patterns)
-    - [Upload](#upload)
-    - [Download](#download)
+    - [Users Without Access to Minio](#users-without-access-to-minio)
+    - [Users With Access to Minio](#users-with-access-to-minio)
+      - [Env Vars for Minio](#env-vars-for-minio)
+      - [Artifacts Types and File Patterns](#artifacts-types-and-file-patterns)
+      - [Upload](#upload)
+      - [Download](#download)
 
 ## General Workflow
 
@@ -143,10 +145,11 @@ check if there's a suitable `.if-<if-anchor-you-need>` anchor
 1. if there is, create a rule following [`rules` Template Naming Rules](#rules-template-naming-rules).For detail information, please refer to [GitLab Documentation `rules-if`](https://docs.gitlab.com/ee/ci/yaml/README.html#rulesif). Here's an example.
 
     ```yaml
-    .rules:dev:
+    .rules:patterns:python-files:
       rules:
-        - <<: *if-trigger
+        - <<: *if-protected
         - <<: *if-dev-push
+          changes: *patterns-python-files
     ```
 
 2. if there isn't

.gitlab/ci/assign-test.yml

@@ -1,39 +0,0 @@
-.assign_test_template:
-  image: $TARGET_TEST_ENV_IMAGE
-  stage: assign_test
-  tags:
-    - assign_test
-  dependencies: []
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-
-assign_integration_test:
-  extends:
-    - .assign_test_template
-    - .rules:test:integration_test
-    - .before_script:minimal
-  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
-  needs:
-    - build_ssc_esp32
-    - build_ssc_esp32c3
-    - build_ssc_esp32c2
-  artifacts:
-    paths:
-      - $TEST_DIR/test_configs
-    expire_in: 1 week
-  variables:
-    TEST_DIR: ${CI_PROJECT_DIR}/tools/ci/integration_test
-    BUILD_DIR: ${CI_PROJECT_DIR}/SSC/ssc_bin
-    INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
-    ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
-    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    GIT_LFS_SKIP_SMUDGE: 1
-  script:
-    - add_gitlab_ssh_keys
-    # clone test script to assign tests
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - cd auto_test_script
-    - ./tools/ci/setup_idfci.sh
-    # assign integration test cases
-    - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs

.gitlab/ci/build.yml

@@ -378,69 +378,6 @@ fast_template_app:
     BUILD_COMMAND_ARGS: "-p"
 #------------------------------------------------------------------------------
 
-.build_ssc_template:
-  extends:
-    - .build_template
-    - .rules:build:integration_test
-  needs:
-    - job: fast_template_app
-      artifacts: false
-  artifacts:
-    paths:
-      - SSC/build/log/*
-      - SSC/build/**/*.log
-      - SSC/build/*log.txt
-      - SSC/ssc_bin
-    when: always
-    expire_in: 1 week
-  script:
-    - retry_failed git clone $SSC_REPOSITORY
-    - python $CHECKOUT_REF_SCRIPT SSC SSC
-    - cd SSC
-    - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME
-
-build_ssc_esp32:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32"
-
-build_ssc_esp32s2:
-  extends: .build_ssc_template
-  parallel: 2
-  variables:
-    TARGET_NAME: "ESP32S2"
-
-build_ssc_esp32c2:
-  extends: .build_ssc_template
-  parallel: 2
-  variables:
-    TARGET_NAME: "ESP32C2"
-
-build_ssc_esp32c3:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32C3"
-
-build_ssc_esp32s3:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32S3"
-
-build_ssc_esp32c6:
-  extends: .build_ssc_template
-  parallel: 3
-  variables:
-    TARGET_NAME: "ESP32C6"
-
-build_ssc_esp32h2:
-  extends: .build_ssc_template
-  parallel: 2
-  variables:
-    TARGET_NAME: "ESP32H2"
-
 build_examples_cmake_esp32:
   extends:
     - .build_cmake_template

.gitlab/ci/common.yml

@@ -45,8 +45,6 @@ variables:
   PYTHON_VER: 3.8.17
 
   # Docker images
-  BOT_DOCKER_IMAGE_TAG: ":latest"
-
   ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.2:2"
   ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.2:2-1"
   QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.2:2-20230522"
@@ -56,9 +54,6 @@ variables:
 
   PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
 
-  # target test config file, used by assign test job
-  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
-
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
 

.gitlab/ci/deploy.yml

@@ -32,7 +32,7 @@ push_to_github:
   extends:
     - .deploy_job_template
     - .before_script:minimal
-    - .rules:protected-no_label
+    - .rules:push_to_github
   dependencies: []
   script:
     - add_github_ssh_keys
@@ -48,43 +48,3 @@ deploy_update_SHA_in_esp-dockerfiles:
     - 'curl --header "PRIVATE-TOKEN: ${ESPCI_SCRIPTS_TOKEN}" -o create_MR_in_esp_dockerfile.sh $GITLAB_HTTP_SERVER/api/v4/projects/1260/repository/files/create_MR_in_esp_dockerfile%2Fcreate_MR_in_esp_dockerfile.sh/raw\?ref\=master'
     - chmod +x create_MR_in_esp_dockerfile.sh
     - ./create_MR_in_esp_dockerfile.sh
-
-deploy_test_result:
-  extends:
-    - .deploy_job_template
-    - .before_script:minimal
-    - .rules:ref:master-always
-  image: $CI_DOCKER_REGISTRY/bot-env:1
-  dependencies: []
-  tags:
-    - deploy_test
-  artifacts:
-    when: always
-    paths:
-      - ${CI_PROJECT_DIR}/test-management/*.log
-    expire_in: 1 week
-  variables:
-    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    TEST_RESULTS_PATH: "${CI_PROJECT_DIR}/TEST_RESULTS"
-  script:
-    - add_gitlab_ssh_keys
-    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
-    - export REV_COUNT=$(git rev-list --count ${GIT_SHA} --)
-    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
-    # Download test result
-    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-    - python3 ${IDF_PATH}/tools/ci/get_all_test_results.py --path ${TEST_RESULTS_PATH} --include_retried
-    - if [[ -z $(find ${TEST_RESULTS_PATH} -name "*.xml") ]]; then exit 0; fi
-    # Clone test-management repo
-    - retry_failed git clone $TEST_MANAGEMENT_REPO
-    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
-    - cd test-management
-    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
-    # Make sure all requirements are installed
-    - pip3 install -r requirements.txt
-    # Update test cases
-    - python3 ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT from_xml -d ${TEST_RESULTS_PATH} -r $GIT_SHA -l IDFCI
-    # update test results
-    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l IDFCI -p ${TEST_RESULTS_PATH} --pipeline_url ${CI_PIPELINE_URL}
-  # May need a long time to upload all test results.
-  timeout: 4 hours

.gitlab/ci/docs.yml

@@ -25,6 +25,9 @@
 .if-protected-no_label: &if-protected-no_label
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
 
+.if-qa-test-tag: &if-qa-test-tag
+  if: '$CI_COMMIT_TAG =~ /^qa-test/'
+
 .if-label-build_docs: &if-label-build_docs
   if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
 
@@ -32,10 +35,12 @@
   if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
 
 .if-dev-push: &if-dev-push
-  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
 
 .doc-rules:build:docs-full:
   rules:
+    - <<: *if-qa-test-tag
+      when: never
     - <<: *if-protected
     - <<: *if-label-build_docs
     - <<: *if-label-docs_full
@@ -44,6 +49,8 @@
 
 .doc-rules:build:docs-partial:
   rules:
+    - <<: *if-qa-test-tag
+      when: never
     - <<: *if-dev-push
       changes: *patterns-docs-full
       when: never

.gitlab/ci/integration_test.yml

@@ -0,0 +1,65 @@
+# generate dynamic integration pipeline by `idf-integration-ci` project
+
+.patterns-integration_test: &patterns-integration_test
+  # add all possible patterns to make sure `gen_integration_pipeline` can be triggered.
+  # fine-grained control will be done while generating the pipeline
+  # find `patterns` in `idf-integration-ci` project
+  - "components/**/*"
+  - "tools/**/*"
+  - ".gitlab-ci.yml"
+  - ".gitlab/ci/common.yml"
+  - ".gitlab/ci/integration-test.yml"
+  - ".gitmodules"
+  - "CMakeLists.txt"
+  - "install.sh"
+  - "export.sh"
+  - "Kconfig"
+  - "sdkconfig.rename"
+
+# Simplify the rules
+.integration_test_rules:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      changes: *patterns-integration_test
+
+gen_integration_pipeline:
+  extends:
+    - .before_script:minimal
+    - .integration_test_rules
+  image: ${CI_INTEGRATION_ASSIGN_ENV}
+  stage: assign_test
+  cache: []
+  tags:
+    - assign_test
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+    GIT_LFS_SKIP_SMUDGE: 1
+  needs:
+    - job: fast_template_app
+      artifacts: false
+      optional: true
+  artifacts:
+    paths:
+      - idf-integration-ci/child_pipeline/
+    expire_in: 2 weeks
+  script:
+    - add_gitlab_ssh_keys
+    - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci
+    - python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci
+    - cd idf-integration-ci
+    - python tools/generate_child_pipeline.py -o child_pipeline/
+
+child_integration_test_pipeline:
+  extends:
+    - .integration_test_rules
+  stage: assign_test
+  needs:
+    - gen_integration_pipeline
+  trigger:
+    include:
+      - artifact: idf-integration-ci/child_pipeline/pipeline.yml
+        job: gen_integration_pipeline
+    forward:
+      yaml_variables: false
+    strategy: depend

.gitlab/ci/pre_check.yml

@@ -22,7 +22,7 @@ check_pre_commit_master_release:
 check_pre_commit_MR:
   extends:
     - .check_pre_commit_template
-    - .rules:dev
+    - .rules:mr
   script:
     - fetch_submodules
     - python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs pre-commit run --files

.gitlab/ci/rules.yml

@@ -345,13 +345,19 @@
   if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/'
 
 .if-protected: &if-protected
-  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
 
 .if-protected-no_label: &if-protected-no_label
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
 
+.if-qa-test-tag: &if-qa-test-tag
+  if: '$CI_COMMIT_TAG =~ /^qa-test/'
+
 .if-dev-push: &if-dev-push
-  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.if-merge_request: &if-merge_request
+  if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
 
 .if-schedule: &if-schedule
   if: '$CI_PIPELINE_SOURCE == "schedule"'
@@ -385,38 +391,27 @@
   rules:
     - <<: *if-protected
 
-.rules:protected-no_label:
+.rules:push_to_github:
   rules:
+    - <<: *if-qa-test-tag
+      when: never
     - <<: *if-protected-no_label
 
 .rules:protected-no_label-always:
   rules:
+    - <<: *if-qa-test-tag
+      when: never
     - <<: *if-protected-no_label
       when: always
 
-.rules:dev:
-  rules:
-    - <<: *if-trigger
-    - <<: *if-dev-push
-
 .rules:mr:
   rules:
-    - <<: *if-dev-push
+    - <<: *if-merge_request
 
 .rules:tag:release:
   rules:
     - <<: *if-tag-release
 
-.rules:ref:master-schedule:
-  rules:
-    - <<: *if-ref-master
-    - <<: *if-schedule
-
-.rules:ref:master-always:
-  rules:
-    - <<: *if-ref-master
-      when: always
-
 ### Patterns ###
 .rules:patterns:python-cache:
   rules:
@@ -1535,27 +1530,6 @@
     - <<: *if-dev-push
       changes: *patterns-target_test-wifi
 
-.rules:build:integration_test:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-protected
-    - <<: *if-label-build
-    - <<: *if-label-integration_test
-    - <<: *if-label-integration_test_ble
-    - <<: *if-label-integration_test_wifi
-    - <<: *if-label-target_test
-    - <<: *if-dev-push
-      changes: *patterns-build_components
-    - <<: *if-dev-push
-      changes: *patterns-build_system
-    - <<: *if-dev-push
-      changes: *patterns-integration_test-ble
-    - <<: *if-dev-push
-      changes: *patterns-integration_test-wifi
-    - <<: *if-dev-push
-      changes: *patterns-target_test-wifi
-
 .rules:build:macos:
   rules:
     - <<: *if-revert-branch
@@ -2626,52 +2600,6 @@
     - <<: *if-dev-push
       changes: *patterns-host_test
 
-.rules:test:integration_test:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-protected
-    - <<: *if-label-build-only
-      when: never
-    - <<: *if-label-integration_test
-    - <<: *if-label-integration_test_ble
-    - <<: *if-label-integration_test_wifi
-    - <<: *if-label-target_test
-    - <<: *if-dev-push
-      changes: *patterns-integration_test-ble
-    - <<: *if-dev-push
-      changes: *patterns-integration_test-wifi
-    - <<: *if-dev-push
-      changes: *patterns-target_test-wifi
-
-.rules:test:integration_test_ble:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-protected
-    - <<: *if-label-build-only
-      when: never
-    - <<: *if-label-integration_test
-    - <<: *if-label-integration_test_ble
-    - <<: *if-label-target_test
-    - <<: *if-dev-push
-      changes: *patterns-integration_test-ble
-
-.rules:test:integration_test_wifi:
-  rules:
-    - <<: *if-revert-branch
-      when: never
-    - <<: *if-protected
-    - <<: *if-label-build-only
-      when: never
-    - <<: *if-label-integration_test
-    - <<: *if-label-integration_test_wifi
-    - <<: *if-label-target_test
-    - <<: *if-dev-push
-      changes: *patterns-integration_test-wifi
-    - <<: *if-dev-push
-      changes: *patterns-target_test-wifi
-
 .rules:test:submodule:
   rules:
     - <<: *if-revert-branch

.gitlab/ci/target-test.yml

@@ -1534,319 +1534,3 @@ pytest_test_apps_esp32c3_wifi_two_dut:
     - job: build_pytest_test_apps_esp32c3
       artifacts: false
   tags: [ esp32c3, wifi_two_dut]
-
-# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
-# we need to convert to pattern "job_name_index.yml"
-.define_config_file_name: &define_config_file_name |
-  JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
-  JOB_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
-  CONFIG_FILE="${CONFIG_FILE_PATH}/${JOB_FULL_NAME}.yml"
-
-.integration_test_template:
-  extends:
-    - .target_test_template
-    - .rules:test:integration_test
-    - .before_script:minimal
-  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
-  cache: []
-  needs: # the assign already needs all the build jobs
-    - assign_integration_test
-  artifacts:
-    when: always
-    paths:
-      - "**/*.log"
-      - $LOG_PATH
-    exclude:
-      - .git/**/*
-    reports:
-      junit: $LOG_PATH/*/XUNIT_RESULT.xml
-    expire_in: 1 week
-  variables:
-    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
-    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
-    LOG_PATH: "${CI_PROJECT_DIR}/TEST_LOGS"
-    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/auto_test_script/TestCaseFiles"
-    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/ci/integration_test/test_configs"
-    KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/tools/ci/integration_test/KnownIssues"
-    CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
-    PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/tools/ci/idf_prepare_test_bins.py"
-    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}"
-    INITIAL_CONDITION_RETRY_COUNT: "1"
-    GIT_LFS_SKIP_SMUDGE: 1
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone local test env configs
-    - add_gitlab_ssh_keys
-    - retry_failed git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    # clone test bench
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - cd auto_test_script
-    - ./tools/ci/setup_idfci.sh
-    # Merge known issues
-    - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
-    # run test
-    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
-    - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
-
-.integration_test_esp32c3_template:
-  extends:
-    - .integration_test_template
-  variables:
-    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C3_IDF"
-
-.integration_test_esp32c2_template:
-  extends:
-    - .integration_test_template
-  variables:
-    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32C2_IDF"
-
-nvs_compatible_test:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test
-  artifacts:
-    when: always
-    paths:
-      - $LOG_PATH
-      - nvs_wifi.bin
-    expire_in: 1 mos
-  tags:
-    - ESP32_IDF
-    - NVS_Compatible
-  script:
-    - *define_config_file_name
-    # first test if config file exists, if not exist, exit 0
-    - test -e $CONFIG_FILE || exit 0
-    # clone local test env configs
-    - add_gitlab_ssh_keys
-    - retry_failed git clone $TEST_ENV_CONFIG_REPO
-    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    # clone test bench
-    - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
-    - cd auto_test_script
-    - ./tools/ci/setup_idfci.sh
-    # prepare nvs bins
-    - ./tools/prepare_nvs_bin.sh
-    # run test
-    - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
-    - python ${CI_RUNNER_SCRIPT} -l "$LOG_PATH/$JOB_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
-
-IT_T1_Simple:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  parallel: 2
-  tags:
-    - ESP32_IDF
-    - SSC_T1_Simple
-
-IT_T1_WAP:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T1_WAP
-
-IT_T2_Simple:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  parallel: 9
-  tags:
-    - ESP32_IDF
-    - SSC_T2_Simple
-
-IT_T5_BT_Simple:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_ble
-  parallel: 3
-  tags:
-    - ESP32_IDF
-    - SSC_T5_BT_Simple
-
-IT_T2_BT_Simple:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_ble
-  parallel: 6
-  tags:
-    - ESP32_IDF
-    - SSC_T2_BT_Simple
-
-IT_T1_BT_Dongle:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_ble
-  parallel: 2
-  tags:
-    - ESP32_IDF
-    - SSC_T1_BT_Dongle
-
-IT_T1_AP:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T1_AP
-
-IT_T1_AP_E:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  parallel: 5
-  tags:
-    - ESP32_IDF
-    - SSC_T1_AP_E
-
-IT_T1_WNIC_AP:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T1_WNIC_AP
-
-IT_T1_WNIC_AP_E:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T1_WNIC_AP_E
-
-IT_T1_MESH1:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T1_MESH1
-
-IT_T2_MESH1:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  parallel: 2
-  tags:
-    - ESP32_IDF
-    - SSC_T2_MESH1
-
-IT_T3_MESH1:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T3_MESH1
-
-IT_T6_MESH1:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T6_MESH1
-
-IT_T12_MESH1:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T12_MESH1
-
-IT_T50_MESH1:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  allow_failure: true
-  tags:
-    - ESP32_IDF
-    - SSC_T50_MESH1
-
-IT_T1_MESH2:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T1_MESH2
-
-IT_T2_WNIC:
-  extends:
-   - .integration_test_template
-   - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T2_WNIC
-
-IT_T2_AP:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T2_AP
-
-IT_T3_Simple:
-  extends:
-    - .integration_test_template
-    - .rules:test:integration_test_wifi
-  tags:
-    - ESP32_IDF
-    - SSC_T3_Simple
-
-IT_C3_T2_BT_Simple:
-  extends:
-    - .integration_test_esp32c3_template
-    - .rules:test:integration_test_ble
-  parallel: 11
-  tags:
-    - ESP32C3_IDF
-    - SSC_T2_BT_Simple
-
-IT_C3_T5_BT_Simple:
-  extends:
-    - .integration_test_esp32c3_template
-    - .rules:test:integration_test_ble
-  parallel: 5
-  tags:
-    - ESP32C3_IDF
-    - SSC_T5_BT_Simple
-
-IT_C3_T1_BT_Dongle:
-  extends:
-    - .integration_test_esp32c3_template
-    - .rules:test:integration_test_ble
-  image: gitlab.espressif.cn:5050/qa/dockerfiles/integration-test-env-dbg:1
-  parallel: 2
-  tags:
-    - ESP32C3_IDF
-    - SSC_T1_BT_Dongle
-
-IT_C2_T2_BT_Simple:
-  extends:
-    - .integration_test_esp32c2_template
-    - .rules:test:integration_test_ble
-  parallel: 9
-  tags:
-    - ESP32C2_IDF
-    - SSC_T2_BT_Simple
-
-IT_C2_T1_BT_Dongle:
-  extends:
-    - .integration_test_esp32c2_template
-    - .rules:test:integration_test_ble
-  image: gitlab.espressif.cn:5050/qa/dockerfiles/integration-test-env-dbg:1
-  parallel: 2
-  tags:
-    - ESP32C2_IDF
-    - SSC_T1_BT_Dongle