Merge branch 'ci/add_integration_test_env' into 'master'

CI: add integration test env

Closes TRM-82 and TRM-83

See merge request espressif/esp-idf!19368

.gitlab-ci.yml

@@ -77,8 +77,6 @@ variables:
 
   # target test repo parameters
   TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
-  CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
-  CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
 
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -183,20 +181,10 @@ before_script:
 
 .before_script_integration_test:
   before_script:
-    # need to install esptool, needn't install idf python env
-  - source tools/ci/utils.sh
-  - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-  - *set_include_nightly_run
-  - source tools/ci/setup_python.sh
-  - add_gitlab_ssh_keys
-  - fetch_submodules
-  # download the constraint file manually
-  - mkdir -p ~/.espressif
-  - wget -O ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw
-  - *download_test_python_contraint_file
-  # TODO: remove this, IDFCI-1207
-  - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  - *download_test_python_tool
+    # needn't install idf python env
+    - source tools/ci/utils.sh
+    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+    - add_gitlab_ssh_keys
 
 .before_script_macos:
   before_script:

.gitlab/ci/assign-test.yml

@@ -85,6 +85,7 @@ assign_integration_test:
   extends:
     - .assign_test_template
     - .rules:test:integration_test
+  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
   needs:
     - build_ssc_esp32
     - build_ssc_esp32c3
@@ -98,7 +99,6 @@ assign_integration_test:
     INTEGRATION_TEST_CASE_PATH: "${CI_PROJECT_DIR}/auto_test_script/TestCaseFiles"
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    PYTHON_VER: 3.7.7
   script:
     # clone test script to assign tests
     # can not retry if downing git lfs files failed, so using empty_branch first.

.gitlab/ci/target-test.yml

@@ -1052,6 +1052,8 @@ UT_S3_FLASH:
     - .rules:test:integration_test
     # needn't install idf python env
     - .before_script_integration_test
+  image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
+  cache: []
   needs: # the assign already needs all the build jobs
     - assign_integration_test
   variables:
@@ -1062,10 +1064,9 @@ UT_S3_FLASH:
     KNOWN_ISSUE_FILE: "${CI_PROJECT_DIR}/tools/ci/integration_test/KnownIssues"
     CI_RUNNER_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIRunner.py"
     PREPARE_TEST_BIN_SCRIPT: "${CI_PROJECT_DIR}/tools/ci/integration_test/prepare_test_bins.py"
-    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${PYTHONPATH}"
+    PYTHONPATH: "${CI_PROJECT_DIR}/auto_test_script/packages:${CI_PROJECT_DIR}/tools/ci/python_packages:${PYTHONPATH}"
+    SETUP_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/tools/ci/setup_idfci.sh"
     INITIAL_CONDITION_RETRY_COUNT: "1"
-    # auto_test_script only supports python 3.7.x
-    PYTHON_VER: 3.7.7
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
@@ -1078,6 +1079,7 @@ UT_S3_FLASH:
     - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
+    - if [ -f ${SETUP_SCRIPT} ]; then ${SETUP_SCRIPT}; fi
     - cat ${KNOWN_ISSUE_FILE} >> ${TEST_CASE_FILE_PATH}/KnownIssues
     # run test
     - python ${PREPARE_TEST_BIN_SCRIPT} $CONFIG_FILE
@@ -1104,10 +1106,6 @@ nvs_compatible_test:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
-    # prepare test utilities
-    - source tools/ci/setup_python.sh
-    - add_gitlab_ssh_keys
-    - fetch_submodules
     # clone local test env configs
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
@@ -1115,6 +1113,7 @@ nvs_compatible_test:
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch
     - retry_failed git -C auto_test_script checkout -f ${CI_AUTO_TEST_SCRIPT_REPO_BRANCH}
+    - if [ -f ${SETUP_SCRIPT} ]; then ${SETUP_SCRIPT}; fi
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # prepare nvs bins
     - cd auto_test_script

tools/ci/integration_test/prepare_test_bins.py

@@ -7,7 +7,14 @@ import argparse
 import os
 
 import gitlab
-import gitlab_api
+
+try:
+    import gitlab_api
+except ImportError:
+    import sys
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python_packages'))
+    import gitlab_api
+
 from AutoTestScript.RunnerConfigs.Config import Config
 
 SSC_BUILD_JOB_MAP = {