CI: update test cases from test results

.gitlab/ci/assign-test.yml

@@ -104,40 +104,3 @@ assign_integration_test:
     - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script --customized_only
     # assign integration test cases
     - python ${ASSIGN_TEST_CASE_SCRIPT} -t ${INTEGRATION_TEST_CASE_PATH} -c $CI_TARGET_TEST_CONFIG_FILE -b ${BUILD_DIR} -o $TEST_DIR/test_configs
-
-update_test_cases:
-  extends: .rules:ref:master-schedule
-  stage: assign_test
-  image: $TARGET_TEST_ENV_IMAGE
-  tags:
-    - deploy_test
-  needs:
-    - build_esp_idf_tests_cmake_esp32
-    - build_esp_idf_tests_cmake_esp32s2
-    - build_esp_idf_tests_cmake_esp32s3
-    - build_esp_idf_tests_cmake_esp32c3
-  artifacts:
-    when: always
-    paths:
-      - ${CI_PROJECT_DIR}/test-management/*.log
-    expire_in: 1 week
-  variables:
-    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
-    UNIT_TEST_DIR: "${CI_PROJECT_DIR}/components/idf_test/unit_test"
-    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
-    PYTHON_VER: 3.7.7
-  script:
-    - export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8)
-    - retry_failed git clone $TEST_MANAGEMENT_REPO
-    - python $CHECKOUT_REF_SCRIPT test-management test-management
-    - cd test-management
-    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
-    # update unit test cases
-    - export UNIT_TEST_CASE_FILES=$(find $UNIT_TEST_DIR -maxdepth 1 -name "*.yml" | xargs)
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILES -r $GIT_SHA
-    # update example test cases
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
-    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/tools/test_apps -r $GIT_SHA
-    # organize test cases
-    - python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT

.gitlab/ci/deploy.yml

@@ -46,35 +46,37 @@ deploy_test_result:
     - .deploy_job_template
     - .before_script_minimal
     - .rules:ref:master-schedule-always
-  image: $CI_DOCKER_REGISTRY/bot-env
+  image: $CI_DOCKER_REGISTRY/bot-env:1
+  dependencies: []
   tags:
     - deploy_test
   artifacts:
     when: always
     paths:
       - ${CI_PROJECT_DIR}/test-management/*.log
-      # save all test logs as artifacts, make it easier to track errors
-      - ${CI_PROJECT_DIR}/TEST_LOGS
-    expire_in: 1 mos
+    expire_in: 1 week
   variables:
-    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
-    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
-    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
+    TEST_RESULTS_PATH: "${CI_PROJECT_DIR}/TEST_RESULTS"
   script:
     - add_gitlab_ssh_keys
-    - export GIT_SHA=$(echo ${PIPELINE_COMMIT_SHA} | cut -c 1-8)
-    - export REV_COUNT=$(git rev-list --count ${PIPELINE_COMMIT_SHA} --)
+    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
+    - export REV_COUNT=$(git rev-list --count ${GIT_SHA} --)
     - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
-    # artifacts of job update_test_cases creates test-management folder
-    # we need to remove it so we can clone test-management folder again
-    - rm -rf test-management
+    # Download test result
+    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+    - python3 ${IDF_PATH}/tools/ci/get_all_test_results.py --path ${TEST_RESULTS_PATH} --include_retried
+    - if [[ -z $(find ${TEST_RESULTS_PATH} -name "*.xml") ]]; then exit 0; fi
+    # Clone test-management repo
     - retry_failed git clone $TEST_MANAGEMENT_REPO
     - python3 $CHECKOUT_REF_SCRIPT test-management test-management
     - cd test-management
     - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
+    # Make sure all requirements are installed
+    - pip3 install -r requirements.txt
+    # Update test cases
+    - python3 ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT from_xml -d ${TEST_RESULTS_PATH} -r $GIT_SHA -l IDFCI
     # update test results
-    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS --pipeline_url ${CI_PIPELINE_URL}
+    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l IDFCI -p ${TEST_RESULTS_PATH} --pipeline_url ${CI_PIPELINE_URL}
   # May need a long time to upload all test results.
-  retry: 2
   timeout: 4 hours

tools/ci/exclude_check_tools_files.txt

@@ -5,6 +5,7 @@ tools/ci/check_*.py
 tools/ci/check_*.txt
 tools/ci/check_*.sh
 tools/ci/check_copyright_config.yaml
+tools/ci/get_all_test_results.py
 tools/gdb_panic_server.py
 tools/check_term.py
 tools/check_python_dependencies.py

tools/ci/get_all_test_results.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+
+'''
+Download artifacts from all test jobs
+
+CI_PROJECT_DIR
+└── TEST_RESULTS
+    ├── <job_id>
+    │   └── XUNIT_RESULT.xml
+    ├── <job_id>
+    │   └── XUNIT_RESULT.xml
+    ├── <job_id>
+    │   └── TEST_LOGS
+
+'''
+
+import argparse
+import json
+import os
+import shutil
+
+import gitlab_api
+
+CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
+CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
+IDF_PATH = os.getenv('IDF_PATH', '.')
+TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
+
+
+def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
+    if os.path.exists(result_path):
+        shutil.rmtree(result_path)
+    os.makedirs(result_path, exist_ok=True)
+
+    gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
+    pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
+    _include_retried = 'true' if include_retried else 'false'
+    jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
+
+    job_info_list = []
+    for job in jobs:
+        if job.stage in ['target_test', 'host_test']:
+            log_path = ''
+            if job.status not in ['success', 'failed']:
+                print('Job {}({}) is not finished'.format(job.id, job.name))
+            elif not hasattr(job, 'artifacts_file'):
+                print('Job {}({}) has no artifacts.'.format(job.id, job.name))
+            else:
+                log_path = os.path.join(result_path, 'job_{}'.format(job.id))
+                print('Downloading artifacts from: {}'.format(job.name))
+                os.makedirs(log_path, exist_ok=True)
+                gitlab_inst.download_artifacts(job.id, log_path)
+
+            job_info = {
+                'id': job.id,
+                'name': job.name,
+                'tag_list': job.tag_list,
+                'status': job.status,
+                'stage': job.stage,
+                'web_url': job.web_url,
+                'commit_url': job.commit['web_url'],
+                'log_path': log_path,
+            }
+            job_info_list.append(job_info)
+
+    with open(os.path.join(result_path, 'index.json'), 'w') as f:
+        f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
+    parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
+    args = parser.parse_args()
+
+    download_all_test_results(args.path, args.include_retried)
+
+
+if __name__ == '__main__':
+    main()