ci: simplify the python env to idf venv only

.gitlab-ci.yml

@@ -57,20 +57,21 @@ variables:
   PYTHON_VER: 3.7.10
 
   CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
-  IDF_BUILD_APPS_PROJ: 2818  # fuhanxi/idf-build-apps
+  IDF_BUILD_APPS_PROJ: 2818  # espressif/idf-build-apps
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
 
-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3"
-  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.0:3"
-  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:2-1"
-  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:2-1"
-  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2"
-  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:2-20220802"
+  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.1:1"
+  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.1:1-1"
+  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.1:1-1"
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1"
+  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.1:1-20220802"
+  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.1:1"
 
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
-  LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells-v5.0:2"
+
+  PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
 
   # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
@@ -110,130 +111,101 @@ cache:
       - .cache/submodule_archives
     policy: pull
 
-.setup_tools_unless_target_test: &setup_tools_unless_target_test |
+.common_before_scripts: &common-before_scripts |
+  source tools/ci/utils.sh
+  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-  export IDF_MIRROR_PREFIX_MAP=
+    export IDF_MIRROR_PREFIX_MAP=
   fi
-  if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
-  tools/idf_tools.py --non-interactive install ${SETUP_TOOLS_LIST:-} && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
-    if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
-    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-    wget $OOCD_DISTRO_URL
-    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-    export PATH=$PWD/openocd-esp32/bin:$PATH
-    fi
+
+  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
+    export INCLUDE_NIGHTLY_RUN="1"
   fi
 
-.download_test_python_contraint_file: &download_test_python_contraint_file |
+  # configure cmake related flags
+  source tools/ci/configure_ci_environment.sh
+
+  # add extra python packages
+  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+
+.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
+  # must use after setup_tools_except_target_test
+  # otherwise the export.sh won't work properly
+
+  # download constraint file for dev
   if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
     wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
     mkdir -p ~/.espressif
     mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
   fi
 
-.download_test_python_tool: &download_test_python_tool |
+  # Mirror
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+    export IDF_MIRROR_PREFIX_MAP=
+  fi
+
+  # install latest python packages
+  # target test jobs
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    # ttfw jobs
+    if ! echo "${CI_JOB_NAME}" | egrep ".+_pytest_.+"; then
+      run_cmd bash install.sh --enable-ci --enable-ttfw
+    else
+      run_cmd bash install.sh --enable-ci --enable-pytest
+    fi
+  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-docs
+  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  else
+    run_cmd bash install.sh --enable-ci
+  fi
+
+  source ./export.sh
+
+  # Custom OpenOCD
+  if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
+    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
+    wget $OOCD_DISTRO_URL
+    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
+    tar -x -f $ARCH_NAME
+    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
+    export PATH=$PWD/openocd-esp32/bin:$PATH
+  fi
+
   if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
     git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
     pip install ./${CI_PYTHON_TOOL_REPO}
     rm -rf ${CI_PYTHON_TOOL_REPO}
   fi
 
-.set_include_nightly_run: &set_include_nightly_run |
-  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
-    export INCLUDE_NIGHTLY_RUN="1"
-  fi
-
 before_script:
-  - source tools/ci/utils.sh
-  - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-  - *set_include_nightly_run
-  - source tools/ci/setup_python.sh
+  - *common-before_scripts
+  - *setup_tools_and_idf_python_venv
   - add_gitlab_ssh_keys
-  - source tools/ci/configure_ci_environment.sh
-  - *setup_tools_unless_target_test
   - fetch_submodules
-  - *download_test_python_contraint_file
-  - $IDF_PATH/tools/idf_tools.py install-python-env
-  # install esptool globally, TODO: remove this, IDFCI-1207
-  - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  - *download_test_python_tool
-
-# used for check scripts which we want to run unconditionally
-.before_script_no_sync_submodule:
-  before_script:
-    - echo "Not setting up GitLab key, not fetching submodules"
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - source tools/ci/setup_python.sh
-    - source tools/ci/configure_ci_environment.sh
-    - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env
 
 .before_script_minimal:
   before_script:
-    - echo "Only load utils.sh"
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-
-.before_script_integration_test:
-  before_script:
-    # needn't install idf python env
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - add_gitlab_ssh_keys
+    - *common-before_scripts
 
 .before_script_macos:
   before_script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
-    - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env
+    - *common-before_scripts
     # On macOS, these tools need to be installed
+    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
-    - *setup_tools_unless_target_test
-    # Install packages required by CI scripts into IDF Python environment
-    - pip install -r $IDF_PATH/tools/requirements/requirements.ci.txt
-    - source tools/ci/configure_ci_environment.sh
-    # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
-    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-    - fetch_submodules
-
-.before_script_pytest:
-  before_script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - source tools/ci/setup_python.sh
-    - add_gitlab_ssh_keys
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_tools_unless_target_test
+    - *setup_tools_and_idf_python_venv
     - fetch_submodules
-    - *download_test_python_contraint_file
-    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
-    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
 
 .before_script_build_jobs:
   before_script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - source tools/ci/setup_python.sh
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
     - add_gitlab_ssh_keys
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_tools_unless_target_test
     - fetch_submodules
-    - *download_test_python_contraint_file
-    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
-    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
 

.gitlab/ci/README.md

@@ -116,7 +116,7 @@ curl -X POST --header "PRIVATE-TOKEN: [YOUR PERSONAL ACCESS TOKEN]" [GITLAB_SERV
     ```yaml
     check_docs_lang_sync:
       extends:
-        - .pre_check_job_template
+        - .pre_check_template
         - .rules:build:docs
       script:
         - cd docs

.gitlab/ci/assign-test.yml

@@ -85,6 +85,7 @@ assign_integration_test:
   extends:
     - .assign_test_template
     - .rules:test:integration_test
+    - .before_script_minimal
   image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
   needs:
     - build_ssc_esp32
@@ -100,6 +101,7 @@ assign_integration_test:
     ASSIGN_TEST_CASE_SCRIPT: "${CI_PROJECT_DIR}/auto_test_script/bin/CIAssignTestCases.py"
     PYTHONPATH: ${CI_PROJECT_DIR}/auto_test_script/packages
   script:
+    - add_gitlab_ssh_keys
     # clone test script to assign tests
     # can not retry if downing git lfs files failed, so using empty_branch first.
     - retry_failed git clone ${CI_AUTO_TEST_SCRIPT_REPO_URL} -b empty_branch

.gitlab/ci/deploy.yml

@@ -1,5 +1,4 @@
 .deploy_job_template:
-  extends: .before_script_no_sync_submodule
   stage: deploy
   image: $ESP_ENV_IMAGE
   tags:

.gitlab/ci/docs.yml

@@ -41,7 +41,7 @@
 # stage: pre_check
 check_readme_links:
   extends:
-    - .pre_check_job_template
+    - .pre_check_template
   tags: ["build", "amd64", "internet"]
   allow_failure: true
   script:
@@ -49,7 +49,7 @@ check_readme_links:
 
 check_docs_lang_sync:
   extends:
-    - .pre_check_job_template
+    - .pre_check_template
     - .doc-rules:build:docs
   script:
     - cd docs
@@ -62,7 +62,6 @@ check_docs_lang_sync:
   dependencies: []
   script:
     - cd docs
-    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs -t $DOCTGT -bs $DOC_BUILDERS -l $DOCLANG build
   parallel:
     matrix:
@@ -72,11 +71,10 @@ check_docs_lang_sync:
 check_docs_gh_links:
   image: $ESP_IDF_DOC_ENV_IMAGE
   extends:
-    - .pre_check_job_template
+    - .pre_check_template
     - .doc-rules:build:docs
   script:
     - cd docs
-    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs gh-linkcheck
 
 # stage: build_doc
@@ -148,8 +146,6 @@ build_docs_pdf:
     DOC_BUILDERS: "latex"
 
 .deploy_docs_template:
-  extends:
-    - .before_script_no_sync_submodule
   image: $ESP_IDF_DOC_ENV_IMAGE
   variables:
     DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
@@ -161,7 +157,6 @@ build_docs_pdf:
   script:
     - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
     - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --)
-    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - deploy-docs
 
 # stage: test_deploy
@@ -227,5 +222,4 @@ check_doc_links:
   allow_failure: true
   script:
     - cd docs
-    - pip install -U -r $IDF_PATH/tools/requirements/requirements.docs.txt
     - build-docs -t $DOCTGT -l $DOCLANG linkcheck

.gitlab/ci/host-test.yml

@@ -317,8 +317,8 @@ test_mkuf2:
     - ./test_mkuf2.py
 
 test_autocomplete:
-  extends: .host_test_template
-  image: $LINUX_SHELL_IMAGE
+  extends:
+    - .host_test_template
   artifacts:
     when: on_failure
     paths:
@@ -328,8 +328,8 @@ test_autocomplete:
     - ${IDF_PATH}/tools/ci/test_autocomplete.py
 
 test_detect_python:
-  extends: .host_test_template
-  image: $LINUX_SHELL_IMAGE
+  extends:
+    - .host_test_template
   script:
     - cd ${IDF_PATH}
     - shellcheck -s sh tools/detect_python.sh

.gitlab/ci/pre_check.yml

@@ -1,21 +1,15 @@
-.pre_check_base_template:
+.pre_check_template:
   stage: pre_check
   image: $ESP_ENV_IMAGE
   tags:
     - host_test
   dependencies: []
 
-.pre_check_job_template:
-  extends:
-    - .pre_check_base_template
-    - .before_script_no_sync_submodule
-
 .check_pre_commit_template:
-  extends: .pre_check_job_template
-  image: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"
-  before_script:
-    - source tools/ci/utils.sh
-    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
+  extends:
+    - .pre_check_template
+    - .before_script_minimal
+  image: $PRE_COMMIT_IMAGE
 
 check_pre_commit_master_release:
   extends:
@@ -35,21 +29,21 @@ check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
   # esp_idf_version.h in a branch before tagging the next version.
   extends:
-    - .pre_check_job_template
+    - .pre_check_template
     - .rules:protected
   script:
     - export IDF_PATH=$PWD
     - tools/ci/check_idf_version.sh
 
 check_rom_api_header:
-  extends: .pre_check_job_template
+  extends: .pre_check_template
   script:
     - tools/ci/check_examples_rom_header.sh
     - tools/ci/check_api_violation.sh
 
 check_python_style:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:patterns:python-files
   artifacts:
     when: on_failure
@@ -60,7 +54,7 @@ check_python_style:
     - python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
 test_check_kconfigs:
-  extends: .pre_check_job_template
+  extends: .pre_check_template
   artifacts:
     when: on_failure
     paths:
@@ -75,7 +69,7 @@ test_check_kconfigs:
     - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
 
 check_blobs:
-  extends: .pre_check_base_template
+  extends: .pre_check_template
   tags:
     - build
   variables:
@@ -90,7 +84,7 @@ check_blobs:
     - bash $IDF_PATH/tools/ci/check_blobs.sh
 
 check_fuzzer_compilation:
-   extends: .pre_check_base_template
+   extends: .pre_check_template
    image: $AFL_FUZZER_TEST_IMAGE
    script:
      - cd ${IDF_PATH}/components/lwip/test_afl_host
@@ -100,7 +94,7 @@ check_fuzzer_compilation:
 
 check_public_headers:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:build
   tags:
     - build
@@ -113,7 +107,7 @@ check_public_headers:
 
 check_chip_support_components:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:build
   tags:
     - build
@@ -130,7 +124,7 @@ check_chip_support_components:
 
 check_esp_err_to_name:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:build
   tags:
     - build
@@ -146,7 +140,7 @@ check_esp_err_to_name:
 
 check_esp_system:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:build
   tags:
     - build
@@ -157,19 +151,19 @@ check_esp_system:
 # the version returned by 'git describe'
 check_version_tag:
   extends:
-    - .pre_check_job_template
+    - .pre_check_template
     - .rules:tag:release
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
 
 check_artifacts_expire_time:
-  extends: .pre_check_job_template
+  extends: .pre_check_template
   script:
     # check if we have set expire time for all artifacts
     - python tools/ci/check_artifacts_expire_time.py
 
 check_commit_msg:
-  extends: .pre_check_job_template
+  extends: .pre_check_template
   script:
     - git status
     - git log -n10 --oneline ${PIPELINE_COMMIT_SHA}
@@ -178,7 +172,7 @@ check_commit_msg:
 
 check_test_scripts_build_test_rules:
   extends:
-    - .pre_check_job_template
+    - .pre_check_template
     - .before_script_build_jobs
   script:
     - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components

.gitlab/ci/static-code-analysis.yml

@@ -1,7 +1,7 @@
 # pre_check stage
 clang_tidy_check:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:patterns:clang_tidy
   image: ${CLANG_STATIC_ANALYSIS_IMAGE}
   artifacts:
@@ -14,7 +14,6 @@ clang_tidy_check:
     RULES_FILE: ${CI_PROJECT_DIR}/tools/ci/static-analysis-rules.yml
     OUTPUT_DIR: ${CI_PROJECT_DIR}/clang_tidy_reports
   script:
-    - python -m pip install -U pip
     - internal_pip_install $CLANG_TIDY_RUNNER_PROJ pyclang
     - export PATH=$PATH:$(python -c "import sys; print(sys.executable.rsplit('/', 1)[0])")
     - dirs=$(cat ${CLANG_TIDY_DIRS_TXT} | while read line; do echo ${CI_PROJECT_DIR}/${line}; done | xargs)
@@ -26,7 +25,7 @@ clang_tidy_check:
 
 check_pylint:
   extends:
-    - .pre_check_base_template
+    - .pre_check_template
     - .rules:patterns:python-files
     - .before_script_minimal
   image: $SONARQUBE_SCANNER_IMAGE

.gitlab/ci/target-test.yml

@@ -15,7 +15,6 @@
 .pytest_template:
   extends:
     - .target_test_template
-    - .before_script_pytest
   artifacts:
     when: always
     paths:
@@ -1050,8 +1049,7 @@ UT_S3_FLASH:
   extends:
     - .target_test_job_template
     - .rules:test:integration_test
-    # needn't install idf python env
-    - .before_script_integration_test
+    - .before_script_minimal
   image: ${CI_INTEGRATION_TEST_ENV_IMAGE}
   cache: []
   needs: # the assign already needs all the build jobs
@@ -1072,6 +1070,7 @@ UT_S3_FLASH:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
+    - add_gitlab_ssh_keys
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench
@@ -1107,6 +1106,7 @@ nvs_compatible_test:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
+    - add_gitlab_ssh_keys
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
     # clone test bench

.gitlab/ci/upload_cache.yml

@@ -9,6 +9,7 @@
 upload-pip-cache:
   extends:
     - .upload_cache_template
+    - .before_script_minimal
     - .rules:patterns:python-cache
   tags:
     - $GEO
@@ -18,13 +19,9 @@ upload-pip-cache:
     paths:
       - .cache/pip
     policy: push
-  before_script: []
   script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - source tools/ci/setup_python.sh
     - rm -rf .cache/pip  # clear old packages
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
+    - bash install.sh --enable-ci --enable-pytest
   parallel:
     matrix:
       - GEO: [ 'shiny',  'brew' ]
@@ -32,6 +29,7 @@ upload-pip-cache:
 upload-submodules-cache:
   extends:
     - .upload_cache_template
+    - .before_script_minimal
     - .rules:patterns:submodule
   tags:
     - $GEO
@@ -41,12 +39,9 @@ upload-submodules-cache:
     paths:
       - .cache/submodule_archives
     policy: push
-  before_script: []
   script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - source tools/ci/setup_python.sh
     - rm -rf .cache/submodule_archives  # clear old submodule archives
+    - add_gitlab_ssh_keys
     - fetch_submodules
   parallel:
     matrix:

tools/ci/exclude_check_tools_files.txt

@@ -29,7 +29,6 @@ tools/ci/mypy_ignore_list.txt
 tools/ci/normalize_clangtidy_path.py
 tools/ci/push_to_github.sh
 tools/ci/python_packages/wifi_tools.py
-tools/ci/setup_python.sh
 tools/ci/utils.sh
 tools/eclipse-code-style.xml
 tools/format-minimal.sh

tools/ci/setup_python.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-if [ -z ${PYTHON_VER+x} ]; then
-    # Use this version of the Python interpreter if it was not defined before.
-    # 3.6.13 is the default python3 interpreter in esp32-ci-env
-    # Jobs which doesn't support this version should define PYTHON_VER themselves
-    PYTHON_VER=3.7.10
-fi
-
-if [ -f /opt/pyenv/activate ];
-then
-    source /opt/pyenv/activate
-    pyenv global $PYTHON_VER || {
-                                    echo 'Python' $PYTHON_VER 'is not installed.'
-                                    INSTALLED_PY_VERS=$(pyenv versions --bare)
-
-                                    while [ ${#PYTHON_VER} -gt 0 ]
-                                    do
-                                        echo 'Tring to locate a match for' $PYTHON_VER
-
-                                        for ver in ${INSTALLED_PY_VERS[@]}
-                                        do
-                                            if [[ $ver == $PYTHON_VER* ]];
-                                            then
-                                                pyenv global $ver
-                                                break 2
-                                            fi
-                                        done
-
-                                        # Removing last character and trying to find some match.
-                                        # For example, if 3.4.8 was selected but isn't installed then it will try to
-                                        # find some other installed 3.4.X version, and then some 3.X.X version.
-                                        PYTHON_VER=${PYTHON_VER: : -1}
-                                    done
-                                }
-    python --version || {
-                            echo 'No matching Python interpreter is found!'
-                            exit 1
-                        }
-elif command -v python -V 1>/dev/null 2>&1;
-then
-    python --version
-    echo 'No /opt/pyenv/activate exists and Python from path is used.'
-else
-    echo 'No /opt/pyenv/activate exists and no Python interpreter is found!'
-    exit 1
-fi
-
-# add esp-idf local package path to PYTHONPATH so it can be imported directly
-export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"