Merge branch 'feature/drop_py2_support' into 'master'

Drop support for unsupported Python versions

Closes IDF-1676

See merge request espressif/esp-idf!13622

.gitlab-ci.yml

@@ -48,6 +48,9 @@ variables:
 
   # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v1"
+  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1"
+  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1"
 
   # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
@@ -57,9 +60,6 @@ variables:
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v3.1"
 
-  # Versioned esp-idf-doc env image to use for all document building jobs
-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v10"
-
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
   export IDF_MIRROR_PREFIX_MAP=

.gitlab/ci/build.yml

@@ -1,6 +1,6 @@
 .build_template:
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - build
   variables:
@@ -105,7 +105,7 @@ build_ssc_esp32c3:
     OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
     BUILD_SYSTEM: "cmake"
     TEST_TYPE: "unit_test"
-    PYTHON_VER: 3
+    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
@@ -156,7 +156,7 @@ build_esp_idf_tests_cmake_esp32c3:
     TEST_TYPE: example_test
     LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
     BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
-    PYTHON_VER: 3
+    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"

.gitlab/ci/deploy.yml

@@ -1,7 +1,7 @@
 .deploy_job_template:
   extends: .before_script_no_sync_submodule
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - deploy
 

.gitlab/ci/docs.yml

@@ -55,6 +55,8 @@ check_docs_lang_sync:
 
 .build_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.10
   tags:
     - build_docs
   dependencies: []
@@ -151,6 +153,8 @@ build_docs_pdf:
   extends:
     - .before_script_no_sync_submodule
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.10
   stage: test_deploy
   tags:
     - deploy

.gitlab/ci/host-test.yml

@@ -1,7 +1,9 @@
 .host_test_template:
   extends: .rules:test:host_test
   stage: host_test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.13
   tags:
     - host_test
   dependencies: []
@@ -11,7 +13,7 @@ test_nvs_on_host:
   extends: .host_test_template
   script:
     - cd components/nvs_flash/test_nvs_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh make test
+    - make test
 
 test_nvs_coverage:
   extends:
@@ -33,7 +35,7 @@ test_partition_table_on_host:
     - build
   script:
     - cd components/partition_table/test_gen_esp32part_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./gen_esp32part_tests.py
+    - ./gen_esp32part_tests.py
 
 test_wl_on_host:
   extends: .host_test_template
@@ -55,10 +57,10 @@ test_ldgen_on_host:
   extends: .host_test_template
   script:
     - cd tools/ldgen/test
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_fragments.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_generation.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_entity.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_output_commands.py
+    - ./test_fragments.py
+    - ./test_generation.py
+    - ./test_entity.py
+    - ./test_output_commands.py
   variables:
     LC_ALL: C.UTF-8
 
@@ -66,7 +68,7 @@ test_ldgen_on_host:
   extends:
     - .host_test_template
     - .rules:labels:fuzzer_test-weekend_test
-  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+  image: $AFL_FUZZER_TEST_IMAGE
   artifacts:
     when: always
     paths:
@@ -111,7 +113,7 @@ test_spiffs_on_host:
     - cd components/spiffs/test_spiffs_host/
     - make test
     - cd ../test_spiffsgen
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_spiffsgen.py
+    - ./test_spiffsgen.py
 
 test_multi_heap_on_host:
   extends: .host_test_template
@@ -125,26 +127,26 @@ test_certificate_bundle_on_host:
     - build
   script:
     - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_gen_crt_bundle.py
+    - ./test_gen_crt_bundle.py
 
 test_confserver:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/confserver
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confserver.py
+    - ./test_confserver.py
 
 test_gen_kconfig_doc:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/gen_kconfig_doc/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_target_visibility.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_kconfig_out.py
+    - ./test_target_visibility.py
+    - ./test_kconfig_out.py
 
 test_confgen:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/confgen/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confgen.py
+    - ./test_confgen.py
 
 test_idf_monitor:
   extends: .host_test_template
@@ -168,7 +170,7 @@ test_idf_size:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/test_idf_size
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_idf_py:
   extends: .host_test_template
@@ -176,7 +178,7 @@ test_idf_py:
     LC_ALL: C.UTF-8
   script:
     - cd ${IDF_PATH}/tools/test_idf_py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_py.py
+    - ./test_idf_py.py
 
 test_idf_tools:
   extends: .host_test_template
@@ -189,7 +191,7 @@ test_idf_tools:
     # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
     - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif" | tr "\n" ":"); echo ${p%:})
     - cd ${IDF_PATH}/tools/test_idf_tools
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+    - ./test_idf_tools.py
     # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
     - cd ${IDF_PATH}/tools
     - python3 ./idf_tools.py install-python-env
@@ -203,12 +205,10 @@ test_esp32_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
+    - ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
     - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found for esp32 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
-    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_esp32s2_efuse_table_on_host:
   extends: .host_test_template
@@ -219,12 +219,10 @@ test_esp32s2_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
+    - ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
     - git diff --exit-code -- esp32s2/esp_efuse_table.c || { echo 'Differences found for esp32s2 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s2/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_esp32s3_efuse_table_on_host:
   extends: .host_test_template
@@ -235,12 +233,10 @@ test_esp32s3_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
+    - ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
     - git diff --exit-code -- esp32s3/esp_efuse_table.c || { echo 'Differences found for esp32s3 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s3/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_esp32c3_efuse_table_on_host:
   extends: .host_test_template
@@ -251,12 +247,10 @@ test_esp32c3_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
+    - ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
     - git diff --exit-code -- esp32c3/esp_efuse_table.c || { echo 'Differences found for esp32c3 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32c3/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_espcoredump:
   extends: .host_test_template
@@ -268,7 +262,7 @@ test_espcoredump:
     expire_in: 1 week
   script:
     - cd components/espcoredump/test/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_espcoredump.sh
+    - ./test_espcoredump.sh
 
 test_logtrace_proc:
   extends: .host_test_template
@@ -280,7 +274,7 @@ test_logtrace_proc:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_sysviewtrace_proc:
   extends: .host_test_template
@@ -292,7 +286,7 @@ test_sysviewtrace_proc:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_mkdfu:
   extends: .host_test_template
@@ -300,17 +294,19 @@ test_mkdfu:
     LC_ALL: C.UTF-8
   script:
     - cd ${IDF_PATH}/tools/test_mkdfu
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py
+    - ./test_mkdfu.py
 
 test_mkuf2:
   extends: .host_test_template
   script:
     - cd ${IDF_PATH}/tools/test_mkuf2
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkuf2.py
+    - ./test_mkuf2.py
 
 test_docs:
   extends: .host_test_template
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.10
   artifacts:
     when: on_failure
     paths:
@@ -318,9 +314,9 @@ test_docs:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/docs/test
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
+    - python -m pip install -r ${IDF_PATH}/docs/requirements.txt
+    - ./test_docs.py
+    - ./test_sphinx_idf_extensions.py
 
 test_autocomplete:
   extends: .host_test_template
@@ -331,7 +327,7 @@ test_autocomplete:
       - ${IDF_PATH}/*.out
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_autocomplete.py
+    - ${IDF_PATH}/tools/ci/test_autocomplete.py
 
 test_detect_python:
   extends: .host_test_template

.gitlab/ci/pre_check.yml

@@ -1,6 +1,6 @@
 .pre_check_base_template:
   stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - host_test
   dependencies: []
@@ -56,16 +56,20 @@ check_python_style:
   extends:
     - .pre_check_base_template
     - .rules:patterns:python-files
+  variables:
+    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
       - flake8_output.txt
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+    - python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
 test_check_kconfigs:
   extends: .pre_check_job_template
+  variables:
+    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
@@ -77,7 +81,7 @@ test_check_kconfigs:
       - tools/*/*/*/Kconfig*.new
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py
+    - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
 
 check_wifi_lib_md5:
   extends: .pre_check_base_template
@@ -91,7 +95,7 @@ check_wifi_lib_md5:
 
 check_fuzzer_compilation:
    extends: .pre_check_base_template
-   image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+   image: $AFL_FUZZER_TEST_IMAGE
    script:
      - cd ${IDF_PATH}/components/lwip/test_afl_host
      - make MODE=dhcp_server
@@ -122,10 +126,8 @@ check_esp_err_to_name:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./gen_esp_err_to_name.py
+    - ./gen_esp_err_to_name.py
     - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./gen_esp_err_to_name.py
-    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
 
 scan_tests:
   extends:

.gitlab/merge_request_templates/Mixed Template.md

@@ -17,7 +17,6 @@ _This entire section can be deleted if all items are checked._
 * [ ] Release note entry if this is a new public feature, or a fix for an issue introduced in the previous release.
 * [ ] The commit log is clean and ready to merge.
 * [ ] All relevant CI jobs have been run, i.e. jobs which cover the code changed by the MR.
-* [ ] Pipeline with Python 3 has been successfully run, i.e. `@bot ...; with Python3`, if the MR adds or changes Python code.
 
 ---
 

SUPPORT_POLICY.md

@@ -39,6 +39,10 @@ are not covered by any support period. Sometimes a particular feature is
 marked as "Preview" in a release, which means it is also not covered
 by the support period.
 
+ESP-IDF should be used in an up-to-date software environment. The operating system
+and other third-party tools should be supported by their maintainers.
+ESP-IDF cannot keep compatibility with unsupported third-party tools.
+
 The ESP-IDF Programming Guide has information about the
 [different versions of ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/versions.html)
 (major, minor, bugfix, etc).

components/app_update/otatool.py

@@ -234,10 +234,6 @@ def _erase_ota_partition(target, ota_id):
 
 
 def main():
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     global quiet
 
     parser = argparse.ArgumentParser('ESP-IDF OTA Partitions Tool')

components/efuse/efuse_table_gen.py

@@ -76,24 +76,24 @@ class FuseTable(list):
 
         # fix up missing bit_start
         last_efuse_block = None
-        for e in res:
-            if last_efuse_block != e.efuse_block:
+        for i in res:
+            if last_efuse_block != i.efuse_block:
                 last_end = 0
-            if e.bit_start is None:
-                e.bit_start = last_end
-            last_end = e.bit_start + e.bit_count
-            last_efuse_block = e.efuse_block
+            if i.bit_start is None:
+                i.bit_start = last_end
+            last_end = i.bit_start + i.bit_count
+            last_efuse_block = i.efuse_block
 
         res.verify_duplicate_name()
 
         # fix up missing field_name
         last_field = None
-        for e in res:
-            if e.field_name == '' and last_field is None:
-                raise InputError('Error at line %d: %s missing field name' % (line_no + 1, e))
-            elif e.field_name == '' and last_field is not None:
-                e.field_name = last_field.field_name
-            last_field = e
+        for i in res:
+            if i.field_name == '' and last_field is None:
+                raise InputError('Error at line %d: %s missing field name' % (line_no + 1, i))
+            elif i.field_name == '' and last_field is not None:
+                i.field_name = last_field.field_name
+            last_field = i
 
         # fill group
         names = [p.field_name for p in res]
@@ -479,10 +479,6 @@ def create_output_files(name, output_table, debug):
 
 
 def main():
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     global quiet
     global max_blk_len
     global idf_target

components/spiffs/spiffsgen.py

@@ -23,7 +23,6 @@ import io
 import math
 import os
 import struct
-import sys
 
 try:
     import typing
@@ -510,10 +509,6 @@ class CustomHelpFormatter(argparse.HelpFormatter):
 
 
 def main():  # type: () -> None
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     parser = argparse.ArgumentParser(description='SPIFFS Image Generator',
                                      formatter_class=CustomHelpFormatter)
 

components/ulp/esp32ulp_mapgen.py

@@ -7,7 +7,6 @@
 
 from __future__ import print_function
 
-import sys
 from optparse import OptionParser
 
 BASE_ADDR = 0x50000000
@@ -44,10 +43,6 @@ def gen_ld_h_from_sym_riscv(f_sym, f_ld, f_h):
 
 
 def main():
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     description = ('This application generates .h and .ld files for symbols defined in input file. '
                    'The input symbols file can be generated using nm utility like this: '
                    'esp32-ulp-nm -g -f posix <elf_file> > <symbols_file>')

components/xtensa/trax/traceparse.py

@@ -55,7 +55,7 @@ import sys
 
 # Check if loaded into GDB
 try:
-    assert gdb.__name__ == 'gdb'
+    assert gdb.__name__ == 'gdb'  # type: ignore
     WITH_GDB = True
 except NameError:
     WITH_GDB = False
@@ -329,10 +329,6 @@ def parse_and_dump(filename, disassemble=WITH_GDB):
 
 
 def main():
-    if sys.version_info[0] < 3:
-        print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     if len(sys.argv) < 2:
         sys.stderr.write('Usage: %s <dump_file>\n')
         raise SystemExit(1)

docs/en/get-started/linux-setup-scratch.rst

@@ -100,12 +100,6 @@ You are looking for similar result containing toolchain's path at the beginning
 
 Instead of ``/home/user-name`` there should be a home path specific to your installation.
 
-Python 2 deprecation
-====================
-
-Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above.
-
-
 Next Steps
 ==========
 

docs/en/get-started/linux-setup.rst

@@ -35,19 +35,13 @@ Permission issues /dev/ttyUSB0
 
 With some Linux distributions you may get the ``Failed to open port /dev/ttyUSB0`` error message when flashing the {IDF_TARGET_NAME}. :ref:`This can be solved by adding the current user to the dialout group<linux-dialout-group>`.
 
-Fixing broken pip on Ubuntu 16.04
-=================================
-
-Package ``python3-pip`` could be broken without possibility to upgrade it. 
-Package has to be removed and installed manually using script `get-pip.py <https://bootstrap.pypa.io/get-pip.py>`_.::
-
-    apt remove python3-pip python3-virtualenv; rm -r ~/.local
-    rm -r ~/.espressif/python_env && python get-pip.py
-
-Python 2 deprecation
+Python compatibility
 ====================
 
-Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above.
+ESP-IDF supports Python 3.6 or newer. It is recommended to upgrade your operating system to a recent version
+satisfying this requirement. Other options include the installation of Python from
+`sources <https://www.python.org/downloads/>`_ or the use of a Python version management system such as
+`pyenv <https://github.com/pyenv/pyenv>`_.
 
 Next Steps
 ==========

docs/en/get-started/macos-setup-scratch.rst

@@ -79,11 +79,6 @@ Build the toolchain::
 
 Toolchain will be built in ``~/esp/ctng-volume/crosstool-NG/builds/{IDF_TARGET_TOOLCHAIN_PREFIX}``. To use it, you need to add ``~/esp/ctng-volume/crosstool-NG/builds/{IDF_TARGET_TOOLCHAIN_PREFIX}/bin`` to ``PATH`` environment variable.
 
-Python 2 deprecation
-====================
-
-Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above.
-
 Next Steps
 ==========
 

docs/en/get-started/macos-setup.rst

@@ -57,11 +57,6 @@ Below is an overview of steps to install Python 3.
 
       sudo port install python38
 
-Python 2 deprecation
-====================
-
-Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above.
-
 
 Next Steps
 ==========

docs/zh_CN/get-started/linux-setup-scratch.rst

@@ -100,11 +100,6 @@
 
 注意这里的 ``/home/user-name`` 应该替换成您安装的主路径。
 
-停用 Python 2
-====================
-
-Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。
-
 
 后续步骤
 ==========

docs/zh_CN/get-started/linux-setup.rst

@@ -43,12 +43,6 @@ Linux 平台工具链的标准设置
     apt remove python3-pip python3-virtualenv; rm -r ~/.local
     rm -r ~/.espressif/python_env && python get-pip.py
 
-停用 Python 2 
-====================
-
-Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。
-
-
 后续步骤
 ==========
 

docs/zh_CN/get-started/macos-setup-scratch.rst

@@ -80,12 +80,6 @@ MacPorts 需要完整的 XCode 软件，而 Homebrew 只需要安装 XCode 命
 编译得到的工具链会被保存到 ``~/esp/ctng-volume/crosstool-NG/builds/{IDF_TARGET_TOOLCHAIN_PREFIX}``。使用工具链前，请将 ``~/esp/ctng-volume/crosstool-NG/builds/{IDF_TARGET_TOOLCHAIN_PREFIX}/bin`` 添加至 ``PATH`` 环境变量。
 
 
-停用 Python 2
-====================
-
-Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。
-
-
 后续步骤
 ==========
 

docs/zh_CN/get-started/macos-setup.rst

@@ -57,12 +57,6 @@ ESP-IDF 将使用 macOS 上默认安装的 Python 版本。
 
       sudo port install python38
 
-停用 Python 2 
-====================
-
-Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。
-
-
 后续步骤
 ==========
 

tools/ci/check_build_warnings.py

@@ -35,8 +35,6 @@ IGNORE_WARNS = [
         r'changes choice state',
         r'crosstool_version_check\.cmake',
         r'CryptographyDeprecationWarning',
-        r'Python 3 versions older than 3.6 are not supported.',
-        r'Support for Python 2 is deprecated and will be removed in future versions.',
         r'Warning: \d+/\d+ app partitions are too small for binary'
     ]
 ]

tools/ci/check_public_headers.py

@@ -141,7 +141,7 @@ class PublicHeaderChecker:
 
     def join(self):
         for t in self.check_threads:
-            while t.isAlive and not self.terminate.is_set():
+            while t.is_alive() and not self.terminate.is_set():
                 t.join(1)  # joins with timeout to respond to keyboard interrupt
 
     # Checks one header calling:

tools/ci/setup_python.sh

@@ -2,9 +2,9 @@
 
 if [ -z ${PYTHON_VER+x} ]; then
     # Use this version of the Python interpreter if it was not defined before.
-    # 3.4.8 is the default python3 interpreter in esp32-ci-env
+    # 3.6.13 is the default python3 interpreter in esp32-ci-env
     # Jobs which doesn't support this version should define PYTHON_VER themselves
-    PYTHON_VER=3.4.8
+    PYTHON_VER=3.6.13
 fi
 
 if [ -f /opt/pyenv/activate ];

tools/idf.py

@@ -1,26 +1,13 @@
 #!/usr/bin/env python
 #
+# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD
+#
+# SPDX-License-Identifier: Apache-2.0
+#
 # 'idf.py' is a top-level config/build command line tool for ESP-IDF
 #
 # You don't have to use idf.py, you can use cmake directly
 # (or use cmake in an IDE)
-#
-#
-#
-# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
 
 # WARNING: we don't check for Python build-time dependencies until
 # check_environment() function below. If possible, avoid importing
@@ -44,6 +31,7 @@ from pkgutil import iter_modules
 # idf.py extensions. Therefore, pyc file generation is turned off:
 sys.dont_write_bytecode = True
 
+import python_version_checker  # noqa: E402
 from idf_py_actions.errors import FatalError  # noqa: E402
 from idf_py_actions.tools import executable_exists, idf_version, merge_action_lists, realpath  # noqa: E402
 
@@ -93,11 +81,13 @@ def check_environment():
         print_warning('Setting IDF_PATH environment variable: %s' % detected_idf_path)
         os.environ['IDF_PATH'] = detected_idf_path
 
-    # check Python version
-    if sys.version_info[0] < 3:
-        print_warning('WARNING: Support for Python 2 is deprecated and will be removed in future versions.')
-    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
-        print_warning('WARNING: Python 3 versions older than 3.6 are not supported.')
+    try:
+        # The Python compatibility check could have been done earlier (tools/detect_python.{sh,fish}) but PATH is
+        # not set for import at that time. Even if the check would be done before, the same check needs to be done
+        # here as well (for example one can call idf.py from a not properly set-up environment).
+        python_version_checker.check()
+    except RuntimeError as e:
+        raise FatalError(e)
 
     # check Python dependencies
     checks_output.append('Checking Python dependencies...')

tools/idf_tools.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python
 # coding=utf-8
 #
+# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD
+#
+# SPDX-License-Identifier: Apache-2.0
+#
 # This script helps installing tools required to use the ESP-IDF, and updating PATH
 # to use the installed tools. It can also create a Python virtual environment,
 # and install Python requirements into it.
@@ -23,22 +27,6 @@
 # * To start using the tools, run `eval "$(idf_tools.py export)"` — this will update
 #   the PATH to point to the installed tools and set up other environment variables
 #   needed by the tools.
-#
-###
-#
-# Copyright 2019 Espressif Systems (Shanghai) PTE LTD
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 import argparse
 import contextlib
@@ -60,19 +48,22 @@ from ssl import SSLContext  # noqa: F401
 from tarfile import TarFile  # noqa: F401
 from zipfile import ZipFile
 
+# Important notice: Please keep the lines above compatible with old Pythons so it won't fail with ImportError but with
+# a nice message printed by python_version_checker.check()
 try:
-    from typing import IO, Callable, Optional, Tuple, Union  # noqa: F401
-except ImportError:
-    pass
-
-try:
-    from urllib.error import ContentTooShortError
-    from urllib.request import urlopen
-    # the following is only for typing annotation
-    from urllib.response import addinfourl  # noqa: F401
-except ImportError:
-    # Python 2
-    from urllib import ContentTooShortError, urlopen  # type: ignore
+    import python_version_checker
+
+    # check the Python version before it will fail with an exception on syntax or package incompatibility.
+    python_version_checker.check()
+except RuntimeError as e:
+    print(e)
+    raise SystemExit(1)
+
+from typing import IO, Callable, Optional, Tuple, Union  # noqa: F401
+from urllib.error import ContentTooShortError
+from urllib.request import urlopen
+# the following is only for typing annotation
+from urllib.response import addinfourl  # noqa: F401
 
 try:
     from exceptions import WindowsError

tools/python_version_checker.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Script for checking the compatibility of the Python interpreter with ESP-IDF.
+#
+# There are related tools/detect_python.{sh,fish} scripts which are called earlier when the paths are not properly
+# set-up and they only intend to prefer the use of Python 3 over Python 2. Why not more? All possible executables
+# (python3.6, python3.7, ...) cannot be hardcoded there and at the end, the user is responsible to set-up a system
+# where "python" or "python3" of compatible version is available.
+
+import sys
+
+try:
+    # Python 2 is not supported anymore but still the old way of typing is used here in order to give a nice Python
+    # version failure and not a typing exception.
+    from typing import Iterable
+except ImportError:
+    pass
+
+OLDEST_PYTHON_SUPPORTED = (3, 6)  # keep it as tuple for comparison with sys.version_info
+
+
+def _ver_to_str(it):  # type: (Iterable) -> str
+    return '.'.join(str(x) for x in it)
+
+
+def is_supported():  # type: () -> bool
+    return sys.version_info[:2] >= OLDEST_PYTHON_SUPPORTED[:2]
+
+
+def check():  # type: () -> None
+    if not is_supported():
+        raise RuntimeError('ESP-IDF supports Python {} or newer but you are using Python {}. Please upgrade your '
+                           'installation as described in the documentation.'.format(_ver_to_str(OLDEST_PYTHON_SUPPORTED),
+                                                                                    _ver_to_str(sys.version_info[:3])))