Merge branch 'feature/python2-deprecation-warn' into 'master'

Tools: Add Python 2 deprecation warning

Closes IDF-1677

See merge request espressif/esp-idf!11115

components/app_update/otatool.py

@@ -249,6 +249,10 @@ def _erase_ota_partition(target, ota_id):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.")
     global quiet
 
     parser = argparse.ArgumentParser("ESP-IDF OTA Partitions Tool")

components/efuse/efuse_table_gen.py

@@ -457,6 +457,10 @@ def create_output_files(name, output_table, debug):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr)
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr)
     global quiet
     global max_blk_len
     global idf_target

components/spiffs/spiffsgen.py

@@ -16,8 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import division
+from __future__ import division, print_function
 import os
+import sys
 import io
 import math
 import struct
@@ -451,6 +452,10 @@ class SpiffsFS():
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.")
     parser = argparse.ArgumentParser(description="SPIFFS Image Generator",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 

components/ulp/esp32ulp_mapgen.py

@@ -5,7 +5,9 @@
 # Copyright (c) 2016-2017 Espressif Systems (Shanghai) PTE LTD.
 # Distributed under the terms of Apache License v2.0 found in the top-level LICENSE file.
 
+from __future__ import print_function
 from optparse import OptionParser
+import sys
 
 BASE_ADDR = 0x50000000
 
@@ -41,6 +43,10 @@ def gen_ld_h_from_sym_riscv(f_sym, f_ld, f_h):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr)
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr)
     description = ("This application generates .h and .ld files for symbols defined in input file. "
                    "The input symbols file can be generated using nm utility like this: "
                    "esp32-ulp-nm -g -f posix <elf_file> > <symbols_file>")

components/xtensa/trax/traceparse.py

@@ -49,7 +49,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from __future__ import print_function
 import sys
 
 # Check if loaded into GDB
@@ -328,6 +328,10 @@ def parse_and_dump(filename, disassemble=WITH_GDB):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr)
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr)
     if len(sys.argv) < 2:
         sys.stderr.write("Usage: %s <dump_file>\n")
         raise SystemExit(1)

docs/en/get-started/linux-setup-scratch.rst

@@ -15,15 +15,17 @@ To compile with ESP-IDF you need to get the following packages:
 
 - CentOS 7::
 
-    sudo yum install git wget ncurses-devel flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache dfu-util
+    sudo yum -y update && sudo yum install git wget ncurses-devel flex bison gperf python3 python3-pip cmake ninja-build ccache dfu-util
+
+CentOS 7 is still supported but CentOS version 8 is recommended for a better user experience.
 
 - Ubuntu and Debian::
 
-    sudo apt-get install git wget libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
+    sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util
+    sudo pacman -Sy --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util
 
 .. note::
     CMake version 3.5 or newer is required for use with ESP-IDF. Older Linux distributions may require updating, enabling of a "backports" repository, or installing of a "cmake3" package rather than "cmake".
@@ -35,7 +37,7 @@ Compile the Toolchain from Source
 
   - CentOS 7::
 
-        sudo yum install gawk gperf grep gettext ncurses-devel python python-devel automake bison flex texinfo help2man libtool make
+        sudo yum install gawk gperf grep gettext ncurses-devel python3 python3-devel automake bison flex texinfo help2man libtool make
 
   - Ubuntu pre-16.04::
 
@@ -51,7 +53,7 @@ Compile the Toolchain from Source
 
   - Arch::
 
-        TODO
+        sudo pacman -Sy --needed python-pip
 
 Create the working directory and go into it::
 
@@ -98,6 +100,11 @@ You are looking for similar result containing toolchain's path at the beginning
 
 Instead of ``/home/user-name`` there should be a home path specific to your installation.
 
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above.
+
 
 Next Steps
 ==========

docs/en/get-started/linux-setup.rst

@@ -11,11 +11,13 @@ To compile with ESP-IDF you need to get the following packages:
 
 - CentOS 7::
 
-    sudo yum install git wget flex bison gperf python cmake ninja-build ccache dfu-util
+    sudo yum -y update && sudo yum install git wget flex bison gperf python3 cmake ninja-build ccache dfu-util
+
+CentOS 7 is still supported but CentOS version 8 is recommended for a better user experience.
 
 - Ubuntu and Debian::
 
-    sudo apt-get install git wget flex bison gperf python python-pip python-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
+    sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
@@ -32,20 +34,47 @@ Permission issues /dev/ttyUSB0
 
 With some Linux distributions you may get the ``Failed to open port /dev/ttyUSB0`` error message when flashing the {IDF_TARGET_NAME}. :ref:`This can be solved by adding the current user to the dialout group<linux-dialout-group>`.
 
+Setting up Python 3 as default for CentOS
+-----------------------------------------
+
+CentOS 7 and older is providing Python 2.7 as the default interpreter.
+Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this::
+
+    sudo yum -y update && sudo yum install python3 python3-pip python3-setuptools
+
+Making Python 3 the default interpreter is possible by running::
+
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
+
 Setting up Python 3 as default for Ubuntu and Debian
 ----------------------------------------------------
 
-Ubuntu (distributions before v20.04) and Debian (distributions before v7) are still providing Python 2.7 as the default interpreter. Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this::
+Ubuntu (version 18.04 and older) and Debian (version 9 and older) are still providing Python 2.7 as the default interpreter.
+Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this::
 
     sudo apt-get install python3 python3-pip python3-setuptools
 
 Making Python 3 the default interpreter is possible by running::
 
-    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
 
 .. note::
     This is system-wide change which may affect all of the applications.
 
+Fixing broken pip on Ubuntu 16.04
+=================================
+
+Package ``python3-pip`` could be broken without possibility to upgrade it. 
+Package has to be removed and installed manually using script `get-pip.py <https://bootstrap.pypa.io/get-pip.py>`_.::
+
+    apt remove python3-pip python3-virtualenv; rm -r ~/.local
+    rm -r ~/.espressif/python_env && python get-pip.py
+
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above.
+
 Next Steps
 ==========
 

docs/en/get-started/macos-setup-scratch.rst

@@ -79,6 +79,10 @@ Build the toolchain::
 
 Toolchain will be built in ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-{IDF_TARGET_TOOLCHAIN_NAME}-elf``. To use it, you need to add ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-{IDF_TARGET_TOOLCHAIN_NAME}-elf/bin`` to ``PATH`` environment variable.
 
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above.
 
 Next Steps
 ==========

docs/en/get-started/macos-setup.rst

@@ -37,7 +37,7 @@ ESP-IDF will use the version of Python installed by default on macOS.
 Installing and setting up Python 3 as default
 ---------------------------------------------
 
-Basing on macOS `Catalina 10.15 release notes`_, use of Python 2.7 is not recommended and Python will not be included by default in future versions of macOS. Check what Python you currently have::
+Basing on macOS `Catalina 10.15 release notes`_, use of Python 2.7 is not recommended and Python 2.7 will not be included by default in future versions of macOS. Check what Python you currently have::
 
   python --version
 
@@ -54,7 +54,7 @@ Below is an overview of steps to install Python 3 and making it default interpre
       brew install python3
       ln -s /usr/local/bin/python3.8 /usr/local/bin/python
 
-    Adjust above directory name ``/usr/local/bin/python3.8`` to point where Python 3 has been installed. To check this directory you can run  ``which -a python``.
+    Adjust above directory name ``/usr/local/bin/python3.8`` to point where Python 3 has been installed. To check this directory you can run  ``which -a python3``.
 
   - If you have MacPorts_, you can run::
 
@@ -72,6 +72,12 @@ If the output is similar to ``Python 3.8.5``, your installation has been done su
 
     This is system-wide change which may affect all of the applications.
 
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above.
+
+
 Next Steps
 ==========
 

docs/zh_CN/get-started/linux-setup-scratch.rst

@@ -15,18 +15,21 @@
 
 - CentOS 7::
 
-    sudo yum install git wget ncurses-devel flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache dfu-util
+    sudo yum -y update && sudo yum install git wget ncurses-devel flex bison gperf python3 python3-pip cmake ninja-build ccache dfu-util
+
+目前仍然支持 CentOS 7，但为了更好的用户体验，建议使用 CentOS 8。
 
 - Ubuntu 和 Debian::
 
-    sudo apt-get install git wget libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
+    sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util
+    sudo pacman -Sy --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util
 
 .. 注解::
-    使用 ESP-IDF 需要 CMake 3.5 或以上版本。较早的 Linux 发行版可能需要升级自身的软件源仓库，或开启 backports 套件库，或安装 "cmake3" 软件包（而不是安装 "cmake")。
+
+    使用 ESP-IDF 需要 CMake 3.5 或以上版本。较早的 Linux 发行版可能需要升级自身的软件源仓库，或开启 backports 套件库，或安装 "cmake3" 软件包（不是安装 "cmake")。
 
 从源代码编译工具链
 =================================
@@ -35,7 +38,7 @@
 
 - CentOS 7::
 
-    sudo yum install gawk gperf grep gettext ncurses-devel python python-devel automake bison flex texinfo help2man libtool make
+    sudo yum install gawk gperf grep gettext ncurses-devel python3 python3-devel automake bison flex texinfo help2man libtool make
 
 - Ubuntu pre-16.04::
 
@@ -51,7 +54,7 @@
 
 - Arch::
 
-    TODO
+    sudo pacman -Sy --needed python-pip
 
 创建工作目录，并进入该目录::
 
@@ -97,7 +100,14 @@
 
 注意这里的 ``/home/user-name`` 应该替换成您安装的主路径。
 
+
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。
+
+
 后续步骤
 ==========
 
-请前往 :ref:`get-started-get-esp-idf` 章节继续设置开发环境。
+请前往 :ref:`get-started-get-esp-idf` 章节继续设置开发环境。

docs/zh_CN/get-started/linux-setup.rst

@@ -11,11 +11,13 @@ Linux 平台工具链的标准设置
 
 - CentOS 7::
 
-    sudo yum install git wget flex bison gperf python cmake ninja-build ccache dfu-util
+    sudo yum -y update && sudo yum install git wget flex bison gperf python3 cmake ninja-build ccache dfu-util
+
+目前仍然支持 CentOS 7，但为了更好的用户体验，建议使用 CentOS 8。
 
 - Ubuntu 和 Debian::
 
-    sudo apt-get install git wget flex bison gperf python python-pip python-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
+    sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
@@ -32,20 +34,47 @@ Linux 平台工具链的标准设置
 
 使用某些 Linux 版本向 {IDF_TARGET_NAME} 烧录固件时，可能会出现 ``Failed to open port /dev/ttyUSB0`` 错误消息。此时可以将用户添加至 :ref:`Linux Dialout 组<linux-dialout-group>`。
 
+设置 Python 3 为 CentOS 默认 Python 版本
+----------------------------------------------------
+
+CentOS 7 及更早的版本提供 Python 2.7 作为默认解释器。但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3::
+
+    sudo yum -y update && sudo yum install python3 python3-pip python3-setuptools
+
+设置 Python 3 为默认 Python 版本::
+
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
+
+
 设置 Python 3 为 Ubuntu 和 Debian 默认 Python 版本
 ----------------------------------------------------
 
-Ubuntu v20.04 和 Debian v7 之前的版本默认解释器为 Python 2.7，但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3::
+Ubuntu（v18.04 及之前的版本）和 Debian（v9 及之前的版本）的默认解释器为 Python 2.7，但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3::
 
     sudo apt-get install python3 python3-pip python3-setuptools
 
 设置 Python 3 为默认 Python 版本::
 
-    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
 
 .. 注解::
     上述设置为全局设置，同时会影响到其它应用。
 
+
+修复 Ubuntu 16.04 损坏的 pip 
+=================================
+
+``python3-pip`` 包可能已损坏无法升级。需使用脚本 `get-pip.py <https://bootstrap.pypa.io/get-pip.py>`_ 手动删除并安装该包::
+
+    apt remove python3-pip python3-virtualenv; rm -r ~/.local
+    rm -r ~/.espressif/python_env && python get-pip.py
+
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。
+
+
 后续步骤
 ==========
 

docs/zh_CN/get-started/macos-setup-scratch.rst

@@ -1,5 +1,5 @@
 ***********************************************
-从零开始设置 MacOS 环境下的工具链
+从零开始设置 macOS 环境下的工具链
 ***********************************************
 
 :link_to_translation:`en:[English]`
@@ -80,6 +80,12 @@ MacPorts 需要完整的 XCode 软件，而 Homebrew 只需要安装 XCode 命
 编译得到的工具链会被保存到 ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-{IDF_TARGET_TOOLCHAIN_NAME}-elf``。使用工具链前，请将 ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-{IDF_TARGET_TOOLCHAIN_NAME}-elf/bin`` 添加至 ``PATH`` 环境变量。
 
 
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。
+
+
 后续步骤
 ==========
 

docs/zh_CN/get-started/macos-setup.rst

@@ -38,7 +38,7 @@ ESP-IDF 将使用 macOS 上默认安装的 Python 版本。
 安装并设置 Python 3 为默认版本
 ---------------------------------------------
 
-`Catalina 10.15 发布说明`_ 中表示不推荐使用 Python 2.7 版本，在未来的 macOS 版本中也不会默认包含 Python。执行以下命令来检查您当前使用的 Python 版本::
+`Catalina 10.15 发布说明`_ 中表示不推荐使用 Python 2.7 版本，在未来的 macOS 版本中也不会默认包含 Python 2.7。执行以下命令来检查您当前使用的 Python 版本::
 
   python --version
 
@@ -55,7 +55,7 @@ ESP-IDF 将使用 macOS 上默认安装的 Python 版本。
       brew install python3
       ln -s /usr/local/bin/python3.8 /usr/local/bin/python
 
-    将上述的目录名 ``/usr/local/bin/python3.8`` 修改为 Python 3 所在的目录。您可以运行 ``which -a python`` 来查看 Python 3 所在的目录。
+    将上述的目录名 ``/usr/local/bin/python3.8`` 修改为 Python 3 所在的目录。您可以运行 ``which -a python3`` 来查看 Python 3 所在的目录。
 
   - 使用 MacPorts_ 进行安装的方法如下::
 
@@ -73,6 +73,13 @@ ESP-IDF 将使用 macOS 上默认安装的 Python 版本。
     上述设置为全局设置，同时会影响到其它应用。
 
 
+
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。
+
+
 后续步骤
 ==========
 
@@ -84,4 +91,5 @@ ESP-IDF 将使用 macOS 上默认安装的 Python 版本。
 .. _ccache: https://ccache.samba.org/
 .. _homebrew: https://brew.sh/
 .. _MacPorts: https://www.macports.org/install.php
-.. _Catalina 10.15 发布说明: https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes
+.. _Catalina 10.15 发布说明: https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes
+

tools/ci/check_build_warnings.py

@@ -30,6 +30,7 @@ IGNORE_WARNS = [
         r"changes choice state",
         r"crosstool_version_check\.cmake",
         r"CryptographyDeprecationWarning",
+        r"Python 3 versions older than 3.6 are not supported."
     ]
 ]
 

tools/ci/config/build.yml

@@ -113,6 +113,7 @@ build_ssc_esp32s2:
     OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
     BUILD_SYSTEM: "cmake"
     TEST_TYPE: "unit_test"
+    PYTHON_VER: 3
   script:
     # RISC-V toolchain is optional but ULP may need it, so install:
     - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc
@@ -152,6 +153,7 @@ build_esp_idf_tests_cmake_esp32s3:
     TEST_TYPE: example_test
     LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
     BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
+    PYTHON_VER: 3
   script:
     # RISC-V toolchain is optional but ULP may need it, so install:
     - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc

tools/idf.py

@@ -91,6 +91,12 @@ def check_environment():
         print_warning("Setting IDF_PATH environment variable: %s" % detected_idf_path)
         os.environ["IDF_PATH"] = detected_idf_path
 
+    # check Python version
+    if sys.version_info[0] < 3:
+        print_warning("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print_warning("WARNING: Python 3 versions older than 3.6 are not supported.")
+
     # check Python dependencies
     checks_output.append("Checking Python dependencies...")
     try: