Merge branch 'feat/idf_tools_migrate_to_venv' into 'master'

Tools: Use built-in venv instead of virtualenv for creating Python environments

Closes IDF-4538

See merge request espressif/esp-idf!18249

docs/en/get-started/linux-macos-setup.rst

@@ -34,17 +34,17 @@ To compile using ESP-IDF you will need to get the following packages. The comman
 
 - Ubuntu and Debian::
 
-    sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0
+    sudo apt-get install git wget flex bison gperf python3 python3-venv cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0
 
 - CentOS 7 & 8::
 
-    sudo yum -y update && sudo yum install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache dfu-util libusbx
+    sudo yum -y update && sudo yum install git wget flex bison gperf python3 cmake ninja-build ccache dfu-util libusbx
 
 CentOS 7 is still supported but CentOS version 8 is recommended for a better user experience.
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make flex bison gperf python-pip cmake ninja ccache dfu-util libusb
+    sudo pacman -S --needed gcc git make flex bison gperf python cmake ninja ccache dfu-util libusb
 
 .. note::
     - CMake version 3.5 or newer is required for use with ESP-IDF. Older Linux distributions may require updating, enabling of a "backports" repository, or installing of a "cmake3" package rather than "cmake".
@@ -55,10 +55,6 @@ For macOS Users
 
 ESP-IDF will use the version of Python installed by default on macOS.
 
-- Install pip::
-
-    sudo easy_install pip
-
 - Install CMake & Ninja build:
 
   - If you have HomeBrew_, you can run::

docs/zh_CN/get-started/linux-macos-setup.rst

@@ -34,17 +34,17 @@ Linux 用户
 
 - Ubuntu 和 Debian::
 
-    sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0
+    sudo apt-get install git wget flex bison gperf python3 python3-venv python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0
 
 - CentOS 7 & 8::
 
-    sudo yum -y update && sudo yum install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache dfu-util libusbx
+    sudo yum -y update && sudo yum install git wget flex bison gperf python3 python3-setuptools cmake ninja-build ccache dfu-util libusbx
 
 目前仍然支持 CentOS 7，但为了更好的用户体验，建议使用 CentOS 8。
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make flex bison gperf python-pip cmake ninja ccache dfu-util libusb
+    sudo pacman -S --needed gcc git make flex bison gperf python cmake ninja ccache dfu-util libusb
 
 .. note::
     - 使用 ESP-IDF 需要 CMake 3.5 或以上版本。较早的 Linux 发行版可能需要升级自身的软件源仓库，或开启 backports 套件库，或安装 "cmake3" 软件包（不是安装 "cmake"）。
@@ -55,10 +55,6 @@ macOS 用户
 
 ESP-IDF 将使用 macOS 上默认安装的 Python 版本。
 
-- 安装 pip::
-
-    sudo easy_install pip
-
 - 安装 CMake 和 Ninja 编译工具：
 
     - 若有 HomeBrew_，您可以运行::

tools/docker/Dockerfile

@@ -23,7 +23,7 @@ RUN : \
     make \
     ninja-build \
     python3 \
-    python3-pip \
+    python3-venv \
     unzip \
     wget \
     xz-utils \
@@ -31,9 +31,6 @@ RUN : \
   && apt-get autoremove -y \
   && rm -rf /var/lib/apt/lists/* \
   && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \
-  && python -m pip install --upgrade \
-    pip \
-    virtualenv \
   && :
 
 # To build the image for a branch or a tag of IDF, pass --build-arg IDF_CLONE_BRANCH_OR_TAG=name.

tools/idf_tools.py

@@ -1889,6 +1889,49 @@ def get_constraints(idf_version):  # type: (str) -> str
         raise DownloadError()
 
 
+def install_legacy_python_virtualenv(path):  # type: (str) -> None
+    # Before creating the virtual environment, check if pip is installed.
+    try:
+        subprocess.check_call([sys.executable, '-m', 'pip', '--version'])
+    except subprocess.CalledProcessError:
+        fatal('Python interpreter at {} doesn\'t have pip installed. '
+              'Please check the Getting Started Guides for the steps to install prerequisites for your OS.'.format(sys.executable))
+        raise SystemExit(1)
+
+    virtualenv_installed_via_pip = False
+    try:
+        import virtualenv  # noqa: F401
+    except ImportError:
+        info('Installing virtualenv')
+        subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'],
+                              stdout=sys.stdout, stderr=sys.stderr)
+        virtualenv_installed_via_pip = True
+        # since we just installed virtualenv via pip, we know that version is recent enough
+        # so the version check below is not necessary.
+
+    with_seeder_option = True
+    if not virtualenv_installed_via_pip:
+        # virtualenv is already present in the system and may have been installed via OS package manager
+        # check the version to determine if we should add --seeder option
+        try:
+            major_ver = int(virtualenv.__version__.split('.')[0])
+            if major_ver < 20:
+                warn('Virtualenv version {} is old, please consider upgrading it'.format(virtualenv.__version__))
+                with_seeder_option = False
+        except (ValueError, NameError, AttributeError, IndexError):
+            pass
+
+    info(f'Creating a new Python environment using virtualenv in {path}')
+    virtualenv_options = ['--python', sys.executable]
+    if with_seeder_option:
+        virtualenv_options += ['--seeder', 'pip']
+
+    subprocess.check_call([sys.executable, '-m', 'virtualenv',
+                           *virtualenv_options,
+                           path],
+                          stdout=sys.stdout, stderr=sys.stderr)
+
+
 def action_install_python_env(args):  # type: ignore
     use_constraints = not args.no_constraints
     reinstall = args.reinstall
@@ -1919,46 +1962,24 @@ def action_install_python_env(args):  # type: ignore
         shutil.rmtree(idf_python_env_path)
 
     if not os.path.exists(virtualenv_python):
-        # Before creating the virtual environment, check if pip is installed.
-        try:
-            subprocess.check_call([sys.executable, '-m', 'pip', '--version'])
-        except subprocess.CalledProcessError:
-            fatal('Python interpreter at {} doesn\'t have pip installed. '
-                  'Please check the Getting Started Guides for the steps to install prerequisites for your OS.'.format(sys.executable))
-            raise SystemExit(1)
-
-        virtualenv_installed_via_pip = False
         try:
-            import virtualenv  # noqa: F401
-        except ImportError:
-            info('Installing virtualenv')
-            subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'virtualenv'],
+            import venv  # noqa: F401
+
+            # venv available
+            virtualenv_options = ['--clear']  # delete environment if already exists
+            if sys.version_info[:2] >= (3, 9):
+                # upgrade pip & setuptools
+                virtualenv_options += ['--upgrade-deps']
+
+            info('Creating a new Python environment in {}'.format(idf_python_env_path))
+            subprocess.check_call([sys.executable, '-m', 'venv',
+                                  *virtualenv_options,
+                                  idf_python_env_path],
                                   stdout=sys.stdout, stderr=sys.stderr)
-            virtualenv_installed_via_pip = True
-            # since we just installed virtualenv via pip, we know that version is recent enough
-            # so the version check below is not necessary.
-
-        with_seeder_option = True
-        if not virtualenv_installed_via_pip:
-            # virtualenv is already present in the system and may have been installed via OS package manager
-            # check the version to determine if we should add --seeder option
-            try:
-                major_ver = int(virtualenv.__version__.split('.')[0])
-                if major_ver < 20:
-                    warn('Virtualenv version {} is old, please consider upgrading it'.format(virtualenv.__version__))
-                    with_seeder_option = False
-            except (ValueError, NameError, AttributeError, IndexError):
-                pass
-
-        info('Creating a new Python environment in {}'.format(idf_python_env_path))
-        virtualenv_options = ['--python', sys.executable]
-        if with_seeder_option:
-            virtualenv_options += ['--seeder', 'pip']
+        except ImportError:
+            # The embeddable Python for Windows doesn't have the built-in venv module
+            install_legacy_python_virtualenv(idf_python_env_path)
 
-        subprocess.check_call([sys.executable, '-m', 'virtualenv',
-                               *virtualenv_options,
-                               idf_python_env_path],
-                              stdout=sys.stdout, stderr=sys.stderr)
     env_copy = os.environ.copy()
     if env_copy.get('PIP_USER')  == 'yes':
         warn('Found PIP_USER="yes" in the environment. Disabling PIP_USER in this shell to install packages into a virtual environment.')