tools: IDF Windows installer offline mode support

.gitlab/ci/build.yml

@@ -389,7 +389,7 @@ build_installer:
   extends: .test-on-windows
   # using a different stage here to be able to use artifacts from build_cmdlinerunner job
   stage: host_test
-  image: $CI_DOCKER_REGISTRY/wine-innosetup:1
+  image: $CI_DOCKER_REGISTRY/wine-innosetup:2
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs:
     - build_cmdlinerunner

.gitlab/ci/host-test.yml

@@ -183,6 +183,9 @@ test_idf_tools:
     - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif" | tr "\n" ":"); echo ${p%:})
     - cd ${IDF_PATH}/tools/test_idf_tools
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+    # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
+    - cd ${IDF_PATH}/tools
+    - python3 ./idf_tools.py install-python-env
 
 test_esp_err_to_name_on_host:
   extends: .host_test_template

docs/en/api-guides/tools/idf-windows-installer.rst

@@ -6,7 +6,9 @@ Command-line parameters
 
 Windows Installer `esp-idf-tools-setup` provides the following command-line parameters:
 
-* ``/GITRECURSIVE=[yes|no]`` - Clone recursively all git repository submodules. Default: yes``
+* ``/CONFIG=[PATH]`` - Path to ``ini`` configuration file to override default configuration of the installer. Default: ``config.ini``.
+* ``/GITCLEAN=[yes|no]`` - Perform git clean and remove untracked directories in Offline mode installation. Default: yes.
+* ``/GITRECURSIVE=[yes|no]`` - Clone recursively all git repository submodules. Default: yes
 * ``/GITREPO=[URL|PATH]`` - URL of repository to clone ESP-IDF. Default: https://github.com/espressif/esp-idf.git
 * ``/GITRESET=[yes|no]`` - Enable/Disable git reset of repository during installation. Default: yes.
 * ``/HELP`` - Display command line options provided by Inno Setup installer.
@@ -16,6 +18,7 @@ Windows Installer `esp-idf-tools-setup` provides the following command-line para
 * ``/LOG=[PATH]`` - Store installation log file in specific directory. Default: empty.
 * ``/OFFLINE=[yes|no]`` - Execute installation of Python packages by PIP in offline mode. The same result can be achieved by setting the environment variable PIP_NO_INDEX. Default: no.
 * ``/USEEMBEDDEDPYTHON=[yes|no]`` - Use Embedded Python version for the installation. Set to ``no`` to allow Python selection screen in the installer. Default: yes.
+* ``/PYTHONNOUSERSITE=[yes|no]`` - Set PYTHONNOUSERSITE variable before launching any Python command to avoid loading Python packages from AppData\Roaming. Default: yes.
 * ``/PYTHONWHEELSURL=[URL]`` - Specify URLs to PyPi repositories for resolving binary Python Wheel dependencies. The same result can be achieved by setting the environment variable PIP_EXTRA_INDEX_URL. Default: https://dl.espressif.com/pypi
 * ``/SKIPSYSTEMCHECK=[yes|no]`` - Skip System Check page. Default: no.
 * ``/VERYSILENT /SUPPRESSMSGBOXES /SP- /NOCANCEL`` - Perform silent installation.

docs/en/get-started/windows-setup.rst

@@ -32,12 +32,7 @@ For this Getting Started we're going to use the Command Prompt, but after ESP-ID
 ESP-IDF Tools Installer
 =======================
 
-The easiest way to install ESP-IDF's prerequisites is to download one of ESP-IDF Tools Installers from this URL:
-
-- Online Installer (30 MB): https://dl.espressif.com/dl/esp-idf-tools-setup-online-2.5.exe
-- Offline Installer - includes ESP-IDF v4.2, v4.1.1 (810 MB): https://dl.espressif.com/dl/esp-idf-tools-setup-offline-2.5.exe
-
-.. IMPORTANT: Next time this link is updated, please go to get-started/index.rst and rewrite the section under "Alternative File Downloads ... Windows". Then delete this comment.
+The easiest way to install ESP-IDF's prerequisites is to download one of ESP-IDF Tools Installers from this URL: https://dl.espressif.com/dl/esp-idf/?idf=4.4
 
 What is the usecase for Online and Offline Installer
 ----------------------------------------------------

docs/zh_CN/get-started/windows-setup.rst

@@ -23,7 +23,7 @@ ESP-IDF 需要安装一些必备工具，才能围绕 {IDF_TARGET_NAME} 构建
 
 .. 注解::
     限定条件：Python 或 ESP-IDF 的安装路径中一定不能包含空格或括号。与此同时，除非操作系统配置为支持 Unicode UTF-8，否则 Python 或 ESP-IDF 的安装路径中也不能包括特殊字符（非 ASCII 码字符）
-   
+
     系统管理员可以通过如下方式将操作系统配置为支持 Unicode UTF-8：控制面板-更改日期、时间或数字格式-管理选项卡-更改系统地域-勾选选项 “Beta：使用 Unicode UTF-8 支持全球语言”-点击确定-重启电脑。
 
 .. _get-started-windows-tools-installer:
@@ -32,12 +32,7 @@ ESP-IDF 需要安装一些必备工具，才能围绕 {IDF_TARGET_NAME} 构建
 ESP-IDF 工具安装器
 =======================
 
-安装 ESP-IDF 必备工具最简易的方式是下载 ESP-IDF 工具安装器，请选择下列地址之一：
-
-- 在线安装（30 MB）：https://dl.espressif.com/dl/esp-idf-tools-setup-online-2.5.exe
-- 离线安装（包括 ESP-IDF v4.2、v4.1.1，810 MB）：https://dl.espressif.com/dl/esp-idf-tools-setup-offline-2.5.exe
-
-.. 重要：下次更新此链接时，请重新写文件 get-started/index.rst 中“其它文件下载方式”这一章节，然后将此条注意事项删除。
+安装 ESP-IDF 必备工具最简易的方式是从 https://dl.espressif.com/dl/esp-idf/?idf=4.4 中下载 ESP-IDF 工具安装器。
 
 在线安装与离线安装的区别
 -----------------------------------------

tools/tools.json

@@ -651,12 +651,12 @@
       "version_regex": "Python ([0-9.]+)",
       "versions": [
         {
-          "name": "3.9.1",
+          "name": "3.8.7",
           "status": "recommended",
           "win64": {
-            "sha256": "d5a0e625dd5b2bc6872de90292d71009c17e2396e3d1575d886a94d0dfb00c87",
-            "size": 20362758,
-            "url": "https://dl.espressif.com/dl/idf-python/idf-python-3.9.1-embed-win64.zip"
+            "sha256": "5fdf20350e71570642377fc58e61138f407cf286a87f11416829d94c4d9f2019",
+            "size": 20086226,
+            "url": "https://dl.espressif.com/dl/idf-python/idf-python-3.8.7-embed-win64.zip"
           }
         }
       ]
@@ -680,17 +680,17 @@
       "version_cmd": [
         "cmd",
         "/c",
-        "echo idf4.3_py3.9_2021-01-07"
+        "echo 3.8-2020-01-21"
       ],
-      "version_regex": "(idf4.3_py3.9_2021-01-07)",
+      "version_regex": "(3.8-2020-01-21)",
       "versions": [
         {
-          "name": "idf4.3_py3.9_2021-01-07",
+          "name": "3.8-2020-01-21",
           "status": "recommended",
           "win64": {
-            "sha256": "2a33dbaa106aec9c5098b1af46f04c69923be947291c19855ff355b9707314b6",
-            "size": 8316997,
-            "url": "https://dl.espressif.com/dl/idf-python-wheels/idf-python-wheels-idf4.3_py3.9_2021-01-07-win64.zip"
+            "sha256": "0c2f40b615ce7a3d70c6690b44ef550c7c9fe4a480a92dfeb151ba4952e0212b",
+            "size": 9107660,
+            "url": "https://dl.espressif.com/dl/idf-python-wheels/idf-python-wheels-3.8-2020-01-21-win64.zip"
           }
         }
       ]

tools/windows/tool_setup/.gitignore

@@ -4,3 +4,5 @@ dist
 unzip
 keys
 idf_versions.txt
+releases
+tools

tools/windows/tool_setup/Languages/idf_tool_en-US.islu

@@ -38,3 +38,10 @@ SystemCheckFixesFailed=Failed application of Fixes. Please refer to the Full log
 SystemCheckNotCompleteConsent=System check is not complete. Do you want to proceed by skipping checks?
 SystemCheckRootCertificates=Checking certificates
 SystemCheckRootCertificateWarning=Unable to load data from server dl.espressif.com.
+CreateShortcutStartMenu=Start Menu
+CreateShortcutDesktop=Desktop
+CreateShortcutPowerShell=PowerShell - Create shortcut for the ESP-IDF Tools:
+CreateShortcutCMD=CMD - Create shortcut for the ESP-IDF Tools:
+OptimizationTitle=Optimization:
+OptimizationWindowsDefender=Register the ESP-IDF Tools executables as Windows Defender exclusions. The registration might improve compilation speed. The registration of exclusions requires elevation of privileges.
+OptimizationDownloadMirror=Use Espressif download server instead of downloading tool packages from GitHub.

tools/windows/tool_setup/README.md

@@ -32,7 +32,60 @@ This uses `wine-innosetup` Docker image and `build_installer.sh` script. This is
 docker run --rm -v $IDF_PATH:/idf -w /idf/tools/windows/tool_setup -it $CI_DOCKER_REGISTRY/wine-innosetup:1 /bin/bash build_installer.sh
 ```
 
-### Manually, step by step
+### Windows development env with WSL2 and Windows Docker Containers
+
+The best approach to quickly develop and test all aspects of the build process is to use Windows with WSL2.
+
+Requirements:
+
+* WSL2 and Ubuntu distribution via Microsoft Store
+* Install Windows Terminal - https://github.com/microsoft/terminal
+* Install Docker and switch container runner to Windows
+* Install Visual Studio Code - install plugin for Inno Setup and Docker
+* Install Inno Setup - `choco install innnosetup`
+
+#### The first build of the installer
+
+This step is bootstrapping the whole process. Open Windows Terminal, click + sign and select Ubuntu.
+
+```
+cd tools/windows/tools_setup/
+./build_installer.sh online
+```
+
+The setup will download the necessary dependencies and it will build the installer.
+
+#### Build of offline version of the installer
+
+The offline version is built by setting /DOFFLINE=yes to ISCC on the command-line. To speed up build, it's possible to redirect stdout of ISCC to the file.
+
+```
+./build_installer.sh offline >out.txt
+```
+
+To speed up development build it's possible to disable compression which is set by default to lzma.
+
+```
+./build_installer.sh offline none >out.txt
+```
+
+#### Development work in idf_tool_setup.iss
+
+Open Inno Setup and open file idf_tool_setup.iss. This is the main file of the installer
+
+Press CTRL+F9 to rebuild the whole installer. Inno Setup detects changes only in the main file. If you change anything in include files, you need to explicitly press CTRL+F9 to build and Run.
+
+Press F9 to run the installer.
+
+Additional parameters to speed up development could be passed via Run - Parameters
+
+#### Development work in iss.inc files
+
+The majority of code is store in iss.inc files. The best way to develop it is to open a whole esp-idf directory in Visual Studio Code.
+
+To configure syntax highlight for inc files, open Settings CTRL+, search for `Associations`. In section TextEditor - Files find `File: Associations`. Click `Add Item`, set `item` to `*.inc`, set `value` to `innnosetup`.
+
+#### Manually, step by step
 
 * Build cmdlinerunner DLL.
   - On Linux/Mac, install mingw-w64 toolchain (`i686-w64-mingw32-gcc`). Then build the DLL using CMake:
@@ -53,15 +106,7 @@ docker run --rm -v $IDF_PATH:/idf -w /idf/tools/windows/tool_setup -it $CI_DOCKE
 
 * Build the installer using Inno Setup Compiler: `ISCC.exe idf_tools_setup.iss`.
 
-## Signing the installer
-
-* Obtain the signing key (e.g `key.pem`) and the certificate chain (e.g. `certchain.pem`). Set the environment variables to point to these files:
-  - `export KEYFILE=key.pem`
-  - `export CERTCHAIN=certchain.pem`
-
-* Run `sign_installer.sh` script. This will ask for the `key.pem` password, and produce the signed installer in the Output directory. If you plan to run the script multiple times, you may also set `KEYPASSWORD` environment variable to the `key.pem` password, to avoid the prompt.
-
-## Development and testing of the installer
+### Testing of the installer
 
 Development and testing of the installer can be simplified by using command line parameters which can be passed to the installer.
 
@@ -85,9 +130,9 @@ Documentation of parameters is available in api-guides/tools/idf-windows-install
 
 ### Testing installation in Docker with Windows containers
 
-The testing script is stored in docker-compose.yml. The test perform full silent installation and executes build of get-started example.
+The testing script is stored in docker-compose.yml. The test performs full silent installation and executes the build of get-started example.
 
-Commands for testing multiple versions:
+Commands for testing of `online` and `offline` installer with support for cache of dist and releases:
 
 ```
 $env:IDF_VERSION="v4.1"; docker-compose.exe run idf-setup-test
@@ -100,7 +145,41 @@ $env:IDF_VERSION="release/v3.3"; docker-compose.exe run idf-setup-test
 $env:IDF_VERSION="master"; docker-compose.exe run idf-setup-test
 ```
 
+Command for testing `offline` type of installer which contains everything but kitchen sink.:
+
+```
+$env:IDF_VERSION="v4.2"; docker-compose.exe run idf-setup-offline-test
+$env:IDF_VERSION="release/v4.2"; docker-compose.exe run idf-setup-offline-test
+```
+
 The installation log is not displayed immediately on the screen. It's stored in the file and it's displayed when the installation finishes. The glitch of Inno Setup is that in case of failed installation it won't terminate and it keeps hanging.
 
 Recommendation: Use Visual Studio Code with Docker plugin to work with container.
 The log file is then accessible under Docker - Containers - Container - Files - Temp - install.txt - right click - Open.
+
+### Testing multiple installations at once
+
+Docker compose contains definition of multiple scenarios. The test can be launched by command:
+
+```
+$env:IDF_VERSION="v4.2"; docker-compose up --force-recreate
+```
+
+Note: `--force-recreate` is necessary otherwise the container will be just resumed from previous state.
+### Testing the installation in Hyper-V
+
+Docker does not support the test of installation with GUI and enabled Windows Defender. These tests can be executed in Hyper-V available on Windows. Launch `Hyper-V Manager`, create VM, and connect to it.
+
+Use the following command to copy the installer to Hyper-V machine with the name "win10":
+
+```
+ Copy-VMFile "win10"  -SourcePath C:\projects\esp-idf\tools\windows\tool_setup\Output\esp-idf-tools-setup-unsigned.exe -DestinationPath "C:\Users\Tester\Desktop\esp-idf-tools-setup-unsigned.exe" -CreateFullPath -FileSource Host -Force
+```
+
+## Signing the installer
+
+* Obtain the signing key (e.g `key.pem`) and the certificate chain (e.g. `certchain.pem`). Set the environment variables to point to these files:
+  - `export KEYFILE=key.pem`
+  - `export CERTCHAIN=certchain.pem`
+
+* Run `sign_installer.sh` script. This will ask for the `key.pem` password and produce the signed installer in the Output directory. If you plan to run the script multiple times, you may also set `KEYPASSWORD` environment variable to the `key.pem` password, to avoid the prompt.

tools/windows/tool_setup/Scripts/Install-Idf.ps1

@@ -4,12 +4,21 @@ param (
     [string]$IdfVersion = "v4.1"
 )
 
-$Installer
-$IdfPath
-$IdfVersion
+"Configuration:"
+"* Installer = $Installer"
+"* IdfPath = $IdfPath"
+"* IdfVersion = $IdfVersion"
+
+$ProcessName = (Get-Item $Installer).Basename
+"Waiting for process: $ProcessName"
+
+# Set PYTHONHOME and PYTHONPATH to some directory which is not on the system to test process of creating venv
+# The Installer and IDF shell wrappers contains clearing of variables
+$env:PYTHONPATH="C:\Hvannadalshnúkur"
+$env:PYTHONHOME="C:\Hvannadalshnúkur"
 
 mkdir C:\Temp
-C:\Output\esp-idf-tools-setup-unsigned.exe /VERYSILENT /LOG=C:\Temp\install.txt /SUPPRESSMSGBOXES /SP- /NOCANCEL /NORESTART /IDFVERSION=${IdfVersion}
-$InstallerProcess = Get-Process esp-idf-tools-setup-unsigned
+&$Installer /VERYSILENT /LOG=C:\Temp\install.txt /SUPPRESSMSGBOXES /SP- /NOCANCEL /NORESTART /IDFVERSION=${IdfVersion}
+$InstallerProcess = Get-Process $ProcessName
 Wait-Process -Id $InstallerProcess.id
-Get-Content C:\Temp\install.txt
+Get-Content -Tail 80 C:\Temp\install.txt

tools/windows/tool_setup/Scripts/Test-Idf.ps1

@@ -1,28 +0,0 @@
-param (
-    [string]$PythonPath = "C:\Python38\",
-    [string]$IdfPath = "C:\Users\ContainerAdministrator\Desktop\esp-idf"
-)
-
-$env:PATH+=";${PythonPath}"
-Set-Location "${IdfPath}"
-#.\export.ps1
-$env:PYTHONPATH="C:\Users\ContainerAdministrator\Desktop\esp-idf\tools\"
-# Append build script and launch via link
-#Add-Content -Path ${IdfPath}\export.bat -Value ${BuildCommands}
-# timeout is necessary to fix the problem when installer is writing some final files
-# it seems that installer exits, but locks were not released yet
-Start-Sleep -s 5
-
-$WSShell = New-Object -comObject WScript.Shell
-$Shortcut = $WSShell.CreateShortcut('C:\Users\ContainerAdministrator\Desktop\ESP-IDF Command Prompt (cmd.exe).lnk')
-$Arguments = $Shortcut.Arguments -replace "/k ", "/c '"
-$Command = $Shortcut.TargetPath + ' ' + $Arguments -replace '""', '"'
-$Command += " && cd examples\get-started\blink\ && idf.py build'"
-Invoke-Expression -Command $Command
-#powershell.exe
-#C:\Windows\system32\cmd.exe /c '"C:\Users\ContainerAdministrator\.espressif\idf_cmd_init.bat" "C:\Users\ContainerAdministrator\.espressif\python_env\idf4.1_py3.7_env\Scripts" "C:\Program Files\Git\cmd\" && cd examples\get-started\blink\ && idf.py build'
-#cmd /c "ping -n 4 127.0.0.1 && .\export.bat && cd examples\get-started\blink\ && idf.py build"
-#cmd /c "ping -n 4 127.0.0.1 && .\export.bat && cd examples\get-started\blink\ && idf.py build"
-#cmd /c "timeout 4 && C:\Users\ContainerAdministrator\.espressif\idf_cmd_init.bat 'C:\' && cd examples\get-started\blink\ && idf.py build"
-
-#& "C:\Users\ContainerAdministrator\Desktop\esp-idf\Run ESP-IDF Command Prompt (cmd.exe).lnk"

tools/windows/tool_setup/Scripts/Test-IdfCmd.ps1

@@ -0,0 +1,19 @@
+param (
+    [string]$PythonPath = "C:\Python38\",
+    [string]$IdfPath = "C:\Users\ContainerAdministrator\Desktop\esp-idf"
+)
+
+$env:PATH+=";${PythonPath}"
+Set-Location "${IdfPath}"
+$env:PYTHONPATH="C:\Users\ContainerAdministrator\Desktop\esp-idf\tools\"
+
+# Timeout is necessary to fix the problem when installer is writing some final files
+# it seems that installer exits, but locks were not released yet
+Start-Sleep -s 5
+
+$WSShell = New-Object -comObject WScript.Shell
+$Shortcut = $WSShell.CreateShortcut('C:\Users\ContainerAdministrator\Desktop\ESP-IDF Command Prompt (cmd.exe).lnk')
+$Arguments = $Shortcut.Arguments -replace "/k ", "/c '"
+$Command = $Shortcut.TargetPath + ' ' + $Arguments -replace '""', '"'
+$Command += " && cd examples\get-started\blink\ && idf.py build'"
+Invoke-Expression -Command $Command

tools/windows/tool_setup/Scripts/Test-IdfPowerShell.ps1

@@ -0,0 +1,31 @@
+param (
+    [string]$PythonPath = "C:\Python38\",
+    [string]$IdfPath = "C:\Users\ContainerAdministrator\Desktop\esp-idf"
+)
+
+$env:PATH+=";${PythonPath}"
+Set-Location "${IdfPath}"
+$env:PYTHONPATH="C:\Users\ContainerAdministrator\Desktop\esp-idf\tools\"
+
+# Timeout is necessary to fix the problem when installer is writing some final files
+# it seems that installer exits, but locks were not released yet
+Start-Sleep -s 5
+
+$WSShell = New-Object -comObject WScript.Shell
+$Shortcut = $WSShell.CreateShortcut('C:\Users\ContainerAdministrator\Desktop\ESP-IDF PowerShell.lnk')
+$Command =  '. ' + $Shortcut.Arguments -replace '""', '"'
+$Command = $Command -replace " -ExecutionPolicy Bypass -NoExit -File", ""
+$Command
+Invoke-Expression -Command $Command
+
+cd examples\get-started\blink\
+idf.py build
+
+# Check whether the repository is clean
+$GitChanges=(git status -s).Lenght
+if ($GitChanges -gt 0) {
+    "* Warning! Git repository dirty."
+    $GitChanges
+} else {
+    "Git repository clean."
+}

tools/windows/tool_setup/build_installer.sh

@@ -7,23 +7,166 @@
 # - Downloads 7z and idf_versions.txt
 # - Runs ISCC under wine to compile the installer itself
 
+set -x
 set -e
 set -u
 
-INSTALLER_TYPE="${1-full}"
+INSTALLER_TYPE="${1-online}"
+COMPRESSION="${2-lzma}"
+
+# Default values for IDF installer passed in 'offline' version of build
+IDF_GIT_VERSION="2.30.0.2"
+IDF_GIT_VERSION_DIR="v2.30.0.windows.2"
+IDF_PYTHON_VERSION="3.8.7"
+IDF_PYTHON_WHEELS_VERSION="3.8-2021-01-21"
 
 echo "Selected installer type: $INSTALLER_TYPE"
-echo "Available installer types: full, netinst"
+echo "Selected compresion: $COMPRESSION"
+echo "Available installer types: online, offline, precached, draft"
+echo "Available compressions: lzma, none"
 
-PACKAGES="all"
-if [[ "$INSTALLER_TYPE" == "netinst" ]]; then
-    PACKAGES="idf-python"
-fi
+# Configuration options passed to ISCC compiler
+# OFFLINE [yes|no] - set installer to offline mode, nothing will be retrieved from
+#  internet during installation
+ISCC_PARAMS=""
+
+function prepare_offline_branches()
+{
+    BUNDLE_DIR="releases/esp-idf-bundle"
+
+    if [[ ! -d "$BUNDLE_DIR" ]]; then
+        echo "Performing full clone."
+        git clone --shallow-since=2020-01-01 --jobs 8 --recursive https://github.com/espressif/esp-idf.git "$BUNDLE_DIR"
+
+        # Fix repo mode
+        git -C "$BUNDLE_DIR" config --local core.fileMode false
+        git -C "$BUNDLE_DIR" submodule foreach --recursive git config --local core.fileMode false
+        # Allow deleting directories by git clean --force
+        # Required when switching between versions which does not have a module present in current branch
+        git -C "$BUNDLE_DIR" config --local clean.requireForce false
+        git -C "$BUNDLE_DIR" reset --hard
+        git -C "$BUNDLE_DIR" submodule foreach git reset --hard
+    else
+        git -C "$BUNDLE_DIR" fetch
+    fi
+
+    VERSIONS="idf_versions.txt"
+
+    tac "$VERSIONS" | while read BRANCH; do
+        pushd "$BUNDLE_DIR"
+
+        if [[ -z "$BRANCH" ]]; then
+            continue
+        fi
+
+        echo "processing branch: ($BRANCH)"
+        git fetch origin tag "$BRANCH"
+        git checkout "$BRANCH"
+
+        # Pull changes only for branches, tags does not support pull
+        #https://stackoverflow.com/questions/1593188/how-to-programmatically-determine-whether-the-git-checkout-is-a-tag-and-if-so-w
+        git describe --exact-match HEAD || git pull
+
+        git submodule update --init --recursive
+
+        # Clean up left over submodule directories after switching to other branch
+        git clean --force -d
+        # Some modules are very persistent like cmok and needs 2nd round of cleaning
+        git clean --force -d
+
+        git reset --hard
+        git submodule foreach git reset --hard
+
+        if [[ $(git status -s | wc -l ) -ne 0 ]]; then
+            echo "git status not empty. Repository is dirty. Aborting."
+            git status
+            exit 1
+        fi
+
+        $IDF_PATH/tools/idf_tools.py --tools-json tools/tools.json --non-interactive download --platform Windows-x86_64 all
+        popd
+    done
+
+    # Remove symlinks which are not supported on Windws, unfortunatelly -c core.symlinks=false does not work
+    find "$BUNDLE_DIR" -type l -print -delete;
+}
+
+function install_idf_package()
+{
+    TOOL_URL="$1"
+    TOOL_FILE="$2"
+    TOOL_VERSION="$3"
+
+    if [[ ! -f "${TOOL_FILE}" ]]; then
+        wget --no-verbose -O "${TOOL_FILE}" "${TOOL_URL}"
+    fi
+
+    if [[ ! -d "${TOOL_VERSION}" ]]; then
+        mkdir -p "${TOOL_VERSION}"
+        unzip -q "${TOOL_FILE}" -d "${TOOL_VERSION}"
+    fi
 
-iscc_path=$(which iscc)
+}
+
+function install_idf_python()
+{
+    install_idf_package \
+        "https://dl.espressif.com/dl/idf-python/idf-python-$IDF_PYTHON_VERSION-embed-win64.zip" \
+        "${IDF_TOOLS_PATH}/idf-python-${IDF_PYTHON_VERSION}-embed-win64.zip" \
+        "tools/idf-python/${IDF_PYTHON_VERSION}"
+}
+
+function install_idf_python_wheels()
+{
+    install_idf_package \
+        "https://dl.espressif.com/dl/idf-python-wheels/idf-python-wheels-$IDF_PYTHON_WHEELS_VERSION-win64.zip" \
+        "${IDF_TOOLS_PATH}/idf-python-wheels-${IDF_PYTHON_WHEELS_VERSION}-win64.zip" \
+        "tools/idf-python-wheels/${IDF_PYTHON_WHEELS_VERSION}"
+}
+
+function install_idf_git()
+{
+    IDF_FILE="Git-${IDF_GIT_VERSION}-64-bit.exe"
+    if [[ -f "${IDF_FILE}" ]]; then
+        return
+    fi
+    mkdir -p "${IDF_TOOLS_PATH}/dist/"
+    wget -nc --no-verbose -O "${IDF_TOOLS_PATH}/dist/${IDF_FILE}" "https://github.com/git-for-windows/git/releases/download/${IDF_GIT_VERSION_DIR}/${IDF_FILE}" || echo "exists"
+    if [[ `file -b "${IDF_TOOLS_PATH}/dist/${IDF_FILE}"` != "PE32 executable (GUI) Intel 80386, for MS Windows" ]]; then
+        echo "Git installer is not valid Windows Executable"
+        exit 1
+    fi
+}
+
+function download_idf_versions()
+{
+    echo "Downloading idf_versions.txt..."
+    wget --no-verbose -O idf_versions.txt https://dl.espressif.com/dl/esp-idf/idf_versions.txt
+}
+
+function build_with_wine()
+{
+    xvfb-run-wine /opt/wine/drive_c/Program\ Files\ \(x86\)/Inno\ Setup\ 6/ISCC.exe $ISCC_PARAMS idf_tool_setup.iss
+}
+
+# Check for ISCC on Windows WSL2
+iscc_path=$(which ISCC.exe) || echo "ISCC.exe not found. If running in WSL2, install ISCC by following command: choco install innosetup"
 if [[ -z "$iscc_path" ]]; then
-    echo "Inno setup compiler (iscc) not found. Are you running wine-innosetup Docker image?"
-    exit 1
+    echo "Searching for iscc"
+    iscc_path=$(which iscc)
+    if [[ -z "$iscc_path" ]]; then
+        echo "Inno setup compiler (iscc) not found. Are you running wine-innosetup Docker image?"
+        exit 1
+    fi
+fi
+
+if [ ! -d "unzip" ]; then
+    echo "Downloading 7z..."
+    mkdir -p unzip
+    pushd unzip
+    wget --no-verbose -O 7z1900-extra.7z https://www.7-zip.org/a/7z1900-extra.7z
+    7zr e -y 7z1900-extra.7z
+    popd
 fi
 
 if [[ -z "${IDF_PATH:-}" ]]; then
@@ -31,24 +174,61 @@ if [[ -z "${IDF_PATH:-}" ]]; then
     echo "Assuming IDF_PATH: ${IDF_PATH}"
 fi
 
-echo "Downloading IDF Tools..."
-mkdir -p idf_tools_tmp
-export IDF_TOOLS_PATH=$PWD/idf_tools_tmp
-$IDF_PATH/tools/idf_tools.py --non-interactive download --platform Windows-x86_64 $PACKAGES
-$IDF_PATH/tools/idf_tools.py --tools-json tools_fallback.json --non-interactive download --platform Windows-x86_64 all
+export IDF_TOOLS_PATH="${PWD}/idf_tools_tmp_${INSTALLER_TYPE}"
+mkdir -p "${IDF_TOOLS_PATH}"
+echo "Using IDF_TOOLS_PATH specific for installer type: ${IDF_TOOLS_PATH}"
+
+# Clean up production dist, data will be transferred from helper dist specific for installer type
+if [[ -d "dist" ]]; then
+    rm -rf dist
+fi
 mkdir -p dist
-cp idf_tools_tmp/dist/* dist/
 
-echo "Downloading 7z..."
-mkdir -p unzip
-pushd unzip
-wget --no-verbose -O 7z1900-extra.7z https://www.7-zip.org/a/7z1900-extra.7z
-7zr e -y 7z1900-extra.7z
-popd
+if [[ "$INSTALLER_TYPE" == "precached" ]]; then
+    ISCC_PARAMS="/DOFFLINE=no /DCOMPRESSION=$COMPRESSION /DSOLIDCOMPRESSION=no /DPYTHONWHEELSVERSION= /DGITVERSION=${IDF_GIT_VERSION} /DGITVERSIONDIR=${IDF_GIT_VERSION_DIR}"
+    download_idf_versions
+    $IDF_PATH/tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all
+    $IDF_PATH/tools/idf_tools.py --tools-json tools_fallback.json --non-interactive download --platform Windows-x86_64 all
+    cp $IDF_TOOLS_PATH/dist/* dist/
+elif [[ "$INSTALLER_TYPE" == "online" ]]; then
+    ISCC_PARAMS="/DOFFLINE=no /DCOMPRESSION=$COMPRESSION /DSOLIDCOMPRESSION=no /DPYTHONWHEELSVERSION= /DGITVERSION=${IDF_GIT_VERSION}  /DGITVERSIONDIR=${IDF_GIT_VERSION_DIR}"
+    download_idf_versions
+    install_idf_python
+    rm -rf tools/idf-python-wheels
+elif [[ "$INSTALLER_TYPE" == "offline" ]]; then
+    # Turn off also solid compression - it causes delay in start time of installer.
+    ISCC_PARAMS="/DOFFLINE=yes /DCOMPRESSION=$COMPRESSION /DSOLIDCOMPRESSION=no /DPYTHONWHEELSVERSION=$IDF_PYTHON_WHEELS_VERSION /DGITVERSION=${IDF_GIT_VERSION}  /DGITVERSIONDIR=${IDF_GIT_VERSION_DIR}"
+
+    install_idf_git
+    install_idf_python
+    install_idf_python_wheels
 
-echo "Downloading idf_versions.txt..."
-wget --no-verbose -O idf_versions.txt https://dl.espressif.com/dl/esp-idf/idf_versions.txt
+    cp idf_versions_offline.txt idf_versions.txt
+    prepare_offline_branches
+    cp $IDF_TOOLS_PATH/dist/* dist/
+elif [[ "$INSTALLER_TYPE" == "draft" ]]; then
+    ISCC_PARAMS="/DOFFLINE=yes /DCOMPRESSION=$COMPRESSION /DSOLIDCOMPRESSION=no /DPYTHONWHEELSVERSION=$IDF_PYTHON_WHEELS_VERSION /DGITVERSION=${IDF_GIT_VERSION}  /DGITVERSIONDIR=${IDF_GIT_VERSION_DIR}"
+else
+    echo "Uknown type of installer: $INSTALLER_TYPE"
+    exit 1
+fi
+
+# Check for cmdlinerunner
+if [[ ! -f "cmdlinerunner/build/cmdlinerunner.dll" ]]; then
+    echo "cmdlinerunner not found, downloading"
+    wget --no-verbose -O $IDF_TOOLS_PATH/idf-cmdlinerunner-1.0.zip https://dl.espressif.com/dl/idf-cmdlinerunner/idf-cmdlinerunner-1.0.zip
+    mkdir -p cmdlinerunner/build
+    unzip -q $IDF_TOOLS_PATH/idf-cmdlinerunner-1.0.zip -d cmdlinerunner/build/
+    rm $IDF_TOOLS_PATH/idf-cmdlinerunner-1.0.zip
+fi
 
 echo "Running ISCC..."
 # https://jrsoftware.org/ishelp/index.php?topic=compilercmdline
-iscc idf_tool_setup.iss
+echo "iscc $ISCC_PARAMS idf_tool_setup.iss"
+
+# Check whether we should run wine in case of docker image
+which xvfb-run-wine && \
+    build_with_wine ||
+    iscc $ISCC_PARAMS idf_tool_setup.iss
+
+mv "Output/esp-idf-tools-setup-unsigned.exe" "Output/esp-idf-tools-setup-${INSTALLER_TYPE}-unsigned.exe"

tools/windows/tool_setup/configuration.ini

@@ -0,0 +1,18 @@
+[DEFAULT]
+GITRECURSIVE=yes
+GITRESET=yes
+GITREPO=https://github.com/espressif/esp-idf.git
+; In case of password protected keychain you can use ssh-agent
+; Make sure that the 'Open SSH Agent' service is not Disabled in services list.
+; Add kyes: ssh-add
+; Tell git which ssh command to use, otherwise it might still prompt you for password
+; [Environment]::SetEnvironmentVariable("GIT_SSH", "$((Get-Command ssh).Source)", [System.EnvironmentVariableTarget]::User)
+; Start new process/terminal to load env variable
+; Note: OpenSSH is part of Windows 1803 - https://poshsecurity.com/blog/using-the-openssh-client-included-in-windows-10-1809-as-your-gits-ssh-client
+IDFDIR=
+IDFVERSION=
+IDFVERSIONSURL=https://dl.espressif.com/dl/esp-idf/idf_versions.txt
+OFFLINE=yes
+PYTHONWHEELSURL=https://dl.espressif.com/pypi
+SKIPSYSTEMCHECK=no
+USEEMBEDDEDPYTHON=yes

tools/windows/tool_setup/configuration.iss.inc

@@ -0,0 +1,73 @@
+{ Copyright 2019-2021 Espressif Systems (Shanghai) CO LTD
+  SPDX-License-Identifier: Apache-2.0 }
+
+{ ------------------------------ Load configuration of the installer ------------------------------ }
+
+var
+    ConfigurationFile: String;
+    GitRepository: String;
+    IsGitRecursive: Boolean;
+    IsGitResetAllowed: Boolean;
+    IsGitCleanAllowed: Boolean;
+    IsPythonNoUserSite: Boolean;
+    IsOfflineMode: Boolean;
+    IDFDirectory: String;
+    IDFVersion: String;
+    IDFVersionUrl: String;
+    PythonWheelsUrl: String;
+    PythonWheelsVersion: String;
+    SkipSystemCheck: Boolean;
+    UseEmbeddedPython: Boolean;
+
+function GetConfigurationString(Key: String; Default: String):String;
+var Value: String;
+begin
+    Value := GetIniString('DEFAULT', Key, Default, ConfigurationFile);
+    Value := ExpandConstant('{param:' + Key + '|' + Value + '}');
+    Log('Configuration /' + Key + '=' + Value);
+    Result := Value;
+end;
+
+function GetConfigurationBoolean(Key: String; DefaultString: String):Boolean;
+begin
+    Result := (GetConfigurationString(Key, DefaultString) = 'yes');
+end;
+
+{ Initialize configuration of the installer. }
+{ Default configuration is encoded in installer. }
+{ The configuration can be changed by configuration.ini file. }
+{ The configuration can be changed by command line options which have highest priority. }
+<event('InitializeWizard')>
+procedure InitializeConfiguration();
+begin
+    ConfigurationFile := ExpandConstant('{param:CONFIG|}');
+
+    if (ConfigurationFile <> '') then begin
+        if (not FileExists(ConfigurationFile)) then begin
+            Log('Configuration file does not exist, using default values.');
+        end;
+    end;
+
+    Log('Configuration /CONFIG=' + ConfigurationFile);
+
+    IsGitCleanAllowed := GetConfigurationBoolean('GITCLEAN', 'yes');
+    IsGitRecursive := GetConfigurationBoolean('GITRECURSIVE', 'yes');
+    IsGitResetAllowed := GetConfigurationBoolean('GITRESET', 'yes');
+    GitRepository := GetConfigurationString('GITREPO', 'https://github.com/espressif/esp-idf.git');
+    IDFDirectory := GetConfigurationString('IDFDIR', '');
+    IDFVersion := GetConfigurationString('IDFVERSION', '');
+    IDFVersionUrl := GetConfigurationString('IDFVERSIONSURL', 'https://dl.espressif.com/dl/esp-idf/idf_versions.txt');
+    IsOfflineMode := GetConfigurationBoolean('OFFLINE', '{#OFFLINE}');
+    IsPythonNoUserSite := GetConfigurationBoolean('PYTHONNOUSERSITE', 'yes');
+    PythonWheelsUrl := GetConfigurationString('PYTHONWHEELSURL', 'https://dl.espressif.com/pypi');
+    PythonWheelsVersion := GetConfigurationString('PYTHONWHEELSVERSION', '{#PYTHONWHEELSVERSION}');
+    SkipSystemCheck := GetConfigurationBoolean('SKIPSYSTEMCHECK', 'no');
+    UseEmbeddedPython := GetConfigurationBoolean('USEEMBEDDEDPYTHON', 'yes');
+end;
+
+
+{ Required to display option for installation configuration. }
+function IsOnlineMode():Boolean;
+begin
+    Result := not IsOfflineMode;
+end;

tools/windows/tool_setup/docker-compose.yml

@@ -3,9 +3,39 @@ version: "3"
 # This docker-compose is for testing the installation process.
 # In starts the installation and executes also build of get-started example.
 services:
-  idf-setup-test:
+  idf-setup-online-test:
     image: mcr.microsoft.com/windows/servercore:1809
-    command: powershell -c "C:/Scripts/Prepare-Cache.ps1 -IdfVersion ${IDF_VERSION}; C:/Scripts/Install-Idf.ps1 -Installer 'c:/Output/esp-idf-tools-setup-unsigned.exe' -IdfVersion ${IDF_VERSION}; C:/Scripts/Test-Idf.ps1; powershell ;exit $$LASTEXITCODE"
+    command: powershell -c "C:/Scripts/Install-Idf.ps1 -Installer 'c:/Output/esp-idf-tools-setup-online-unsigned.exe' -IdfVersion ${IDF_VERSION}; C:/Scripts/Test-IdfCmd.ps1; C:/Scripts/Test-IdfPowerShell.ps1;; powershell ;exit $$LASTEXITCODE"
+    tmpfs:
+      - C:\Users\ContainerAdministrator\.espressif
+    volumes:
+      - type: bind
+        source: C:\projects\esp-idf\tools\windows\tool_setup\Output
+        target: C:\Output
+        read_only: true
+      - type: bind
+        source: C:\projects\esp-idf\tools\windows\tool_setup\Scripts
+        target: C:\Scripts
+        read_only: true
+
+  idf-setup-offline-test:
+    image: mcr.microsoft.com/windows/servercore:1809
+    command: powershell -c "C:/Scripts/Install-Idf.ps1 -Installer 'c:/Output/esp-idf-tools-setup-offline-unsigned.exe' -IdfVersion ${IDF_VERSION}; C:/Scripts/Test-IdfCmd.ps1; C:/Scripts/Test-IdfPowerShell.ps1; powershell ;exit $$LASTEXITCODE"
+    tmpfs:
+      - C:\Users\ContainerAdministrator\.espressif
+    volumes:
+      - type: bind
+        source: C:\projects\esp-idf\tools\windows\tool_setup\Output
+        target: C:\Output
+        read_only: true
+      - type: bind
+        source: C:\projects\esp-idf\tools\windows\tool_setup\Scripts
+        target: C:\Scripts
+        read_only: true
+
+  idf-setup-precached-test:
+    image: mcr.microsoft.com/windows/servercore:1809
+    command: powershell -c "C:/Scripts/Prepare-Cache.ps1 -IdfVersion ${IDF_VERSION}; C:/Scripts/Install-Idf.ps1 -Installer 'c:/Output/esp-idf-tools-setup-online-unsigned.exe' -IdfVersion ${IDF_VERSION}; C:/Scripts/Test-IdfCmd.ps1; C:/Scripts/Test-IdfPowerShell.ps1; powershell ;exit $$LASTEXITCODE"
     tmpfs:
       - C:\Users\ContainerAdministrator\.espressif
     volumes:

tools/windows/tool_setup/idf_cmd_init.bat

@@ -31,6 +31,18 @@ if not "%PYTHONPATH%"=="" (
     set PYTHONPATH=
 )
 
+:: Clear PYTHONHOME as it may contain path to other Python versions which can cause crash of Python using virtualenv
+if not "%PYTHONHOME%"=="" (
+    echo Clearing PYTHONHOME, was set to %PYTHONHOME%
+    set PYTHONHOME=
+)
+
+:: Set PYTHONNOUSERSITE to avoid loading of Python packages from AppData\Roaming profile
+if "%PYTHONNOUSERSITE%"=="" (
+    echo Setting PYTHONNOUSERSITE, was not set
+    set PYTHONNOUSERSITE=True
+)
+
 :: Add Python and Git paths to PATH
 set "PATH=%IDF_PYTHON_DIR%;%IDF_GIT_DIR%;%PATH%"
 echo Using Python in %IDF_PYTHON_DIR%

tools/windows/tool_setup/idf_cmd_init.ps1

@@ -17,6 +17,24 @@ if (-not $isEspIdfRoot) {
     Write-Output "This script must be invoked from ESP-IDF directory."
 }
 
+# Clear PYTHONPATH as it may contain libraries of other Python versions
+if ($null -ne $env:PYTHONPATH) {
+    "Clearing PYTHONPATH, was set to $env:PYTHONPATH"
+    $env:PYTHONPATH=$null
+}
+
+# Clear PYTHONHOME as it may contain path to other Python versions which can cause crash of Python using virtualenv
+if ($null -ne $env:PYTHONHOME) {
+    "Clearing PYTHONHOME, was set to $env:PYTHONHOME"
+    $env:PYTHONHOME=$null
+}
+
+# Set PYTHONNOUSERSITE to avoid loading of Python packages from AppData\Roaming profile
+if ($null -eq $env:PYTHONNOUSERSITE) {
+    "Setting PYTHONNOUSERSITE, was not set"
+    $env:PYTHONNOUSERSITE="True"
+}
+
 # Strip quotes
 $IdfGitDir = $IdfGitDir.Trim("`"")
 $IdfPythonDir = $IdfPythonDir.Trim("`"")

tools/windows/tool_setup/idf_download_page.iss.inc

@@ -13,6 +13,11 @@ var
 BaseName: String;
 RepeatIndex: Integer;
 begin
+  if (IDFDirectory <> '') then begin
+    Result := IDFDirectory
+    Exit;
+  end;
+
   { Start with Desktop\esp-idf name and if it already exists,
     keep trying with Desktop\esp-idf-N for N=2 and above. }
   BaseName := ExpandConstant('{userdesktop}\esp-idf');
@@ -41,19 +46,22 @@ begin
     Result := '';
 end;
 
+procedure ExtractIDFVersionList();
+begin
+  ExtractTemporaryFile('idf_versions.txt');
+end;
+
 procedure DownloadIDFVersionsList();
 var
-  Url: String;
   VersionFile: String;
 begin
-  Url := ExpandConstant('{param:IDFVERSIONSURL|https://dl.espressif.com/dl/esp-idf/idf_versions.txt}')
   VersionFile := ExpandConstant('{tmp}\idf_versions.txt');
-  if idpDownloadFile(Url, VersionFile) then
+  if idpDownloadFile(IDFVersionUrl, VersionFile) then
   begin
-    Log('Downloaded ' + Url + ' to ' + VersionFile);
+    Log('Downloaded ' + IDFVersionUrl + ' to ' + VersionFile);
   end else begin
-    Log('Download of ' + Url + ' failed, using a fallback versions list');
-    ExtractTemporaryFile('idf_versions.txt');
+    Log('Download of ' + IDFVersionUrl + ' failed, using a fallback versions list');
+    ExtractIDFVersionList();
   end;
 end;
 
@@ -62,14 +70,18 @@ var
   Page: TInputOptionWizardPage;
   VersionFile: String;
   i: Integer;
-  SuggestedIDFDirectory: String;
 begin
   Page := TInputOptionWizardPage(Sender);
   Log('OnIDFDownloadPagePrepare');
   if Page.CheckListBox.Items.Count > 0 then
     exit;
 
-  DownloadIDFVersionsList();
+  if (IsOfflineMode) then begin
+    Log('Offline Mode: using embedded idf_versions.txt')
+    ExtractIDFVersionList();
+  end else begin
+    DownloadIDFVersionsList();
+  end;
 
   VersionFile := ExpandConstant('{tmp}\idf_versions.txt');
   if not LoadStringsFromFile(VersionFile, IDFDownloadAvailableVersions) then
@@ -87,8 +99,7 @@ begin
   end;
   Page.SelectedValueIndex := 0;
 
-  SuggestedIDFDirectory := ExpandConstant('{param:IDFDIR|' + GetSuggestedIDFDirectory() + '}');
-  ChoicePageSetInputText(Page, SuggestedIDFDirectory);
+  ChoicePageSetInputText(Page, GetSuggestedIDFDirectory());
 end;
 
 { Validation of PATH for IDF releases which does not support special characters. }
@@ -143,15 +154,15 @@ begin
 
   IDFDownloadPath := IDFPath;
 
-  { Use parameter /IDFVE=x to override selection in the box. }
-  IDFDownloadVersion := ExpandConstant('{param:IDFVERSION|}');
+  { Use parameter /IDFVERSION=x to override selection in the box. }
+  IDFDownloadVersion := IDFVersion;
   if (IDFDownloadVersion = '') then begin
     IDFDownloadVersion := IDFDownloadAvailableVersions[Page.SelectedValueIndex];
   end;
 
   { Following ZIP versions of IDF does not support installation on path with special characters. }
   { Issue: https://github.com/espressif/esp-idf/issues/5996 }
-  if ((IDFDownloadVersion = 'v4.1') or (IDFDownloadVersion = 'v4.0.2') or
+  if ((IDFDownloadVersion = 'v4.2') or (IDFDownloadVersion = 'v4.0.2') or
     (IDFDownloadVersion = 'v3.3.4')) then begin
     if (not IsDirNameValid(IDFPath)) then begin
       MsgBox('The installation of selected version of IDF is not supported on path with special characters.' + #13#10
@@ -175,10 +186,10 @@ procedure CreateIDFDownloadPage();
 begin
   IDFDownloadPage := ChoicePageCreate(
     IDFPage.ID,
-    'Download ESP-IDF', 'Please choose ESP-IDF version to download',
+    'Version of ESP-IDF', 'Please choose ESP-IDF version to install',
     'For more information about ESP-IDF versions, see' + #13#10 +
       'https://docs.espressif.com/projects/esp-idf/en/latest/versions.html',
-    'Choose a directory to download ESP-IDF to',
+    'Choose a directory to install ESP-IDF to',
     True,
     @OnIDFDownloadPagePrepare,
     @OnIDFDownloadSelectionChange,

tools/windows/tool_setup/idf_page.iss.inc

@@ -103,6 +103,16 @@ begin
   end;
 end;
 
+<event('ShouldSkipPage')>
+function ShouldSkipIDFPage(PageID: Integer): Boolean;
+begin
+  { The page does not make sense in offline mode }
+  if (PageID = IDFPage.ID) and IsOfflineMode then begin
+    Result := True;
+  end;
+end;
+
+
 <event('InitializeWizard')>
 procedure CreateIDFPage();
 begin

tools/windows/tool_setup/idf_setup.iss.inc

@@ -51,19 +51,24 @@ var
   Url, MirrorUrl: String;
 begin
   IDFZIPFileVersion := GetIDFZIPFileVersion(IDFDownloadVersion);
+
+  Log('IDFZIPFileVersion: ' + IDFZIPFileVersion);
+
   if IDFZIPFileVersion <> '' then
   begin
     Url := 'https://github.com/espressif/esp-idf/releases/download/' + IDFZIPFileVersion + '/esp-idf-' + IDFZIPFileVersion + '.zip';
     MirrorUrl := 'https://dl.espressif.com/github_assets/espressif/esp-idf/releases/download/' + IDFZIPFileVersion + '/esp-idf-' + IDFZIPFileVersion + '.zip';
-    IDFZIPFileName := ExpandConstant('{app}\releases\esp-idf-' + IDFZIPFileVersion + '.zip')
+    IDFZIPFileName := ExpandConstant('{app}\releases\esp-idf-' + IDFZIPFileVersion + '.zip');
+
     if not FileExists(IDFZIPFileName) then
     begin
+      Log('IDFZIPFileName: ' + IDFZIPFileName + ' exists');
       ForceDirectories(ExpandConstant('{app}\releases'))
       Log('Adding download: ' + Url + ', mirror: ' + MirrorUrl + ', destination: ' + IDFZIPFileName);
       idpAddFile(Url, IDFZIPFileName);
       idpAddMirror(Url, MirrorUrl);
     end else begin
-      Log(IDFZIPFileName + ' already exists')
+      Log('IDFZIPFileName: ' + IDFZIPFileName + ' does not exist');
     end;
   end;
 end;
@@ -90,6 +95,19 @@ begin
   FindFileRecursive(Path + '\.git', 'alternates', @RemoveAlternatesFile);
 end;
 
+{
+  Initialize submodules - required to call when switching branches in existing repo.
+  E.g. created by offline installer
+}
+procedure GitUpdateSubmodules(Path: String);
+var
+  CmdLine: String;
+begin
+  CmdLine := GitExecutablePath + ' -C ' + Path + ' submodule update --init --recursive';
+  Log('Updating submodules: ' + CmdLine);
+  DoCmdlineInstall('Finishing ESP-IDF installation', 'Updating submodules', CmdLine);
+end;
+
 {
   Run git config fileMode is repairing problem when git repo was zipped on Linux and extracted on Windows.
   The repo and submodules are marked as dirty which confuses users that fresh installation already contains changes.
@@ -109,10 +127,15 @@ begin
 end;
 
 { Run git reset --hard in the repo and in the submodules, to fix the newlines. }
-procedure GitRepoFixNewlines(Path: String);
+procedure GitResetHard(Path: String);
 var
   CmdLine: String;
 begin
+  if (not IsGitResetAllowed) then begin
+    Log('Git reset disabled by command line option /GITRESET=no.');
+    Exit;
+  end;
+
   CmdLine := GitExecutablePath + ' -C ' + Path + ' reset --hard';
   Log('Resetting the repository: ' + CmdLine);
   DoCmdlineInstall('Finishing ESP-IDF installation', 'Updating newlines', CmdLine);
@@ -122,6 +145,39 @@ begin
   DoCmdlineInstall('Finishing ESP-IDF installation', 'Updating newlines in submodules', CmdLine);
 end;
 
+{ Run git clean - clean leftovers after switching between tags }
+{ The repo should be created with: git config --local clean.requireForce false}
+procedure GitCleanForceDirectory(Path: String);
+var
+  CmdLine: String;
+begin
+  if (not IsGitCleanAllowed) then begin
+    Log('Git clean disabled by command line option /GITCLEAN=no.');
+    Exit;
+  end;
+
+  CmdLine := GitExecutablePath + ' -C ' + Path + ' clean --force -d';
+  Log('Resetting the repository: ' + CmdLine);
+  DoCmdlineInstall('Finishing ESP-IDF installation', 'Cleaning untracked directories', CmdLine);
+end;
+
+
+{
+  Switch to different branch. Used in offline installation.
+}
+procedure GitSwitchBranch(Path: String; BranchName: String);
+var
+  CmdLine: String;
+begin
+  CmdLine := GitExecutablePath + ' -C ' + Path + ' checkout ' + BranchName;
+  Log('Updating submodules: ' + CmdLine);
+  DoCmdlineInstall('Switching branch', 'Switching to branch', CmdLine);
+
+  GitUpdateSubmodules(Path);
+  GitResetHard(Path);
+  GitCleanForceDirectory(Path);
+end;
+
 {
   There are 3 possible ways how an ESP-IDF copy can be obtained:
   - Download the .zip archive with submodules included, extract to destination directory,
@@ -134,15 +190,25 @@ end;
     as a '--reference'. This is done for other versions (such as release branches).
 }
 
-procedure IDFDownload();
+procedure IDFOfflineInstall();
+var
+  IDFTempPath: String;
+  IDFPath: String;
+begin
+  IDFPath := IDFDownloadPath;
+
+  IDFTempPath := ExpandConstant('{app}\releases\esp-idf-bundle');
+  Log('IDFTempPath - location of bundle: ' + IDFTempPath);
+
+  GitSwitchBranch(IDFPath, IDFDownloadVersion);
+end;
+
+procedure IDFDownloadInstall();
 var
   CmdLine: String;
   IDFTempPath: String;
   IDFPath: String;
   NeedToClone: Boolean;
-  GitRepoParam: String;
-  GitResetParam: String;
-  GitRecursiveParam: String;
 begin
   IDFPath := IDFDownloadPath;
   { If there is a release archive to download, IDFZIPFileName and IDFZIPFileVersion will be set.
@@ -175,17 +241,14 @@ begin
   begin
     CmdLine := GitExecutablePath + ' clone --progress -b ' + IDFDownloadVersion;
 
-    GitRecursiveParam := ExpandConstant('{param:GITRECURSIVE|yes}');
-    if (GitRecursiveParam = 'yes') then begin
+    if (IsGitRecursive) then begin
       CmdLine := CmdLine + ' --recursive ';
     end;
 
     if IDFTempPath <> '' then
       CmdLine := CmdLine + ' --reference ' + IDFTempPath;
 
-    GitRepoParam := ExpandConstant('{param:GITREPO|https://github.com/espressif/esp-idf.git}');
-
-    CmdLine := CmdLine + ' ' + GitRepoParam +' ' + IDFPath;
+    CmdLine := CmdLine + ' ' + GitRepository +' ' + IDFPath;
     Log('Cloning IDF: ' + CmdLine);
     DoCmdlineInstall('Downloading ESP-IDF', 'Using git to clone ESP-IDF repository', CmdLine);
 
@@ -193,6 +256,7 @@ begin
       GitRepoDissociate(IDFPath);
 
   end else begin
+
     Log('Copying ' + IDFTempPath + ' to ' + IDFPath);
     if DirExists(IDFPath) then
     begin
@@ -202,21 +266,18 @@ begin
         RaiseException('Failed to copy ESP-IDF')
       end;
     end;
+
     { If cmd.exe command argument starts with a quote, the first and last quote chars in the command
       will be removed by cmd.exe.
       Keys explanation: /s+/e includes all subdirectories, /i assumes that destination is a directory,
       /h copies hidden files, /q disables file name logging (making copying faster!)
     }
+
     CmdLine := ExpandConstant('cmd.exe /c ""xcopy" /s /e /i /h /q "' + IDFTempPath + '" "' + IDFPath + '""');
     DoCmdlineInstall('Extracting ESP-IDF', 'Copying ESP-IDF into the destination directory', CmdLine);
-    GitRepoFixFileMode(IDFPath);
 
-    GitResetParam := ExpandConstant('{param:GITRESET|yes}');
-    if (GitResetParam = 'yes') then begin
-      GitRepoFixNewlines(IDFPath);
-    end else begin
-      Log('Git reset disabled by command line option /GITRESET=yes.');
-    end;
+    GitRepoFixFileMode(IDFPath);
+    GitResetHard(IDFPath);
 
     DelTree(IDFTempPath, True, True, True);
   end;
@@ -318,6 +379,22 @@ begin
   Result := 'idf' + IDFVersionString + '_py' + GetShortVersion(PythonVersion);
 end;
 
+function GetPythonVirtualEnvPath(): String;
+var
+  PythonVirtualEnvPath: String;
+begin
+  { The links should contain reference to Python vitual env }
+  PythonVirtualEnvPath := ExpandConstant('{app}\python_env\') + GetIDFPythonEnvironmentVersion() + '_env\Scripts';
+  Log('Path to Python in virtual env: ' + PythonVirtualEnvPath);
+
+  { Fallback in case of not existing environment. }
+  if (not FileExists(PythonVirtualEnvPath + '\python.exe')) then begin
+    PythonVirtualEnvPath := PythonPath;
+    Log('python.exe not found, reverting to:' + PythonPath);
+  end;
+  Result := PythonVirtualEnvPath;
+end;
+
 procedure IDFToolsSetup();
 var
   CmdLine: String;
@@ -326,8 +403,7 @@ var
   IDFToolsPyCmd: String;
   BundledIDFToolsPyPath: String;
   JSONArg: String;
-  OfflineParameter: String;
-  PythonWheelsUrlParameter: String;
+  PythonVirtualEnvPath: String;
 begin
   IDFPath := GetIDFPath('');
   IDFToolsPyPath := IDFPath + '\tools\idf_tools.py';
@@ -355,54 +431,54 @@ begin
 
   SetEnvironmentVariable('PYTHONUNBUFFERED', '1');
 
-  OfflineParameter := ExpandConstant('{param:OFFLINE|no}');
-  if (OfflineParameter = 'yes') then begin
+  if (IsOfflineMode) then begin
     SetEnvironmentVariable('PIP_NO_INDEX', 'true');
     Log('Offline installation selected. Setting environment variable PIP_NO_INDEX=1');
+    SetEnvironmentVariable('PIP_FIND_LINKS', ExpandConstant('{app}\tools\idf-python-wheels\' + PythonWheelsVersion));
   end else begin
-    PythonWheelsUrlParameter := ExpandConstant('{param:PYTHONWHEELSURL|https://dl.espressif.com/pypi}');
-    SetEnvironmentVariable('PIP_EXTRA_INDEX_URL', PythonWheelsUrlParameter);
-    Log('Adding extra Python wheels location. Setting environment variable PIP_EXTRA_INDEX_URL=' + PythonWheelsUrlParameter);
+    SetEnvironmentVariable('PIP_EXTRA_INDEX_URL', PythonWheelsUrl);
+    Log('Adding extra Python wheels location. Setting environment variable PIP_EXTRA_INDEX_URL=' + PythonWheelsUrl);
   end;
 
   Log('idf_tools.py command: ' + IDFToolsPyCmd);
   CmdLine := IDFToolsPyCmd + ' install';
+
   Log('Installing tools:' + CmdLine);
   DoCmdlineInstall('Installing ESP-IDF tools', '', CmdLine);
 
-  CmdLine := IDFToolsPyCmd + ' install-python-env';
+  CmdLine := PythonExecutablePath + ' -m virtualenv --version';
+  Log('Checking Python virtualenv support:' + CmdLine)
+  DoCmdlineInstall('Checking Python virtualenv support', '', CmdLine);
+
+  PythonVirtualEnvPath := ExpandConstant('{app}\python_env\')  + GetIDFPythonEnvironmentVersion() + '_env';
+  CmdLine := PythonExecutablePath + ' -m virtualenv "' + PythonVirtualEnvPath + '" -p ' + '"' + PythonExecutablePath + '"';
+  if (DirExists(PythonVirtualEnvPath)) then begin
+    Log('ESP-IDF Python Virtual environment exists, refreshing the environment: ' + CmdLine);
+  end else begin
+    Log('ESP-IDF Python Virtual environment does not exist, creating the environment: ' + CmdLine);
+  end;
+  DoCmdlineInstall('Creating Python environment', '', CmdLine);
 
+  CmdLine := IDFToolsPyCmd + ' install-python-env';
   Log('Installing Python environment:' + CmdLine);
   DoCmdlineInstall('Installing Python environment', '', CmdLine);
 end;
 
-
 { ------------------------------ Start menu shortcut ------------------------------ }
 
 procedure CreateIDFCommandPromptShortcut(LnkString: String);
 var
   Destination: String;
   Description: String;
-  PythonVirtualEnvPath: String;
   Command: String;
 begin
   ForceDirectories(ExpandConstant(LnkString));
   Destination := ExpandConstant(LnkString + '\{#IDFCmdExeShortcutFile}');
   Description := '{#IDFCmdExeShortcutDescription}';
 
-  { The links should contain reference to Python vitual env }
-  PythonVirtualEnvPath := ExpandConstant('{app}\python_env\') + GetIDFPythonEnvironmentVersion() + '_env\Scripts';
-  Log('Path to Python in virtual env: ' + PythonVirtualEnvPath);
-
-  { Fallback in case of not existing environment. }
-  if (not FileExists(PythonVirtualEnvPath + '\python.exe')) then begin
-    PythonVirtualEnvPath := PythonPath;
-    Log('python.exe not found, reverting to:' + PythonPath);
-  end;
-
   { If cmd.exe command argument starts with a quote, the first and last quote chars in the command
     will be removed by cmd.exe; each argument needs to be surrounded by quotes as well. }
-  Command := ExpandConstant('/k ""{app}\idf_cmd_init.bat" "') + PythonVirtualEnvPath + '" "' + GitPath + '""';
+  Command := ExpandConstant('/k ""{app}\idf_cmd_init.bat" "') + GetPythonVirtualEnvPath() + '" "' + GitPath + '""';
   Log('CreateShellLink Destination=' + Destination + ' Description=' + Description + ' Command=' + Command)
   try
     CreateShellLink(
@@ -430,7 +506,8 @@ begin
   Destination := ExpandConstant(LnkString + '\{#IDFPsShortcutFile}');
   Description := '{#IDFPsShortcutDescription}';
   GitPathWithForwardSlashes := GitPath;
-  PythonPathWithForwardSlashes := PythonPath;
+
+  PythonPathWithForwardSlashes := GetPythonVirtualEnvPath();
   StringChangeEx(GitPathWithForwardSlashes, '\', '/', True);
   StringChangeEx(PythonPathWithForwardSlashes, '\', '/', True);
   Command := ExpandConstant('-ExecutionPolicy Bypass -NoExit -File ""{app}\idf_cmd_init.ps1"" ') + '"' + GitPathWithForwardSlashes + '" "' + PythonPathWithForwardSlashes + '"'
@@ -448,38 +525,3 @@ begin
     RaiseException('Failed to create the shortcut');
   end;
 end;
-
-
-{ ------------------------------ WD exclusion registration ------------------------------ }
-
-procedure RegisterIDFToolsExecutablesInWD();
-var
-  CmdLine: String;
-begin
-  CmdLine := ExpandConstant('powershell -ExecutionPolicy ByPass -File "{app}\dist\tools_WD_excl.ps1" -AddExclPath "{app}\*.exe"');
-  Log('Registering IDF Tools executables in Windows Defender: ' + CmdLine);
-  DoCmdlineInstall('Finishing ESP-IDF installation', 'Registering IDF Tools executables in Windows Defender', CmdLine);
-end;
-
-
-
-<event('CurPageChanged')>
-procedure CheckWinDefenderAvailable(CurPageID: Integer);
-var
-  x: Integer;
-begin
-  if CurPageID = wpSelectTasks then
-  begin
-    { WD registration checkbox is identified by 'Windows Defender' substring anywhere in its caption.
-      Please, keep this in mind when making changes }
-    for x:=0 to (WizardForm.TasksList.Items.Count-1) do
-    begin
-      if Pos('Windows Defender',  WizardForm.TasksList.ItemCaption[x]) > 0 then
-      begin
-        WizardForm.TasksList.ItemEnabled[x] := isWindowsDefenderEnabled;
-        WizardForm.TasksList.Checked[x] := isWindowsDefenderEnabled;
-        break;
-      end;
-    end;
-  end;
-end;

tools/windows/tool_setup/idf_tool_setup.iss

@@ -5,26 +5,55 @@
 #include <idp.iss>
 
 #define MyAppName "ESP-IDF Tools"
-#define MyAppVersion "2.4"
+#define MyAppVersion "2.5"
 #define MyAppPublisher "Espressif Systems (Shanghai) Co. Ltd."
 #define MyAppURL "https://github.com/espressif/esp-idf"
 
-#define PythonVersion "3.9.1"
-#define PythonInstallerName "idf-python-3.9.1-embed-win64.zip"
-#define PythonInstallerDownloadURL "https://dl.espressif.com/dl/idf-python/idf-python-3.9.1-embed-win64.zip"
-
-#define GitVersion "2.28.0"
-#define GitInstallerName "Git-2.28.0-64-bit.exe"
-#define GitInstallerDownloadURL "https://github.com/git-for-windows/git/releases/download/v2.28.0.windows.1/Git-2.28.0-64-bit.exe"
+#ifndef PYTHONVERSION
+  #define PYTHONVERSION "3.8.7"
+#endif
+#define PythonInstallerName "idf-python-" + PYTHONVERSION + "-embed-win64.zip"
+#define PythonInstallerDownloadURL "https://dl.espressif.com/dl/idf-python/idf-python-" + PYTHONVERSION + "-embed-win64.zip"
+
+#ifndef GITVERSION
+  #define GITVERSION "2.30.0.2"
+#endif
+; The URL where git is stored is not equal to it's version. Minor build has prefixes with windows
+#ifndef GITVERSIONDIR
+  #define GITVERSIONDIR "v2.30.0.windows.2"
+#endif
+#define GitInstallerName "Git-" + GITVERSION + "-64-bit.exe"
+#define GitInstallerDownloadURL "https://github.com/git-for-windows/git/releases/download/" + GITVERSIONDIR + "/Git-" + GITVERSION + "-64-bit.exe"
 
 #define IDFVersionsURL "https://dl.espressif.com/dl/esp-idf/idf_versions.txt"
 
 #define IDFCmdExeShortcutDescription "Open ESP-IDF Command Prompt (cmd.exe) Environment"
-#define IDFCmdExeShortcutFile "ESP-IDF CMD.lnk"
+#define IDFCmdExeShortcutFile "ESP-IDF Command Prompt (cmd.exe).lnk"
 
 #define IDFPsShortcutDescription "Open ESP-IDF PowerShell Environment"
 #define IDFPsShortcutFile "ESP-IDF PowerShell.lnk"
 
+; List of default values
+;  Default values can be set by command-line option when startig installer
+;  or it can be stored in .INI file which can be passed to installer by /CONFIG=[PATH].
+;  Code for evaluating configuration is in the file configuration.inc.iss.
+#ifndef COMPRESSION
+  #define COMPRESSION = 'lzma';
+#endif
+; In case of large installer set it to 'no' to avoid problem delay during starting installer
+; In case of 1 GB installer it could be 30+ seconds just to start installer.
+#ifndef SOLIDCOMPRESSION
+  #define SOLIDCOMPRESSION = 'yes';
+#endif
+
+; Offline installation specific options
+#ifndef OFFLINE
+  #define OFFLINE = 'no';
+#endif
+#ifndef PYTHONWHEELSVERSION
+  #define PYTHONWHEELSVERSION = '3.8-2021-01-21'
+#endif
+
 [Setup]
 ; NOTE: The value of AppId uniquely identifies this application.
 ; Do not use the same AppId value in installers for other applications.
@@ -43,8 +72,8 @@ DirExistsWarning=no
 DefaultGroupName=ESP-IDF
 DisableProgramGroupPage=yes
 OutputBaseFilename=esp-idf-tools-setup-unsigned
-Compression=lzma
-SolidCompression=yes
+Compression={#COMPRESSION}
+SolidCompression={#SOLIDCOMPRESSION}
 ArchitecturesAllowed=x64
 ArchitecturesInstallIn64BitMode=x64
 LicenseFile=license.txt
@@ -53,6 +82,12 @@ SetupLogging=yes
 ChangesEnvironment=yes
 WizardStyle=modern
 
+; https://jrsoftware.org/ishelp/index.php?topic=setup_touchdate
+; Default values are set to 'no' which might result in files that are installed on the machine
+; in the 'future'. This creates a problem for Ninja/CMake which may end up in a neverending loop.
+; Setting this flag to 'yes' should avoid the problem.
+TimeStampsInUTC=yes
+
 [Languages]
 Name: "english"; MessagesFile: "compiler:Default.isl,Languages/idf_tool_en-US.islu"
 
@@ -60,6 +95,7 @@ Name: "english"; MessagesFile: "compiler:Default.isl,Languages/idf_tool_en-US.is
 Name: "{app}\dist"
 
 [Files]
+Source: "configuration.ini"; Flags: dontcopy noencryption
 Source: "cmdlinerunner\build\cmdlinerunner.dll"; Flags: dontcopy
 Source: "unzip\7za.exe"; Flags: dontcopy
 Source: "idf_versions.txt"; Flags: dontcopy
@@ -68,7 +104,13 @@ Source: "..\..\idf_tools.py"; DestDir: "{app}"; DestName: "idf_tools_fallback.py
 Source: "tools_fallback.json"; DestDir: "{app}"; DestName: "tools_fallback.json"
 Source: "idf_cmd_init.bat"; DestDir: "{app}"
 Source: "idf_cmd_init.ps1"; DestDir: "{app}"
-Source: "dist\*"; DestDir: "{app}\dist"
+Source: "dist\*"; DestDir: "{app}\dist"; Flags: skipifsourcedoesntexist;
+
+; esp-idf-bundle - bundle only in case it exists, it's used only in offline installer
+Source: "releases\esp-idf-bundle\*"; DestDir: "{code:GetIDFPath}"; Flags: recursesubdirs skipifsourcedoesntexist;
+
+Source: "tools\idf-python\*"; DestDir: "{app}\tools\idf-python\"; Flags: recursesubdirs;
+Source: "tools\idf-python-wheels\*"; DestDir: "{app}\tools\idf-python-wheels\"; Flags: recursesubdirs skipifsourcedoesntexist;
 ; Helper Python files for sanity check of Python environment - used by system_check_page
 Source: "system_check\system_check_download.py"; Flags: dontcopy
 Source: "system_check\system_check_subprocess.py"; Flags: dontcopy
@@ -88,19 +130,22 @@ Type: files; Name: "{autodesktop}\{#IDFCmdExeShortcutFile}"
 Type: files; Name: "{autodesktop}\{#IDFPsShortcutFile}"
 
 [Tasks]
-Name: CreateLnkStartCmd; Description: "Create Start Menu shortcut for the ESP-IDF Tools Command Prompt Environment";
-Name: CreateLnkStartPs; Description: "Create Start Menu shortcut for the ESP-IDF Tools Powershell Environment";
-Name: CreateLnkDeskCmd; Description: "Create Desktop shortcut for the ESP-IDF Tools Command Prompt Environment";
-Name: CreateLnkDeskPs; Description: "Create Desktop shortcut for the ESP-IDF Tools Powershell Environment";
-; WD registration checkbox is identified by 'Windows Defender' substring anywhere in its caption, not by the position index in WizardForm.TasksList.Items
-; Please, keep this in mind when making changes to the item's description - WD checkbox is to be disabled on systems without the Windows Defender installed
-Name: wdexcl; Description: "Register the ESP-IDF Tools executables as Windows Defender exclusions (improves compilation speed, requires elevation)";
-Name: idf_tools_use_mirror; Description: "Use Espressif download server instead of downloading tool packages from Github"; Flags: unchecked;
+Name: CreateLinkStartPowerShell; GroupDescription: "{cm:CreateShortcutPowerShell}"; Description: "{cm:CreateShortcutStartMenu}";
+Name: CreateLinkDeskPowerShell; GroupDescription: "{cm:CreateShortcutPowerShell}"; Description: "{cm:CreateShortcutDesktop}";
+
+Name: CreateLinkStartCmd; GroupDescription: "{cm:CreateShortcutCMD}"; Description: "{cm:CreateShortcutStartMenu}";
+Name: CreateLinkDeskCmd; GroupDescription: "{cm:CreateShortcutCMD}"; Description: "{cm:CreateShortcutDesktop}";
+
+; Optimization for Online mode
+Name: UseMirror;  GroupDescription:"{cm:OptimizationTitle}"; Description: "{cm:OptimizationDownloadMirror}"; Flags: unchecked; Check: IsOnlineMode
 
 [Run]
 Filename: "{app}\dist\{#GitInstallerName}"; Parameters: "/silent /tasks="""" /norestart"; Description: "Installing Git"; Check: GitInstallRequired
 Filename: "{group}\{#IDFPsShortcutFile}"; Flags: postinstall shellexec unchecked; Description: "Run ESP-IDF PowerShell Environment"; Check: IsPowerShellInstalled
 Filename: "{group}\{#IDFCmdExeShortcutFile}"; Flags: postinstall shellexec unchecked; Description: "Run ESP-IDF Command Prompt Environment"; Check: IsCmdInstalled
+; WD registration checkbox is identified by 'Windows Defender' substring anywhere in its caption, not by the position index in WizardForm.TasksList.Items
+; Please, keep this in mind when making changes to the item's description - WD checkbox is to be disabled on systems without the Windows Defender installed
+Filename: "powershell"; Parameters: "-ExecutionPolicy ByPass -File ""{app}\dist\tools_WD_excl.ps1"" -AddExclPath ""{app}\*.exe"""; Flags: postinstall shellexec runhidden; Description: "{cm:OptimizationWindowsDefender}"; Check: GetIsWindowsDefenderEnabled
 
 
 [UninstallRun]
@@ -113,6 +158,7 @@ Root: HKCU; Subkey: "Environment"; ValueType: string; ValueName: "IDF_TOOLS_PATH
     ValueData: "{app}"; Flags: preservestringtype createvalueifdoesntexist uninsdeletevalue deletevalue;
 
 [Code]
+#include "configuration.iss.inc"
 #include "utils.iss.inc"
 #include "choice_page.iss.inc"
 #include "cmdline_page.iss.inc"

tools/windows/tool_setup/idf_versions_offline.txt

@@ -0,0 +1,2 @@
+v4.2
+v4.1.1

tools/windows/tool_setup/main.iss.inc

@@ -40,11 +40,19 @@ var
   DestPath: String;
 begin
   Result := True;
-  if CurPageID <> wpReady then
-    exit;
+  if CurPageID <> wpReady then begin
+    Exit;
+  end;
 
   ForceDirectories(ExpandConstant('{app}\dist'));
 
+  if (IsOfflineMode) then begin
+    ForceDirectories(ExpandConstant('{app}\releases'));
+    IDFZIPFileVersion := IDFDownloadVersion;
+    IDFZIPFileName := ExpandConstant('{app}\releases\esp-idf-bundle.zip');
+    Exit;
+  end;
+
   if not GitUseExisting then
   begin
     DestPath := ExpandConstant('{app}\dist\{#GitInstallerName}');
@@ -68,6 +76,8 @@ procedure AddPythonGitToPath();
 var
   EnvPath: String;
   PythonLibPath: String;
+  EnvPythonHome: String;
+  PythonNoUserSite: String;
 begin
   EnvPath := GetEnv('PATH');
 
@@ -83,10 +93,28 @@ begin
     new value to be visible to this process. }
   SetEnvironmentVariable('IDF_TOOLS_PATH', ExpandConstant('{app}'))
 
+  { Set PYTHONNOUSERSITE variable True to avoid loading packages from AppData\Roaming. }
+  { https://doc.pypy.org/en/latest/man/pypy.1.html#environment }
+  { If set to a non-empty value, equivalent to the -s option. Don’t add the user site directory to sys.path. }
+  if (IsPythonNoUserSite) then begin
+    PythonNoUserSite := 'True';
+  end else begin
+    PythonNoUserSite := '';
+  end;
+  Log('PYTHONNOUSERSITE=' + PythonNoUserSite);
+  SetEnvironmentVariable('PYTHONNOUSERSITE', PythonNoUserSite);
+
   { Log and clear PYTHONPATH variable, as it might point to libraries of another Python version}
   PythonLibPath := GetEnv('PYTHONPATH')
   Log('PYTHONPATH=' + PythonLibPath)
   SetEnvironmentVariable('PYTHONPATH', '')
+
+  { Log and clear PYTHONHOME, the existence of PYTHONHOME might cause trouble when creating virtualenv. }
+  { The error message when creating virtualenv: }
+  {   Fatal Python error: init_fs_encoding: failed to get the Python codec of the filesystem encoding. }
+  EnvPythonHome := GetEnv('PYTHONHOME')
+  Log('PYTHONHOME=' + EnvPythonHome)
+  SetEnvironmentVariable('PYTHONHOME', '')
 end;
 
 procedure InstallEmbeddedPython();
@@ -102,10 +130,11 @@ begin
   UpdatePythonVariables(EmbeddedPythonPath);
   Log('Checking existence of Embedded Python: ' + EmbeddedPythonPath);
   if (FileExists(EmbeddedPythonPath)) then begin
+    Log('Embedded Python found.');
     Exit;
   end;
 
-  CmdLine := ExpandConstant('{tmp}\7za.exe x -o{app}\tools\python\' + PythonVersion + '\ -r -aoa "{app}\dist\idf-python-' + PythonVersion + '-embed-win64.zip"');
+  CmdLine := ExpandConstant('{tmp}\7za.exe x -o{app}\tools\idf-python\' + PythonVersion + '\ -r -aoa "{app}\dist\idf-python-' + PythonVersion + '-embed-win64.zip"');
   DoCmdlineInstall('Extracting Python Interpreter', 'Using Embedded Python', CmdLine);
 end;
 
@@ -117,16 +146,22 @@ begin
   if CurStep <> ssPostInstall then
     exit;
 
-  ExtractTemporaryFile('7za.exe')
+  ExtractTemporaryFile('7za.exe');
+
   InstallEmbeddedPython();
 
   try
     AddPythonGitToPath();
 
-    if not IDFUseExisting then
-      IDFDownload();
+    if not IDFUseExisting then begin
+      if (IsOfflineMode) then begin
+        IDFOfflineInstall();
+      end else begin
+        IDFDownloadInstall();
+      end;
+    end;
 
-    if WizardIsTaskSelected('idf_tools_use_mirror') then
+    if WizardIsTaskSelected('UseMirror') then
     begin
       SetEnvironmentVariable('IDF_GITHUB_ASSETS', 'dl.espressif.com/github_assets')
     end;
@@ -134,31 +169,26 @@ begin
     IDFToolsSetup();
 
 
-  if WizardIsTaskSelected('CreateLnkStartCmd') then
+  if WizardIsTaskSelected('CreateLinkStartCmd') then
   begin
     CreateIDFCommandPromptShortcut('{autostartmenu}\Programs\ESP-IDF');
   end;
 
-  if WizardIsTaskSelected('CreateLnkStartPs') then
+  if WizardIsTaskSelected('CreateLinkStartPowerShell') then
   begin
     CreateIDFPowershellShortcut('{autostartmenu}\Programs\ESP-IDF' );
   end;
 
-  if WizardIsTaskSelected('CreateLnkDeskCmd') then
+  if WizardIsTaskSelected('CreateLinkDeskCmd') then
   begin
     CreateIDFCommandPromptShortcut('{autodesktop}');
   end;
 
-  if WizardIsTaskSelected('CreateLnkDeskPs') then
+  if WizardIsTaskSelected('CreateLinkDeskPowerShell') then
   begin
     CreateIDFPowershellShortcut('{autodesktop}');
   end;
 
-  if WizardIsTaskSelected('wdexcl') then
-  begin
-    RegisterIDFToolsExecutablesInWD();
-  end;
-
   except
     SetupAborted := True;
     if MsgBox('Installation log has been created, it may contain more information about the problem.' + #13#10
@@ -184,10 +214,10 @@ end;
 
 function IsPowerShellInstalled(): Boolean;
 begin
-  Result := ((not SetupAborted) and (WizardIsTaskSelected('CreateLnkDeskPs') or WizardIsTaskSelected('CreateLnkStartPs')));
+  Result := ((not SetupAborted) and (WizardIsTaskSelected('CreateLinkDeskPowerShell') or WizardIsTaskSelected('CreateLinkStartPowerShell')));
 end;
 
 function IsCmdInstalled(): Boolean;
 begin
-  Result := ((not SetupAborted) and (WizardIsTaskSelected('CreateLnkDeskCmd') or WizardIsTaskSelected('CreateLnkStartCmd')));
+  Result := ((not SetupAborted) and (WizardIsTaskSelected('CreateLinkDeskCmd') or WizardIsTaskSelected('CreateLinkStartCmd')));
 end;

tools/windows/tool_setup/python_page.iss.inc

@@ -128,13 +128,10 @@ end;
 
 <event('ShouldSkipPage')>
 function ShouldSkipPythonPage(PageID: Integer): Boolean;
-var
-  UseEmbeddedPythonParam: String;
 begin
   if (PageID = PythonPage.ID) then begin
     { Skip in case of embedded Python. }
-    UseEmbeddedPythonParam := ExpandConstant('{param:USEEMBEDDEDPYTHON|yes}');
-    if (UseEmbeddedPythonParam = 'yes') then begin
+    if (UseEmbeddedPython) then begin
       ApplyPythonConfigurationByIndex(0);
       Result := True;
     end;

tools/windows/tool_setup/sign_installer.sh

@@ -6,6 +6,8 @@
 set -e
 set -u
 
+INSTALLER_TYPE="${1-online}"
+
 if [[ -z "${KEYFILE:-}" || -z "${CERTCHAIN:-}" ]]; then
     echo "To sign the installer, set the following environment variables:"
     echo "  KEYFILE - private key file"
@@ -19,8 +21,8 @@ umask 770  # for the process substitution FIFO
 VERSION=`grep "#define MyAppVersion " idf_tool_setup.iss | cut -d ' ' -f3 | tr -d '"'`
 echo "Installer version ${VERSION}"
 
-IN_FILE="Output/esp-idf-tools-setup-unsigned.exe"
-OUT_FILE="Output/esp-idf-tools-setup-${VERSION}.exe"
+IN_FILE="Output/esp-idf-tools-setup-${INSTALLER_TYPE}-unsigned.exe"
+OUT_FILE="Output/esp-idf-tools-setup-${INSTALLER_TYPE}-${VERSION}.exe"
 
 if [[ -n "${KEYPASSWORD:-}" ]]; then
     PASSARG="-readpass <(echo \"$KEYPASSWORD\")"

tools/windows/tool_setup/summary.iss.inc

@@ -29,7 +29,7 @@ begin
     Result := Result + 'Using existing ESP-IDF copy: ' + NewLine
               + Space + IDFExistingPath + NewLine + NewLine;
   end else begin
-    Result := Result + 'Will download ESP-IDF ' + IDFDownloadVersion + ' into:' + NewLine
+    Result := Result + 'Will install ESP-IDF ' + IDFDownloadVersion + ' into:' + NewLine
               + Space + IDFDownloadPath + NewLine + NewLine;
   end;
 

tools/windows/tool_setup/system_check_page.iss.inc

@@ -472,10 +472,14 @@ begin
   end;
 end;
 
+{ Wrapper function for Run task. Run tasks requires calling function. }
+function GetIsWindowsDefenderEnabled(): Boolean;
+begin
+  Result := IsWindowsDefenderEnabled;
+end;
+
 { Execute system check }
 procedure ExecuteSystemCheck();
-var
-  UseEmbeddedPythonParam: String;
 begin
   { Execute system check only once. Avoid execution in case of back button. }
   if (SystemCheckState <> SYSTEM_CHECK_STATE_INIT) then begin
@@ -486,11 +490,12 @@ begin
   SystemLogTitle(CustomMessage('SystemCheckStart'));
   StopSystemCheckButton.Enabled := True;
 
-  VerifyRootCertificates();
+  if (not IsOfflineMode) then begin
+    VerifyRootCertificates();
+  end;
 
   { Search for the installed Python version only on explicit user request. }
-  UseEmbeddedPythonParam := ExpandConstant('{param:USEEMBEDDEDPYTHON|yes}');
-  if (UseEmbeddedPythonParam <> 'yes') then begin
+  if (not UseEmbeddedPython) then begin
     FindInstalledPythonVersions();
   end;
 
@@ -525,7 +530,6 @@ end;
 
 { Invoke scan of system environment. }
 procedure OnSystemCheckActivate(Sender: TWizardPage);
-var SystemCheckParam:String;
 begin
   { Display special controls. For some reason the first call of the page does not invoke SystemCheckOnCurPageChanged. }
   FullLogButton.Visible := True;
@@ -533,8 +537,7 @@ begin
   StopSystemCheckButton.Visible := True;
   SystemCheckViewer.Visible := True;
 
-  SystemCheckParam := ExpandConstant('{param:SKIPSYSTEMCHECK|no}');
-  if (SystemCheckParam = 'yes') then begin
+  if (SkipSystemCheck) then begin
     SystemCheckState := SYSTEM_CHECK_STATE_STOPPED;
     SystemLog('System Check disabled by command line option /SKIPSYSTEMCHECK.');
   end;
@@ -598,7 +601,7 @@ begin
   InstalledPythonVersions := TStringList.Create();
   InstalledPythonDisplayNames := TStringList.Create();
   InstalledPythonExecutables := TStringList.Create();
-  PythonVersionAdd('{#PythonVersion}', 'Use Python {#PythonVersion} Embedded (Recommended)', 'tools\python\{#PythonVersion}\python.exe');
+  PythonVersionAdd('{#PythonVersion}', 'Use Python {#PythonVersion} Embedded (Recommended)', 'tools\idf-python\{#PythonVersion}\python.exe');
 
   { Create Spinner animation. }
   Spinner := TStringList.Create();