Merge branch 'feature/idf_tools_installer_v2.1_v4.0' into 'release/v4.0'

IDF tools installer: extra fixes, bump version to v2.1 (backport 4.0)

See merge request espressif/esp-idf!6401

docs/en/get-started/windows-setup.rst

@@ -24,7 +24,7 @@ ESP-IDF Tools Installer
 
 The easiest way to install ESP-IDF's prerequisites is to download the ESP-IDF Tools installer from this URL:
 
-https://dl.espressif.com/dl/esp-idf-tools-setup-2.0.exe
+https://dl.espressif.com/dl/esp-idf-tools-setup-2.1.exe
 
 The installer includes the cross-compilers, OpenOCD, cmake_ and Ninja_ build tool, and a configuration tool called mconf-idf_. The installer can also download and run installers for Python_ 3.7 and `Git For Windows`_ if they are not already installed on the computer.
 

tools/ci/config/build.yml

@@ -345,19 +345,5 @@ build_installer:
     - build_cmdlinerunner
   before_script: []
   script:
-    - mkdir idf_tools_tmp
-    - export IDF_TOOLS_PATH=$PWD/idf_tools_tmp
-    - tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all
-    - tools/idf_tools.py --tools-json tools/windows/tool_setup/tools_fallback.json --non-interactive download --platform Windows-x86_64 all
-    - mkdir tools/windows/tool_setup/dist
-    - mv idf_tools_tmp/dist/* tools/windows/tool_setup/dist/
-
     - cd tools/windows/tool_setup/
-    - mkdir unzip
-    - cd unzip
-    - wget --no-verbose https://www.7-zip.org/a/7z1900-extra.7z
-    - 7zr e -y 7z1900-extra.7z
-    - cd ..
-
-    - wget --no-verbose https://dl.espressif.com/dl/esp-idf/idf_versions.txt
-    - iscc idf_tool_setup.iss
+    - ./build_installer.sh

tools/ci/executable-list.txt

@@ -81,3 +81,4 @@ tools/test_idf_tools/test_idf_tools.py
 tools/unit-test-app/unit_test.py
 tools/windows/eclipse_make.sh
 tools/windows/tool_setup/build_installer.sh
+tools/windows/tool_setup/sign_installer.sh

tools/idf_tools.py

@@ -61,6 +61,12 @@ try:
 except ImportError:
     from urllib import urlretrieve
 
+try:
+    from exceptions import WindowsError
+except ImportError:
+    class WindowsError(OSError):
+        pass
+
 
 TOOLS_FILE = 'tools/tools.json'
 TOOLS_SCHEMA_FILE = 'tools/tools_schema.json'
@@ -254,13 +260,34 @@ def unpack(filename, destination):
     archive_obj.extractall(destination)
 
 
+# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError.
+# This is confirmed to be a workaround:
+# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118
+# https://github.com/espressif/esp-idf/issues/4063#issuecomment-531490140
+# https://stackoverflow.com/a/43046729
+def rename_with_retry(path_from, path_to):
+    if sys.platform.startswith('win'):
+        retry_count = 100
+    else:
+        retry_count = 1
+
+    for retry in range(retry_count):
+        try:
+            os.rename(path_from, path_to)
+            return
+        except (OSError, WindowsError):       # WindowsError until Python 3.3, then OSError
+            if retry == retry_count - 1:
+                raise
+            warn('Rename {} to {} failed, retrying...'.format(path_from, path_to))
+
+
 def strip_container_dirs(path, levels):
     assert levels > 0
     # move the original directory out of the way (add a .tmp suffix)
     tmp_path = path + '.tmp'
     if os.path.exists(tmp_path):
         shutil.rmtree(tmp_path)
-    os.rename(path, tmp_path)
+    rename_with_retry(path, tmp_path)
     os.mkdir(path)
     base_path = tmp_path
     # walk given number of levels down
@@ -276,7 +303,7 @@ def strip_container_dirs(path, levels):
     for name in contents:
         move_from = os.path.join(base_path, name)
         move_to = os.path.join(path, name)
-        os.rename(move_from, move_to)
+        rename_with_retry(move_from, move_to)
     shutil.rmtree(tmp_path)
 
 
@@ -544,7 +571,7 @@ class IDFTool(object):
             if not self.check_download_file(download_obj, local_temp_path):
                 warn('Failed to download file {}'.format(local_temp_path))
                 continue
-            os.rename(local_temp_path, local_path)
+            rename_with_retry(local_temp_path, local_path)
             downloaded = True
             break
         if not downloaded:
@@ -810,7 +837,7 @@ def get_python_env_path():
         with open(version_file_path, "r") as version_file:
             idf_version_str = version_file.read()
     else:
-        idf_version_str = subprocess.check_output(['git', '-C', global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode()
+        idf_version_str = subprocess.check_output(['git', '--work-tree=' + global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode()
     match = re.match(r'^v([0-9]+\.[0-9]+).*', idf_version_str)
     idf_version = match.group(1)
 

tools/windows/tool_setup/README.md

@@ -14,7 +14,17 @@ Some functionality of the installer depends on additional programs:
 
 * [cmdlinerunner](cmdlinerunner/cmdlinerunner.c) — a helper DLL used to run external command line programs from the installer, capture live console output, and get the exit code.
 
-## Steps required to build the installer
+## Building the installer 
+
+### In Docker
+
+This uses `wine-innosetup` Docker image and `build_installer.sh` script. This is how the installer is built in CI.
+
+```
+docker run --rm -v $IDF_PATH:/idf -w /idf/tools/windows/tool_setup -it $CI_DOCKER_REGISTRY/wine-innosetup:1 /bin/bash build_installer.sh
+```
+
+### Manually, step by step
 
 * Build cmdlinerunner DLL.
   - On Linux/Mac, install mingw-w64 toolchain (`i686-w64-mingw32-gcc`). Then build the DLL using CMake:
@@ -35,5 +45,10 @@ Some functionality of the installer depends on additional programs:
 
 * Build the installer using Inno Setup Compiler: `ISCC.exe idf_tools_setup.iss`.
 
-* Obtain the signing keys, then sign `Output/esp-idf-tools-setup-unsigned.exe`.
+## Signing the installer
+
+* Obtain the signing key (e.g `key.pem`) and the certificate chain (e.g. `certchain.pem`). Set the environment variables to point to these files:
+  - `export KEYFILE=key.pem`
+  - `export CERTCHAIN=certchain.pem`
 
+* Run `sign_installer.sh` script. This will ask for the `key.pem` password, and produce the signed installer in the Output directory. If you plan to run the script multiple times, you may also set `KEYPASSWORD` environment variable to the `key.pem` password, to avoid the prompt.

tools/windows/tool_setup/build_installer.sh

@@ -1,67 +1,43 @@
 #!/bin/bash
 #
-# Setup script to build Windows tool installer with Inno Setup
+# Script to build the IDF Tools installer for Windows with Inno Setup.
+# This script should be executed inside wine-innosetup docker image.
 #
-# Designed to be run on Linux (with wine) but could be adapted to run under MSYS2 on Windows
-# pretty easily...
-#
-# - Downloads (if necessary) all tools to install to the "dl/" directory
-# - Deletes the "input" directory contains and copies everything under there
+# - Downloads all tools to install into the "dist/" directory
+# - Downloads 7z and idf_versions.txt
 # - Runs ISCC under wine to compile the installer itself
+
 set -e
+set -u
 
-if [ -z "${KEYPASSWORD}" ]; then
-    echo "KEYPASSWORD should be set"
+iscc_path=$(which iscc)
+if [[ -z "$iscc_path" ]]; then
+    echo "Inno setup compiler (iscc) not found. Are you running wine-innosetup Docker image?"
     exit 1
 fi
 
-if [ "$1" != "--no-download" ]; then
-
-    mkdir -p dl input
-
-    cd `dirname $0`
-    pushd dl
-    wget --continue "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
-    wget --continue "https://github.com/espressif/binutils-esp32ulp/releases/download/v2.28.51-esp32ulp-20180809/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip"
-    wget --continue "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20180920/openocd-esp32-win32-0.10.0-esp32-20180920.zip"
-    wget --continue "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20180525/mconf-v4.6.0.0-idf-20180525-win32.zip"
-    wget --continue "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip"
-    popd
-
-    rm -rf input/*
-    pushd input
-    unzip ../dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip
-    unzip ../dl/mconf-v4.6.0.0-idf-20180525-win32.zip
-    unzip ../dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip
-    unzip ../dl/openocd-esp32-win32-0.10.0-esp32-20180920.zip
-    unzip ../dl/ninja-win.zip
-    popd
+if [[ -z "${IDF_PATH:-}" ]]; then
+    export IDF_PATH=$(cd ../../../; pwd)
+    echo "Assuming IDF_PATH: ${IDF_PATH}"
 fi
 
-wine "C:\Program Files\Inno Setup 5\ISCC.exe" "`winepath -w ./idf_tool_setup.iss`"
-
-# sign the installer with osslsigncode, parsing the version number out of the
-# installer config
-
-VERSION=`grep "^AppVersion=" idf_tool_setup.iss | cut -d'=' -f2`
-
-echo "Signing installer..."
-
-# Note: The cert chain passed to -certs needs to contain the intermediate
-# cert(s) as well, appended after the code signing cert, or Windows may see
-# it as "Unknown Publisher"
-#
-# See https://stackoverflow.com/a/52637050 for full details
-#
-umask 770  # for the process substitution FIFO
-
-osslsigncode -certs ./keys/certchain.pem -key ./keys/key.pem \
-             -readpass <(echo "$KEYPASSWORD") \
-             -in Output/esp-idf-tools-setup-unsigned.exe \
-             -out Output/esp-idf-tools-setup-${VERSION}.exe \
-             -h sha256 \
-             -n "Espressif Systems (Shanghai) Pte. Ltd." \
-             -i "https://www.espressif.com/" \
-             -ts http://timestamp.digicert.com
-
-chmod 644 Output/esp-idf-tools-setup-${VERSION}.exe  # make up for the umask
+echo "Downloading IDF Tools..."
+mkdir -p idf_tools_tmp
+export IDF_TOOLS_PATH=$PWD/idf_tools_tmp
+$IDF_PATH/tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all
+$IDF_PATH/tools/idf_tools.py --tools-json tools_fallback.json --non-interactive download --platform Windows-x86_64 all
+mkdir -p dist
+cp idf_tools_tmp/dist/* dist/
+
+echo "Downloading 7z..."
+mkdir -p unzip
+pushd unzip
+wget --no-verbose -O 7z1900-extra.7z https://www.7-zip.org/a/7z1900-extra.7z
+7zr e -y 7z1900-extra.7z
+popd
+
+echo "Downloading idf_versions.txt..."
+wget --no-verbose -O idf_versions.txt https://dl.espressif.com/dl/esp-idf/idf_versions.txt
+
+echo "Running ISCC..."
+iscc idf_tool_setup.iss

tools/windows/tool_setup/idf_tool_setup.iss

@@ -5,7 +5,7 @@
 #include <idp.iss>
 
 #define MyAppName "ESP-IDF Tools"
-#define MyAppVersion "2.0"
+#define MyAppVersion "2.1"
 #define MyAppPublisher "Espressif Systems (Shanghai) Co. Ltd."
 #define MyAppURL "https://github.com/espressif/esp-idf"
 

tools/windows/tool_setup/sign_installer.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Script to sign the IDF Tools installer for Windows, built with build_installer.sh.
+#
+
+set -e
+set -u
+
+if [[ -z "${KEYFILE:-}" || -z "${CERTCHAIN:-}" ]]; then
+    echo "To sign the installer, set the following environment variables:"
+    echo "  KEYFILE - private key file"
+    echo "  KEYPASSWORD - password for the private key file (optional, will prompt for password if not set)"
+    echo "  CERTCHAIN - certificate chain file"
+    exit 1
+fi
+
+umask 770  # for the process substitution FIFO
+
+VERSION=`grep "#define MyAppVersion " idf_tool_setup.iss | cut -d ' ' -f3 | tr -d '"'`
+echo "Installer version ${VERSION}"
+
+IN_FILE="Output/esp-idf-tools-setup-unsigned.exe"
+OUT_FILE="Output/esp-idf-tools-setup-${VERSION}.exe"
+
+if [[ -n "${KEYPASSWORD:-}" ]]; then
+    PASSARG="-readpass <(echo \"$KEYPASSWORD\")"
+else
+    PASSARG="-askpass"
+fi
+
+echo "Signing the installer (${IN_FILE})..."
+# Note: The cert chain passed to -certs needs to contain the intermediate
+# cert(s) as well, appended after the code signing cert, or Windows may see
+# it as "Unknown Publisher"
+#
+# See https://stackoverflow.com/a/52637050 for full details
+#
+osslsigncode -certs ${CERTCHAIN} -key ${KEYFILE} \
+             ${PASSARG} \
+             -in ${IN_FILE} \
+             -out ${OUT_FILE} \
+             -h sha256 \
+             -n "Espressif Systems (Shanghai) Co., Ltd." \
+             -i "https://www.espressif.com/" \
+             -ts http://timestamp.digicert.com
+
+chmod 644 ${OUT_FILE}  # make up for the umask
+
+echo "Generated ${OUT_FILE}"