Merge branch 'feature/ci_mirror_for_submodules' into 'master'

Use custom fetch strategy. Use local mirrors for submodules



See merge request !890

.gitlab-ci.yml

@@ -18,6 +18,11 @@ variables:
   GET_SOURCES_ATTEMPTS: "10"
   ARTIFACT_DOWNLOAD_ATTEMPTS: "10"
 
+  # We use get_sources.sh script to fetch the submodules and/or re-fetch the repo
+  # if it was corrupted (if submodule update fails this can happen)
+  GIT_STRATEGY: fetch
+  GIT_SUBMODULE_STRATEGY: none
+
 # IDF environment
 
   IDF_PATH: "$CI_PROJECT_DIR"
@@ -34,10 +39,27 @@ before_script:
   # Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
   #
   # (the same regular expressions are used to set these are used in 'only:' sections below
-  - source make/configure_ci_environment.sh
+  - source tools/ci/configure_ci_environment.sh
+
+  # fetch the submodules (& if necessary re-fetch repo) from gitlab
+  - time ./tools/ci/get-full-sources.sh
+
+.do_nothing_before:
+  before_script: &do_nothing_before
+    - echo "Not setting up GitLab key, not fetching submodules"
+    - source tools/ci/configure_ci_environment.sh
 
-  # fetch all submodules
-  - git submodule update --init --recursive
+.add_gitlab_key_before:
+  before_script: &add_gitlab_key_before
+    - echo "Not fetching submodules"
+    - source tools/ci/configure_ci_environment.sh
+    # add gitlab ssh key
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
 
 build_template_app:
   stage: build
@@ -45,7 +67,6 @@ build_template_app:
   tags:
     - build
   variables:
-    GIT_STRATEGY: clone
     BATCH_BUILD: "1"
     IDF_CI_BUILD: "1"
   script:
@@ -73,7 +94,6 @@ build_template_app:
   tags:
     - build
   variables:
-    GIT_STRATEGY: clone
     BATCH_BUILD: "1"
     V: "0"
 
@@ -128,14 +148,13 @@ build_esp_idf_tests:
     expire_in: 1 week
   variables:
     IDF_CI_BUILD: "1"
-    GIT_STRATEGY: fetch
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
     # mechanism work, but this is the next best thing
     - mkdir build_examples
     - cd build_examples
     # build some of examples
-    - ${IDF_PATH}/make/build_examples.sh "${CI_JOB_NAME}"
+    - ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}"
 
 build_examples_00:
   <<: *build_examples_template
@@ -209,8 +228,8 @@ test_build_system:
     - build_test
   dependencies: []
   script:
-    - ./make/test_configure_ci_environment.sh
-    - ./make/test_build_system.sh
+    - ./tools/ci/test_configure_ci_environment.sh
+    - ./tools/ci/test_build_system.sh
 
 test_report:
   stage: test_report
@@ -266,8 +285,6 @@ test_report:
     - test "${TEST_RESULT}" = "Pass" || exit 1
 
 push_master_to_github:
-  before_script:
-    - echo "Not setting up GitLab key, not fetching submodules"
   stage: deploy
   image: $CI_DOCKER_REGISTRY/esp32-ci-env
   tags:
@@ -279,8 +296,8 @@ push_master_to_github:
   when: on_success
   dependencies: []
   variables:
-    GIT_STRATEGY: clone
     GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master
+  before_script: *do_nothing_before
   script:
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
@@ -297,8 +314,6 @@ push_master_to_github:
 
 
 deploy_docs:
-  before_script:
-    - echo "Not setting up GitLab key, not fetching submodules"
   stage: deploy
   image: $CI_DOCKER_REGISTRY/esp32-ci-env
   tags:
@@ -310,6 +325,7 @@ deploy_docs:
    - triggers
   dependencies:
     - build_docs
+  before_script: *do_nothing_before
   script:
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
@@ -354,12 +370,12 @@ check_commit_msg:
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
   dependencies: []
-  before_script:
-    - echo "skip update submodule"
+  before_script: *do_nothing_before
   script:
-    - git checkout ${CI_COMMIT_REF_NAME}
+    - git status
+    - git log -n10 --oneline
     # commit start with "WIP: " need to be squashed before merge
-    - 'git log --pretty=%s master..${CI_COMMIT_REF_NAME} | grep "^WIP: " || exit 0 && exit 1'
+    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
 
 check_submodule_sync:
   stage: deploy
@@ -371,8 +387,9 @@ check_submodule_sync:
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
   dependencies: []
-  before_script:
-    - echo "do not use gitlab submodule repository"
+  variables:
+    GIT_STRATEGY: clone
+  before_script: *do_nothing_before
   script:
     # check if all submodules are correctly synced to public repostory
     - git submodule update --init --recursive
@@ -389,6 +406,7 @@ assign_test:
       - components/idf_test/*/CIConfigs
       - components/idf_test/*/TC.sqlite
     expire_in: 1 mos
+  before_script: *add_gitlab_key_before
   script:
     # first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
     - mkdir -p test_bins/ESP32_IDF/UT
@@ -404,8 +422,6 @@ assign_test:
     - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins
 
 .test_template: &test_template
-  before_script:
-    - echo "Skip cloning submodule here"
   stage: test
   when: on_success
   only:
@@ -422,25 +438,15 @@ assign_test:
       - $LOG_PATH
     expire_in: 6 mos
   variables:
-    # set git strategy to fetch so we can get esptool without update submodule
-    GIT_STRATEGY: fetch
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
     MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
     CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
+  before_script: *add_gitlab_key_before
   script:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
-    # remove artifacts from the 'unit_test' stage
-    - rm -rf "$LOG_PATH"
-    # add gitlab ssh key
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
     # clone local test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
     # clone test bench
@@ -456,7 +462,6 @@ assign_test:
   allow_failure: false
   stage: unit_test
   variables:
-    GIT_STRATEGY: fetch
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test"
@@ -475,15 +480,6 @@ nvs_compatible_test:
     - ESP32_IDF
     - NVS_Compatible
   script:
-    # remove artifacts from the 'unit_test' stage
-    - rm -rf "$LOG_PATH"
-    # add gitlab ssh key
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
     # clone local test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
     # clone test bench

make/configure_ci_environment.sh → tools/ci/configure_ci_environment.sh

@@ -1,16 +1,15 @@
-#!/bin/bash
-#
-# Short script that is sourced in to the CI environment
+# This file is sourced in to the CI environment
 # in .gitlab-ci.yml
 #
-# Sets IS_PUBLIC and IS_PRIVATE based on branch type
+
+# Sets the error behaviour options for shell throughout the CI environment
 #
-# Tweaks .gitmodules file for private builds
+set -o errexit # Exit if command failed.
 
 [ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1
 
-REF=$CI_COMMIT_REF_NAME
-
+# Sets IS_PUBLIC and IS_PRIVATE based on branch type
+#
 # Public branches are:
 # release branches - start with release/
 # release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end
@@ -18,18 +17,10 @@ REF=$CI_COMMIT_REF_NAME
 #
 # These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file
 #
+REF=$CI_COMMIT_REF_NAME
 if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then
     export IS_PUBLIC=1
 else
     export IS_PRIVATE=1
 fi
-
 unset REF
-
-set -e
-
-if [[ $IS_PRIVATE ]]; then
-   # Redirect git submodules from public github to our private gitlab server
-   sed -i "s%https://github.com/espressif/esp32-wifi-lib%${GITLAB_SSH_SERVER}/idf/esp32-wifi-lib%" .gitmodules
-   sed -i "s%https://github.com/espressif/esp32-bt-lib%${GITLAB_SSH_SERVER}/idf/esp32-bt-lib%" .gitmodules
-fi

tools/ci/get-full-sources.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Short script that is run as part of the CI environment
+# in .gitlab-ci.yml
+#
+# Sets up submodules in the ESP-IDF source tree
+# - Ideally, this just means doing a "git submodule update"
+# - But if something goes wrong we re-clone the repo from scratch
+#
+# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone
+#
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
+[[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment."
+
+SCRIPT_DIR=$(dirname -- "${0}")
+update_submodules() {
+    if [ "${IS_PRIVATE}" ]; then
+        ${SCRIPT_DIR}/mirror-submodule-update.sh
+    else
+        git submodule foreach "git submodule deinit --force ."
+        git submodule deinit --force .
+        git submodule update --init --recursive
+    fi
+}
+
+DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
+del_files() {
+    # if non-empty
+    [ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" )
+}
+del_files_confirm() {
+    rm -rf "${DELETED_FILES}"
+}
+
+RETRIES=10
+# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here...
+
+# For the first time, we try the fastest way.
+for try in `seq $RETRIES`; do
+    echo "Trying to add submodules to existing repo..."
+    update_submodules &&
+        echo "Fetch strategy submodules succeeded" &&
+        exit 0
+done
+
+# Then we use the clean way.
+for try in `seq $RETRIES`; do
+    cd ${CI_PROJECT_DIR}  # we are probably already here but pays to be certain
+    echo "Trying a clean clone of IDF..."
+    del_files
+    git clone ${CI_REPOSITORY_URL} . &&
+        git checkout ${CI_COMMIT_SHA} &&
+        update_submodules &&
+        echo "Clone strategy succeeded" &&
+        del_files_confirm &&
+        exit 0
+
+    echo "Clean clone failed..."
+done
+
+die "Failed to clone repo & submodules together"

tools/ci/mirror-list.txt

@@ -0,0 +1,10 @@
+components/esp32/lib                                @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
+components/bt/lib                                   @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
+components/aws_iot/aws-iot-device-sdk-embedded-C    @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git   ALLOW_TO_SYNC_FROM_PUBLIC
+components/coap/libcoap                             @GENERAL_MIRROR_SERVER@/idf/libcoap.git                         ALLOW_TO_SYNC_FROM_PUBLIC
+components/esptool_py/esptool                       @GENERAL_MIRROR_SERVER@/idf/esptool.git                         ALLOW_TO_SYNC_FROM_PUBLIC
+components/libsodium/libsodium                      @GENERAL_MIRROR_SERVER@/idf/libsodium.git                       ALLOW_TO_SYNC_FROM_PUBLIC
+components/micro-ecc/micro-ecc                      @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git                       ALLOW_TO_SYNC_FROM_PUBLIC
+components/nghttp/nghttp2                           @GENERAL_MIRROR_SERVER@/idf/nghttp2.git                         ALLOW_TO_SYNC_FROM_PUBLIC
+third-party/mruby                                   @GENERAL_MIRROR_SERVER@/idf/mruby.git                           ALLOW_TO_SYNC_FROM_PUBLIC
+third-party/neverbleed                              @GENERAL_MIRROR_SERVER@/idf/neverbleed.git                      ALLOW_TO_SYNC_FROM_PUBLIC

tools/ci/mirror-submodule-update.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Redirects git submodules to the specified local mirrors and updates these recursively.
+#
+# To revert the changed URLs use 'git submodule deinit .'
+#
+
+# -----------------------------------------------------------------------------
+# Common bash
+
+if [[ ! -z ${DEBUG} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+# -----------------------------------------------------------------------------
+
+[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
+
+REPO_DIR=${1:-"${PWD}"}
+REPO_DIR=$(readlink -f -- "${REPO_DIR}")
+
+SCRIPT_DIR=$(dirname -- "${0}")
+SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
+
+SCRIPT_SH=$(readlink -f -- "${0}")
+
+MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
+
+[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
+[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
+[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
+
+pushd ${REPO_DIR} >/dev/null
+
+# 0
+[ -f ".gitmodules" ] || exit 0
+
+# 1
+git submodule init
+
+
+# 2
+# Replacing each submodule URL of the current repository
+# according to the one found in the MIRRORLIST
+
+#   SED parses the strings like:
+#
+#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
+#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
+#
+for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
+do
+    SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
+    [ ${SUBMIRROR} ] || continue
+    SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
+    echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR"
+
+    git config submodule.${SUBPATH}.url ${SUBMIRROR}
+done
+
+# 3
+# Getting submodules of the current repository from the local mirrors
+git submodule update
+
+# 4
+# Replacing URLs for each sub-submodule.
+# The script runs recursively
+git submodule foreach "${SCRIPT_SH}" # No '--recursive'
+
+popd >/dev/null

tools/ci/mirror-synchronize.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Script for automate mirroring
+# You can run this script manually
+#
+
+# -----------------------------------------------------------------------------
+# Common bash
+
+if [[ ! -z ${DEBUG} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+set -o nounset # Exit if variable not set.
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+# -----------------------------------------------------------------------------
+
+[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
+
+REPO_DIR=${1:-"${PWD}"}
+REPO_DIR=$(readlink -f -- "${REPO_DIR}")
+
+SCRIPT_DIR=$(dirname -- "${0}")
+SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
+
+SCRIPT_SH=$(readlink -f -- "${0}")
+
+MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
+
+[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
+[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
+[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
+
+pushd ${REPO_DIR} >/dev/null
+
+# 0
+[ -f ".gitmodules" ] || exit 0
+
+# 1
+# Recursion
+git submodule update --init
+git submodule foreach "${SCRIPT_SH}" # No '--recursive'
+
+# 2
+git submodule deinit --force .
+git submodule update --init
+
+
+# 3
+# Mirroring from original URLs to ones listed in the MIRRORLIST
+
+#   SED parses the strings like:
+#
+#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
+#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
+#
+for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
+do
+    SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url)
+    SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
+    [ ${SUBMIRROR} ] || continue
+    SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
+
+    ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
+    if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ]
+    then
+        echo "[should not to sync mirror] ${SUBMIRROR}"
+        continue
+    fi
+
+    echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}"
+
+    pushd ${SUBPATH} >/dev/null
+        git fetch --prune
+        git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/*
+    popd >/dev/null
+done
+
+popd >/dev/null