Merge branch 'feature/ccache_in_ci' into 'master'

ci: enable ccache for build jobs

Closes IDF-776

See merge request espressif/esp-idf!15828

.gitlab/ci/build.yml

@@ -5,6 +5,11 @@
     - build
   variables:
     SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
+    # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
+    IDF_CCACHE_ENABLE: "1"
+  after_script:
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
   dependencies: []
 
 .build_pytest_template:

tools/ci/configure_ci_environment.sh

@@ -19,3 +19,34 @@ DEBUG_SHELL=${DEBUG_SHELL:-"0"}
 PEDANTIC_FLAGS="-DIDF_CI_BUILD -Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
 export PEDANTIC_CFLAGS="${PEDANTIC_FLAGS} -Wstrict-prototypes"
 export PEDANTIC_CXXFLAGS="${PEDANTIC_FLAGS}"
+
+# ccache related settings.
+# IDF_CCACHE_ENABLE may be enabled at job level (see build.yml). However it is possible to override it
+# globally via CI_CCACHE_DISABLE, in case there are any ccache related issues.
+if [ "$CI_CCACHE_DISABLE" = 1 ]; then
+    export IDF_CCACHE_ENABLE=0
+    echo "INFO: ccache disabled globally using CI_CCACHE_DISABLE=0"
+fi
+
+# Set ccache base directory to the project checkout path, to cancel out differences between runners
+export CCACHE_BASEDIR="${CI_PROJECT_DIR}"
+
+# In tools/ci/find_apps_build_apps.sh, we use --work-dir argument to copy apps to a separate location
+# before building them. This results in cache misses, even though the same code is compiled.
+# To solve this issue, we can disable 'hash_dir' option of ccache by setting CCACHE_NOHASHDIR env variable.
+# Note, this can result in issues with debug information, see:
+#   https://ccache.dev/manual/4.5.html#_compiling_in_different_directories
+#
+# 'CI_CCACHE_DISABLE_NOHASHDIR' variable can be used at project level to revert to hash_dir=true, in
+# case we start seeing failures due to false cache hits.
+if [ "${CI_CCACHE_DISABLE_NOHASHDIR}" != "1" ]; then
+    export CCACHE_NOHASHDIR="1"
+    echo "INFO: ccache CCACHE_NOHASHDIR option is enabled"
+fi
+
+# If 'REDIS_CACHE' variable is set at runner (or project) level, use that as secondary ccache storage.
+# This can be disabled at project level using 'CI_CCACHE_DISABLE_SECONDARY', in case of issues.
+if [ "${CI_CCACHE_DISABLE_SECONDARY}" != "1" ] && [ -n "${REDIS_CACHE}" ]; then
+    export CCACHE_SECONDARY_STORAGE="redis://${REDIS_CACHE}"
+    echo "INFO: Using CCACHE_SECONDARY_STORAGE=${CCACHE_SECONDARY_STORAGE}"
+fi

tools/ci/test_build_system_cmake.sh

@@ -35,6 +35,9 @@
 
 export PATH="$IDF_PATH/tools:$PATH"  # for idf.py
 
+# Some tests assume that ccache is not enabled
+unset IDF_CCACHE_ENABLE
+
 function run_tests()
 {
     FAILURES=