tools: move target guessing into cmake

The _guess_or_check_idf_target() function has sdkconfig and sdkconfig.defaults
file names hardcoded. Since config file names may be specified with SDKCONFIG
or SDKCONFIG_DEFAULTS cmake vars, directly in CMakeLists.txt or passed in with
the -D cmake option, they are not respected.

Problem is when SDKCONFIG or SDKCONFIG_DEFAULTS is set in
CMakeLists.txt. While idf can detect cmake vars passed through it
to cmake via the -D option, detecting SDKCONFIG and SDKCONFIG_DEFAULTS
vars settings in CMakeLists.txt would require to parse it. This seems
like error prone approach. Also if the vars defined by the -D option
are passed directly to cmake, not via idf, they will not be visible to idf.

It seems reasonable to move the logic into cmake, where we know the correct
SDKCONFIG and SDKCONFIG_DEFAULTS values. So the IDF_TARGET detection/guessing
is moved into targets.cmake, where the IDF_TARGET is actually set. The target
is guessed based on the following precendence.

1) $ENV{IDF_TARGET}
2) IDF_TARGET
3) SDKCONFIG
4) sdkconfig
5) SDKCONFIG_DEFAULTS if non-empty or
   $ENV{SDKCONFIG_DEFAULTS} if non-empty or
   sdkconfig.defaults
6) esp32

All config files referred in $ENV{SDKCONFIG_DEFAULTS} and SDKCONFIG_DEFAULTS
are searched, compared to the current behaviour. First target found in the
above chain is used.

The original _guess_or_check_idf_target() is renamed to _check_idf_target() and
used for the target consistency checks only.

The get_sdkconfig_filename() helper is now used to get the sdkconfig file
for consistency checks. It looks in SDKCONFIG specified with the -D
option and project_description.json.

With this change config full paths are reported in messages, so it's clear
e.g. from which config the target was guessed from or which config has
consistency problem. test_non_default_target.py was adjusted to this
change and also new test for testing the IDF_TARGET guessing was added.

Signed-off-by: Frantisek Hrbata <frantisek.hrbata@espressif.com>

tools/cmake/targets.cmake

@@ -1,3 +1,72 @@
+#
+# Get target from single sdkconfig file
+#
+function(__target_from_config config target_out file_out)
+    set(${target_out} NOTFOUND PARENT_SCOPE)
+    set(${file_out} NOTFOUND PARENT_SCOPE)
+
+    if(NOT EXISTS "${config}")
+        return()
+    endif()
+
+    file(STRINGS "${config}" lines)
+    foreach(line ${lines})
+        if(NOT "${line}" MATCHES "^CONFIG_IDF_TARGET=\"[^\"]+\"$")
+            continue()
+        endif()
+
+        string(REGEX REPLACE "CONFIG_IDF_TARGET=\"([^\"]+)\"" "\\1" target "${line}")
+        set(${target_out} ${target} PARENT_SCOPE)
+        set(${file_out} ${config} PARENT_SCOPE)
+        return()
+    endforeach()
+endfunction()
+
+#
+# Get target from list of sdkconfig files
+#
+function(__target_from_configs configs target_out file_out)
+    set(target NOTFOUND)
+    set(file NOTFOUND)
+
+    foreach(config ${configs})
+        message(DEBUG "Searching for target in '${config}'")
+        get_filename_component(config "${config}" ABSOLUTE)
+        __target_from_config("${config}" target file)
+        if(target)
+            break()
+        endif()
+    endforeach()
+
+    set(${target_out} ${target} PARENT_SCOPE)
+    set(${file_out} ${file} PARENT_SCOPE)
+endfunction()
+
+#
+# Search for target in config files in the following order.
+# SDKCONFIG cmake var, default sdkconfig, SDKCONFIG_DEFAULTS cmake var
+# if non-empty or SDKCONFIG_DEFAULTS env var if non-empty or
+# sdkconfig.defaults.
+#
+function(__target_guess target_out file_out)
+    # Select sdkconfig_defaults to look for target
+    if(SDKCONFIG_DEFAULTS)
+        set(defaults "${SDKCONFIG_DEFAULTS}")
+    elseif(DEFINED ENV{SDKCONFIG_DEFAULTS})
+        set(defaults "$ENV{SDKCONFIG_DEFAULTS}")
+    endif()
+
+    if(NOT defaults)
+        set(defaults "${CMAKE_SOURCE_DIR}/sdkconfig.defaults")
+    endif()
+
+    set(configs "${SDKCONFIG}" "${CMAKE_SOURCE_DIR}/sdkconfig" "${defaults}")
+    message(DEBUG "Searching for target in '${configs}'")
+    __target_from_configs("${configs}" target file)
+    set(${target_out} ${target} PARENT_SCOPE)
+    set(${file_out} ${file} PARENT_SCOPE)
+endfunction()
+
 #
 # Set the target used for the standard project build.
 #
@@ -10,8 +79,15 @@ macro(__target_init)
         if(IDF_TARGET)
             set(env_idf_target ${IDF_TARGET})
         else()
-            set(env_idf_target esp32)
-            message(STATUS "IDF_TARGET not set, using default target: ${env_idf_target}")
+            # Try to guess IDF_TARGET from sdkconfig files while honoring
+            # SDKCONFIG and SDKCONFIG_DEFAULTS values
+            __target_guess(env_idf_target where)
+            if(env_idf_target)
+                message(STATUS "IDF_TARGET is not set, guessed '${env_idf_target}' from sdkconfig '${where}'")
+            else()
+                set(env_idf_target esp32)
+                message(STATUS "IDF_TARGET not set, using default target: ${env_idf_target}")
+            endif()
         endif()
     else()
         # IDF_TARGET set both in environment and in cache, must be the same

tools/idf_py_actions/tools.py

@@ -457,8 +457,8 @@ def ensure_build_directory(args: 'PropertyDict', prog_name: str, always_run_cmak
 
     cache_cmdl = _parse_cmdl_cmakecache(args.define_cache_entry)
 
-    # Validate or set IDF_TARGET
-    _guess_or_check_idf_target(args, prog_name, cache)
+    # Validate IDF_TARGET
+    _check_idf_target(args, prog_name, cache, cache_cmdl)
 
     if always_run_cmake or _new_cmakecache_entries(cache, cache_cmdl):
         if args.generator is None:
@@ -581,36 +581,22 @@ def is_target_supported(project_path: str, supported_targets: List) -> bool:
     return get_target(project_path) in supported_targets
 
 
-def _guess_or_check_idf_target(args: 'PropertyDict', prog_name: str, cache: Dict) -> None:
+def _check_idf_target(args: 'PropertyDict', prog_name: str, cache: Dict, cache_cmdl: Dict) -> None:
     """
-    If CMakeCache.txt doesn't exist, and IDF_TARGET is not set in the environment, guess the value from
-    sdkconfig or sdkconfig.defaults, and pass it to CMake in IDF_TARGET variable.
-
-    Otherwise, cross-check the three settings (sdkconfig, CMakeCache, environment) and if there is
+    Cross-check the three settings (sdkconfig, CMakeCache, environment) and if there is
     mismatch, fail with instructions on how to fix this.
     """
-    # Default locations of sdkconfig files.
-    # FIXME: they may be overridden in the project or by a CMake variable (IDF-1369).
-    # These are used to guess the target from sdkconfig, or set the default target by sdkconfig.defaults.
-    idf_target_from_sdkconfig = get_target(args.project_dir)
-    idf_target_from_sdkconfig_defaults = get_target(args.project_dir, 'sdkconfig.defaults')
+    sdkconfig = get_sdkconfig_filename(args, cache_cmdl)
+    idf_target_from_sdkconfig = get_sdkconfig_value(sdkconfig, 'CONFIG_IDF_TARGET')
     idf_target_from_env = os.environ.get('IDF_TARGET')
     idf_target_from_cache = cache.get('IDF_TARGET')
 
-    if not cache and not idf_target_from_env:
-        # CMakeCache.txt does not exist yet, and IDF_TARGET is not set in the environment.
-        guessed_target = idf_target_from_sdkconfig or idf_target_from_sdkconfig_defaults
-        if guessed_target:
-            if args.verbose:
-                print("IDF_TARGET is not set, guessed '%s' from sdkconfig" % (guessed_target))
-            args.define_cache_entry.append('IDF_TARGET=' + guessed_target)
-
-    elif idf_target_from_env:
+    if idf_target_from_env:
         # Let's check that IDF_TARGET values are consistent
         if idf_target_from_sdkconfig and idf_target_from_sdkconfig != idf_target_from_env:
-            raise FatalError("Project sdkconfig was generated for target '{t_conf}', but environment variable IDF_TARGET "
+            raise FatalError("Project sdkconfig '{cfg}' was generated for target '{t_conf}', but environment variable IDF_TARGET "
                              "is set to '{t_env}'. Run '{prog} set-target {t_env}' to generate new sdkconfig file for target {t_env}."
-                             .format(t_conf=idf_target_from_sdkconfig, t_env=idf_target_from_env, prog=prog_name))
+                             .format(cfg=sdkconfig, t_conf=idf_target_from_sdkconfig, t_env=idf_target_from_env, prog=prog_name))
 
         if idf_target_from_cache and idf_target_from_cache != idf_target_from_env:
             raise FatalError("Target settings are not consistent: '{t_env}' in the environment, '{t_cache}' in CMakeCache.txt. "
@@ -619,10 +605,10 @@ def _guess_or_check_idf_target(args: 'PropertyDict', prog_name: str, cache: Dict
 
     elif idf_target_from_cache and idf_target_from_sdkconfig and idf_target_from_cache != idf_target_from_sdkconfig:
         # This shouldn't happen, unless the user manually edits CMakeCache.txt or sdkconfig, but let's check anyway.
-        raise FatalError("Project sdkconfig was generated for target '{t_conf}', but CMakeCache.txt contains '{t_cache}'. "
+        raise FatalError("Project sdkconfig '{cfg}' was generated for target '{t_conf}', but CMakeCache.txt contains '{t_cache}'. "
                          "To keep the setting in sdkconfig ({t_conf}) and re-generate CMakeCache.txt, run '{prog} fullclean'. "
                          "To re-generate sdkconfig for '{t_cache}' target, run '{prog} set-target {t_cache}'."
-                         .format(t_conf=idf_target_from_sdkconfig, t_cache=idf_target_from_cache, prog=prog_name))
+                         .format(cfg=sdkconfig, t_conf=idf_target_from_sdkconfig, t_cache=idf_target_from_cache, prog=prog_name))
 
 
 class TargetChoice(click.Choice):

tools/test_build_system/test_non_default_target.py

@@ -8,6 +8,9 @@ from pathlib import Path
 import pytest
 from test_build_system_helpers import EnvDict, IdfPyFunc, check_file_contains, run_cmake
 
+ESP32C3_TARGET = 'esp32c3'
+ESP32C2_TARGET = 'esp32c2'
+ESP32S3_TARGET = 'esp32s3'
 ESP32S2_TARGET = 'esp32s2'
 ESP32_TARGET = 'esp32'
 
@@ -36,9 +39,12 @@ def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvD
     idf_py('set-target', ESP32S2_TARGET)
     default_idf_env.update({'IDF_TARGET': ESP32_TARGET})
 
+    cfg_path = test_app_copy.joinpath('sdkconfig')
+
     logging.info("idf.py fails if IDF_TARGET settings don't match the environment")
-    reconfigure_and_check_return_values("Project sdkconfig was generated for target '{}', but environment "
-                                        "variable IDF_TARGET is set to '{}'.".format(ESP32S2_TARGET, ESP32_TARGET))
+    reconfigure_and_check_return_values("Project sdkconfig '{}' was generated for target '{}', but environment "
+                                        "variable IDF_TARGET is set to '{}'.".format(cfg_path, ESP32S2_TARGET,
+                                                                                     ESP32_TARGET))
 
     logging.info("idf.py fails if IDF_TARGET settings in CMakeCache.txt don't match the environment")
     (test_app_copy / 'sdkconfig').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET))
@@ -47,8 +53,8 @@ def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvD
 
     logging.info("idf.py fails if IDF_TARGET settings in CMakeCache.txt don't match the sdkconfig")
     default_idf_env.pop('IDF_TARGET')
-    reconfigure_and_check_return_values("Project sdkconfig was generated for target '{}', but CMakeCache.txt "
-                                        "contains '{}'.".format(ESP32_TARGET, ESP32S2_TARGET))
+    reconfigure_and_check_return_values("Project sdkconfig '{}' was generated for target '{}', but CMakeCache.txt "
+                                        "contains '{}'.".format(cfg_path, ESP32_TARGET, ESP32S2_TARGET))
 
 
 def test_target_precedence(idf_py: IdfPyFunc, default_idf_env: EnvDict, test_app_copy: Path) -> None:
@@ -102,3 +108,41 @@ def test_target_using_sdkconfig(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     idf_py('reconfigure')
     check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
     check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET_{}=y'.format(ESP32S2_TARGET.upper()))
+
+
+def test_target_guessing(idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
+    """
+    Tests are performed from the lowest to the highest priority, while
+    configs, except from the sdkconfig, and parameters of previous tests are
+    preserved.
+    """
+
+    logging.info('Can guess target from sdkconfig.defaults')
+    (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET))
+    idf_py('reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32_TARGET))
+
+    logging.info('Can guess target from SDKCONFIG_DEFAULTS environment variable')
+    (test_app_copy / 'sdkconfig1').write_text('NOTHING HERE')
+    (test_app_copy / 'sdkconfig2').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    clean_app(test_app_copy)
+    default_idf_env.update({'SDKCONFIG_DEFAULTS': 'sdkconfig1;sdkconfig2'})
+    idf_py('reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S2_TARGET))
+
+    logging.info('Can guess target from SDKCONFIG_DEFAULTS using -D')
+    (test_app_copy / 'sdkconfig3').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S2_TARGET))
+    (test_app_copy / 'sdkconfig4').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32S3_TARGET))
+    clean_app(test_app_copy)
+    idf_py('-D', 'SDKCONFIG_DEFAULTS=sdkconfig4;sdkconfig3', 'reconfigure')
+    check_file_contains('sdkconfig', 'CONFIG_IDF_TARGET="{}"'.format(ESP32S3_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32S3_TARGET))
+
+    logging.info('Can guess target from custom sdkconfig')
+    (test_app_copy / 'sdkconfig5').write_text('CONFIG_IDF_TARGET="{}"'.format(ESP32C3_TARGET))
+    clean_app(test_app_copy)
+    idf_py('-D', 'SDKCONFIG=sdkconfig5', '-D', 'SDKCONFIG_DEFAULTS=sdkconfig4;sdkconfig3', 'reconfigure')
+    check_file_contains('sdkconfig5', 'CONFIG_IDF_TARGET="{}"'.format(ESP32C3_TARGET))
+    check_file_contains('build/CMakeCache.txt', 'IDF_TARGET:STRING={}'.format(ESP32C3_TARGET))