Merge branch 'bugfix/confgen_windows_command_line' into 'master'

cmake kconfig: Pass environment variables to confgen.py via a file

Closes IDF-711

See merge request idf/esp-idf!5356

tools/cmake/kconfig.cmake

@@ -107,6 +107,17 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
     idf_build_set_property(KCONFIG_PROJBUILDS "${kconfig_projbuilds}")
 
     idf_build_get_property(idf_target IDF_TARGET)
+    idf_build_get_property(idf_path IDF_PATH)
+
+    string(REPLACE ";" " " kconfigs "${kconfigs}")
+    string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}")
+
+    # Place config-related environment arguments into config.env file
+    # to work around command line length limits for execute_process
+    # on Windows & CMake < 3.11
+    set(config_env_path "${CMAKE_CURRENT_BINARY_DIR}/config.env")
+    configure_file("${idf_path}/tools/kconfig_new/config.env.in" ${config_env_path})
+    idf_build_set_property(CONFIG_ENV_PATH ${config_env_path})
 
     if(sdkconfig_defaults)
         set(defaults_arg --defaults "${sdkconfig_defaults}")
@@ -116,22 +127,15 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
         list(APPEND defaults_arg --defaults "${sdkconfig_defaults}.${idf_target}")
     endif()
 
-    idf_build_get_property(idf_path IDF_PATH)
     idf_build_get_property(root_kconfig __ROOT_KCONFIG)
     idf_build_get_property(python PYTHON)
 
-    string(REPLACE ";" " " kconfigs "${kconfigs}")
-    string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}")
-
     set(confgen_basecommand
         ${python} ${idf_path}/tools/kconfig_new/confgen.py
         --kconfig ${root_kconfig}
         --config ${sdkconfig}
         ${defaults_arg}
-        --env "COMPONENT_KCONFIGS=${kconfigs}"
-        --env "COMPONENT_KCONFIGS_PROJBUILD=${kconfig_projbuilds}"
-        --env "IDF_CMAKE=y"
-        --env "IDF_TARGET=${idf_target}")
+        --env-file ${config_env_path})
 
     idf_build_get_property(build_dir BUILD_DIR)
     set(config_dir ${build_dir}/config)
@@ -217,10 +221,10 @@ function(__kconfig_generate_config sdkconfig sdkconfig_defaults)
 
     # Custom target to run confserver.py from the build tool
     add_custom_target(confserver
-        COMMAND ${CMAKE_COMMAND} -E env
-        "COMPONENT_KCONFIGS=${kconfigs}"
-        "COMPONENT_KCONFIGS_PROJBUILD=${kconfig_projbuilds}"
-        ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py --kconfig ${IDF_PATH}/Kconfig --config ${sdkconfig}
+        COMMAND ${PYTHON} ${IDF_PATH}/tools/kconfig_new/confserver.py
+        --env-file ${config_env_path}
+        --kconfig ${IDF_PATH}/Kconfig
+        --config ${sdkconfig}
         VERBATIM
         USES_TERMINAL)
 endfunction()

tools/cmake/ldgen.cmake

@@ -54,6 +54,8 @@ function(__ldgen_process_template template output)
     string(REPLACE ";" " " kconfigs "${kconfigs}")
     string(REPLACE ";" " " kconfig_projbuilds "${kconfig_projbuilds}")
 
+    idf_build_get_property(config_env_path CONFIG_ENV_PATH)
+
     add_custom_command(
         OUTPUT ${output}
         COMMAND ${python} ${idf_path}/tools/ldgen/ldgen.py
@@ -62,11 +64,7 @@ function(__ldgen_process_template template output)
         --input     ${template}
         --output    ${output}
         --kconfig   ${root_kconfig}
-        --env       "COMPONENT_KCONFIGS=${kconfigs}"
-        --env       "COMPONENT_KCONFIGS_PROJBUILD=${kconfig_projbuilds}"
-        --env       "IDF_CMAKE=y"
-        --env       "IDF_PATH=${idf_path}"
-        --env       "IDF_TARGET=${idf_target}"
+        --env-file  "${config_env_path}"
         --libraries-file ${build_dir}/ldgen_libraries
         --objdump   ${CMAKE_OBJDUMP}
         DEPENDS     ${template} ${ldgen_fragment_files} ${ldgen_depends} ${SDKCONFIG}
@@ -76,4 +74,4 @@ function(__ldgen_process_template template output)
     add_custom_target(__ldgen_output_${_name} DEPENDS ${output})
     add_dependencies(__idf_build_target __ldgen_output_${_name})
     idf_build_set_property(__LINK_DEPENDS ${output} APPEND)
-endfunction()
+endfunction()

tools/kconfig_new/confgen.py

@@ -198,6 +198,10 @@ def main():
     parser.add_argument('--env', action='append', default=[],
                         help='Environment to set when evaluating the config file', metavar='NAME=VAL')
 
+    parser.add_argument('--env-file', type=argparse.FileType('r'),
+                        help='Optional file to load environment variables from. Contents '
+                             'should be a JSON object where each key/value pair is a variable.')
+
     args = parser.parse_args()
 
     for fmt, filename in args.output:
@@ -214,6 +218,10 @@ def main():
     for name, value in args.env:
         os.environ[name] = value
 
+    if args.env_file is not None:
+        env = json.load(args.env_file)
+        os.environ.update(env)
+
     config = kconfiglib.Kconfig(args.kconfig)
     config.disable_redun_warnings()
     config.disable_override_warnings()

tools/kconfig_new/config.env.in

@@ -0,0 +1,7 @@
+{
+    "COMPONENT_KCONFIGS": "${kconfigs}",
+    "COMPONENT_KCONFIGS_PROJBUILD": "${kconfig_projbuilds}",
+    "IDF_CMAKE": "y",
+    "IDF_TARGET": "${idf_target}",
+    "IDF_PATH": "${idf_path}"
+}

tools/kconfig_new/confserver.py

@@ -32,6 +32,10 @@ def main():
     parser.add_argument('--env', action='append', default=[],
                         help='Environment to set when evaluating the config file', metavar='NAME=VAL')
 
+    parser.add_argument('--env-file', type=argparse.FileType('r'),
+                        help='Optional file to load environment variables from. Contents '
+                             'should be a JSON object where each key/value pair is a variable.')
+
     parser.add_argument('--version', help='Set protocol version to use on initial status',
                         type=int, default=MAX_PROTOCOL_VERSION)
 
@@ -54,6 +58,10 @@ def main():
     for name, value in args.env:
         os.environ[name] = value
 
+    if args.env_file is not None:
+        env = json.load(args.env_file)
+        os.environ.update(env)
+
     run_server(args.kconfig, args.config)
 
 

tools/ldgen/ldgen.py

@@ -16,6 +16,7 @@
 #
 
 import argparse
+import json
 import sys
 import tempfile
 import subprocess
@@ -30,6 +31,17 @@ from pyparsing import ParseException, ParseFatalException
 from io import StringIO
 
 
+def _update_environment(args):
+    env = [(name, value) for (name,value) in (e.split("=",1) for e in args.env)]
+    for name, value in env:
+        value = " ".join(value.split())
+        os.environ[name] = value
+
+    if args.env_file is not None:
+        env = json.load(args.env_file)
+        os.environ.update(env)
+
+
 def main():
 
     argparser = argparse.ArgumentParser(description="ESP-IDF linker script generator")
@@ -68,6 +80,10 @@ def main():
         action='append', default=[],
         help='Environment to set when evaluating the config file', metavar='NAME=VAL')
 
+    argparser.add_argument('--env-file', type=argparse.FileType('r'),
+                           help='Optional file to load environment variables from. Contents '
+                           'should be a JSON object where each key/value pair is a variable.')
+
     argparser.add_argument(
         "--objdump",
         help="Path to toolchain objdump")
@@ -93,7 +109,9 @@ def main():
 
         generation_model = GenerationModel()
 
-        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
+        _update_environment(args)  # assign args.env and args.env_file to os.environ
+
+        sdkconfig = SDKConfig(kconfig_file, config_file)
 
         for fragment_file in fragment_files:
             try:

tools/ldgen/sdkconfig.py

@@ -46,13 +46,7 @@ class SDKConfig:
     # Operators supported by the expression evaluation
     OPERATOR = oneOf(["=", "!=", ">", "<", "<=", ">="])
 
-    def __init__(self, kconfig_file, sdkconfig_file, env=[]):
-        env = [(name, value) for (name,value) in (e.split("=",1) for e in env)]
-
-        for name, value in env:
-            value = " ".join(value.split())
-            os.environ[name] = value
-
+    def __init__(self, kconfig_file, sdkconfig_file):
         self.config = kconfiglib.Kconfig(kconfig_file)
         self.config.load_config(sdkconfig_file)