Merge branch 'feature/use_libraries_directly_with_ldgen' into 'master'

Pass libraries as arguments to ldgen directly

See merge request idf/esp-idf!4887

make/ldgen.mk

@@ -1,7 +1,6 @@
 # Makefile to support the linker script generation mechanism
-
-LDGEN_SECTIONS_INFO_FILES = $(foreach lib, $(COMPONENT_LIBRARIES), $(BUILD_DIR_BASE)/$(lib)/lib$(lib).a.sections_info)
 LDGEN_FRAGMENT_FILES = $(COMPONENT_LDFRAGMENTS)
+LDGEN_LIBRARIES=$(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp)/lib$(libcomp).a)
 
 ON_WINDOWS:=n
 
@@ -15,54 +14,45 @@ endif
 # the components
 ifeq ($(ON_WINDOWS),y)
 define ldgen_process_template
-$(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_SECTIONS_INFO_FILES) $(IDF_PATH)/make/ldgen.mk
-	printf "$(foreach info,$(LDGEN_SECTIONS_INFO_FILES),$(subst \,/,$(shell cygpath -w $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen.section_infos
+$(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk
+	printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -w $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries
 
-$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos
+$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries
 	@echo 'Generating $(notdir $(2))'
 	$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
 		--input         $(1) \
 		--config        $(SDKCONFIG) \
 		--fragments     $(LDGEN_FRAGMENT_FILES) \
+		--libraries-file $(BUILD_DIR_BASE)/ldgen_libraries \
 		--output        $(2) \
-		--sections      $(BUILD_DIR_BASE)/ldgen.section_infos \
 		--kconfig       $(IDF_PATH)/Kconfig \
 		--env           "COMPONENT_KCONFIGS=$(foreach k, $(COMPONENT_KCONFIGS), $(shell cygpath -w $(k)))" \
 		--env           "COMPONENT_KCONFIGS_PROJBUILD=$(foreach k, $(COMPONENT_KCONFIGS_PROJBUILD), $(shell cygpath -w $(k)))" \
-		--env           "IDF_CMAKE=n"
+		--env           "IDF_CMAKE=n" \
+		--objdump		$(OBJDUMP)
 endef
 else # ON_WINDOWS
 define ldgen_process_template
-$(BUILD_DIR_BASE)/ldgen.section_infos: $(LDGEN_SECTIONS_INFO_FILES) $(IDF_PATH)/make/ldgen.mk
-	printf "$(foreach info,$(LDGEN_SECTIONS_INFO_FILES),$(info)\n)" > $(BUILD_DIR_BASE)/ldgen.section_infos
+$(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk
+	printf "$(foreach library,$(LDGEN_LIBRARIES),$(library)\n)" > $(BUILD_DIR_BASE)/ldgen_libraries
 
-$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen.section_infos 
+$(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries
 	@echo 'Generating $(notdir $(2))'
 	$(PYTHON) $(IDF_PATH)/tools/ldgen/ldgen.py \
 		--input         $(1) \
 		--config        $(SDKCONFIG) \
 		--fragments     $(LDGEN_FRAGMENT_FILES) \
+		--libraries-file $(BUILD_DIR_BASE)/ldgen_libraries \
 		--output        $(2) \
-		--sections      $(BUILD_DIR_BASE)/ldgen.section_infos \
 		--kconfig       $(IDF_PATH)/Kconfig \
 		--env           "COMPONENT_KCONFIGS=$(COMPONENT_KCONFIGS)" \
 		--env           "COMPONENT_KCONFIGS_PROJBUILD=$(COMPONENT_KCONFIGS_PROJBUILD)" \
-		--env           "IDF_CMAKE=n"
+		--env           "IDF_CMAKE=n" \
+		--objdump		$(OBJDUMP)
 endef
 endif # ON_WINDOWS
 
 define ldgen_create_commands
-$(foreach lib, $(COMPONENT_LIBRARIES), \
-	$(eval $(call ldgen_generate_target_sections_info, $(BUILD_DIR_BASE)/$(lib)/lib$(lib).a)))
-
 ldgen-clean:
-	rm -f $(LDGEN_SECTIONS_INFO_FILES)
-	rm -f $(BUILD_DIR_BASE)/ldgen.section_infos
-endef
-
-# Target to generate sections info file from objdump of component archive
-define ldgen_generate_target_sections_info
-$(1).sections_info: $(1)
-	@echo 'Generating $(notdir $(1).sections_info)'
-	$(OBJDUMP) -h $(1) > $(1).sections_info
-endef
+	rm -f $(BUILD_DIR_BASE)/ldgen_libraries
+endef

tools/cmake/components.cmake

@@ -64,7 +64,7 @@ function(register_component)
 
         set_property(TARGET ${COMPONENT_TARGET} PROPERTY OUTPUT_NAME ${COMPONENT_NAME})
 
-        ldgen_generate_sections_info(${COMPONENT_TARGET})
+        ldgen_component_add(${COMPONENT_TARGET})
     else()
         add_library(${COMPONENT_TARGET} INTERFACE) # header-only component
         set(include_type INTERFACE)

tools/cmake/ldgen.cmake

@@ -1,45 +1,34 @@
 # Utilities for supporting linker script generation in the build system
 
-# ldgen_create_target
+# ldgen_set_variables
 #
 # Create the custom target to attach the fragment files and template files
 # for the build to.
 function(ldgen_set_variables)
-    add_custom_target(ldgen_section_infos)
-    add_custom_target(ldgen DEPENDS ldgen_section_infos)
+    add_custom_target(ldgen)
 endfunction()
 
-# ldgen_add_fragment_file
+# ldgen_add_fragment_files
 #
 # Add one or more linker fragment files, and append it to the list of fragment
 # files found so far.
-function(ldgen_add_fragment_files target fragment_files)
+function(ldgen_add_fragment_files component fragment_files)
     spaces2list(fragment_files)
 
     foreach(fragment_file ${fragment_files})
-        get_filename_component(fragment_file_abs_dir ${fragment_file} ABSOLUTE BASE_DIR ${component_dir})
-        list(APPEND fragment_files_full_path ${fragment_file_abs_dir})
+        get_filename_component(_fragment_file ${fragment_file} ABSOLUTE)
+        list(APPEND _fragment_files ${_fragment_file})
     endforeach()
 
-    set_property(TARGET ldgen APPEND PROPERTY FRAGMENT_FILES ${fragment_files_full_path})
+    set_property(TARGET ldgen APPEND PROPERTY FRAGMENT_FILES ${_fragment_files})
 endfunction()
 
-# ldgen_generate_sections_info
+# ldgen_component_add
 #
-# Generate sections info for specified target to be used in linker script generation
-function(ldgen_generate_sections_info target)
-    get_filename_component(target_sections_info ${CMAKE_CURRENT_BINARY_DIR}/${target}.sections_info ABSOLUTE)
-
-    add_custom_command(
-        OUTPUT ${target_sections_info}
-        COMMAND ${CMAKE_OBJDUMP} $<TARGET_FILE:${target}> -h >  ${target_sections_info}
-        DEPENDS ${target}
-        )
-
-    add_custom_target(${target}_sections_info DEPENDS ${target_sections_info})
-    add_dependencies(ldgen_section_infos ${target}_sections_info)
-
-    set_property(TARGET ldgen_section_infos APPEND PROPERTY SECTIONS_INFO_FILES ${target_sections_info})
+# Add component to known libraries for linker script generation
+function(ldgen_component_add component_lib)
+    set_property(TARGET ldgen APPEND PROPERTY OUTPUT_LIBRARIES "$<TARGET_FILE:${component_lib}>")
+    set_property(TARGET ldgen APPEND PROPERTY LIBRARIES ${component_lib})
 endfunction()
 
 # ldgen_process_template
@@ -47,10 +36,17 @@ endfunction()
 # Passes a linker script template to the linker script generation tool for
 # processing
 function(ldgen_process_template template output)
-    file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/ldgen.section_infos
-        CONTENT "$<JOIN:$<TARGET_PROPERTY:ldgen_section_infos,SECTIONS_INFO_FILES>,\n>")
+    get_property(output_libraries TARGET ldgen PROPERTY OUTPUT_LIBRARIES)
+    file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/ldgen_libraries.in CONTENT "$<JOIN:${output_libraries},\n>")
+    file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/ldgen_libraries INPUT ${CMAKE_BINARY_DIR}/ldgen_libraries.in)
+
+    get_filename_component(filename "${template}" NAME_WE)
+
+    set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES
+        "${CMAKE_BINARY_DIR}/ldgen_libraries.in"
+        "${CMAKE_BINARY_DIR}/ldgen_libraries")
 
-    # Create command to invoke the linker script generator tool.
     add_custom_command(
         OUTPUT ${output}
         COMMAND ${PYTHON} ${IDF_PATH}/tools/ldgen/ldgen.py
@@ -58,15 +54,16 @@ function(ldgen_process_template template output)
         --fragments "$<JOIN:$<TARGET_PROPERTY:ldgen,FRAGMENT_FILES>,\t>"
         --input     ${template}
         --output    ${output}
-        --sections  ${CMAKE_BINARY_DIR}/ldgen.section_infos
-        --kconfig   ${IDF_PATH}/Kconfig
+        --kconfig   ${ROOT_KCONFIG}
         --env       "COMPONENT_KCONFIGS=${COMPONENT_KCONFIGS}"
         --env       "COMPONENT_KCONFIGS_PROJBUILD=${COMPONENT_KCONFIGS_PROJBUILD}"
         --env       "IDF_CMAKE=y"
         --env       "IDF_PATH=${IDF_PATH}"
         --env       "IDF_TARGET=${IDF_TARGET}"
-        DEPENDS     ${template} $<TARGET_PROPERTY:ldgen,FRAGMENT_FILES> ${SDKCONFIG}
-                    ldgen_section_infos
+        --libraries-file ${CMAKE_BINARY_DIR}/ldgen_libraries
+        --objdump   ${CMAKE_OBJDUMP}
+        DEPENDS     ${template} $<TARGET_PROPERTY:ldgen,FRAGMENT_FILES>
+                    $<TARGET_PROPERTY:ldgen,LIBRARIES> ${SDKCONFIG}
     )
 
     get_filename_component(output_name ${output} NAME)
@@ -74,11 +71,11 @@ function(ldgen_process_template template output)
     add_dependencies(ldgen ldgen_${output_name}_script)
 endfunction()
 
-# ldgen_create_commands
+# ldgen_add_dependencies
 #
-# Create the command to generate the output scripts from templates presented.
+# Add dependency of project executable to ldgen custom target.
 function(ldgen_add_dependencies)
     if(IDF_PROJECT_EXECUTABLE)
         add_dependencies(${IDF_PROJECT_EXECUTABLE} ldgen)
     endif()
-endfunction()
+endfunction()

tools/ldgen/generation.py

@@ -573,8 +573,8 @@ class SectionsInfo(dict):
     def __init__(self):
         self.sections = dict()
 
-    def add_sections_info(self, sections_info_file):
-        first_line = sections_info_file.readline()
+    def add_sections_info(self, sections_info_dump):
+        first_line = sections_info_dump.readline()
 
         archive_path = (Literal("In archive").suppress() +
                         # trim the last character from archive_path, :
@@ -587,10 +587,10 @@ class SectionsInfo(dict):
         try:
             results = parser.parseString(first_line)
         except ParseException as p:
-            raise ParseException("File " + sections_info_file.name + " is not a valid sections info file. " + p.message)
+            raise ParseException("Parsing sections info for library " + sections_info_dump.name + " failed. " + p.message)
 
         archive = os.path.basename(results.archive_path)
-        self.sections[archive] = SectionsInfo.__info(sections_info_file.name, sections_info_file.read())
+        self.sections[archive] = SectionsInfo.__info(sections_info_dump.name, sections_info_dump.read())
 
     def _get_infos_from_file(self, info):
         # Object file line: '{object}:  file format elf32-xtensa-le'

tools/ldgen/ldgen.py

@@ -18,12 +18,14 @@
 import argparse
 import sys
 import tempfile
+import subprocess
 
 from fragments import FragmentFile
 from sdkconfig import SDKConfig
 from generation import GenerationModel, TemplateModel, SectionsInfo
 from ldgen_common import LdGenFailure
 from pyparsing import ParseException, ParseFatalException
+from io import StringIO
 
 
 def main():
@@ -42,9 +44,9 @@ def main():
         nargs="+")
 
     argparser.add_argument(
-        "--sections", "-s",
+        "--libraries-file",
         type=argparse.FileType("r"),
-        help="Library sections info")
+        help="File that contains the list of libraries in the build")
 
     argparser.add_argument(
         "--output", "-o",
@@ -64,27 +66,28 @@ def main():
         action='append', default=[],
         help='Environment to set when evaluating the config file', metavar='NAME=VAL')
 
+    argparser.add_argument(
+        "--objdump",
+        help="Path to toolchain objdump")
+
     args = argparser.parse_args()
 
     input_file = args.input
     fragment_files = [] if not args.fragments else args.fragments
+    libraries_file = args.libraries_file
     config_file = args.config
     output_path = args.output
     kconfig_file = args.kconfig
-    sections = args.sections
+    objdump = args.objdump
 
     try:
         sections_infos = SectionsInfo()
-
-        if sections:
-            section_info_contents = [s.strip() for s in sections.read().split("\n")]
-            section_info_contents = [s for s in section_info_contents if s]
-        else:
-            section_info_contents = []
-
-        for sections_info_file in section_info_contents:
-            with open(sections_info_file) as sections_info_file_obj:
-                sections_infos.add_sections_info(sections_info_file_obj)
+        for library in libraries_file:
+            library = library.strip()
+            if library:
+                dump = StringIO(subprocess.check_output([objdump, "-h", library]).decode())
+                dump.name = library
+                sections_infos.add_sections_info(dump)
 
         generation_model = GenerationModel()