Merge branch 'bugfix/ldgen_ignore_nonexistent_archives_and_obj' into 'master'

ldgen: check mappings

Closes IDF-1624

See merge request espressif/esp-idf!8557

components/app_trace/linker.lf

@@ -3,10 +3,11 @@ archive: libapp_trace.a
 entries:
     app_trace (noflash)
     app_trace_util (noflash)
-    SEGGER_SYSVIEW (noflash)
-    SEGGER_RTT_esp32 (noflash)
-    SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
-    SEGGER_SYSVIEW_FreeRTOS (noflash)
+    if SYSVIEW_ENABLE = y:
+      SEGGER_SYSVIEW (noflash)
+      SEGGER_RTT_esp32 (noflash)
+      SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
+      SEGGER_SYSVIEW_FreeRTOS (noflash)
 
 [mapping:driver]
 archive: libdriver.a

components/esp_hw_support/linker.lf

@@ -6,8 +6,8 @@ entries:
         regi2c_ctrl (noflash)
     rtc_clk (noflash)
     rtc_init:rtc_vddsdio_set_config (noflash)
-    rtc_periph (noflash_text)
     rtc_pm (noflash_text)
     rtc_sleep (noflash_text)
     rtc_time (noflash_text)
-    rtc_wdt (noflash_text)
+    if IDF_TARGET_ESP32C3 = n:
+        rtc_wdt (noflash_text)

components/esp_hw_support/port/esp32s3/CMakeLists.txt

@@ -7,7 +7,8 @@ set(srcs
     "rtc_init.c"
     "rtc_pm.c"
     "rtc_sleep.c"
-    "rtc_time.c")
+    "rtc_time.c"
+    "rtc_wdt.c")
 
 add_prefix(srcs "${CMAKE_CURRENT_LIST_DIR}/" "${srcs}")
 target_sources(${COMPONENT_LIB} PRIVATE "${srcs}")

components/soc/esp32s2/rtc_periph.c → components/esp_hw_support/port/esp32s3/rtc_wdt.c

@@ -11,5 +11,3 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
-#include "soc/rtc_periph.h"

components/esp_pm/linker.lf

@@ -36,12 +36,14 @@ entries:
 archive: libesp_timer.a
 entries:
     if PM_SLP_IRAM_OPT = y:
-        esp_timer_impl_lac:esp_timer_impl_lock (noflash)
-        esp_timer_impl_lac:esp_timer_impl_unlock (noflash)
-        esp_timer_impl_lac:esp_timer_impl_advance (noflash)
-        esp_timer_impl_systimer:esp_timer_impl_lock (noflash)
-        esp_timer_impl_systimer:esp_timer_impl_unlock (noflash)
-        esp_timer_impl_systimer:esp_timer_impl_advance (noflash)
+        if ESP_TIMER_IMPL_TG0_LAC = y:
+            esp_timer_impl_lac:esp_timer_impl_lock (noflash)
+            esp_timer_impl_lac:esp_timer_impl_unlock (noflash)
+            esp_timer_impl_lac:esp_timer_impl_advance (noflash)
+        elif ESP_TIMER_IMPL_SYSTIMER = y:
+            esp_timer_impl_systimer:esp_timer_impl_lock (noflash)
+            esp_timer_impl_systimer:esp_timer_impl_unlock (noflash)
+            esp_timer_impl_systimer:esp_timer_impl_advance (noflash)
 
 [mapping:newlib_pm]
 archive: libnewlib.a

components/esp_system/linker.lf

@@ -6,10 +6,6 @@ entries:
     panic_arch (noflash)
     reset_reason (noflash)
     system_api:esp_system_abort (noflash)
-    startup:do_core_init (default)
-    startup:do_secondary_init (default)
-    startup:start_cpu0_default (default)
-
 
     if ESP_CONSOLE_USB_CDC_SUPPORT_ETS_PRINTF:
         usb_console:esp_usb_console_write_char (noflash)

components/espcoredump/linker.lf

@@ -19,13 +19,11 @@ entries:
     else:
         * (default)
 
-[mapping:mbedtls]
-archive: libmbedtls.a
+[mapping:sha256_coredump]
+archive: libmbedcrypto.a
 entries:
     if ESP_COREDUMP_CHECKSUM_SHA256 = y :
         if MBEDTLS_HARDWARE_SHA = n:
             sha256 (noflash_text)
         else:
             esp_sha256 (noflash_text)
-    else:
-        * (default)

components/freertos/linker.lf

@@ -2,25 +2,25 @@
 archive: libfreertos.a
 entries:
     * (noflash_text)
-    queue:xQueueGenericCreateStatic (default)
     if FREERTOS_PLACE_FUNCTIONS_INTO_FLASH = y:
         port: pxPortInitialiseStack (default)
         port: xPortStartScheduler (default)
-        port: vPortStoreTaskMPUSettings (default)
-        port: vPortReleaseTaskMPUSettings (default)
         if ESP_PANIC_HANDLER_IRAM != y:
             tasks: uxTaskGetSnapshotAll (default)
             tasks: prvTaskGetSnapshot (default)
             tasks: prvTaskGetSnapshotsFromList (default)
+        if IDF_TARGET_ESP32S2 =n && IDF_TARGET_ESP32C3 = n :
+            port: vPortReleaseTaskMPUSettings (default)
+            tasks: xTaskCreateRestricted (default)
+            port: vPortStoreTaskMPUSettings (default)
+            tasks: vTaskAllocateMPURegions (default)
         tasks: prvTaskCheckFreeStackSpace (default)
-        tasks: prvListTaskWithinSingleList (default)
         tasks: prvInitialiseNewTask (default)
         tasks: prvInitialiseTaskLists (default)
         tasks: prvDeleteTCB (default)
         tasks: prvCheckTasksWaitingTermination (default)
         tasks: prvIdleTask (default)
         tasks: prvAddNewTaskToReadyList (default)
-        tasks: xTaskCreateStaticPinnedToCore (default)
         tasks: xTaskCreatePinnedToCore (default)
         tasks: vTaskResume (default)
         tasks: vTaskStartScheduler (default)
@@ -44,16 +44,14 @@ entries:
         tasks: uxTaskResetEventItemValue (default)
         tasks: ulTaskNotifyTake (default)
         tasks: xTaskNotifyWait (default)
-        tasks: xTaskNotify (default)
+        tasks: xTaskGenericNotify (default)
         tasks: eTaskGetState (default)
         tasks: pxTaskGetStackStart (default)
         tasks: uxTaskGetStackHighWaterMark (default)
-        tasks: vTaskAllocateMPURegions (default)
         tasks: vTaskEndScheduler (default)
         tasks: vTaskList (default)
         tasks: vTaskMissedYield (default)
         tasks: vTaskSetThreadLocalStoragePointer (default)
-        tasks: xTaskCreateRestricted (default)
         tasks: xTaskGetAffinity (default)
         tasks: xTaskGetIdleTaskHandleForCPU (default)
         timers: prvInsertTimerInActiveList (default)
@@ -67,7 +65,6 @@ entries:
         timers: prvProcessReceivedCommands (default)
         timers: xTimerCreateTimerTask (default)
         timers: xTimerCreate (default)
-        timers: xTimerCreateStatic (default)
         timers: xTimerGenericCommand (default)
         timers: xTimerGetPeriod (default)
         timers: xTimerGetExpiryTime (default)
@@ -76,7 +73,6 @@ entries:
         timers: vTimerSetTimerID (default)
         timers: prvGetNextExpireTime (default)
         event_groups: prvTestWaitCondition (default)
-        event_groups: xEventGroupCreateStatic (default)
         event_groups: xEventGroupCreate (default)
         event_groups: xEventGroupWaitBits (default)
         event_groups: xEventGroupClearBits (default)
@@ -90,11 +86,9 @@ entries:
         queue: xQueueGenericReset (default)
         queue: xQueueGenericCreate (default)
         queue: xQueueGetMutexHolder (default)
-        queue: xQueueCreateCountingSemaphoreStatic (default)
         queue: xQueueCreateCountingSemaphore (default)
         queue: xQueueGenericSend (default)
         queue: xQueueCreateMutex (default)
-        queue: xQueueCreateMutexStatic (default)
         queue: xQueueGiveMutexRecursive (default)
         queue: xQueueTakeMutexRecursive (default)
         queue: uxQueueMessagesWaiting (default)
@@ -112,4 +106,16 @@ entries:
         queue: xQueueSelectFromSet (default)
     port_common:main_task (default)
     port:esp_startup_start_app (default)
-    port:esp_startup_start_app_other_cores (default)
+    if ESP_SYSTEM_SINGLE_CORE_MODE = n:
+        port:esp_startup_start_app_other_cores (default)
+
+if FREERTOS_SUPPORT_STATIC_ALLOCATION = y && FREERTOS_PLACE_FUNCTIONS_INTO_FLASH = y:
+    [mapping:freertos_static]
+    archive: libfreertos.a
+    entries:
+        queue:xQueueGenericCreateStatic (default)
+        queue: xQueueCreateCountingSemaphoreStatic (default)
+        tasks: xTaskCreateStaticPinnedToCore (default)
+        timers: xTimerCreateStatic (default)
+        event_groups: xEventGroupCreateStatic (default)
+        queue: xQueueCreateMutexStatic (default)

components/hal/linker.lf

@@ -10,12 +10,14 @@ entries:
     spi_flash_hal_iram (noflash)
     ledc_hal_iram (noflash)
     i2c_hal_iram (noflash)
-    spi_flash_hal_gpspi (noflash)
     cpu_hal (noflash)
     soc_hal (noflash)
     wdt_hal_iram (noflash)
-    systimer_hal (noflash)
-    if TWAI_ISR_IN_IRAM = y:
-        twai_hal_iram (noflash)
-    else:
-        twai_hal_iram (default)
+    if IDF_TARGET_ESP32C3 = n:
+        if TWAI_ISR_IN_IRAM = y:
+            twai_hal_iram (noflash)
+        else:
+            twai_hal_iram (default)
+    if IDF_TARGET_ESP32 = n:
+        spi_flash_hal_gpspi (noflash)
+        systimer_hal (noflash)

components/heap/linker.lf

@@ -3,4 +3,5 @@ archive: libheap.a
 entries:
     heap_tlsf (noflash)
     multi_heap (noflash)
-    multi_heap_poisoning (noflash)
+    if HEAP_POISONING_DISABLED = n:
+        multi_heap_poisoning (noflash)

components/lwip/linker.lf

@@ -62,7 +62,6 @@ entries:
     pbuf:pbuf_header_impl (noflash_text)
     pbuf:pbuf_header (noflash_text)
     pbuf:pbuf_free (noflash_text)
-    timeouts:sys_timeouts_mbox_fetch (noflash_text)
     udp:udp_input_local_match (noflash_text)
     udp:udp_input (noflash_text)
     udp:udp_send (noflash_text)

components/soc/esp32/CMakeLists.txt

@@ -9,7 +9,6 @@ set(srcs
     "pcnt_periph.c"
     "rmt_periph.c"
     "rtc_io_periph.c"
-    "rtc_periph.c"
     "sdio_slave_periph.c"
     "sdmmc_periph.c"
     "sigmadelta_periph.c"

components/soc/esp32s2/CMakeLists.txt

@@ -10,7 +10,6 @@ set(srcs
     "pcnt_periph.c"
     "rmt_periph.c"
     "rtc_io_periph.c"
-    "rtc_periph.c"
     "sigmadelta_periph.c"
     "soc_memory_layout.c"
     "spi_periph.c"

components/soc/esp32s3/CMakeLists.txt

@@ -11,7 +11,6 @@ set(srcs
     "pcnt_periph.c"
     "rmt_periph.c"
     "rtc_io_periph.c"
-    "rtc_periph.c"
     "sdio_slave_periph.c"
     "sdmmc_periph.c"
     "sigmadelta_periph.c"

components/xtensa/linker.lf

@@ -2,7 +2,8 @@
 archive: libxtensa.a
 entries:
     eri (noflash_text)
-    stdatomic (noflash)
+    if IDF_TARGET_ESP32S2 = y:
+        stdatomic (noflash)
 
 [mapping:xt_hal]
 archive: libxt_hal.a

tools/ci/check_ldgen_mapping_exceptions.txt

@@ -0,0 +1,3 @@
+libc
+sha256_coredump
+gcc

tools/ci/config/build.yml

@@ -120,6 +120,7 @@ build_ssc_esp32c3:
     BUILD_SYSTEM: "cmake"
     TEST_TYPE: "unit_test"
     PYTHON_VER: 3
+    LDGEN_CHECK_MAPPING: 1
   script:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
     - cd $CI_PROJECT_DIR/tools/unit-test-app
@@ -162,6 +163,7 @@ build_esp_idf_tests_cmake_esp32c3:
     LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
     BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
     PYTHON_VER: 3
+    LDGEN_CHECK_MAPPING: 1
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"
     # mechanism work, but this is the next best thing

tools/cmake/ldgen.cmake

@@ -56,6 +56,12 @@ function(__ldgen_process_template template output)
 
     idf_build_get_property(config_env_path CONFIG_ENV_PATH)
 
+    if($ENV{LDGEN_CHECK_MAPPING})
+        set(ldgen_check "--check-mapping"
+            "--check-mapping-exceptions=${idf_path}/tools/ci/check_ldgen_mapping_exceptions.txt")
+        message(STATUS "Mapping check enabled in ldgen")
+    endif()
+
     add_custom_command(
         OUTPUT ${output}
         COMMAND ${python} ${idf_path}/tools/ldgen/ldgen.py
@@ -67,6 +73,7 @@ function(__ldgen_process_template template output)
         --env-file  "${config_env_path}"
         --libraries-file ${build_dir}/ldgen_libraries
         --objdump   ${CMAKE_OBJDUMP}
+        ${ldgen_check}
         DEPENDS     ${template} ${ldgen_fragment_files} ${ldgen_depends} ${SDKCONFIG}
     )
 

tools/ldgen/generation.py

@@ -21,7 +21,7 @@ import fnmatch
 
 from fragments import Sections, Scheme, Mapping, Fragment
 from pyparsing import Suppress, White, ParseException, Literal, Group, ZeroOrMore
-from pyparsing import Word, OneOrMore, nums, alphanums, alphas, Optional, restOfLine
+from pyparsing import Word, OneOrMore, nums, alphas, restOfLine, SkipTo
 from ldgen_common import LdGenFailure
 
 
@@ -80,7 +80,7 @@ class PlacementRule():
         def do_section_expansion(rule, section):
             if section in rule.get_section_names():
                 sections_in_obj = sections_infos.get_obj_sections(rule.archive, rule.obj)
-                expansions = [n for n in sections_in_obj or [] if fnmatch.fnmatch(n, section)]
+                expansions = fnmatch.filter(sections_in_obj, section)
                 return expansions
 
         def remove_section_expansions(rule, section, expansions):
@@ -253,11 +253,18 @@ class GenerationModel:
 
     DEFAULT_SCHEME = "default"
 
-    def __init__(self):
+    def __init__(self, check_mappings=False, check_mapping_exceptions=None):
         self.schemes = {}
         self.sections = {}
         self.mappings = {}
 
+        self.check_mappings = check_mappings
+
+        if check_mapping_exceptions:
+            self.check_mapping_exceptions = check_mapping_exceptions
+        else:
+            self.check_mapping_exceptions = []
+
     def _add_mapping_rules(self, archive, obj, symbol, scheme_name, scheme_dict, rules):
         # Use an ordinary dictionary to raise exception on non-existing keys
         temp_dict = dict(scheme_dict)
@@ -337,6 +344,19 @@ class GenerationModel:
                 try:
                     if not (obj == Mapping.MAPPING_ALL_OBJECTS and symbol is None and
                             scheme_name == GenerationModel.DEFAULT_SCHEME):
+                        if self.check_mappings and mapping.name not in self.check_mapping_exceptions:
+                            if not obj == Mapping.MAPPING_ALL_OBJECTS:
+                                obj_sections = sections_infos.get_obj_sections(archive, obj)
+                                if not obj_sections:
+                                    message = "'%s:%s' not found" % (archive, obj)
+                                    raise GenerationException(message, mapping)
+
+                                if symbol:
+                                    obj_sym = fnmatch.filter(obj_sections, "*%s" % symbol)
+                                    if not obj_sym:
+                                        message = "'%s:%s %s' not found" % (archive, obj, symbol)
+                                        raise GenerationException(message, mapping)
+
                         self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
                 except KeyError:
                     message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
@@ -588,7 +608,7 @@ class SectionsInfo(dict):
         results = None
 
         try:
-            results = parser.parseString(first_line)
+            results = parser.parseString(first_line, parseAll=True)
         except ParseException as p:
             raise ParseException("Parsing sections info for library " + sections_info_dump.name + " failed. " + p.msg)
 
@@ -596,44 +616,57 @@ class SectionsInfo(dict):
         self.sections[archive] = SectionsInfo.__info(sections_info_dump.name, sections_info_dump.read())
 
     def _get_infos_from_file(self, info):
-        # Object file line: '{object}:  file format elf32-xtensa-le'
-        obj = Fragment.ENTITY.setResultsName("object") + Literal(":").suppress() + \
-            (Literal("file format elf32-") + (Literal("xtensa-le") | Literal("littleriscv"))).suppress()
+        # {object}:  file format elf32-xtensa-le
+        object_line = SkipTo(":").setResultsName("object") + Suppress(restOfLine)
 
-        # Sections table
-        header = Suppress(Literal("Sections:") + Literal("Idx") + Literal("Name") + Literal("Size") + Literal("VMA") +
-                          Literal("LMA") + Literal("File off") + Literal("Algn"))
-        entry = Word(nums).suppress() + Fragment.ENTITY + Suppress(OneOrMore(Word(alphanums, exact=8)) +
-                                                                   Word(nums + "*") + ZeroOrMore(Word(alphas.upper()) +
-                                                                   Optional(Literal(","))))
+        # Sections:
+        # Idx Name ...
+        section_start = Suppress(Literal("Sections:"))
+        section_header = Suppress(OneOrMore(Word(alphas)))
 
-        # Content is object file line + sections table
-        content = Group(obj + header + Group(ZeroOrMore(entry)).setResultsName("sections"))
+        # 00 {section} 0000000 ...
+        #              CONTENTS, ALLOC, ....
+        section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \
+            Suppress(ZeroOrMore(Word(alphas) + Literal(",")) + Word(alphas))
 
+        content = Group(object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName("sections"))
         parser = Group(ZeroOrMore(content)).setResultsName("contents")
 
-        sections_info_text = info.content
         results = None
 
         try:
-            results = parser.parseString(sections_info_text)
+            results = parser.parseString(info.content, parseAll=True)
         except ParseException as p:
             raise ParseException("Unable to parse section info file " + info.filename + ". " + p.msg)
 
         return results
 
     def get_obj_sections(self, archive, obj):
-        stored = self.sections[archive]
-
-        # Parse the contents of the sections file
-        if not isinstance(stored, dict):
-            parsed = self._get_infos_from_file(stored)
-            stored = dict()
-            for content in parsed.contents:
-                sections = list(map(lambda s: s, content.sections))
-                stored[content.object] = sections
-            self.sections[archive] = stored
-
-        for obj_key in stored.keys():
-            if obj_key == obj + ".o" or obj_key == obj + ".c.obj":
-                return stored[obj_key]
+        res = []
+        try:
+            stored = self.sections[archive]
+
+            # Parse the contents of the sections file on-demand,
+            # save the result for later
+            if not isinstance(stored, dict):
+                parsed = self._get_infos_from_file(stored)
+                stored = dict()
+                for content in parsed.contents:
+                    sections = list(map(lambda s: s, content.sections))
+                    stored[content.object] = sections
+                self.sections[archive] = stored
+
+            try:
+                res = stored[obj + ".o"]
+            except KeyError:
+                try:
+                    res = stored[obj + ".c.obj"]
+                except KeyError:
+                    try:
+                        res = stored[obj + ".cpp.obj"]
+                    except KeyError:
+                        res = stored[obj + ".S.obj"]
+        except KeyError:
+            pass
+
+        return res

tools/ldgen/ldgen.py

@@ -83,6 +83,18 @@ def main():
         "--kconfig", "-k",
         help="IDF Kconfig file")
 
+    argparser.add_argument(
+        "--check-mapping",
+        help="Perform a check if a mapping (archive, obj, symbol) exists",
+        action='store_true'
+    )
+
+    argparser.add_argument(
+        "--check-mapping-exceptions",
+        help="Mappings exempted from check",
+        type=argparse.FileType("r")
+    )
+
     argparser.add_argument(
         "--env", "-e",
         action='append', default=[],
@@ -106,6 +118,12 @@ def main():
     kconfig_file = args.kconfig
     objdump = args.objdump
 
+    check_mapping = args.check_mapping
+    if args.check_mapping_exceptions:
+        check_mapping_exceptions = [line.strip() for line in args.check_mapping_exceptions]
+    else:
+        check_mapping_exceptions = None
+
     try:
         sections_infos = SectionsInfo()
         for library in libraries_file:
@@ -115,7 +133,7 @@ def main():
                 dump.name = library
                 sections_infos.add_sections_info(dump)
 
-        generation_model = GenerationModel()
+        generation_model = GenerationModel(check_mapping, check_mapping_exceptions)
 
         _update_environment(args)  # assign args.env and args.env_file to os.environ
 

tools/ldgen/test/data/sections.info

@@ -1,4 +1,4 @@
-In archive /home/user/ãóç+ěščřžýáíé/build/esp-idf/freertos/libfreertos.a:
+In archive /home/user/build/esp-idf/freertos/libfreertos.a:
 
 FreeRTOS-openocd.c.obj:     file format elf32-xtensa-le
 

tools/ldgen/test/data/sections_parse.info

@@ -0,0 +1,19 @@
+In archive /home/user/ãóç+ěščřžýáíé/build/esp-idf/freertos/libsections_parse.a:
+
+croutine.c.obj:     file format elf32-littleriscv
+
+Sections:
+Idx Name          Size      VMA       LMA       File off  Algn
+  0 .text         00000000  00000000  00000000  00000034  2**0
+                  CONTENTS, ALLOC, LOAD, READONLY, CODE
+  1 .data         00000000  00000000  00000000  00000034  2**0
+                  CONTENTS, ALLOC, LOAD, DATA
+  2 .bss          00000000  00000000  00000000  00000034  2**0
+                  ALLOC
+
+FreeRTOS-openocd.c.obj:     file format elf32-xtensa-le // 'F' should not get included in match for 'CONTENTS, ALLOC, LOAD ...' prior
+
+Sections:
+Idx Name          Size      VMA       LMA       File off  Algn
+  0 .literal.prvCheckPendingReadyList 00000018  00000000  00000000  00000034  2**2
+                  CONTENTS, ALLOC, LOAD, RELOC, READONLY, CODE

tools/ldgen/test/test_generation.py

@@ -1422,6 +1422,19 @@ entries:
         self.assertListEqual(actual["flash_text"], expected["flash_text"])
         self.assertListEqual(actual["iram0_text"], expected["iram0_text"])
 
+    def test_sections_info_parsing(self):
+
+        self.sections_info = SectionsInfo()
+
+        with open("data/sections_parse.info") as sections_info_file_obj:
+            self.sections_info.add_sections_info(sections_info_file_obj)
+
+        sections = self.sections_info.get_obj_sections("libsections_parse.a", "croutine")
+        self.assertEqual(set(sections), set([".text", ".data", ".bss"]))
+
+        sections = self.sections_info.get_obj_sections("libsections_parse.a", "FreeRTOS-openocd")
+        self.assertEqual(set(sections), set([".literal.prvCheckPendingReadyList"]))
+
 
 if __name__ == "__main__":
     unittest.main()