Merge branch 'feature/common_fragment_grammar' into 'master'

ldgen: Common fragment grammar

See merge request idf/esp-idf!4576

components/app_trace/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:app_trace]
 archive: libapp_trace.a
 entries: 
     app_trace (noflash)
@@ -8,10 +8,11 @@ entries:
     SEGGER_SYSVIEW_Config_FreeRTOS (noflash)
     SEGGER_SYSVIEW_FreeRTOS (noflash)
 
-[mapping]
+[mapping:driver]
 archive: libdriver.a
 entries:
-    : SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y 
-      || SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y
-    timer (noflash)
-
+    if SYSVIEW_TS_SOURCE_TIMER_00 = y || SYSVIEW_TS_SOURCE_TIMER_01 = y
+       || SYSVIEW_TS_SOURCE_TIMER_10 = y || SYSVIEW_TS_SOURCE_TIMER_11 = y:
+      timer (noflash)
+    else:
+      * (default)

components/esp32/linker.lf

@@ -1,14 +1,14 @@
-[mapping]
+[mapping:esp32]
 archive: libesp32.a
 entries:
     panic (noflash)
 
-[mapping]
+[mapping:gcc]
 archive: libgcc.a
 entries:
     lib2funcs (noflash_text)
 
-[mapping]
+[mapping:gcov]
 archive: libgcov.a
 entries:
     * (noflash)

components/esp_ringbuf/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:esp_ringbuf]
 archive: libesp_ringbuf.a
 entries:
     * (noflash_text)

components/esp_wifi/linker.lf

@@ -1,22 +1,21 @@
-[mapping]
+[mapping:phy]
 archive: libphy.a
 entries:
     * (noflash_data)
 
-[mapping]
+[mapping:rtc]
 archive: librtc.a
 entries:
     * (noflash_text)
 
-[mapping]
+[mapping:pp]
 archive: libpp.a
 entries:
-    : ESP32_WIFI_IRAM_OPT = y
-    * (wifi_iram)
+    if ESP32_WIFI_IRAM_OPT = y:
+        * (wifi_iram)
 
-[mapping]
+[mapping:net80211]
 archive: libnet80211.a
 entries:
-    : ESP32_WIFI_IRAM_OPT = y
-    * (wifi_iram)
-
+    if ESP32_WIFI_IRAM_OPT = y:
+        * (wifi_iram)

components/espcoredump/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:espcoredump]
 archive: libespcoredump.a
 entries: 
     core_dump_uart (noflash_text)

components/freertos/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:freertos]
 archive: libfreertos.a
 entries:
     * (noflash_text)

components/heap/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:heap]
 archive: libheap.a
 entries:
     multi_heap (noflash)

components/lwip/linker.lf

@@ -1,7 +1,7 @@
-[mapping]
+[mapping:lwip]
 archive: liblwip.a
 entries:
-    :LWIP_IRAM_OPTIMIZATION = y
+  if LWIP_IRAM_OPTIMIZATION = y:
     ethernetif:ethernet_low_level_output (noflash_text)
     ethernetif:ethernetif_input (noflash_text)
     wlanif:low_level_output (noflash_text)
@@ -55,3 +55,5 @@ entries:
     pbuf:pbuf_header_impl (noflash_text)
     pbuf:pbuf_header (noflash_text)
     ethernet:ethernet_input (noflash_text)
+  else:
+    * (default)

components/newlib/esp32-spiram-rom-functions.lf

@@ -8,137 +8,139 @@
 # Note: We currently never link libg-psram-workaround.a, so no rules
 # are generated for this library
 
-[mapping]
+[mapping:libc_psram_workaround]
 archive: libc-psram-workaround.a
 entries:
-  : SPIRAM_CACHE_WORKAROUND = y
-  lib_a-utoa (noflash)
-  lib_a-longjmp (noflash)
-  lib_a-setjmp (noflash)
-  lib_a-abs (noflash)
-  lib_a-div (noflash)
-  lib_a-labs (noflash)
-  lib_a-ldiv (noflash)
-  lib_a-quorem (noflash)
-  lib_a-utoa (noflash)
-  lib_a-itoa (noflash)
-  lib_a-atoi (noflash)
-  lib_a-atol (noflash)
-  lib_a-strtol (noflash)
-  lib_a-strtoul (noflash)
-  lib_a-wcrtomb (noflash)
-  lib_a-fvwrite (noflash)
-  lib_a-wbuf (noflash)
-  lib_a-wsetup (noflash)
-  lib_a-fputwc (noflash)
-  lib_a-wctomb_r (noflash)
-  lib_a-ungetc (noflash)
-  lib_a-makebuf (noflash)
-  lib_a-fflush (noflash)
-  lib_a-refill (noflash)
-  lib_a-s_fpclassify (noflash)
-  lib_a-asctime (noflash)
-  lib_a-ctime (noflash)
-  lib_a-ctime_r (noflash)
-  lib_a-lcltime (noflash)
-  lib_a-lcltime_r (noflash)
-  lib_a-gmtime (noflash)
-  lib_a-gmtime_r (noflash)
-  lib_a-strftime (noflash)
-  lib_a-mktime (noflash)
-  lib_a-syswrite (noflash)
-  lib_a-tzset_r (noflash)
-  lib_a-tzset (noflash)
-  lib_a-toupper (noflash)
-  lib_a-tolower (noflash)
-  lib_a-toascii (noflash)
-  lib_a-systimes (noflash)
-  lib_a-time (noflash)
-  lib_a-gettzinfo (noflash)
-  lib_a-strupr (noflash)
-  lib_a-asctime_r (noflash)
-  lib_a-bzero (noflash)
-  lib_a-close (noflash)
-  lib_a-creat (noflash)
-  lib_a-environ (noflash)
-  lib_a-fclose (noflash)
-  lib_a-isalnum (noflash)
-  lib_a-isalpha (noflash)
-  lib_a-isascii (noflash)
-  lib_a-isblank (noflash)
-  lib_a-iscntrl (noflash)
-  lib_a-isdigit (noflash)
-  lib_a-isgraph (noflash)
-  lib_a-islower (noflash)
-  lib_a-isprint (noflash)
-  lib_a-ispunct (noflash)
-  lib_a-isspace (noflash)
-  lib_a-isupper (noflash)
-  lib_a-memccpy (noflash)
-  lib_a-memchr (noflash)
-  lib_a-memcmp (noflash)
-  lib_a-memcpy (noflash)
-  lib_a-memmove (noflash)
-  lib_a-memrchr (noflash)
-  lib_a-memset (noflash)
-  lib_a-open (noflash)
-  lib_a-rand (noflash)
-  lib_a-rand_r (noflash)
-  lib_a-read (noflash)
-  lib_a-rshift (noflash)
-  lib_a-sbrk (noflash)
-  lib_a-srand (noflash)
-  lib_a-strcasecmp (noflash)
-  lib_a-strcasestr (noflash)
-  lib_a-strcat (noflash)
-  lib_a-strchr (noflash)
-  lib_a-strcmp (noflash)
-  lib_a-strcoll (noflash)
-  lib_a-strcpy (noflash)
-  lib_a-strcspn (noflash)
-  lib_a-strdup (noflash)
-  lib_a-strlcat (noflash)
-  lib_a-strlcpy (noflash)
-  lib_a-strlen (noflash)
-  lib_a-strlwr (noflash)
-  lib_a-strncasecmp (noflash)
-  lib_a-strncat (noflash)
-  lib_a-strncmp (noflash)
-  lib_a-strncpy (noflash)
-  lib_a-strndup (noflash)
-  lib_a-strnlen (noflash)
-  lib_a-strrchr (noflash)
-  lib_a-strsep (noflash)
-  lib_a-strspn (noflash)
-  lib_a-strstr (noflash)
-  lib_a-strtok_r (noflash)
-  lib_a-strupr (noflash)
-  lib_a-stdio (noflash)
-  lib_a-syssbrk (noflash)
-  lib_a-sysclose (noflash)
-  lib_a-sysopen (noflash)
-  creat (noflash)
-  lib_a-sysread (noflash)
-  lib_a-syswrite (noflash)
-  lib_a-impure (noflash)
-  lib_a-tzvars (noflash)
-  lib_a-sf_nan (noflash)
-  lib_a-tzcalc_limits (noflash)
-  lib_a-month_lengths (noflash)
-  lib_a-timelocal (noflash)
-  lib_a-findfp (noflash)
-  lock (noflash)
-  lib_a-getenv_r (noflash)
-  isatty (noflash)
-  lib_a-fwalk (noflash)
-  lib_a-getenv_r (noflash)
-  lib_a-tzlock (noflash)
-  lib_a-ctype_ (noflash)
-  lib_a-sccl (noflash)
-  lib_a-strptime (noflash)
-  lib_a-envlock (noflash)
-  lib_a-raise (noflash)
-  lib_a-strdup_r (noflash)
-  lib_a-system (noflash)
-  lib_a-strndup_r (noflash)
+  if SPIRAM_CACHE_WORKAROUND = y:
+    lib_a-utoa (noflash)
+    lib_a-longjmp (noflash)
+    lib_a-setjmp (noflash)
+    lib_a-abs (noflash)
+    lib_a-div (noflash)
+    lib_a-labs (noflash)
+    lib_a-ldiv (noflash)
+    lib_a-quorem (noflash)
+    lib_a-utoa (noflash)
+    lib_a-itoa (noflash)
+    lib_a-atoi (noflash)
+    lib_a-atol (noflash)
+    lib_a-strtol (noflash)
+    lib_a-strtoul (noflash)
+    lib_a-wcrtomb (noflash)
+    lib_a-fvwrite (noflash)
+    lib_a-wbuf (noflash)
+    lib_a-wsetup (noflash)
+    lib_a-fputwc (noflash)
+    lib_a-wctomb_r (noflash)
+    lib_a-ungetc (noflash)
+    lib_a-makebuf (noflash)
+    lib_a-fflush (noflash)
+    lib_a-refill (noflash)
+    lib_a-s_fpclassify (noflash)
+    lib_a-asctime (noflash)
+    lib_a-ctime (noflash)
+    lib_a-ctime_r (noflash)
+    lib_a-lcltime (noflash)
+    lib_a-lcltime_r (noflash)
+    lib_a-gmtime (noflash)
+    lib_a-gmtime_r (noflash)
+    lib_a-strftime (noflash)
+    lib_a-mktime (noflash)
+    lib_a-syswrite (noflash)
+    lib_a-tzset_r (noflash)
+    lib_a-tzset (noflash)
+    lib_a-toupper (noflash)
+    lib_a-tolower (noflash)
+    lib_a-toascii (noflash)
+    lib_a-systimes (noflash)
+    lib_a-time (noflash)
+    lib_a-gettzinfo (noflash)
+    lib_a-strupr (noflash)
+    lib_a-asctime_r (noflash)
+    lib_a-bzero (noflash)
+    lib_a-close (noflash)
+    lib_a-creat (noflash)
+    lib_a-environ (noflash)
+    lib_a-fclose (noflash)
+    lib_a-isalnum (noflash)
+    lib_a-isalpha (noflash)
+    lib_a-isascii (noflash)
+    lib_a-isblank (noflash)
+    lib_a-iscntrl (noflash)
+    lib_a-isdigit (noflash)
+    lib_a-isgraph (noflash)
+    lib_a-islower (noflash)
+    lib_a-isprint (noflash)
+    lib_a-ispunct (noflash)
+    lib_a-isspace (noflash)
+    lib_a-isupper (noflash)
+    lib_a-memccpy (noflash)
+    lib_a-memchr (noflash)
+    lib_a-memcmp (noflash)
+    lib_a-memcpy (noflash)
+    lib_a-memmove (noflash)
+    lib_a-memrchr (noflash)
+    lib_a-memset (noflash)
+    lib_a-open (noflash)
+    lib_a-rand (noflash)
+    lib_a-rand_r (noflash)
+    lib_a-read (noflash)
+    lib_a-rshift (noflash)
+    lib_a-sbrk (noflash)
+    lib_a-srand (noflash)
+    lib_a-strcasecmp (noflash)
+    lib_a-strcasestr (noflash)
+    lib_a-strcat (noflash)
+    lib_a-strchr (noflash)
+    lib_a-strcmp (noflash)
+    lib_a-strcoll (noflash)
+    lib_a-strcpy (noflash)
+    lib_a-strcspn (noflash)
+    lib_a-strdup (noflash)
+    lib_a-strlcat (noflash)
+    lib_a-strlcpy (noflash)
+    lib_a-strlen (noflash)
+    lib_a-strlwr (noflash)
+    lib_a-strncasecmp (noflash)
+    lib_a-strncat (noflash)
+    lib_a-strncmp (noflash)
+    lib_a-strncpy (noflash)
+    lib_a-strndup (noflash)
+    lib_a-strnlen (noflash)
+    lib_a-strrchr (noflash)
+    lib_a-strsep (noflash)
+    lib_a-strspn (noflash)
+    lib_a-strstr (noflash)
+    lib_a-strtok_r (noflash)
+    lib_a-strupr (noflash)
+    lib_a-stdio (noflash)
+    lib_a-syssbrk (noflash)
+    lib_a-sysclose (noflash)
+    lib_a-sysopen (noflash)
+    creat (noflash)
+    lib_a-sysread (noflash)
+    lib_a-syswrite (noflash)
+    lib_a-impure (noflash)
+    lib_a-tzvars (noflash)
+    lib_a-sf_nan (noflash)
+    lib_a-tzcalc_limits (noflash)
+    lib_a-month_lengths (noflash)
+    lib_a-timelocal (noflash)
+    lib_a-findfp (noflash)
+    lock (noflash)
+    lib_a-getenv_r (noflash)
+    isatty (noflash)
+    lib_a-fwalk (noflash)
+    lib_a-getenv_r (noflash)
+    lib_a-tzlock (noflash)
+    lib_a-ctype_ (noflash)
+    lib_a-sccl (noflash)
+    lib_a-strptime (noflash)
+    lib_a-envlock (noflash)
+    lib_a-raise (noflash)
+    lib_a-strdup_r (noflash)
+    lib_a-system (noflash)
+    lib_a-strndup_r (noflash)
+  else:
+    * (default)
 

components/soc/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:soc]
 archive: libsoc.a
 entries: 
     cpu_util (noflash_text)

components/spi_flash/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:spi_flash]
 archive: libspi_flash.a
 entries: 
     spi_flash_rom_patch (noflash_text)

components/vfs/test/linker.lf

@@ -1,4 +1,4 @@
-[mapping]
+[mapping:vfs]
 archive: libvfs.a
 entries:
     vfs:esp_vfs_open (noflash)

components/xtensa/linker.lf

@@ -1,9 +1,9 @@
-[mapping]
+[mapping:xtensa]
 archive: libxtensa.a
 entries:
     eri (noflash_text)
 
-[mapping]
+[mapping:hal]
 archive: libhal.a
 entries:
     * (noflash_text)

docs/en/api-guides/linker-script-generation.rst

@@ -5,131 +5,131 @@ Linker Script Generation
 Overview
 --------
 
-There are several :ref:`memory regions<memory-layout>` where code and data can be placed. Usually, code and read-only data are placed in flash regions,
-writable data in RAM, etc. A common action is changing where code/data are mapped by default, say placing critical code/rodata in RAM for performance
-reasons or placing code/data/rodata in RTC memory for use in a wake stub or the ULP coprocessor.
+There are several :ref:`memory regions<memory-layout>` where code and data can be placed. Code and read-only data are placed by default in flash,
+writable data in RAM, etc. However, it is sometimes necessary to change these default placements. For example, it may
+be necessary to place critical code in RAM for performance reasons or to place code in RTC memory for use in a wake stub or the ULP coprocessor.
 
-IDF provides the ability for defining these placements at the component level using the linker script generation mechanism. The component presents
-how it would like to map the input sections of its object files (or even functions/data) through :ref:`linker fragment files<ldgen-fragment-files>`. During app build, 
-the linker fragment files are collected, parsed and processed; and the :ref:`linker script template<ldgen-script-templates>` is augmented with
-information generated from the fragment files to produce the final linker script. This linker script is then used for the linking
-the final app binary.
+With the linker script generation mechanism, it is possible to specify these placements at the component level within ESP-IDF. The component presents
+information on how it would like to place its symbols, objects or the entire archive. During build the information presented by the components are collected,
+parsed and processed; and the placement rules generated is used to link the app.
 
 Quick Start
 ------------
 
-This section presents a guide for quickly placing code/data to RAM and RTC memory; as well as demonstrating how to make these placements 
-dependent on project configuration values. In a true quick start fashion, this section glosses over terms and concepts that will be discussed 
-at a later part of the document. However, whenever it does so, it provides a link to the relevant section on the first mention.
+This section presents a guide for quickly placing code/data to RAM and RTC memory - placements ESP-IDF provides out-of-the-box.
 
-.. _ldgen-add-fragment-file :
+For this guide, suppose we have the following::
 
-Preparation
-^^^^^^^^^^^
+    - components/
+                    - my_component/                              
+                                    - CMakeLists.txt
+                                    - component.mk
+                                    - Kconfig                                
+                                    - src/
+                                          - my_src1.c                          
+                                          - my_src2.c                          
+                                          - my_src3.c                       
+                                    - my_linker_fragment_file.lf          
+
+
+- a component named ``my_component`` that is archived as library ``libmy_component.a`` during build
+- three source files archived under the library, ``my_src1.c``, ``my_src2.c`` and ``my_src3.c`` which are compiled as ``my_src1.o``, ``my_src2.o`` and ``my_src3.o``, respectively
+- under ``my_src1.o``, the function ``my_function1`` is defined; under ``my_src2.o``, the function ``my_function2`` is defined
+- there exist bool-type config ``PERFORMANCE_MODE`` (y/n) and int type config ``PERFORMANCE_LEVEL`` (with range 0-3) in my_component's Kconfig
+
+
+Creating and Specifying a Linker Fragment File
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Before anything else, a linker fragment file needs to be created. A linker fragment file
+is simply a text file with a ``.lf`` extension upon which the desired placements will be written.
+After creating the file, it is then necessary to present it to the build system. The instructions for the build systems
+supported by ESP-IDF are as follows:
 
 Make
 """"
 
-Create a linker fragment file inside the component directory, which is just a text file with a .lf extension. In order for the build system to collect your fragment file,
-add an entry to it from the component, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to your linker file/s before the ``register_component`` call.
+In the component's ``component.mk`` file, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to the path of the created linker
+fragment file. The path can either be an absolute path or a relative path from the component directory.
 
 .. code-block:: make
 
-    # file paths relative to component Makefile 
-    COMPONENT_ADD_LDFRAGMENTS += "path/to/linker_fragment_file.lf" "path/to/another_linker_fragment_file.lf"
+    COMPONENT_ADD_LDFRAGMENTS += "my_linker_fragment_file.lf"
 
 CMake
 """""
 
-For CMake set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to your linker file/s before the ``register_component`` call.
+In the component's ``CMakeLists.txt`` file, set the variable ``COMPONENT_ADD_LDFRAGMENTS`` to the path of the created linker
+fragment file before the ``register_component`` call. The path can either be an absolute path or a relative path from the component directory.
 
 .. code-block:: cmake
 
-    # file paths relative to CMakeLists.txt  
-    set(COMPONENT_ADD_LDFRAGMENTS "path/to/linker_fragment_file.lf" "path/to/another_linker_fragment_file.lf")
-
+    set(COMPONENT_ADD_LDFRAGMENTS "my_linker_fragment_file.lf")
     register_component()
 
-It is also possible to specify fragment files from the project CMakeLists.txt or component project_include.cmake using the function `ldgen_add_fragment_files`::
-
-    ldgen_add_fragment_files(target files ...)
-
 
 Specifying placements
 ^^^^^^^^^^^^^^^^^^^^^
 
-This mechanism allows specifying placement of the following entities:
-
-    - one or multiple object files within the component
-    - one or multiple function/variable using their names
-    - the entire component library
-
-For the following text, suppose we have the following:
-
-    - a component named ``component`` that is archived as library ``libcomponent.a`` during build
-    - three object files archived under the library, ``object1.o``, ``object2.o`` and ``object3.o``
-    - under ``object1.o``, the function ``function1`` is defined; under ``object2.o``, the function ``function2`` is defined
-    - there exist configuration ``PERFORMANCE_MODE`` and ``PERFORMANCE_LEVEL`` in one of the IDF KConfig files, with the set value indicated by entries ``CONFIG_PERFORMANCE_MODE`` and ``CONFIG_PERFORMANCE_LEVEL`` in the project sdkconfig
-
-In the created linker fragment file, we write:
-
-.. code-block:: none
-
-    [mapping]
-    archive: libcomponent.a
-    entries:
+It is possible to specify placements at the following levels of granularity:
 
-This creates an empty :ref:`mapping fragment<ldgen-mapping-fragment>`, which doesn't do anything yet. During linking the :ref:`default placements<ldgen-default-placements>` 
-will still be used for ``libcomponent.a``, unless the ``entries`` key is populated.
+    - object file (``.obj`` or ``.o`` files)
+    - symbol (function/variable)
+    - archive (``.a`` files)
 
 .. _ldgen-placing-object-files :
 
 Placing object files
 """"""""""""""""""""
 
-Suppose the entirety of ``object1.o``  is performance-critical, so it is desirable to place it in RAM. On the other hand, suppose all of ``object2.o`` contains things to be executed coming out of deep sleep, so it needs to be put under RTC memory. We can write:
+Suppose the entirety of ``my_src1.o`` is performance-critical, so it is desirable to place it in RAM. 
+On the other hand, the entirety of ``my_src2.o`` contains symbols needed coming out of deep sleep, so it needs to be put under RTC memory.
+In the the linker fragment file, we can write:
 
 .. code-block:: none
 
-    [mapping]
-    archive: libcomponent.a
+    [mapping:my_component]
+    archive: libmy_component.a
     entries:
-        object1 (noflash)     # places all code / read-only data under IRAM/ DRAM
-        object2 (rtc)         # places all code/ data and read-only data under RTC fast memory/ RTC slow memory
+        my_src1 (noflash)     # places all my_src1 code/read-only data under IRAM/DRAM
+        my_src2 (rtc)         # places all my_src2 code/ data and read-only data under RTC fast memory/RTC slow memory
 
-What happens to ``object3.o``? Since it is not specified, default placements are used for ``object3.o``.
+What happens to ``my_src3.o``? Since it is not specified, default placements are used for ``my_src3.o``. More on default placements
+:ref:`here<ldgen-default-placements>`. 
 
-Placing functions/data using their names
-""""""""""""""""""""""""""""""""""""""""
+Placing symbols
+""""""""""""""""
 
-Continuing our example, suppose that among functions defined under ``object1.o``, only ``function1`` is performance-critical; and under ``object2.o``,
-only ``function2`` needs to execute after the chip comes out of deep sleep. This could be accomplished by writing:
+Continuing our example, suppose that among functions defined under ``object1.o``, only ``my_function1`` is performance-critical; and under ``object2.o``,
+only ``my_function2`` needs to execute after the chip comes out of deep sleep. This could be accomplished by writing:
 
 .. code-block:: none
 
-    [mapping]
-    archive: libcomponent.a
+    [mapping:my_component]
+    archive: libmy_component.a
     entries:
-        object1:function1 (noflash) 
-        object2:function2 (rtc) 
+        my_src1:my_function1 (noflash)
+        my_src2:my_function2 (rtc)
+
+The default placements are used for the rest of the functions in ``my_src1.o`` and ``my_src2.o`` and the entire ``object3.o``. Something similar
+can be achieved for placing data by writing the variable name instead of the function name, like so::
 
-The default placements are used for the rest of the functions in ``object1.o`` and ``object2.o`` and the entire ``object3.o``. Something similar
-can be achieved for placing data by writing the variable name instead of the function name after ``:``.
+       my_src1:my_variable (noflash)
 
 .. warning::
 
-    There are :ref:`limitations<ldgen-type1-limitations>` in placing code/data using their symbol names. In order to ensure proper placements, an alternative would be to group
-    relevant code and data into source files, and :ref:`use object file placement<ldgen-placing-object-files>`.
+    There are :ref:`limitations<ldgen-symbol-granularity-placements>` in placing code/data at symbol granularity. In order to ensure proper placements, an alternative would be to group
+    relevant code and data into source files, and :ref:`use object-granularity placements<ldgen-placing-object-files>`.
 
-Placing entire component
-""""""""""""""""""""""""
+Placing entire archive
+"""""""""""""""""""""""
 
-In this example, suppose that the entire component needs to be placed in RAM. This can be written as:
+In this example, suppose that the entire component archive needs to be placed in RAM. This can be written as:
 
 .. code-block:: none
 
-    [mapping]
-    archive: libcomponent.a
+    [mapping:my_component]
+    archive: libmy_component.a
     entries:
         * (noflash)
 
@@ -137,122 +137,226 @@ Similarly, this places the entire component in RTC memory:
 
 .. code-block:: none
 
-    [mapping]
-    archive: libcomponent.a
+    [mapping:my_component]
+    archive: libmy_component.a
     entries:
         * (rtc)
 
 Configuration-dependent placements
 """"""""""""""""""""""""""""""""""
 
-Suppose that the entire component library should only be placed when ``CONFIG_PERFORMANCE_MODE == y`` in the sdkconfig. This could be written as:
+Suppose that the entire component library should only have special placement when a certain condition is true; for example, when ``CONFIG_PERFORMANCE_MODE == y``.
+This could be written as:
 
 .. code-block:: none
 
-    [mapping]
-    archive: libcomponent.a
+    [mapping:my_component]
+    archive: libmy_component.a
     entries:
-        : PERFORMANCE_MODE = y
-        * (noflash)
-
-In pseudocode, this translates to:
+        if PERFORMANCE_MODE = y:
+            * (noflash)
+        else:
+            * (default)
 
-.. code-block:: none
-
-    if PERFORMANCE_MODE = y
-        place entire libcomponent.a in RAM
-    else
-        use default placements
-
-It is also possible to have multiple conditions to test. Suppose the following requirements: when ``CONFIG_PERFORMANCE_LEVEL == 1``, only ``object1.o`` is put in RAM;
+For a more complex config-dependent placement, suppose the following requirements: when ``CONFIG_PERFORMANCE_LEVEL == 1``, only ``object1.o`` is put in RAM;
 when ``CONFIG_PERFORMANCE_LEVEL == 2``, ``object1.o`` and ``object2.o``; and when ``CONFIG_PERFORMANCE_LEVEL == 3`` all object files under the archive
-are to be put into RAM. When these three are false however, put entire library in RTC memory. This scenario is a bit contrived, but, 
+are to be put into RAM. When these three are false however, put entire library in RTC memory. This scenario is a bit contrived, but,
 it can be written as:
 
 .. code-block:: none
 
-    [mapping]
-    archive: libcomponent.a
+    [mapping:my_component]
+    archive: libmy_component.a
     entries:
-        : PERFORMANCE_LEVEL = 3
-        * (noflash)
-        : PERFORMANCE_LEVEL = 2
-        object1 (noflash)
-        object2 (noflash)
-        : PERFORMANCE_LEVEL = 1
-        object1 (noflash)
-        : default
-        * (rtc)
-
-Which reads:
+        if PERFORMANCE_LEVEL = 1:
+            my_src1 (noflash)
+        elif PERFORMANCE_LEVEL = 2:
+            my_src1 (noflash)
+            my_src2 (noflash)
+        elif PERFORMANCE_LEVEL = 3:
+            my_src1 (noflash)
+            my_src2 (noflash)
+            my_src3 (noflash)
+        else:
+            * (rtc)
+
+Nesting condition-checking is also possible. The following is equivalent to the snippet above:
 
 .. code-block:: none
 
-    if CONFIG_PERFORMANCE_LEVEL == 3
-        place entire libcomponent.a in RAM
-    else if CONFIG_PERFORMANCE_LEVEL == 2
-        only place object1.o and object2.o in RAM
-    else if CONFIG_PERFORMANCE_LEVEL == 1
-        only place object1.o in RAM
-    else
-        place entire libcomponent.a in RTC memory 
-
-The conditions test :ref:`support other operations<ldgen-condition-entries>`.
+    [mapping:my_component]
+    archive: libmy_component.a
+    entries:
+        if PERFORMANCE_LEVEL <= 3 && PERFORMANCE_LEVEL > 0:
+            if PERFORMANCE_LEVEL >= 1:
+                object1 (noflash)
+                if PERFORMANCE_LEVEL >= 2:
+                    object2 (noflash)
+                    if PERFORMANCE_LEVEL >= 3:
+                        object2 (noflash)
+        else:
+            * (rtc)
 
 .. _ldgen-default-placements:
 
 The 'default' placements
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-Up until this point, the term  'default placements' has been mentioned as fallback placements for when the 
-placement rules ``rtc`` and ``noflash`` are not specified. The tokens ``noflash`` or ``rtc`` are not merely keywords known by the mechanism, but are actually 
-objects called :ref:`scheme fragments<ldgen-scheme-fragment>` that are specified by the user. Due to the commonness of these placement use cases,
-they are pre-defined in IDF.
+Up until this point, the term  'default placements' has been mentioned as fallback placements for when the
+placement rules ``rtc`` and ``noflash`` are not specified. It is important to note that the tokens ``noflash`` or ``rtc`` are not merely keywords, but are actually
+entities called fragments, specifically :ref:`schemes<ldgen-scheme-fragment>`.
 
-Similarly, there exists a ``default`` scheme fragment which defines what the default placement rules should be, which is discussed :ref:`here<ldgen-default-scheme>`.
+In the same manner as ``rtc`` and ``noflash`` are schemes, there exists a ``default`` scheme which defines what the default placement rules should be.
+As the name suggests, it is where code and data are usually placed, i.e. code/constants is placed in flash, variables
+placed in RAM, etc.  More on the default scheme :ref:`here<ldgen-default-scheme>`.
 
 .. note::
-    For an example of an IDF component using this feature, see :component_file:`freertos/CMakeLists.txt`. The ``freertos`` component uses this 
-    mechanism to place all code, literal and rodata of all of its object files to the instruction RAM memory region for performance reasons.
+    For an example of an ESP-IDF component using the linker script generation mechanism, see :component_file:`freertos/CMakeLists.txt`. 
+    ``freertos`` uses this to place its object files to the instruction RAM for performance reasons.
+
+This marks the end of the quick start guide. The following text discusses the internals of the mechanism in a little bit more detail.
+The following sections should be helpful in creating custom placements or modifying default behavior.
 
-This marks the end of the quick start guide. The following text discusses this mechanism in a little bit more detail, such its components, essential concepts,
-the syntax, how it is integrated with the build system, etc. The following sections should be helpful in creating custom mappings or modifying default 
-behavior.
+Linker Script Generation Internals
+----------------------------------
 
-Components
-----------
+Linking is the last step in the process of turning C/C++ source files into an executable. It is performed by the toolchain's linker, and accepts
+linker scripts which specify code/data placements, among other things. With the linker script generation mechanism, this process is no different, except
+that the linker script passed to the linker is dynamically generated from: (1) the collected :ref:`linker fragment files<ldgen-linker-fragment-files>` and 
+(2) :ref:`linker script template<ldgen-linker-script-template>`.
+
+.. note::
 
-.. _ldgen-fragment-files :
+    The tool that implements the linker script generation mechanism lives under :idf:`tools/ldgen`.
+
+.. _ldgen-linker-fragment-files :
 
 Linker Fragment Files
 ^^^^^^^^^^^^^^^^^^^^^
 
-The fragment files contain objects called 'fragments'. These fragments contain pieces of information which, when put together, form
-placement rules that tell where to place sections of object files in the output binary.
+As mentioned in the quick start guide, fragment files are simple text files with the ``.lf`` extension containing the desired placements. This is a simplified
+description of what fragment files contain, however. What fragment files actually contain are 'fragments'. Fragments are entities which contain pieces of information which, when put together, form
+placement rules that tell where to place sections of object files in the output binary. There are three types of fragments: :ref:`sections<ldgen-sections-fragment>`, 
+:ref:`scheme<ldgen-scheme-fragment>` and :ref:`mapping<ldgen-mapping-fragment>`.
+
+Grammar
+"""""""
 
-Another way of putting it is that processing linker fragment files aims to create the section placement rules inside GNU LD ``SECTIONS`` command. 
-Where to collect and put these section placement rules is represented internally as a ``target`` token.
+The three fragment types share a common grammar:
 
-The three types of fragments are discussed below.
+.. code-block:: none
+
+    [type:name]
+    key: value
+    key:
+        value
+        value
+        value
+        ...
+
+- type: Corresponds to the fragment type, can either be ``sections``, ``scheme`` or ``mapping``.
+- name: The name of the fragment, should be unique for the specified fragment type.
+- key, value: Contents of the fragment; each fragment type may support different keys and different grammars for the key values.
 
 .. note::
 
-    Fragments have a name property (except mapping fragments) and are known globally. 
-    Fragment naming follows C variable naming rules, i.e. case sensitive, must begin with a letter or underscore, alphanumeric/underscore after
-    initial characters are allowed, no spaces/special characters. Each type of fragment has its own namespace. In cases where multiple fragments
-    of the same type and name are encountered, an exception is thrown.
+    In cases where multiple fragments of the same type and name are encountered, an exception is thrown.
 
-.. _ldgen-sections-fragment :
+.. note::
+
+    The only valid characters for fragment names and keys are alphanumeric characters and underscore.
 
-I. Sections
-"""""""""""
 
-Sections fragments defines a list of object file sections that the GCC compiler emits. It may be a default section (e.g. ``.text``, ``.data``) or 
-it may be user defined section through the ``__attribute__`` keyword. 
+.. _ldgen-condition-checking :
 
-The use of an optional '+' indicates the inclusion of the section in the list, as well as sections that start with it. This is the preferred method over listing both explicitly. 
+**Condition Checking**
 
-**Syntax**
+Condition checking enable the linker script generation to be configuration-aware. Depending on whether expressions involving configuration values
+are true or not, a particular set of values for a key can be used. The evaluation uses ``eval_string`` from :idf_file:`tools/kconfig_new/kconfiglib.py` 
+and adheres to its required syntax and limitations. Supported operators are as follows:
+
+    - comparison
+        - LessThan ``<``
+        - LessThanOrEqualTo ``<=``
+        - MoreThan ``>``
+        - MoreThanOrEqualTo ``>=``
+        - Equal ``=``
+        - NotEqual ``!=``
+    - logical
+        - Or ``||``
+        - And ``&&``
+        - Negation ``!``
+    - grouping
+        - Parenthesis ``()``
+
+Condition checking behaves as you would expect an ``if...elseif/elif...else`` block in other languages. Condition-checking is possible
+for both key values and entire fragments. The two sample fragments below are equivalent:
+
+.. code-block:: none
+
+    # Value for keys is dependent on config
+    [type:name]
+    key_1:
+        if CONDITION = y:
+            value_1
+        else:
+            value_2
+    key_2:
+        if CONDITION = y:
+            value_a
+        else:
+            value_b
+
+.. code-block:: none
+
+    # Entire fragment definition is dependent on config
+    if CONDITION = y:
+        [type:name]
+        key_1:
+            value_1
+        key_2:
+            value_b
+    else:
+        [type:name]
+        key_1:
+            value_2
+        key_2:
+            value_b
+
+
+**Comments**
+
+Comment in linker fragment files begin with ``#``. Like in other languages, comment are used to provide helpful descriptions and documentation
+and are ignored during processing. 
+
+Compatibility with ESP-IDF v3.x Linker Script Fragment Files
+""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+ESP-IDF v4.0 brings some changes to the linker script fragment file grammar:
+
+- indentation is enforced and improperly indented fragment files generate a parse exception; this was not enforced in the old version but previous documentation
+    and examples demonstrates properly indented grammar
+- move to ``if...elif...else`` structure for conditionals, with the ability to nest checks and place entire fragments themselves inside conditionals
+- mapping fragments now requires a name like other fragment types
+
+Linker script generator should be able to parse ESP-IDF v3.x linker fragment files that are indented properly (as demonstrated by
+the ESP-IDF v3.x version of this document). Backward compatibility with the previous mapping fragment grammar (optional
+name and the old grammar for conditionals) has also been retained but with a deprecation warning. Users should switch to the newer grammar discussed
+in this document as support for the old grammar is planned to be removed in the future.
+
+Note that linker fragment files using the new ESP-IDF v4.0 grammar is not supported on ESP-IDF v3.x, however.
+
+Types
+"""""
+
+.. _ldgen-sections-fragment :
+
+**Sections**
+
+Sections fragments defines a list of object file sections that the GCC compiler emits. It may be a default section (e.g. ``.text``, ``.data``) or
+it may be user defined section through the ``__attribute__`` keyword.
+
+The use of an optional '+' indicates the inclusion of the section in the list, as well as sections that start with it. This is the preferred method over listing both explicitly.
 
 .. code-block:: none
 
@@ -262,7 +366,7 @@ The use of an optional '+' indicates the inclusion of the section in the list, a
         .section
         ...
 
-**Example**
+Example:
 
 .. code-block:: none
 
@@ -282,12 +386,9 @@ The use of an optional '+' indicates the inclusion of the section in the list, a
 
 .. _ldgen-scheme-fragment :
 
-II. Scheme
-""""""""""
-
-Scheme fragments define what ``target`` a sections fragment is assigned to. 
+**Scheme**
 
-**Syntax**
+Scheme fragments define what ``target`` a sections fragment is assigned to.
 
 .. code-block:: none
 
@@ -297,7 +398,7 @@ Scheme fragments define what ``target`` a sections fragment is assigned to.
         sections -> target
         ...
 
-**Example**
+Example:
 
 .. code-block:: none
 
@@ -308,69 +409,46 @@ Scheme fragments define what ``target`` a sections fragment is assigned to.
 
 .. _ldgen-default-scheme:
 
-**The** ``default`` **scheme**
+The ``default`` scheme
 
 There exists a special scheme with the name ``default``. This scheme is special because catch-all placement rules are generated from
-its entries. This means that, if one of its entries is ``text -> flash_text``, the placement rule 
+its entries. This means that, if one of its entries is ``text -> flash_text``, the placement rule
 
 .. code-block:: none
 
     *(.literal .literal.* .text .text.*)
 
-will be generated for the target ``flash_text``. 
+will be generated for the target ``flash_text``.
 
-These catch-all rules then effectively serve as fallback rules for those whose mappings were not specified. 
+These catch-all rules then effectively serve as fallback rules for those whose mappings were not specified.
 
 .. note::
 
-    The ``default scheme`` is defined in :component:`esp32/ld/esp32_fragments.lf`. The ``noflash`` and ``rtc`` scheme fragments which are 
+    The ``default scheme`` is defined in :component:`esp32/ld/esp32_fragments.lf`. The ``noflash`` and ``rtc`` scheme fragments which are
     built-in schemes referenced in the quick start guide are also defined in this file.
 
 .. _ldgen-mapping-fragment :
 
-III. Mapping
-""""""""""""
+**Mapping**
 
-Mapping fragments define what scheme fragment to use for mappable entities, i.e. object files, function names, variable names. There are two types of entries
-for this fragment: mapping entries and condition entries.
-
-.. note::
-
-    Mapping fragments have no explicit name property. Internally, the name is constructed from the value of the archive entry. 
-
-**Syntax**
+Mapping fragments define what scheme fragment to use for mappable entities, i.e. object files, function names, variable names, archives.
 
 .. code-block:: none
 
-    [mapping]
+    [mapping:name]
     archive: archive                # output archive file name, as built (i.e. libxxx.a)
     entries:
-        : condition                 # condition entry, non-default
-        object:symbol (scheme)      # mapping entry, Type I
-        object (scheme)             # mapping entry, Type II
-        * (scheme)                  # mapping entry, Type III
-
-        # optional separation/comments, for readability
-
-        : default                   # condition entry, default
-        * (scheme)                  # mapping entry, Type III
-
-.. _ldgen-mapping-entries :
+        object:symbol (scheme)      # symbol granularity
+        object (scheme)             # object granularity
+        * (scheme)                  # archive granularity
 
-**Mapping Entries**
+There are three levels of placement granularity:
 
-There are three types of mapping entries:
+    - symbol: The object file name and symbol name are specified. The symbol name can be a function name or a variable name.
+    - object: Only the object file name is specified.
+    - archive: ``*`` is specified, which is a short-hand for all the object files under the archive.
 
-    ``Type I``
-        The object file name and symbol name are specified. The symbol name can be a function name or a variable name.
-
-    ``Type II``
-        Only the object file name is specified.
-
-    ``Type III``
-        ``*`` is specified, which is a short-hand for all the object files under the archive.
-
-To know what a mapping entry means, let us expand a ``Type II`` entry. Originally: 
+To know what an entry means, let us expand a sample object-granularity placement:
 
 .. code-block:: none
 
@@ -380,8 +458,8 @@ Then expanding the scheme fragment from its entries definitions, we have:
 
 .. code-block:: none
 
-    object (sections -> target, 
-            sections -> target, 
+    object (sections -> target,
+            sections -> target,
             ...)
 
 Expanding the sections fragment with its entries definition:
@@ -391,59 +469,38 @@ Expanding the sections fragment with its entries definition:
     object (.section,      # given this object file
             .section,      # put its sections listed here at this
             ... -> target, # target
-            
+
             .section,
             .section,      # same should be done for these sections
-            ... -> target, 
-            
-            ...)           # and so on
-
-.. _ldgen-type1-limitations :
+            ... -> target,
 
-**On** ``Type I`` **Mapping Entries**
-
-``Type I`` mapping entry is possible due to compiler flags ``-ffunction-sections`` and ``-ffdata-sections``. If the user opts to remove these flags, then
-the ``Type I`` mapping will not work. Furthermore, even if the user does not opt to compile without these flags, there are still limitations 
-as the implementation is dependent on the emitted output sections.
-
-For example, with ``-ffunction-sections``, separate sections are emitted for each function; with section names predictably constructed i.e. ``.text.{func_name}``
-and ``.literal.{func_name}``. This is not the case for string literals within the function, as they go to pooled or generated section names.
-
-With ``-fdata-sections``, for global scope data the compiler predictably emits either ``.data.{var_name}``, ``.rodata.{var_name}`` or ``.bss.{var_name}``; and so ``Type I`` mapping entry works for these. 
-However, this is not the case for static data declared in function scope, as the generated section name is a result of mangling the variable name with some other information.
-
-.. _ldgen-condition-entries :
+            ...)           # and so on
 
-**Condition Entries**
+Example:
 
-Condition entries enable the linker script generation to be configuration-aware. Depending on whether expressions involving configuration values
-are true or not, a particular set of mapping entries can be used. The evaluation uses ``eval_string`` from :idf_file:`tools/kconfig_new/kconfiglib.py` and adheres to its required syntax and limitations.
+.. code-block:: none
 
-All mapping entries defined after a condition entry until the next one or the end of the mapping fragment belongs to that condition entry. During processing 
-conditions are tested sequentially, and the mapping entries under the first condition that evaluates to ``TRUE`` are used.
+    [mapping:map]
+    archive: libfreertos.a
+    entries:
+        * (noflash)
 
-A default condition can be defined (though every mapping contains an implicit, empty one), whose mapping entries get used in the event no conditions evaluates to ``TRUE``.
+.. _ldgen-symbol-granularity-placements :
 
-**Example**
+On Symbol-Granularity Placements
+""""""""""""""""""""""""""""""""
 
-.. code-block:: none
+Symbol granularity placements is possible due to compiler flags ``-ffunction-sections`` and ``-ffdata-sections``. ESP-IDF compiles with these flags by default.
+If the user opts to remove these flags, then the symbol-granularity placements will not work. Furthermore, even with the presence of these flags, there are still other limitations to keep in mind
+due to the dependence on the compiler's emitted output sections.
 
-    [scheme:noflash]
-    entries:
-        text -> iram0_text
-        rodata -> dram0_data
+For example, with ``-ffunction-sections``, separate sections are emitted for each function; with section names predictably constructed i.e. ``.text.{func_name}``
+and ``.literal.{func_name}``. This is not the case for string literals within the function, as they go to pooled or generated section names.
 
-    [mapping:lwip]
-    archive: liblwip.a
-    entries:
-        : LWIP_IRAM_OPTIMIZATION = y         # if CONFIG_LWIP_IRAM_OPTIMIZATION is set to 'y' in sdkconfig
-        ip4:ip4_route_src_hook (noflash)     # map ip4.o:ip4_route_src_hook, ip4.o:ip4_route_src and
-        ip4:ip4_route_src (noflash)          # ip4.o:ip4_route using the noflash scheme, which puts 
-        ip4:ip4_route (noflash)              # them in RAM
-        
-        : default                            # else no special mapping rules apply
+With ``-fdata-sections``, for global scope data the compiler predictably emits either ``.data.{var_name}``, ``.rodata.{var_name}`` or ``.bss.{var_name}``; and so ``Type I`` mapping entry works for these.
+However, this is not the case for static data declared in function scope, as the generated section name is a result of mangling the variable name with some other information.
 
-.. _ldgen-script-templates :
+.. _ldgen-linker-script-template :
 
 Linker Script Template
 ^^^^^^^^^^^^^^^^^^^^^^
@@ -451,15 +508,13 @@ Linker Script Template
 The linker script template is the skeleton in which the generated placement rules are put into. It is an otherwise ordinary linker script, with a specific marker syntax
 that indicates where the generated placement rules are placed.
 
-**Syntax**
-
 To reference the placement rules collected under a ``target`` token, the following syntax is used:
 
 .. code-block:: none
 
     mapping[target]
 
-**Example**
+Example:
 
 The example below is an excerpt from a possible linker script template. It defines an output section ``.iram0.text``, and inside is a marker referencing
 the target ``iram0_text``.
@@ -530,19 +585,9 @@ Then the corresponding excerpt from the generated linker script will be as follo
     it too is placed wherever ``iram0_text`` is referenced by a marker. Since it is a rule generated from the default scheme, it comes first
     among all other rules collected under the same target name.
 
+.. note::
 
-Integration with Build System
------------------------------
-
-The linker script generation occurs during application build, before the final output binary is linked. The tool that implements the mechanism 
-lives under ``$(IDF_PATH)/tools/ldgen``.
+    The linker script template currently used is :component:`esp32/ld/esp32.project.ld.in`, specified by the ``esp32`` component; the
+    generated output script is put under its build directory.
 
-Linker Script Template
-^^^^^^^^^^^^^^^^^^^^^^
-Currently, the linker script template used is :component:`esp32/ld/esp32.project.ld.in`, and is used only for the app build. The generated output script is 
-put under the build directory of the same component. Modifying this linker script template triggers a re-link of the app binary. 
 
-Linker Fragment File
-^^^^^^^^^^^^^^^^^^^^
-Any component can add a fragment file to the build. In order to add a fragment file to process, set COMPONENT_ADD_LDFRAGMENTS or use the function ``ldgen_add_fragment_files`` (CMake only) as mentioned :ref:`here <ldgen-add-fragment-file>`.
-Modifying any fragment file presented to the build system triggers a re-link of the app binary.

tools/ldgen/fragments.py

@@ -13,9 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-import re
 import os
+import re
 
 from sdkconfig import SDKConfig
 from pyparsing import OneOrMore
@@ -23,49 +22,181 @@ from pyparsing import restOfLine
 from pyparsing import alphanums
 from pyparsing import Word
 from pyparsing import alphas
-from pyparsing import ParseBaseException
+from pyparsing import ParseFatalException
 from pyparsing import Suppress
 from pyparsing import Group
 from pyparsing import Literal
 from pyparsing import ZeroOrMore
 from pyparsing import Optional
 from pyparsing import originalTextFor
-from common import LdGenFailure
+from pyparsing import Forward
+from pyparsing import indentedBlock
+from collections import namedtuple
+import abc
+
 
+KeyGrammar = namedtuple("KeyGrammar", "grammar min max required")
 
-class FragmentFileModel():
+
+class FragmentFile():
     """
     Fragment file internal representation. Parses and stores instances of the fragment definitions
     contained within the file.
     """
 
-    def __init__(self, fragment_file):
-        path = os.path.realpath(fragment_file.name)
+    def __init__(self, fragment_file, sdkconfig):
+        try:
+            fragment_file = open(fragment_file, "r")
+        except TypeError:
+            pass
 
-        sections = Sections.get_fragment_grammar()
-        scheme = Scheme.get_fragment_grammar()
-        mapping = Mapping.get_fragment_grammar()
+        path = os.path.realpath(fragment_file.name)
 
-        # Each fragment file is composed of sections, scheme or mapping fragments. The grammar
-        # for each of those objects are defined it the respective classes
-        parser = OneOrMore(sections | scheme | mapping)
+        indent_stack = [1]
+
+        class parse_ctx:
+            fragment = None  # current fragment
+            key = ""  # current key
+            keys = list()  # list of keys parsed
+            key_grammar = None  # current key grammar
+
+            @staticmethod
+            def reset():
+                parse_ctx.fragment_instance = None
+                parse_ctx.key = ""
+                parse_ctx.keys = list()
+                parse_ctx.key_grammar = None
+
+        def fragment_type_parse_action(toks):
+            parse_ctx.reset()
+            parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]()  # create instance of the fragment
+            return None
+
+        def expand_conditionals(toks, stmts):
+            try:
+                stmt = toks["value"]
+                stmts.append(stmt)
+            except KeyError:
+                try:
+                    conditions = toks["conditional"]
+                    for condition in conditions:
+                        try:
+                            _toks = condition[1]
+                            _cond = condition[0]
+                            if sdkconfig.evaluate_expression(_cond):
+                                expand_conditionals(_toks, stmts)
+                                break
+                        except IndexError:
+                            expand_conditionals(condition[0], stmts)
+                except KeyError:
+                    for tok in toks:
+                        expand_conditionals(tok, stmts)
+
+        def key_body_parsed(pstr, loc, toks):
+            stmts = list()
+            expand_conditionals(toks, stmts)
+
+            if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min:
+                raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" %
+                                          (parse_ctx.key_grammar.min, parse_ctx.key))
+
+            if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max:
+                raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" %
+                                          (parse_ctx.key_grammar.max, parse_ctx.key))
+
+            try:
+                parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
+            except Exception as e:
+                raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, e.message))
+            return None
+
+        key = Word(alphanums + "_") + Suppress(":")
+        key_stmt = Forward()
+
+        condition_block = indentedBlock(key_stmt, indent_stack)
+        key_stmts = OneOrMore(condition_block)
+        key_body = Suppress(key) + key_stmts
+        key_body.setParseAction(key_body_parsed)
+
+        condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName("condition")
+        if_condition = Group(Suppress("if") + condition + Suppress(":") + condition_block)
+        elif_condition = Group(Suppress("elif") + condition + Suppress(":") + condition_block)
+        else_condition = Group(Suppress("else") + Suppress(":") + condition_block)
+        conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName("conditional")
+
+        def key_parse_action(pstr, loc, toks):
+            key = toks[0]
+
+            if key in parse_ctx.keys:
+                raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key)
+
+            parse_ctx.key = key
+            parse_ctx.keys.append(key)
+
+            try:
+                parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key]
+                key_grammar = parse_ctx.key_grammar.grammar
+            except KeyError:
+                raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key)
+            except Exception as e:
+                raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, e.message))
+
+            key_stmt << (conditional | Group(key_grammar).setResultsName("value"))
+
+            return None
+
+        def name_parse_action(pstr, loc, toks):
+            parse_ctx.fragment.name = toks[0]
+
+        key.setParseAction(key_parse_action)
+
+        ftype = Word(alphas).setParseAction(fragment_type_parse_action)
+        fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name")
+        fid.setParseAction(name_parse_action)
+        header = Suppress("[") + ftype + fid + Suppress("]")
+
+        def fragment_parse_action(pstr, loc, toks):
+            key_grammars = parse_ctx.fragment.get_key_grammars()
+            required_keys = set([k for (k,v) in key_grammars.items() if v.required])
+            present_keys = required_keys.intersection(set(parse_ctx.keys))
+            if present_keys != required_keys:
+                raise ParseFatalException(pstr, loc, "required keys %s for fragment not found" %
+                                          list(required_keys - present_keys))
+            return parse_ctx.fragment
+
+        fragment_stmt = Forward()
+        fragment_block = indentedBlock(fragment_stmt, indent_stack)
+
+        fragment_if_condition = Group(Suppress("if") + condition + Suppress(":") + fragment_block)
+        fragment_elif_condition = Group(Suppress("elif") + condition + Suppress(":") + fragment_block)
+        fragment_else_condition = Group(Suppress("else") + Suppress(":") + fragment_block)
+        fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) +
+                                Optional(fragment_else_condition)).setResultsName("conditional")
+
+        fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName("value")
+        fragment.setParseAction(fragment_parse_action)
+        fragment.ignore("#" + restOfLine)
+
+        deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName("value")
+
+        fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional))
+
+        def fragment_stmt_parsed(pstr, loc, toks):
+            stmts = list()
+            expand_conditionals(toks, stmts)
+            return stmts
 
-        # Set any text beginnning with # as comment
-        parser.ignore("#" + restOfLine)
+        parser = ZeroOrMore(fragment_stmt)
+        parser.setParseAction(fragment_stmt_parsed)
 
-        try:
-            self.fragments = parser.parseFile(fragment_file, parseAll=True)
-        except ParseBaseException as e:
-            # the actual parse error is kind of useless for normal users, so just point to the location of
-            # the error
-            raise LdGenFailure("Parse error in linker fragment %s: error at line %d col %d (char %d)" % (
-                fragment_file.name, e.lineno, e.column, e.loc))
+        self.fragments = parser.parseFile(fragment_file, parseAll=True)
 
         for fragment in self.fragments:
             fragment.path = path
 
 
-class Fragment:
+class Fragment():
+    __metaclass__ = abc.ABCMeta
     """
     Encapsulates a fragment as defined in the generator syntax. Sets values common to all fragment and performs processing
     such as checking the validity of the fragment name and getting the entry values.
@@ -74,22 +205,20 @@ class Fragment:
     IDENTIFIER = Word(alphas + "_", alphanums + "_")
     ENTITY = Word(alphanums + ".-_$")
 
-    def __init__(self, name, entries):
-        self.path = None
-        self.name = name
-        self.entries = entries
+    @abc.abstractmethod
+    def set_key_value(self, key, parse_results):
+        pass
 
+    @abc.abstractmethod
+    def get_key_grammars(self):
+        pass
 
-class Sections(Fragment):
 
-    def __init__(self, name, entries):
-        Fragment.__init__(self, name, entries)
-        self._process_entries()
+class Sections(Fragment):
 
-    def _process_entries(self):
-        # Quietly ignore duplicate entries
-        self.entries = set(self.entries)
-        self.entries = list(self.entries)
+    grammars = {
+        "entries": KeyGrammar(Word(alphanums + "+.").setResultsName("section"), 1, None, True)
+    }
 
     """
     Utility function that returns a list of sections given a sections fragment entry,
@@ -110,20 +239,14 @@ class Sections(Fragment):
             else:
                 return (sections_entry, None)
 
-    @staticmethod
-    def get_fragment_grammar():
-        name = Fragment.IDENTIFIER
-        header = Suppress("[") + Suppress("sections") + Suppress(":") + name.setResultsName("name") + Suppress("]")
-        entry = Word(alphanums + "+" + ".")
-        entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
-
-        sections = Group(header + entries)
+    def set_key_value(self, key, parse_results):
+        if key == "entries":
+            self.entries = set()
+            for result in parse_results:
+                self.entries.add(result["section"])
 
-        sections.setParseAction(lambda t: Sections(t[0].name, t[0].entries))
-
-        sections.ignore("#" + restOfLine)
-
-        return sections
+    def get_key_grammars(self):
+        return self.__class__.grammars
 
 
 class Scheme(Fragment):
@@ -131,38 +254,19 @@ class Scheme(Fragment):
     Encapsulates a scheme fragment, which defines what target input sections are placed under.
     """
 
-    def __init__(self, name, items):
-        Fragment.__init__(self, name, items)
-        self._process_entries()
-
-    def _process_entries(self):
-        processed = set()
-
-        # Store entries as a set of tuples. Quietly ignores duplicate entries.
-        for entry in self.entries:
-            processed.add((entry.sections, entry.target))
-
-        self.entries = processed
-
-    @staticmethod
-    def get_fragment_grammar():
-        name = Fragment.IDENTIFIER
-        header = Suppress("[") + Suppress("scheme") + Suppress(":") + name.setResultsName("name") + Suppress("]")
-
-        # Scheme entry in the form 'sections -> target'
-        sections = Fragment.IDENTIFIER
-        target = Fragment.IDENTIFIER
-        entry = Group(sections.setResultsName("sections") + Suppress("->") + target.setResultsName("target"))
-
-        entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
+    grammars = {
+        "entries": KeyGrammar(Fragment.IDENTIFIER.setResultsName("sections") + Suppress("->") +
+                              Fragment.IDENTIFIER.setResultsName("target"), 1, None, True)
+    }
 
-        scheme = Group(header + entries)
+    def set_key_value(self, key, parse_results):
+        if key == "entries":
+            self.entries = set()
+            for result in parse_results:
+                self.entries.add((result["sections"], result["target"]))
 
-        scheme.setParseAction(lambda t: Scheme(t[0].name, t[0].entries))
-
-        scheme.ignore("#" + restOfLine)
-
-        return scheme
+    def get_key_grammars(self):
+        return self.__class__.grammars
 
 
 class Mapping(Fragment):
@@ -170,63 +274,70 @@ class Mapping(Fragment):
     Encapsulates a mapping fragment, which defines what targets the input sections of mappable entties are placed under.
     """
 
-    # Name of the default condition entry
-    DEFAULT_CONDITION = "default"
     MAPPING_ALL_OBJECTS = "*"
 
-    def __init__(self, archive, entries):
-        self.archive = archive
-
-        # Generate name from archive value by replacing all non-alphanumeric
-        # characters with underscore
-        name = Mapping.get_mapping_name_from_archive(self.archive)
-        Fragment.__init__(self, name, entries)
-
-        self._process_entries()
+    def set_key_value(self, key, parse_results):
+        if key == "archive":
+            self.archive = parse_results[0]["archive"]
+        elif key == "entries":
+            self.entries = set()
 
-    def _create_mappings_set(self, mappings):
-        mapping_set = set()
+            for result in parse_results:
+                obj = None
+                symbol = None
+                scheme = None
 
-        for mapping in mappings:
-            obj = mapping.object
-            symbol = mapping.symbol
-            scheme = mapping.scheme
+                try:
+                    obj = result["object"]
+                except KeyError:
+                    pass
 
-            if symbol == "":
-                symbol = None
+                try:
+                    symbol = result["symbol"]
+                except KeyError:
+                    pass
 
-            # Quietly handle duplicate definitions under the same condition
-            mapping_set.add((obj, symbol, scheme))
+                try:
+                    scheme = result["scheme"]
+                except KeyError:
+                    pass
 
-        return mapping_set
+                self.entries.add((obj, symbol, scheme))
 
-    def _process_entries(self):
-        processed = []
+    def get_key_grammars(self):
+        # There are three possible patterns for mapping entries:
+        #       obj:symbol (scheme)
+        #       obj (scheme)
+        #       * (scheme)
+        obj = Fragment.ENTITY.setResultsName("object")
+        symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
+        scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")")
 
-        for normal_group in self.entries[0]:
-            # Get the original string of the condition
-            condition  = next(iter(normal_group.condition.asList())).strip()
-            mappings = self._create_mappings_set(normal_group[1])
+        pattern1 = obj + symbol + scheme
+        pattern2 = obj + scheme
+        pattern3 = Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme
 
-            processed.append((condition, mappings))
+        entry = pattern1 | pattern2 | pattern3
 
-        default_group = self.entries[1]
+        grammars = {
+            "archive": KeyGrammar(Fragment.ENTITY.setResultsName("archive"), 1, 1, True),
+            "entries": KeyGrammar(entry, 1, None, True)
+        }
 
-        if len(default_group) > 1:
-            mappings = self._create_mappings_set(default_group[1])
-        else:
-            mappings = self._create_mappings_set(default_group[0])
+        return grammars
 
-        processed.append(("default", mappings))
 
-        self.entries = processed
+class DeprecatedMapping():
+    """
+    Encapsulates a mapping fragment, which defines what targets the input sections of mappable entties are placed under.
+    """
 
-    @staticmethod
-    def get_mapping_name_from_archive(archive):
-        return re.sub(r"[^0-9a-zA-Z]+", "_", archive)
+    # Name of the default condition entry
+    DEFAULT_CONDITION = "default"
+    MAPPING_ALL_OBJECTS = "*"
 
     @staticmethod
-    def get_fragment_grammar():
+    def get_fragment_grammar(sdkconfig, fragment_file):
 
         # Match header [mapping]
         header = Suppress("[") + Suppress("mapping") + Suppress("]")
@@ -251,7 +362,7 @@ class Mapping(Fragment):
         mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings")
 
         normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar())
-        default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION))
+        default_condition = Optional(Suppress(":") + Literal(DeprecatedMapping.DEFAULT_CONDITION))
 
         normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries)
         default_group = Group(default_condition + mapping_entries).setResultsName("default_group")
@@ -263,9 +374,48 @@ class Mapping(Fragment):
         entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries")
 
         mapping = Group(header + archive + entries)
-
-        mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries))
-
         mapping.ignore("#" + restOfLine)
 
+        def parsed_deprecated_mapping(pstr, loc, toks):
+            fragment = Mapping()
+            fragment.archive = toks[0].archive
+            fragment.name = re.sub(r"[^0-9a-zA-Z]+", "_", fragment.archive)
+
+            fragment.entries = set()
+            condition_true = False
+            for entries in toks[0].entries[0]:
+                condition  = next(iter(entries.condition.asList())).strip()
+                condition_val = sdkconfig.evaluate_expression(condition)
+
+                if condition_val:
+                    for entry in entries[1]:
+                        fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
+                    condition_true = True
+                    break
+
+            if not fragment.entries and not condition_true:
+                try:
+                    entries = toks[0].entries[1][1]
+                except IndexError:
+                    entries = toks[0].entries[1][0]
+                for entry in entries:
+                    fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
+
+            if not fragment.entries:
+                fragment.entries.add(("*", None, "default"))
+
+            dep_warning = str(ParseFatalException(pstr, loc,
+                              "Warning: Deprecated old-style mapping fragment parsed in file %s." % fragment_file))
+
+            print(dep_warning)
+            return fragment
+
+        mapping.setParseAction(parsed_deprecated_mapping)
         return mapping
+
+
+FRAGMENT_TYPES = {
+    "sections": Sections,
+    "scheme": Scheme,
+    "mapping": Mapping
+}

tools/ldgen/generation.py

@@ -321,7 +321,7 @@ class GenerationModel:
 
         return scheme_dictionary
 
-    def generate_rules(self, sdkconfig, sections_infos):
+    def generate_rules(self, sections_infos):
         placement_rules = collections.defaultdict(list)
 
         scheme_dictionary = self._build_scheme_dictionary()
@@ -334,33 +334,18 @@ class GenerationModel:
 
         # Generate rules based on mapping fragments
         for mapping in self.mappings.values():
-            for (condition, entries) in mapping.entries:
-                condition_true = False
-
-                # Only non-default condition are evaluated agains sdkconfig model
-                if condition != Mapping.DEFAULT_CONDITION:
-                    try:
-                        condition_true = sdkconfig.evaluate_expression(condition)
-                    except Exception as e:
-                        raise GenerationException(e.message, mapping)
-                else:
-                    condition_true = True
-
-                if condition_true:
-
-                    mapping_rules = list()
-
-                    archive = mapping.archive
-                    for (obj, symbol, scheme_name) in entries:
-                        try:
-                            self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
-                        except KeyError:
-                            message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
-                            raise GenerationException(message, mapping)
-
-                    all_mapping_rules[mapping.name] = mapping_rules
+            mapping_rules = list()
+            archive = mapping.archive
+            for (obj, symbol, scheme_name) in mapping.entries:
+                try:
+                    if not (obj == Mapping.MAPPING_ALL_OBJECTS and symbol is None and
+                            scheme_name == GenerationModel.DEFAULT_SCHEME):
+                        self._add_mapping_rules(archive, obj, symbol, scheme_name, scheme_dictionary, mapping_rules)
+                except KeyError:
+                    message = GenerationException.UNDEFINED_REFERENCE + " to scheme '" + scheme_name + "'."
+                    raise GenerationException(message, mapping)
 
-                    break   # Exit on first condition that evaluates to true
+            all_mapping_rules[mapping.name] = mapping_rules
 
         # Detect rule conflicts
         for mapping_rules in all_mapping_rules.items():
@@ -397,8 +382,7 @@ class GenerationModel:
                 if intersections and rule_a.maps_same_entities_as(rule_b):
                     rules_string = str([str(rule_a), str(rule_b)])
                     message = "Rules " + rules_string + " map sections " + str(list(intersections)) + " into multiple targets."
-                    mapping = self.mappings[Mapping.get_mapping_name_from_archive(archive)]
-                    raise GenerationException(message, mapping)
+                    raise GenerationException(message)
 
     def _create_extra_rules(self, rules):
         # This function generates extra rules for symbol specific rules. The reason for generating extra rules is to isolate,
@@ -523,7 +507,7 @@ class TemplateModel:
                 # Does not match marker syntax
                 self.members.append(line)
 
-    def fill(self, mapping_rules, sdkconfig):
+    def fill(self, mapping_rules):
         for member in self.members:
             target = None
             try:

tools/ldgen/ldgen.py

@@ -19,10 +19,11 @@ import argparse
 import sys
 import tempfile
 
-from fragments import FragmentFileModel
+from fragments import FragmentFile
 from sdkconfig import SDKConfig
 from generation import GenerationModel, TemplateModel, SectionsInfo
 from common import LdGenFailure
+from pyparsing import ParseException, ParseFatalException
 
 
 def main():
@@ -52,13 +53,11 @@ def main():
 
     argparser.add_argument(
         "--config", "-c",
-        help="Project configuration",
-        type=argparse.FileType("r"))
+        help="Project configuration")
 
     argparser.add_argument(
         "--kconfig", "-k",
-        help="IDF Kconfig file",
-        type=argparse.FileType("r"))
+        help="IDF Kconfig file")
 
     argparser.add_argument(
         "--env", "-e",
@@ -89,15 +88,22 @@ def main():
 
         generation_model = GenerationModel()
 
+        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
+
         for fragment_file in fragment_files:
-            fragment_file = FragmentFileModel(fragment_file)
+            try:
+                fragment_file = FragmentFile(fragment_file, sdkconfig)
+            except (ParseException, ParseFatalException) as e:
+                # ParseException is raised on incorrect grammar
+                # ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
+                # keys, key unsupported by fragment, unexpected number of values, etc.)
+                raise LdGenFailure("failed to parse %s\n%s" % (fragment_file.name, str(e)))
             generation_model.add_fragments_from_file(fragment_file)
 
-        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
-        mapping_rules = generation_model.generate_rules(sdkconfig, sections_infos)
+        mapping_rules = generation_model.generate_rules(sections_infos)
 
         script_model = TemplateModel(input_file)
-        script_model.fill(mapping_rules, sdkconfig)
+        script_model.fill(mapping_rules)
 
         with tempfile.TemporaryFile("w+") as output:
             script_model.write(output)

tools/ldgen/samples/mappings.lf

@@ -1,62 +1,62 @@
 
-[mapping]
+[mapping:heap]
 archive: libheap.a
 entries:
     multi_heap (noflash)
     multi_heap_poisoning (noflash)
 
-[mapping]
+[mapping:soc]
 archive: libsoc.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:freertos]
 archive: libfreertos.a
 entries:
     * (noflash)
 
-[mapping]
+[mapping:esp32]
 archive: libesp32.a
 entries: 
     core_dump (noflash)
     panic (noflash)
 
-[mapping]
+[mapping:app_trace]
 archive: libapp_trace.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:xtensa_debug_module]
 archive: libxtensa-debug-module.a
 entries: 
     eri (noflash)
 
-[mapping]
+[mapping:phy]
 archive: libphy.a
 entries: 
     * (noflash_data)
 
-[mapping]
+[mapping:rtc]
 archive: librtc.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:hal]
 archive: libhal.a
 entries: 
     * (noflash)
 
-[mapping]
+[mapping:gcc]
 archive: libgcc.a
 entries: 
     lib2funcs (noflash)
 
-[mapping]
+[mapping:spi_flash]
 archive: libspi_flash.a
 entries: 
     spi_flash_rom_patch (noflash)
 
-[mapping]
+[mapping:gcov]
 archive: libgcov.a
 entries: 
     * (noflash)

tools/ldgen/sdkconfig.py

@@ -15,7 +15,7 @@
 #
 
 import os
-from pyparsing import Word, printables, Combine, Literal, hexnums, quotedString, Optional, nums, removeQuotes, oneOf, Group, infixNotation, opAssoc
+from pyparsing import Word, alphanums, printables, Combine, Literal, hexnums, quotedString, Optional, nums, removeQuotes, oneOf, Group, infixNotation, opAssoc
 
 import sys
 try:
@@ -34,11 +34,11 @@ class SDKConfig:
     """
 
     # A configuration entry is in the form CONFIG=VALUE. Definitions of components of that grammar
-    IDENTIFIER = Word(printables.upper())
+    IDENTIFIER = Word(alphanums.upper() + "_")
 
     HEX = Combine("0x" + Word(hexnums)).setParseAction(lambda t:int(t[0], 16))
     DECIMAL = Combine(Optional(Literal("+") | Literal("-")) + Word(nums)).setParseAction(lambda t:int(t[0]))
-    LITERAL = Word(printables)
+    LITERAL = Word(printables.replace(":", ""))
     QUOTED_LITERAL = quotedString.setParseAction(removeQuotes)
 
     VALUE = HEX | DECIMAL | LITERAL | QUOTED_LITERAL
@@ -53,8 +53,8 @@ class SDKConfig:
             value = " ".join(value.split())
             os.environ[name] = value
 
-        self.config = kconfiglib.Kconfig(kconfig_file.name)
-        self.config.load_config(sdkconfig_file.name)
+        self.config = kconfiglib.Kconfig(kconfig_file)
+        self.config.load_config(sdkconfig_file)
 
     def evaluate_expression(self, expression):
         result = self.config.eval_string(expression)
@@ -64,7 +64,7 @@ class SDKConfig:
         elif result == 2:  # y
             return True
         else:  # m
-            raise Exception("Unsupported config expression result.")
+            raise Exception("unsupported config expression result")
 
     @staticmethod
     def get_expression_grammar():

tools/ldgen/test/data/Kconfig

@@ -1,7 +1,19 @@
 menu "Test config"
 
     config PERFORMANCE_LEVEL
-        int "Performance level"
+        int
         range 0 3
+        prompt "Performance level"
 
+    config A
+        bool
+        default "y"
+
+    config B
+        bool
+        default "n"
+
+    config C
+        bool
+        default "y"
 endmenu

tools/ldgen/test/test_fragments.py

@@ -14,625 +14,966 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-import unittest
 import sys
-from pyparsing import ParseException
-from pyparsing import restOfLine
+import unittest
+
+from io import StringIO
+from pyparsing import Word, ParseException, ParseFatalException, alphanums
 
 try:
-    import fragments
+    from fragments import FragmentFile, FRAGMENT_TYPES, Fragment, KeyGrammar
+    from sdkconfig import SDKConfig
 except ImportError:
     sys.path.append('../')
-    import fragments
-
-from sdkconfig import SDKConfig
+    from fragments import FragmentFile, FRAGMENT_TYPES, Fragment, KeyGrammar
+    from sdkconfig import SDKConfig
 
 
-class FragmentTest(unittest.TestCase):
-
-    def parse(self, text):
-        self.parser.ignore("#" + restOfLine)
-        fragment = self.parser.parseString(text, parseAll=True)
-        return fragment[0]
+class SampleFragment(Fragment):
 
+    grammars = {
+        "key_1": KeyGrammar(Word(alphanums + "_").setResultsName("value"), 0, None, True),
+        "key_2": KeyGrammar(Word(alphanums + "_").setResultsName("value"), 0, None, False),
+        "key_3": KeyGrammar(Word(alphanums + "_").setResultsName("value"), 3, 5, False)
+    }
 
-class SectionsTest(FragmentTest):
-
-    def setUp(self):
-        self.parser = fragments.Sections.get_fragment_grammar()
+    def set_key_value(self, key, parse_results):
+        if key == "key_1":
+            self.key_1 = list()
+            for result in parse_results:
+                self.key_1.append(result["value"])
+        elif key == "key_2":
+            self.key_2 = list()
+            for result in parse_results:
+                self.key_2.append(result["value"])
 
-    def test_valid_entries(self):
-        valid_entries = """
-        [sections:test]
-        entries:
-            .section1
-                .section2
+    def get_key_grammars(self):
+        return self.__class__.grammars
 
-            # Section 3 should not exist
-            # section3
-            .section4
 
-            # This is a comment
+FRAGMENT_TYPES["test"] = SampleFragment
 
-            .section5
-        """
-
-        sections = self.parse(valid_entries)
-
-        self.assertEqual("test", sections.name)
-
-        entries = sections.entries
-
-        expected = {
-            ".section1",
-            ".section2",
-            ".section4",
-            ".section5"
-        }
-
-        self.assertEqual(set(entries), expected)
-
-    def test_blank_entries(self):
-        blank_entries = """
-        [sections:test]
-        entries:
-        """
-
-        with self.assertRaises(ParseException):
-            self.parse(blank_entries)
 
-    def test_invalid_names(self):
-        with_spaces = """
-        [sections:invalid name 1]
-        entries:
-        """
-
-        begins_with_number = """
-        [sections:2invalid_name]
-        entries:
-        """
-
-        with_special_character = """
-        [sections:invalid_name~]
-        entries:
-        """
-
-        with self.assertRaises(ParseException):
-            self.parse(with_spaces)
+class FragmentTest(unittest.TestCase):
 
+    def setUp(self):
+        self.sdkconfig = SDKConfig("data/Kconfig", "data/sdkconfig")
+
+    @staticmethod
+    def create_fragment_file(contents, name="test_fragment.lf"):
+        f = StringIO(contents)
+        f.name = name
+        return f
+
+    def test_basic(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_1
+    value_2 # comments should be ignored
+    value_3
+# this is a comment as well
+key_2: value_a
+
+# this is the last comment
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+
+        self.assertEqual(len(fragment_file.fragments[0].key_1), 3)
+        self.assertEqual(fragment_file.fragments[0].key_1[0], "value_1")
+        self.assertEqual(fragment_file.fragments[0].key_1[1], "value_2")
+        self.assertEqual(fragment_file.fragments[0].key_1[2], "value_3")
+        self.assertEqual(len(fragment_file.fragments[0].key_2), 1)
+        self.assertEqual(fragment_file.fragments[0].key_2[0], "value_a")
+
+    def test_duplicate_keys(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1: value_1
+key_1: value_a
+""")
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_empty_key(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+""")
         with self.assertRaises(ParseException):
-            self.parse(begins_with_number)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_conditional(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_1
+    if A = y:
+        value_2
+    value_3
+    if A = n:
+        value_4
+    if B = n:
+        value_5
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].key_1[0], "value_1")
+        self.assertEqual(fragment_file.fragments[0].key_1[1], "value_2")
+        self.assertEqual(fragment_file.fragments[0].key_1[2], "value_3")
+        self.assertEqual(fragment_file.fragments[0].key_1[3], "value_5")
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_1
+    if B = y:
+        value_2
+    elif C = y:
+        value_3
+    elif A = y:
+        value_4
+    else:
+        value_5
+    value_6
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].key_1[0], "value_1")
+        self.assertEqual(fragment_file.fragments[0].key_1[1], "value_3")
+        self.assertEqual(fragment_file.fragments[0].key_1[2], "value_6")
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_1
+    if A = y:
+        value_2
+        if B = y:
+            value_3
+        else:
+            value_4
+            if C = y:
+                value_5
+            value_6
+        value_7
+key_2:
+    value_a
+    if B != y:
+        value_b
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].key_1[0], "value_1")
+        self.assertEqual(fragment_file.fragments[0].key_1[1], "value_2")
+        self.assertEqual(fragment_file.fragments[0].key_1[2], "value_4")
+        self.assertEqual(fragment_file.fragments[0].key_1[3], "value_5")
+        self.assertEqual(fragment_file.fragments[0].key_1[4], "value_6")
+        self.assertEqual(fragment_file.fragments[0].key_1[5], "value_7")
+        self.assertEqual(fragment_file.fragments[0].key_2[0], "value_a")
+        self.assertEqual(fragment_file.fragments[0].key_2[1], "value_b")
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    if A = n:
+        value_2
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(len(fragment_file.fragments[0].key_1), 0)
+
+    def test_empty_file(self):
+        test_fragment = self.create_fragment_file(u"""
+
+
+
+
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(len(fragment_file.fragments), 0)
+
+    def test_setting_indent(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+ value_1
+ value_2
+ value_3
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+
+        self.assertEqual(len(fragment_file.fragments[0].key_1), 3)
+        self.assertEqual(fragment_file.fragments[0].key_1[0], "value_1")
+        self.assertEqual(fragment_file.fragments[0].key_1[1], "value_2")
+        self.assertEqual(fragment_file.fragments[0].key_1[2], "value_3")
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+ value_1
+  value_2 # first element dictates indent
+  value_3
+""")
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_values_num_limit(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_a
+key_3:
+    value_1
+    value_2
+    value_3
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_a
+key_3:
+    value_1
+    value_2
+    value_3
+    value_4
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(len(fragment_file.fragments), 1)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_a
+key_3:
+    value_1
+    value_2
+    value_3
+    value_4
+    value_5
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(len(fragment_file.fragments), 1)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_a
+key_3:
+    value_1
+    value_2
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_a
+key_3:
+    value_1
+    value_2
+    value_3
+    value_4
+    value_5
+    value_6
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_unsupported_key(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    value_a
+key_4:
+    value_1
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_empty_fragment(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(with_special_character)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_empty_conditional(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    if B = y:
+    else:
+        value_1
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    if B = y:
+        value_1
+    else B = y:
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    if B = y:
+        value_1
+    elif B = y:
+    else:
+        value_2
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_out_of_order_conditional(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    elif B = y:
+        value_1
+    else:
+        value_2
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_1:
+    else:
+        value_2
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_required_keys(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test]
+key_2:
+    value_1
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_multiple_fragments(self):
+        test_fragment = self.create_fragment_file(u"""
+[test:test1]
+key_1:
+    value_1
+
+[test:test2]
+key_1:
+    value_2
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+
+        self.assertEqual(len(fragment_file.fragments), 2)
+        self.assertEqual(fragment_file.fragments[0].key_1[0], "value_1")
+        self.assertEqual(fragment_file.fragments[1].key_1[0], "value_2")
+
+    def test_whole_conditional_fragment(self):
+        test_fragment = self.create_fragment_file(u"""
+if B = y:
+    [test:test1]
+    key_1:
+        value_1
+else:
+    [test:test2]
+    key_1:
+        value_2
+
+    if A = y:
+        [test:test3]
+        key_1:
+            value_3
+            if C = y:
+                value_6
+
+    [test:test4]
+    key_1:
+        value_4
+
+[test:test5]
+key_1:
+    value_5
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(len(fragment_file.fragments), 4)
+        self.assertEqual(fragment_file.fragments[0].name, "test2")
+        self.assertEqual(fragment_file.fragments[1].name, "test3")
+        self.assertEqual(fragment_file.fragments[1].key_1[1], "value_6")
+        self.assertEqual(fragment_file.fragments[2].name, "test4")
+        self.assertEqual(fragment_file.fragments[3].name, "test5")
+
+    def test_equivalent_conditional_fragment(self):
+        test_fragment1 = self.create_fragment_file(u"""
+if A = y:
+    [test:test1]
+    key_1:
+        value_1
+else:
+    [test:test2]
+    key_1:
+        value_2
+""")
+
+        fragment_file1 = FragmentFile(test_fragment1, self.sdkconfig)
+        self.assertEqual(len(fragment_file1.fragments), 1)
+        self.assertEqual(fragment_file1.fragments[0].key_1[0], "value_1")
+
+        test_fragment2 = self.create_fragment_file(u"""
+[test:test1]
+key_1:
+    if A = y:
+        value_1
+    else:
+        value_2
+""")
+
+        fragment_file2 = FragmentFile(test_fragment2, self.sdkconfig)
+        self.assertEqual(len(fragment_file2.fragments), 1)
+        self.assertEqual(fragment_file2.fragments[0].key_1[0], "value_1")
 
-    def test_non_existent_entries(self):
-        misspelled_entries_field = """
-        [sections:test]
-        entrie:
-            .section1
-        """
 
-        missing_entries_field = """
-        [sections:test]
-        """
+class SectionsTest(FragmentTest):
 
-        with self.assertRaises(ParseException):
-            self.parse(misspelled_entries_field)
+    def test_basic(self):
+        test_fragment = self.create_fragment_file(u"""
+[sections:test]
+entries:
+    .section1
+    .section2
+""")
 
-        with self.assertRaises(ParseException):
-            self.parse(missing_entries_field)
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].entries, {".section1", ".section2"})
 
     def test_duplicate_entries(self):
-        duplicate_entries = """
-        [sections:test]
-        entries:
-            .section1
-            .section3
-            .section1
-            .section1
-            .section2
-            .section3
-            .section1
-        """
+        test_fragment = self.create_fragment_file(u"""
+[sections:test]
+entries:
+    .section1
+    .section2
+    .section3
+    .section2
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].entries, {".section1", ".section2", ".section3"})
+
+    def test_empty_entries(self):
+        test_fragment = self.create_fragment_file(u"""
+[sections:test]
+entries:
+""")
 
-        sections = self.parse(duplicate_entries)
-
-        entries = sections.entries
+        with self.assertRaises(ParseException):
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        expected = {
-            ".section1",
-            ".section2",
-            ".section3",
-        }
+        test_fragment = self.create_fragment_file(u"""
+[sections:test]
+entries:
+    if B = y:
+        .section1
+""")
 
-        self.assertEqual(set(entries), expected)
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
 
 
 class SchemeTest(FragmentTest):
 
-    def setUp(self):
-        self.parser = fragments.Scheme.get_fragment_grammar()
-
-    def test_valid_entries(self):
-        valid_entries = """
-        [scheme:test]
-        entries:
-            sections1 -> target1
-            sections2   ->    target2
-        """
-
-        scheme = self.parse(valid_entries)
-
-        entries = scheme.entries
-
-        expected = {
-            ("sections1", "target1"),
-            ("sections2", "target2")
-        }
-
-        self.assertEqual(entries, expected)
-
-    def test_duplicate_same_mapping(self):
-        duplicate_entries = """
-        [scheme:duplicate_same_mapping]
-        entries:
-            sections1 -> target1
-            sections2 -> target2
-            sections1 -> target1
-        """
+    def test_basic(self):
+        test_fragment = self.create_fragment_file(u"""
+[scheme:test]
+entries:
+    sections1 -> target1
+    sections2 -> target2
+""")
 
-        scheme = self.parse(duplicate_entries)
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].entries,
+                         {("sections1", "target1"),
+                          ("sections2", "target2")})
 
-        entries = scheme.entries
+    def test_duplicate_entries(self):
+        test_fragment = self.create_fragment_file(u"""
+[scheme:test]
+entries:
+    sections1 -> target1
+    sections2 -> target2
+    sections2 -> target2
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(fragment_file.fragments[0].entries,
+                         {("sections1", "target1"),
+                          ("sections2", "target2")})
+
+    def test_empty_entries(self):
+        test_fragment = self.create_fragment_file(u"""
+[scheme:test]
+entries:
+""")
 
-        expected = {
-            ("sections1", "target1"),
-            ("sections2", "target2")
-        }
+        with self.assertRaises(ParseException):
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        self.assertEqual(len(entries), 2)
-        self.assertEqual(entries, expected)
+        test_fragment = self.create_fragment_file(u"""
+[scheme:test]
+entries:
+    if B = y:
+        sections1 -> target1
+""")
 
-    def test_invalid_separator(self):
-        wrong_character = """
-        [scheme:test]
-        entries:
-            sections1, target1
-        """
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        single_word = """
-        [scheme:test]
-        entries:
-            sections1
-        """
+    def test_improper_grammar(self):
+        test_fragment = self.create_fragment_file(u"""
+[scheme:test]
+entries:
+    sections1, target1 # improper separator
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(wrong_character)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(single_word)
 
-    def test_blank_entries(self):
-        blank_entries = """
-        [scheme:test]
-        entries:
-        """
+class MappingTest(FragmentTest):
 
-        with self.assertRaises(ParseException):
-            self.parse(blank_entries)
+    def test_basic(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    obj:symbol (noflash)
+    obj (noflash)
+    obj:symbol_2 (noflash)
+    obj_2 (noflash)
+    * (noflash)
+""")
+
+        expected = {("obj", "symbol", "noflash"),
+                    ("obj", None, "noflash"),
+                    ("obj", "symbol_2", "noflash"),
+                    ("obj_2", None, "noflash"),
+                    ("*", None, "noflash")}
 
-    def test_non_existent_entries(self):
-        misspelled_entries_field = """
-        [scheme:test]
-        entrie:
-            section -> target
-        """
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
-        missing_entries_field = """
-        [scheme:test]
-        """
+    def test_archive(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive:
+entries:
+    * (default)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(misspelled_entries_field)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive:
+    lib1.a
+    lib2.a
+entries:
+    * (default)
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_empty_entries(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive:
+    lib.a
+entries:
+    if B = y:
+        * (noflash) # if condition is false, then no 'entries' key value
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive:
+    lib.a
+entries:
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(missing_entries_field)
-
+            FragmentFile(test_fragment, self.sdkconfig)
 
-class MappingTest(FragmentTest):
-
-    def setUp(self):
-        self.parser = fragments.Mapping.get_fragment_grammar()
+    def test_duplicate_entries(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive:
+    lib.a
+entries:
+    obj:symbol (noflash)
+    obj:symbol (noflash)
+""")
 
-    def parse_expression(self, expression):
-        parser = SDKConfig.get_expression_grammar()
-        return parser.parseString(expression, parseAll=True)
+        expected = {("obj", "symbol", "noflash")}
 
-    def test_valid_grammar(self):
-        valid_entries = """
-        [mapping]
-        archive: lib.a
-        entries:
-            obj:symbol (noflash)
-            # Comments should not matter
-            obj (noflash)
-            # Nor should whitespace
-                            obj  :     symbol_2 (    noflash )
-              obj_2  (    noflash )
-            * (noflash)
-        """
-
-        mapping = self.parse(valid_entries)
-
-        self.assertEqual("lib.a", mapping.archive)
-        self.assertEqual("lib_a", mapping.name)
-
-        entries = mapping.entries
-
-        expected = [("default", {
-                    ("obj", "symbol", "noflash"),
-                    ("obj", None, "noflash"),
-                    ("obj", "symbol_2", "noflash"),
-                    ("obj_2", None, "noflash"),
-                    ("*", None, "noflash")
-                    })]
-
-        self.assertEqual(entries, expected)
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_invalid_grammar(self):
-        with_fragment_name = """
-        [mapping:name]
-        archive: lib.a
-        entries:
-            obj:symbol (noflash)
-        """
-
-        missing_archive = """
-        [mapping:name]
-        entries:
-            obj:symbol (noflash)
-        """
-
-        misspelled_archive = """
-        [mapping:name]
-        archi: lib.a
-        entries:
-            obj:symbol (noflash)
-        """
-
-        missing_entries = """
-        [mapping]
-        archive: lib.a
-        """
-
-        misspelled_entries = """
-        [mapping]
-        archive: lib.a
-        entrie:
-            obj:symbol (noflash)
-        """
-
-        missing_symbols = """
-        [mapping]
-        archive: lib.a
-        entries:
-            obj: (noflash)
-        """
-
-        missing_scheme_1 = """
-        [mapping]
-        archive: lib.a
-        entries:
-            obj: ()
-        """
-
-        missing_scheme_2 = """
-        [mapping]
-        archive: lib.a
-        entries:
-            obj:symbol
-        """
-
-        missing_entity = """
-        [mapping]
-        archive: lib.a
-        entries:
-            (noflash)
-        """
-
-        wilcard_symbol = """
-        [mapping]
-        archive: lib.a
-        entries:
-            obj:* (noflash)
-        """
-
-        empty_object_with_symbol = """
-        [mapping]
-        archive: lib.a
-        entries:
-            :symbol (noflash)
-        """
-
-        wildcard_object_with_symbol = """
-        [mapping]
-        archive: lib.a
-        entries:
-            *:symbol (noflash)
-        """
-
-        empty_definition = """
-        [mapping]
-        """
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive:
+    lib.a
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+entries:
+    * (default)
+""")
+
+        with self.assertRaises(ParseFatalException):
+            FragmentFile(test_fragment, self.sdkconfig)
+
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    obj: (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(with_fragment_name)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(missing_archive)
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    obj: ()
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(misspelled_archive)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(missing_entries)
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    obj:symbol
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(misspelled_entries)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(missing_symbols)
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(missing_scheme_1)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(missing_scheme_2)
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    obj:* (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(missing_entity)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(wilcard_symbol)
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    :symbol (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(empty_object_with_symbol)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-        with self.assertRaises(ParseException):
-            self.parse(wildcard_object_with_symbol)
+        test_fragment = self.create_fragment_file(u"""
+[mapping:test]
+archive: lib.a
+entries:
+    *:symbol (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(empty_definition)
+            FragmentFile(test_fragment, self.sdkconfig)
 
-    def test_explicit_blank_default_w_others(self):
-        expl_blnk_w_oth = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_a (noflash)
-            : default
-        """
 
-        mapping = self.parse(expl_blnk_w_oth)
+class DeprecatedMappingTest(FragmentTest):
 
-        entries = mapping.entries
+    def test_valid_grammar(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    obj:symbol (noflash)
+    # Comments should not matter
+    obj (noflash)
+    # Nor should whitespace
+                    obj  :     symbol_2 (    noflash )
+        obj_2  (    noflash )
+    * (noflash)
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual("lib.a", fragment_file.fragments[0].archive)
+        self.assertEqual("lib_a", fragment_file.fragments[0].name)
+
+        expected = {("obj", "symbol", "noflash"),
+                    ("obj", None, "noflash"),
+                    ("obj", "symbol_2", "noflash"),
+                    ("obj_2", None, "noflash"),
+                    ("*", None, "noflash")
+                    }
 
-        expected = [(entries[0][0], {
-                    ("obj_a", None, "noflash"),
-                    }),
-                    ("default", set())]
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
-        self.assertEqual(entries, expected)
+    def test_explicit_blank_default_w_others(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = n
+    obj_a (noflash)
+    : default
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        expected = {("*", None, "default")}
+
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_implicit_blank_default_w_others(self):
-        impl_blnk_w_oth = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_a (noflash)
-        """
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = n
+    obj_a (noflash)
+""")
 
-        mapping = self.parse(impl_blnk_w_oth)
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        expected = {("*", None, "default")}
 
-        entries = mapping.entries
-
-        expected = [(entries[0][0], {
-                    ("obj_a", None, "noflash"),
-                    }),
-                    ("default", set())]
-
-        self.assertEqual(entries, expected)
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_explicit_blank_default(self):
-        expl_blnk_def = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : default
-        """
-        mapping = self.parse(expl_blnk_def)
-        entries = mapping.entries
-        expected = [("default", set())]
-
-        self.assertEqual(entries, expected)
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : default
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        expected = {("*", None, "default")}
+
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_implicit_blank_default(self):
-        impl_blnk_def = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : default
-        """
-        mapping = self.parse(impl_blnk_def)
-        entries = mapping.entries
-        expected = [("default", set())]
-
-        self.assertEqual(entries, expected)
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : default
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        expected = {("*", None, "default")}
+
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_multiple_entries(self):
-        multiple_entries = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_a1 (noflash)
-            obj_a2 (noflash)
-            : CONFIG_B = y
-            obj_b1 (noflash)
-            obj_b2 (noflash)
-            obj_b3 (noflash)
-            : CONFIG_C = y
-            obj_c1 (noflash)
-        """
-
-        mapping = self.parse(multiple_entries)
-
-        entries = mapping.entries
-
-        expected = [(entries[0][0], {
-                    ("obj_a1", None, "noflash"),
-                    ("obj_a2", None, "noflash"),
-                    }),
-                    (entries[1][0], {
-                        ("obj_b1", None, "noflash"),
-                        ("obj_b2", None, "noflash"),
-                        ("obj_b3", None, "noflash"),
-                    }),
-                    (entries[2][0], {
-                        ("obj_c1", None, "noflash"),
-                    }),
-                    ("default", set())]
-
-        self.assertEqual(entries, expected)
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = n
+    obj_a1 (noflash)
+    obj_a2 (noflash)
+    : B = n
+    obj_b1 (noflash)
+    obj_b2 (noflash)
+    obj_b3 (noflash)
+    : C = n
+    obj_c1 (noflash)
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        expected = {("obj_b1", None, "noflash"),
+                    ("obj_b2", None, "noflash"),
+                    ("obj_b3", None, "noflash")}
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_blank_entries(self):
-        blank_entries = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_a (noflash)
-            : CONFIG_B = y
-            : CONFIG_C = y
-            obj_c (noflash)
-            : CONFIG_D = y
-            : CONFIG_E = y
-            : default
-            obj (noflash)
-        """
-
-        mapping = self.parse(blank_entries)
-
-        entries = mapping.entries
-
-        expected = [(entries[0][0], {
-                    ("obj_a", None, "noflash")
-                    }),
-                    (entries[1][0], set()),
-                    (entries[2][0], {
-                        ("obj_c", None, "noflash")
-                    }),
-                    (entries[3][0], set()),
-                    (entries[4][0], set()),
-                    ("default", {
-                        ("obj", None, "noflash")
-                    })]
-
-        self.assertEqual(entries, expected)
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = n
+    obj_a (noflash)
+    : B = n
+    : C = n
+    obj_c (noflash)
+    : default
+    obj (noflash)
+""")
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        expected = {("*", None, "default")}
+        self.assertEqual(expected, fragment_file.fragments[0].entries)
 
     def test_blank_first_condition(self):
-        blank_first_condition = """
-        [mapping]
-        archive: lib.a
-        entries:
-            obj_a (noflash)
-            : CONFIG_B = y
-            obj_b (noflash)
-        """
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    obj_a (noflash)
+    : CONFIG_B = y
+    obj_b (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(blank_first_condition)
-
-    def test_nonlast_default(self):
-        nonlast_default_1 = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : default
-            obj_a (noflash)
-            : CONFIG_A = y
-            obj_A (noflash)
-        """
-
-        nonlast_default_2 = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_A (noflash)
-            : default
-            obj_a (noflash)
-            : CONFIG_B = y
-            obj_B (noflash)
-        """
-
-        nonlast_default_3 = """
-        [mapping]
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_A (noflash)
-            :
-            obj_a (noflash)
-            : CONFIG_B = y
-            obj_B (noflash)
-        """
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_nonlast_default_1(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : default
+    obj_a (noflash)
+    : CONFIG_A = y
+    obj_A (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(nonlast_default_1)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_nonlast_default_2(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = y
+    obj_A (noflash)
+    : default
+    obj_a (noflash)
+    : B = y
+    obj_B (noflash
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(nonlast_default_2)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_nonlast_default_3(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = y
+    obj_A (noflash)
+    :
+    obj_a (noflash)
+    : B = y
+    obj_B (noflash
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(nonlast_default_3)
-
-    def test_duplicate_default(self):
-        duplicate_default_1 = """
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_A (noflash)
-            : default
-            obj_a (noflash)
-            : CONFIG_B = y
-            obj_B (noflash)
-            : default
-            obj_a (noflash)
-        """
-
-        duplicate_default_2 = """
-        archive: lib.a
-        entries:
-            : CONFIG_A = y
-            obj_A (noflash)
-            : CONFIG_B = y
-            obj_a (noflash)
-            : default
-            obj_B (noflash)
-            :
-            obj_a (noflash)
-        """
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_duplicate_default_1(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : CONFIG_A = y
+    obj_A (noflash)
+    : default
+    obj_a (noflash)
+    : CONFIG_B = y
+    obj_B (noflash)
+    : default
+    obj_a (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(duplicate_default_1)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_duplicate_default_2(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : CONFIG_A = y
+    obj_A (noflash)
+    : CONFIG_B = y
+    obj_a (noflash)
+    : default
+    obj_B (noflash)
+    :
+    obj_a (noflash)
+""")
 
         with self.assertRaises(ParseException):
-            self.parse(duplicate_default_2)
+            FragmentFile(test_fragment, self.sdkconfig)
+
+    def test_mixed_deprecated_mapping(self):
+        test_fragment = self.create_fragment_file(u"""
+[mapping]
+archive: lib.a
+entries:
+    : A = n
+    obj_A (noflash)
+    : default
+    obj_B (noflash)
+
+
+[mapping:test]
+archive: lib.a
+entries:
+    if A = n:
+        obj_A (noflash)
+    else:
+        obj_B (noflash)
+""")
+
+        fragment_file = FragmentFile(test_fragment, self.sdkconfig)
+        self.assertEqual(2, len(fragment_file.fragments))
+
+        self.assertEqual(fragment_file.fragments[0].entries,
+                         fragment_file.fragments[1].entries)
 
 
 if __name__ == "__main__":