Merge branch 'feature/follow_symlinks_in_spiffsgen' into 'master'

spiffs: follow symlinks in spiffsgen

See merge request espressif/esp-idf!6280

components/spiffs/Kconfig

@@ -76,6 +76,13 @@ menu "SPIFFS Configuration"
             SPIFFS_OBJ_NAME_LEN + SPIFFS_META_LENGTH should not exceed
             SPIFFS_PAGE_SIZE - 64.
 
+    config SPIFFS_FOLLOW_SYMLINKS
+        bool "Enable symbolic links for image creation"
+        default "n"
+        help
+            If this option is enabled, symbolic links are taken into account
+            during partition image creation.
+
     config SPIFFS_USE_MAGIC
         bool "Enable SPIFFS Filesystem Magic"
         default "y"

components/spiffs/Makefile.projbuild

@@ -13,6 +13,12 @@ else
 USE_MAGIC_LEN = ""
 endif
 
+ifdef CONFIG_SPIFFS_FOLLOW_SYMLINKS
+FOLLOW_SYMLINKS = "--follow-symlinks" 
+else
+FOLLOW_SYMLINKS = ""
+endif
+
 # spiffs_create_partition_image
 #
 # Create a spiffs image of the specified directory on the host during build and optionally
@@ -27,6 +33,7 @@ $(1)_bin: $(PARTITION_TABLE_BIN) $(SPIFFS_IMAGE_DEPENDS) | check_python_dependen
 	--page-size=$(CONFIG_SPIFFS_PAGE_SIZE) \
 	--obj-name-len=$(CONFIG_SPIFFS_OBJ_NAME_LEN) \
 	--meta-len=$(CONFIG_SPIFFS_META_LENGTH) \
+	$(FOLLOW_SYMLINKS) \
 	$(USE_MAGIC) \
 	$(USE_MAGIC_LEN) 
 
@@ -43,4 +50,4 @@ endef
 
 ESPTOOL_ALL_FLASH_ARGS += $(foreach partition,$(SPIFFSGEN_FLASH_IN_PROJECT), \
 $(shell $(GET_PART_INFO) --partition-table-file $(PARTITION_TABLE_BIN) \
-get_partition_info --partition-name $(partition) --info offset) $(BUILD_DIR_BASE)/$(partition).bin)
+get_partition_info --partition-name $(partition) --info offset) $(BUILD_DIR_BASE)/$(partition).bin)

components/spiffs/project_include.cmake

@@ -25,6 +25,10 @@ function(spiffs_create_partition_image partition base_dir)
         set(use_magic_len "--use-magic-len")
     endif()
 
+    if(CONFIG_SPIFFS_FOLLOW_SYMLINKS)
+        set(follow_symlinks "--follow-symlinks")
+    endif()
+
     # Execute SPIFFS image generation; this always executes as there is no way to specify for CMake to watch for
     # contents of the base dir changing.
     add_custom_target(spiffs_${partition}_bin ALL
@@ -32,6 +36,7 @@ function(spiffs_create_partition_image partition base_dir)
         --page-size=${CONFIG_SPIFFS_PAGE_SIZE}
         --obj-name-len=${CONFIG_SPIFFS_OBJ_NAME_LEN}
         --meta-len=${CONFIG_SPIFFS_META_LENGTH}
+        ${follow_symlinks}
         ${use_magic}
         ${use_magic_len}
         DEPENDS ${arg_DEPENDS}
@@ -46,4 +51,4 @@ function(spiffs_create_partition_image partition base_dir)
     else()
         esptool_py_flash_project_args(${partition} ${offset} ${image_file})
     endif()
-endfunction()
+endfunction()

components/spiffs/spiffsgen.py

@@ -488,6 +488,11 @@ def main():
                         action="store_true",
                         default=True)
 
+    parser.add_argument("--follow-symlinks",
+                        help="Take into account symbolic links during partition image creation.",
+                        action="store_true",
+                        default=False)
+
     parser.add_argument("--use-magic-len",
                         help="Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.",
                         action="store_true",
@@ -513,7 +518,7 @@ def main():
 
         spiffs = SpiffsFS(image_size, spiffs_build_default)
 
-        for root, dirs, files in os.walk(args.base_dir):
+        for root, dirs, files in os.walk(args.base_dir, followlinks=args.follow_symlinks):
             for f in files:
                 full_path = os.path.join(root, f)
                 spiffs.create_file("/" + os.path.relpath(full_path, args.base_dir).replace("\\", "/"), full_path)

components/spiffs/test_spiffs_host/Makefile

@@ -87,7 +87,11 @@ $(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_BUILD_DIR)/$(SPI_FLASH_SI
 
 # Use spiffs source directory as the test image
 spiffs_image: ../spiffs $(shell find ../spiffs -type d) $(shell find ../spiffs -type -f -name '*')
-	../spiffsgen.py 2097152 ../spiffs image.bin 
+	# Creation of test symlinks unfortunately causes rerun of spiffsgen.py every make invoke
+	rm -f ../spiffs/include ../spiffs/CMakeLists.txt 
+	ln -s ../include ../spiffs/include
+	ln -s ../CMakeLists.txt ../spiffs/CMakeLists.txt
+	../spiffsgen.py --follow-symlinks 2097152 ../spiffs image.bin 
 
 test: $(TEST_PROGRAM) spiffs_image 
 	./$(TEST_PROGRAM)

components/spiffs/test_spiffs_host/test_spiffs.cpp

@@ -1,9 +1,13 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdint.h>
+#include <stdlib.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <dirent.h>
 #include <limits.h>
+#include <unistd.h>
 
 #include "esp_partition.h"
 #include "spiffs.h"
@@ -96,7 +100,18 @@ static void check_spiffs_files(spiffs *fs, const char *base_path, char* cur_path
 
     while ((entry = readdir(dir)) != NULL) {
         char *name = entry->d_name;
-        if (entry->d_type == DT_DIR) {
+
+        char path[PATH_MAX] = { 0 };
+
+        // Read the file from host FS
+        strcpy(path, cur_path);
+        strcat(path, "/");
+        strcat(path, name);
+
+        struct stat sb;
+        stat(path, &sb);
+
+        if (S_ISDIR(sb.st_mode)) {
             if (!strcmp(name, ".") || !strcmp(name, ".."))
                 continue;
             cur_path[len] = '/';
@@ -104,13 +119,6 @@ static void check_spiffs_files(spiffs *fs, const char *base_path, char* cur_path
             check_spiffs_files(fs, base_path, cur_path);
             cur_path[len] = '\0';
         } else {
-            char path[PATH_MAX];
-
-            // Read the file from host FS
-            strcpy(path, cur_path);
-            strcat(path, "/");
-            strcat(path, name);
-
             FILE* f = fopen(path , "r");
             REQUIRE(f);
             fseek(f, 0, SEEK_END);
@@ -126,6 +134,7 @@ static void check_spiffs_files(spiffs *fs, const char *base_path, char* cur_path
             // Read the file from SPIFFS
             char *spiffs_path = path + strlen(base_path);
             spiffs_res = SPIFFS_open(fs, spiffs_path, SPIFFS_RDONLY, 0);
+
             REQUIRE(spiffs_res > SPIFFS_OK);
 
             spiffs_file fd = spiffs_res;

tools/ldgen/samples/sdkconfig

@@ -516,6 +516,7 @@ CONFIG_SPIFFS_PAGE_SIZE=256
 CONFIG_SPIFFS_OBJ_NAME_LEN=32
 CONFIG_SPIFFS_USE_MAGIC=y
 CONFIG_SPIFFS_USE_MAGIC_LENGTH=y
+CONFIG_SPIFFS_FOLLOW_SYMLINKS=
 CONFIG_SPIFFS_META_LENGTH=4
 CONFIG_SPIFFS_USE_MTIME=y