Merge branch 'contrib/github_pr_8169' into 'master'

fix fatfs_create_spiflash_image() feature

Closes IDFGH-6383

See merge request espressif/esp-idf!16601

components/fatfs/fatfsgen.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -107,7 +107,7 @@ class FATFS:
             parent_dir = self.root_directory.recursive_search(path_from_root, self.root_directory)
         self.root_directory.new_directory(name=name, parent=parent_dir, path_from_root=path_from_root)
 
-    def write_content(self, path_from_root: List[str], content: str) -> None:
+    def write_content(self, path_from_root: List[str], content: bytes) -> None:
         """
         fat fs invokes root directory to recursively find the required file and writes the content
         """
@@ -167,7 +167,7 @@ class FATFS:
         normal_path = os.path.normpath(folder_relative_path)
         split_path = normal_path.split(os.sep)
         if os.path.isfile(real_path):
-            with open(real_path) as file:
+            with open(real_path, 'rb') as file:
                 content = file.read()
             file_name, extension = os.path.splitext(split_path[-1])
             extension = extension[1:]  # remove the dot from the extension

components/fatfs/fatfsgen_utils/fs_object.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -42,12 +42,12 @@ class File:
     def name_equals(self, name: str, extension: str) -> bool:
         return self.name == name and self.extension == extension
 
-    def write(self, content: str) -> None:
+    def write(self, content: bytes) -> None:
         self.entry.update_content_size(len(content))
         # we assume that the correct amount of clusters is allocated
         current_cluster = self._first_cluster
         for content_part in split_content_into_sectors(content, self.fatfs_state.sector_size):
-            content_as_list = content_part.encode()
+            content_as_list = content_part
             if current_cluster is None:
                 raise FatalError('No free space left!')
 
@@ -207,7 +207,7 @@ class Directory:
         directory.init_directory()
         target_dir.entities.append(directory)
 
-    def write_to_file(self, path: List[str], content: str) -> None:
+    def write_to_file(self, path: List[str], content: bytes) -> None:
         """
         Writes to file existing in the directory structure.
 

components/fatfs/fatfsgen_utils/utils.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
@@ -18,7 +18,7 @@ def crc32(input_values: List[int], crc: int) -> int:
     return binascii.crc32(bytearray(input_values), crc)
 
 
-def required_clusters_count(cluster_size: int, content: str) -> int:
+def required_clusters_count(cluster_size: int, content: bytes) -> int:
     # compute number of required clusters for file text
     return (len(content) + cluster_size - 1) // cluster_size
 
@@ -66,7 +66,7 @@ def clean_second_half_byte(bytes_array: bytearray, address: int) -> None:
     bytes_array[address] &= 0x0f
 
 
-def split_content_into_sectors(content: str, sector_size: int) -> List[str]:
+def split_content_into_sectors(content: bytes, sector_size: int) -> List[bytes]:
     result = []
     clusters_cnt = required_clusters_count(cluster_size=sector_size, content=content)
 

components/fatfs/test_fatfsgen/test_fatfsgen.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -49,14 +49,13 @@ class FatFSGen(unittest.TestCase):
         fatfs.create_file('TESTF', extension='TXT')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
-
         self.assertEqual(file_system[0x2000:0x200c], b'TESTF   TXT\x20')  # check entry name and type
         self.assertEqual(file_system[0x1000:0x1006], b'\xf8\xff\xff\xff\x0f\x00')  # check fat
 
     def test_write_to_file_with_extension_sn_fat12(self) -> None:
         fatfs = fatfsgen.FATFS()
         fatfs.create_file('WRITEF', extension='TXT')
-        fatfs.write_content(path_from_root=['WRITEF.TXT'], content='testcontent')
+        fatfs.write_content(path_from_root=['WRITEF.TXT'], content=b'testcontent')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
 
@@ -69,7 +68,7 @@ class FatFSGen(unittest.TestCase):
         fatfs = fatfsgen.FATFS()
         fatfs.create_directory('TESTFOLD')
         fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD'])
-        fatfs.write_content(path_from_root=['TESTFOLD', 'WRITEF.TXT'], content='testcontent')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'WRITEF.TXT'], content=b'testcontent')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
 
@@ -100,7 +99,7 @@ class FatFSGen(unittest.TestCase):
     def test_full_sector_file(self) -> None:
         fatfs = fatfsgen.FATFS()
         fatfs.create_file('WRITEF', extension='TXT')
-        fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * 'a')
+        fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
         self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\xff\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00')
@@ -109,7 +108,7 @@ class FatFSGen(unittest.TestCase):
     def test_file_chaining(self) -> None:
         fatfs = fatfsgen.FATFS()
         fatfs.create_file('WRITEF', extension='TXT')
-        fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * 'a' + 'a')
+        fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a' + b'a')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
         self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\x03\xf0\xff\x00\x00\x00\x00\x00\x00\x00\x00')
@@ -121,8 +120,8 @@ class FatFSGen(unittest.TestCase):
 
         for i in range(CFG['sector_size'] // CFG['entry_size']):
             fatfs.create_file(f'A{str(i).upper()}', path_from_root=['TESTFOLD'])
-        fatfs.write_content(path_from_root=['TESTFOLD', 'A0'], content='first')
-        fatfs.write_content(path_from_root=['TESTFOLD', 'A126'], content='later')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'A0'], content=b'first')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'A126'], content=b'later')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
         self.assertEqual(file_system[0x1000: 0x10d0],
@@ -136,13 +135,13 @@ class FatFSGen(unittest.TestCase):
         fatfs.create_directory('TESTFOLD')
         fatfs.create_directory('TESTFOLL', path_from_root=['TESTFOLD'])
         self.assertRaises(WriteDirectoryException, fatfs.write_content, path_from_root=['TESTFOLD', 'TESTFOLL'],
-                          content='testcontent')
+                          content=b'testcontent')
 
     def test_write_non_existing_file_in_folder_sn_fat12(self) -> None:
         fatfs = fatfsgen.FATFS()
         fatfs.create_directory('TESTFOLD')
         self.assertRaises(FileNotFoundError, fatfs.write_content, path_from_root=['TESTFOLD', 'AHOJ'],
-                          content='testcontent')
+                          content=b'testcontent')
 
     @staticmethod
     def create_too_many_files() -> None:
@@ -160,8 +159,8 @@ class FatFSGen(unittest.TestCase):
 
         for i in range(2 * CFG['sector_size'] // CFG['entry_size']):
             fatfs.create_file(f'A{str(i).upper()}', path_from_root=['TESTFOLD'])
-        fatfs.write_content(path_from_root=['TESTFOLD', 'A253'], content='later')
-        fatfs.write_content(path_from_root=['TESTFOLD', 'A255'], content='last')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'A253'], content=b'later')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'A255'], content=b'last')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
         self.assertEqual(file_system[0x105000:0x105010], b'later\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
@@ -192,7 +191,7 @@ class FatFSGen(unittest.TestCase):
         fatfs.create_directory('TESTFOLL', path_from_root=['TESTFOLD'])
         fatfs.create_directory('TESTFOLO', path_from_root=['TESTFOLD', 'TESTFOLL'])
         fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO'])
-        fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO', 'WRITEF.TXT'], content='later')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO', 'WRITEF.TXT'], content=b'later')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
 
@@ -204,7 +203,7 @@ class FatFSGen(unittest.TestCase):
         fatfs.create_directory('TESTFOLD', path_from_root=['TESTFOLD'])
         fatfs.create_directory('TESTFOLD', path_from_root=['TESTFOLD', 'TESTFOLD'])
         fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD'])
-        fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD', 'WRITEF.TXT'], content='later')
+        fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD', 'WRITEF.TXT'], content=b'later')
         fatfs.write_filesystem(CFG['output_file'])
         file_system = fatfs.read_filesystem(CFG['output_file'])
 

components/fatfs/test_fatfsgen/test_wl_fatfsgen.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -132,6 +132,7 @@ class WLFatFSGen(unittest.TestCase):
         fatfs.wl_write_filesystem(CFG['output_file'])
         with open(CFG['output_file'], 'rb') as fs_file:
             file_system = bytearray(fs_file.read())
+
         self.assertEqual(file_system[0x7060:0x7070], b'TESTFIL2    \x00\x00\x01\x00')
         self.assertEqual(file_system[0x7070:0x7080], b'!\x00\x00\x00\x00\x00\x01\x00\x01\x00\x05\x00\x0b\x00\x00\x00')
         self.assertEqual(file_system[0x8040:0x8050], b'LASTFILE    \x00\x00\x01\x00')

components/fatfs/wl_fatfsgen.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 from typing import List, Optional
@@ -180,7 +180,7 @@ class WLFATFS:
     def wl_create_directory(self, name: str, path_from_root: Optional[List[str]] = None) -> None:
         self.plain_fatfs.create_directory(name, path_from_root)
 
-    def wl_write_content(self, path_from_root: List[str], content: str) -> None:
+    def wl_write_content(self, path_from_root: List[str], content: bytes) -> None:
         self.plain_fatfs.write_content(path_from_root, content)