fatfsgen.py: enabled extension of the FAT table

Closes IDF-5688

components/fatfs/fatfs_utils/boot_sector.py

@@ -107,14 +107,11 @@ class BootSector:
             # uncomment for FAT32 implementation
             # sectors_count_ = self._parsed_header['BPB_TotSec32']
             # possible_fat_types = [FAT32]
-            assert self._parsed_header['BPB_FATSz16'] == 0
+            assert self._parsed_header['BPB_TotSec16'] == 0
             raise NotImplementedError('FAT32 not implemented!')
         else:
             raise InconsistentFATAttributes('The number of FS sectors cannot be zero!')
 
-        # in the current code assigning self._parsed_header['BPB_TotSec32'] is not reachable
-        # the option to assign it is kept for possibility to implement FAT32
-        sectors_per_fat_cnt_ = self._parsed_header['BPB_FATSz16'] or self._parsed_header['BPB_TotSec32']
         if self._parsed_header['BPB_BytsPerSec'] not in ALLOWED_SECTOR_SIZES:
             raise InconsistentFATAttributes(f'The number of bytes '
                                             f"per sector is {self._parsed_header['BPB_BytsPerSec']}! "
@@ -134,7 +131,6 @@ class BootSector:
                                                  root_dir_sectors_cnt=root_dir_sectors_cnt_,
                                                  sectors_count=sectors_count_,
                                                  media_type=self._parsed_header['BPB_Media'],
-                                                 sectors_per_fat_cnt=sectors_per_fat_cnt_,
                                                  sec_per_track=self._parsed_header['BPB_SecPerTrk'],
                                                  num_heads=self._parsed_header['BPB_NumHeads'],
                                                  hidden_sectors=self._parsed_header['BPB_HiddSec'],

components/fatfs/fatfs_utils/fatfs_state.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 from .exceptions import InconsistentFATAttributes
 from .utils import (ALLOWED_SECTOR_SIZES, FAT12, FAT12_MAX_CLUSTERS, FAT16, FAT16_MAX_CLUSTERS, FATDefaults,
-                    get_fatfs_type, get_non_data_sectors_cnt, number_of_clusters)
+                    get_fat_sectors_count, get_fatfs_type, get_non_data_sectors_cnt, number_of_clusters)
 
 
 class FATFSState:
@@ -20,7 +20,6 @@ class FATFSState:
                  root_dir_sectors_cnt: int,
                  size: int,
                  media_type: int,
-                 sectors_per_fat: int,
                  sectors_per_cluster: int,
                  volume_label: str,
                  oem_name: str,
@@ -40,18 +39,18 @@ class FATFSState:
                                                  root_dir_sectors_cnt=root_dir_sectors_cnt,
                                                  sectors_count=size // sector_size,
                                                  media_type=media_type,
-                                                 sectors_per_fat_cnt=sectors_per_fat,
                                                  sec_per_track=sec_per_track,
                                                  num_heads=num_heads,
                                                  hidden_sectors=hidden_sectors,
                                                  volume_label=volume_label,
                                                  file_sys_type=file_sys_type,
                                                  volume_uuid=-1)
+
         self._explicit_fat_type: Optional[int] = explicit_fat_type
         self.long_names_enabled: bool = long_names_enabled
         self.use_default_datetime: bool = use_default_datetime
 
-        if self.boot_sector_state.clusters in (FAT12_MAX_CLUSTERS, FAT16_MAX_CLUSTERS):
+        if (size // sector_size) * sectors_per_cluster in (FAT12_MAX_CLUSTERS, FAT16_MAX_CLUSTERS):
             print('WARNING: It is not recommended to create FATFS with bounding '
                   f'count of clusters: {FAT12_MAX_CLUSTERS} or {FAT16_MAX_CLUSTERS}')
         self.check_fat_type()
@@ -86,7 +85,6 @@ class BootSectorState:
                  root_dir_sectors_cnt: int,
                  sectors_count: int,
                  media_type: int,
-                 sectors_per_fat_cnt: int,
                  sec_per_track: int,
                  num_heads: int,
                  hidden_sectors: int,
@@ -102,7 +100,7 @@ class BootSectorState:
         self.root_dir_sectors_cnt: int = root_dir_sectors_cnt
         self.sectors_count: int = sectors_count
         self.media_type: int = media_type
-        self.sectors_per_fat_cnt: int = sectors_per_fat_cnt
+        self.sectors_per_fat_cnt = get_fat_sectors_count(self.size // self.sector_size, self.sector_size)
         self.sec_per_track: int = sec_per_track
         self.num_heads: int = num_heads
         self.hidden_sectors: int = hidden_sectors
@@ -161,4 +159,5 @@ class BootSectorState:
 
     @property
     def root_directory_start(self) -> int:
-        return (self.reserved_sectors_cnt + self.sectors_per_fat_cnt) * self.sector_size
+        root_dir_start: int = (self.reserved_sectors_cnt + self.sectors_per_fat_cnt) * self.sector_size
+        return root_dir_start

components/fatfs/fatfs_utils/utils.py

@@ -70,6 +70,17 @@ def get_fatfs_type(clusters_count: int) -> int:
     return FAT32
 
 
+def get_fat_sectors_count(clusters_count: int, sector_size: int) -> int:
+    fatfs_type_ = get_fatfs_type(clusters_count)
+    if fatfs_type_ == FAT32:
+        raise NotImplementedError('FAT32 is not supported!')
+    # number of byte halves
+    cluster_s: int = fatfs_type_ // 4
+    fat_size_bytes: int = (
+        clusters_count * 2 + cluster_s) if fatfs_type_ == FAT16 else (clusters_count * 3 + 1) // 2 + cluster_s
+    return (fat_size_bytes + sector_size - 1) // sector_size
+
+
 def required_clusters_count(cluster_size: int, content: bytes) -> int:
     # compute number of required clusters for file text
     return (len(content) + cluster_size - 1) // cluster_size
@@ -253,7 +264,6 @@ class FATDefaults:
     FAT_TABLES_COUNT: int = 1
     SECTORS_PER_CLUSTER: int = 1
     SECTOR_SIZE: int = 0x1000
-    SECTORS_PER_FAT: int = 1
     HIDDEN_SECTORS: int = 0
     ENTRY_SIZE: int = 32
     NUM_HEADS: int = 0xff

components/fatfs/fatfsgen.py

@@ -27,7 +27,6 @@ class FATFS:
                  fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT,
                  sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER,
                  sector_size: int = FATDefaults.SECTOR_SIZE,
-                 sectors_per_fat: int = FATDefaults.SECTORS_PER_FAT,
                  hidden_sectors: int = FATDefaults.HIDDEN_SECTORS,
                  long_names_enabled: bool = False,
                  use_default_datetime: bool = True,
@@ -54,7 +53,6 @@ class FATFS:
                                             file_sys_type=file_sys_type,
                                             num_heads=num_heads,
                                             fat_tables_cnt=fat_tables_cnt,
-                                            sectors_per_fat=sectors_per_fat,
                                             sectors_per_cluster=sectors_per_cluster,
                                             media_type=media_type,
                                             hidden_sectors=hidden_sectors,

components/fatfs/test_fatfsgen/test_fatfsgen.py

@@ -272,7 +272,7 @@ class FatFSGen(unittest.TestCase):
         fatfs.write_filesystem(CFG['output_file'])
         file_system = read_filesystem(CFG['output_file'])
         self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\xff\x03\x00\xff\xff\x00\x00\x00\x00\x00\x00')
-        self.assertEqual(file_system[0x7000: 0x8000], b'a' + (CFG['sector_size'] - 1) * b'\x00')
+        self.assertEqual(file_system[0x9000: 0xa000], b'a' + (CFG['sector_size'] - 1) * b'\x00')
 
     def test_full_sector_folder_fat16(self) -> None:
         fatfs = fatfsgen.FATFS(size=17 * 1024 * 1024)
@@ -285,9 +285,9 @@ class FatFSGen(unittest.TestCase):
         file_system = read_filesystem(CFG['output_file'])
         self.assertEqual(file_system[0x1000: 0x1110],
                          b'\xf8\xff\xff\xff\x82\x00' + 258 * b'\xff' + 8 * b'\x00')
-        self.assertEqual(file_system[0x85000:0x85005], b'later')
-        self.assertEqual(file_system[0x86000:0x86010], b'A126        \x00\x00\x00\x00')
-        self.assertEqual(file_system[0x86020:0x86030], b'A127        \x00\x00\x00\x00')
+        self.assertEqual(file_system[0x87000:0x87005], b'later')
+        self.assertEqual(file_system[0x88000:0x88010], b'A126        \x00\x00\x00\x00')
+        self.assertEqual(file_system[0x88020:0x88030], b'A127        \x00\x00\x00\x00')
 
     def test_empty_lfn_short_name(self) -> None:
         fatfs = fatfsgen.FATFS(long_names_enabled=True)
@@ -416,14 +416,14 @@ class FatFSGen(unittest.TestCase):
         self.assertEqual(file_system[0x7000: 0x7010], b'this is a test\x00\x00')
 
     def test_boundary_clusters12(self) -> None:
-        output: bytes = check_output(['python', '../fatfsgen.py', '--partition_size', '16756736', 'test_dir'],
+        output: bytes = check_output(['python', '../fatfsgen.py', '--partition_size', '16732160', 'test_dir'],
                                      stderr=STDOUT)
         self.assertEqual(
             output,
             b'WARNING: It is not recommended to create FATFS with bounding count of clusters: 4085 or 65525\n')
 
     def test_boundary_clusters16(self) -> None:
-        output: bytes = check_output(['python', '../fatfsgen.py', '--partition_size', '268415097', 'test_dir'],
+        output: bytes = check_output(['python', '../fatfsgen.py', '--partition_size', '268390400', 'test_dir'],
                                      stderr=STDOUT)
         self.assertEqual(
             output,
@@ -433,7 +433,7 @@ class FatFSGen(unittest.TestCase):
         self.assertRaises(NotImplementedError, fatfsgen.FATFS, size=268419193)
 
     def test_inconsistent_fat12(self) -> None:
-        self.assertRaises(InconsistentFATAttributes, fatfsgen.FATFS, size=268411001, explicit_fat_type=FAT12)
+        self.assertRaises(InconsistentFATAttributes, fatfsgen.FATFS, size=20480000, explicit_fat_type=FAT12)
 
     def test_lfn_increasing(self) -> None:
         fatfs: fatfsgen.FATFS = fatfsgen.FATFS(long_names_enabled=True)

components/fatfs/wl_fatfsgen.py

@@ -85,7 +85,6 @@ class WLFATFS:
                  reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT,
                  fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT,
                  sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER,
-                 sectors_per_fat: int = FATDefaults.SECTORS_PER_FAT,
                  explicit_fat_type: int = None,
                  hidden_sectors: int = FATDefaults.HIDDEN_SECTORS,
                  long_names_enabled: bool = False,
@@ -117,7 +116,6 @@ class WLFATFS:
         wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
                       self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
         self.plain_fat_sectors = self.total_sectors - wl_sectors
-
         self.plain_fatfs = FATFS(
             explicit_fat_type=explicit_fat_type,
             size=self.plain_fat_sectors * FATDefaults.WL_SECTOR_SIZE,
@@ -125,7 +123,6 @@ class WLFATFS:
             fat_tables_cnt=fat_tables_cnt,
             sectors_per_cluster=sectors_per_cluster,
             sector_size=FATDefaults.WL_SECTOR_SIZE,
-            sectors_per_fat=sectors_per_fat,
             root_entry_count=root_entry_count,
             hidden_sectors=hidden_sectors,
             long_names_enabled=long_names_enabled,