Merge branch 'feature/add_optional_immediate_fsync' into 'master'

fatfs: add optional immediate fsync

See merge request espressif/esp-idf!26097

components/fatfs/Kconfig

@@ -230,4 +230,13 @@ menu "FAT Filesystem support"
             accessing target media for given file descriptor!
             See 'Improving I/O performance' section of 'Maximizing Execution Speed' documentation page
             for more details.
+
+    config FATFS_IMMEDIATE_FSYNC
+        bool "Enable automatic f_sync"
+        default n
+        help
+            Enables automatic calling of f_sync() to flush recent file changes after each call of vfs_fat_write(),
+            vfs_fat_pwrite(), vfs_fat_link(), vfs_fat_truncate() and vfs_fat_ftruncate() functions.
+            This feature improves file-consistency and size reporting accuracy for the FatFS,
+            at a price on decreased performance due to frequent disk operations
 endmenu

components/fatfs/test_apps/flash_wl/main/test_fatfs_flash_wl.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -10,6 +10,7 @@
 #include <time.h>
 #include <sys/time.h>
 #include <sys/unistd.h>
+#include <sys/stat.h>
 #include "unity.h"
 #include "esp_partition.h"
 #include "esp_log.h"
@@ -274,3 +275,14 @@ TEST_CASE("FATFS prefers SPI RAM for allocations", "[fatfs]")
     test_teardown();
 }
 #endif // CONFIG_SPIRAM
+
+#if CONFIG_FATFS_IMMEDIATE_FSYNC
+
+TEST_CASE("Size is correct after write when immediate fsync is enabled", "[fatfs]")
+{
+    test_setup();
+    test_fatfs_size("/spiflash/size.txt", "random text\n preferably something relatively long\n");
+    test_teardown();
+}
+
+#endif // CONFIG_FATFS_IMMEDIATE_FSYNC

components/fatfs/test_apps/flash_wl/sdkconfig.ci.auto_fsync

@@ -0,0 +1 @@
+CONFIG_FATFS_IMMEDIATE_FSYNC=y

components/fatfs/test_apps/test_fatfs_common/test_fatfs_common.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -434,6 +434,40 @@ void test_fatfs_stat(const char* filename, const char* root_dir)
     TEST_ASSERT_FALSE(st.st_mode & S_IFREG);
 }
 
+void test_fatfs_size(const char* filename, const char* content) {
+    size_t expected_size = strlen(content);
+
+    int fd = open(filename, O_CREAT | O_WRONLY);
+    TEST_ASSERT_NOT_EQUAL(-1, fd);
+
+    ssize_t wr = write(fd, content, expected_size);
+    TEST_ASSERT_NOT_EQUAL(-1, wr);
+
+    struct stat st;
+    TEST_ASSERT_EQUAL(0, stat(filename, &st));
+    TEST_ASSERT_EQUAL(wr, st.st_size);
+
+    ssize_t wr2 = pwrite(fd, content, expected_size, expected_size);
+    TEST_ASSERT_NOT_EQUAL(-1, wr2);
+
+    TEST_ASSERT_EQUAL(0, stat(filename, &st));
+    TEST_ASSERT_EQUAL(wr + wr2, st.st_size);
+
+    TEST_ASSERT_EQUAL(0, ftruncate(fd, wr));
+
+    TEST_ASSERT_EQUAL(0, stat(filename, &st));
+    TEST_ASSERT_EQUAL(wr, st.st_size);
+
+    TEST_ASSERT_EQUAL(0, close(fd));
+
+    wr /= 2;
+
+    TEST_ASSERT_EQUAL(0, truncate(filename, wr));
+
+    TEST_ASSERT_EQUAL(0, stat(filename, &st));
+    TEST_ASSERT_EQUAL(wr, st.st_size);
+}
+
 void test_fatfs_mtime_dst(const char* filename, const char* root_dir)
 {
     struct timeval tv = { 1653638041, 0 };

components/fatfs/test_apps/test_fatfs_common/test_fatfs_common.h

@@ -51,6 +51,8 @@ void test_fatfs_ftruncate_file(const char* path);
 
 void test_fatfs_stat(const char* filename, const char* root_dir);
 
+void test_fatfs_size(const char* filename, const char* content);
+
 void test_fatfs_mtime_dst(const char* filename, const char* root_dir);
 
 void test_fatfs_utime(const char* filename, const char* root_dir);

components/fatfs/vfs/vfs_fat.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -416,6 +416,20 @@ static ssize_t vfs_fat_write(void* ctx, int fd, const void * data, size_t size)
             return -1;
         }
     }
+
+#if CONFIG_FATFS_IMMEDIATE_FSYNC
+    _lock_acquire(&fat_ctx->lock);
+    if (written > 0) {
+        res = f_sync(file);
+        if (res != FR_OK) {
+            ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
+            errno = fresult_to_errno(res);
+            return -1;
+        }
+     }
+    _lock_release(&fat_ctx->lock);
+#endif
+
     return written;
 }
 
@@ -514,6 +528,18 @@ static ssize_t vfs_fat_pwrite(void *ctx, int fd, const void *src, size_t size, o
         ret = -1; // in case the write was successful but the seek wasn't
     }
 
+#if CONFIG_FATFS_IMMEDIATE_FSYNC
+    if (wr > 0) {
+        FRESULT f_res2 = f_sync(file); // We need new result to check whether we can overwrite errno
+        if (f_res2 != FR_OK) {
+            ESP_LOGD(TAG, "%s: fresult=%d", __func__, f_res2);
+            if (f_res == FR_OK)
+                errno = fresult_to_errno(f_res2);
+            ret = -1;
+        }
+    }
+#endif
+
 pwrite_release:
     _lock_release(&fat_ctx->lock);
     return ret;
@@ -727,6 +753,13 @@ static int vfs_fat_link(void* ctx, const char* n1, const char* n2)
         }
         size_left -= will_copy;
     }
+
+#if CONFIG_FATFS_IMMEDIATE_FSYNC
+    _lock_acquire(&fat_ctx->lock);
+    res = f_sync(pf2);
+    _lock_release(&fat_ctx->lock);
+#endif
+
 fail3:
     f_close(pf2);
 fail2:
@@ -985,13 +1018,24 @@ static int vfs_fat_truncate(void* ctx, const char *path, off_t length)
     }
 
     res = f_truncate(file);
-    _lock_release(&fat_ctx->lock);
 
+    if (res != FR_OK) {
+        ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
+        errno = fresult_to_errno(res);
+        ret = -1;
+        goto close;
+    }
+
+#if CONFIG_FATFS_IMMEDIATE_FSYNC
+    res = f_sync(file);
     if (res != FR_OK) {
         ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
         errno = fresult_to_errno(res);
         ret = -1;
     }
+#endif
+
+    _lock_release(&fat_ctx->lock);
 
 close:
     res = f_close(file);
@@ -1055,8 +1099,18 @@ static int vfs_fat_ftruncate(void* ctx, int fd, off_t length)
         ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
         errno = fresult_to_errno(res);
         ret = -1;
+        goto out;
     }
 
+#if CONFIG_FATFS_IMMEDIATE_FSYNC
+    res = f_sync(file);
+    if (res != FR_OK) {
+        ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
+        errno = fresult_to_errno(res);
+        ret = -1;
+    }
+#endif
+
 out:
     _lock_release(&fat_ctx->lock);
     return ret;

docs/en/api-reference/storage/fatfs.rst

@@ -26,21 +26,23 @@ Most applications use the following workflow when working with ``esp_vfs_fat_``
 
 2. Call :cpp:func:`ff_diskio_register` to register the disk I/O driver for the drive number used in Step 1.
 
-3. Call the FatFs function ``f_mount``, and optionally ``f_fdisk``, ``f_mkfs``, to mount the filesystem using the same drive number which was passed to :cpp:func:`esp_vfs_fat_register`. For more information, see `FatFs documentation <http://elm-chan.org/fsw/ff/doc/mount.html>`_.
+3. Call the FatFs function :cpp:func:`f_mount`, and optionally :cpp:func:`f_fdisk`, :cpp:func:`f_mkfs`, to mount the filesystem using the same drive number which was passed to :cpp:func:`esp_vfs_fat_register`. For more information, see `FatFs documentation <http://elm-chan.org/fsw/ff/doc/mount.html>`_.
 
 4. Call the C standard library and POSIX API functions to perform such actions on files as open, read, write, erase, copy, etc. Use paths starting with the path prefix passed to :cpp:func:`esp_vfs_register` (for example, ``"/sdcard/hello.txt"``). The filesystem uses `8.3 filenames <https://en.wikipedia.org/wiki/8.3_filename>`_ format (SFN) by default. If you need to use long filenames (LFN), enable the :ref:`CONFIG_FATFS_LONG_FILENAMES` option. More details on the FatFs filenames are available `here <http://elm-chan.org/fsw/ff/doc/filename.html>`_.
 
 5. Optionally, by enabling the option :ref:`CONFIG_FATFS_USE_FASTSEEK`, you can use the POSIX lseek function to perform it faster. The fast seek does not work for files in write mode, so to take advantage of fast seek, you should open (or close and then reopen) the file in read-only mode.
 
-6. Optionally, call the FatFs library functions directly. In this case, use paths without a VFS prefix (for example, ``"/hello.txt"``).
+6. Optionally, by enabling the option :ref:`CONFIG_FATFS_IMMEDIATE_FSYNC`, you can enable automatic calling of :cpp:func:`f_sync` to flush recent file changes after each call of :cpp:func:`vfs_fat_write`, :cpp:func:`vfs_fat_pwrite`, :cpp:func:`vfs_fat_link`, :cpp:func:`vfs_fat_truncate` and :cpp:func:`vfs_fat_ftruncate` functions. This feature improves file-consistency and size reporting accuracy for the FatFs, at a price on decreased performance due to frequent disk operations.
 
-7. Close all open files.
+7. Optionally, call the FatFs library functions directly. In this case, use paths without a VFS prefix, for example, ``"/hello.txt"``.
 
-8. Call the FatFs function ``f_mount`` for the same drive number with NULL ``FATFS*`` argument to unmount the filesystem.
+8. Close all open files.
 
-9. Call the FatFs function :cpp:func:`ff_diskio_register` with NULL ``ff_diskio_impl_t*`` argument and the same drive number to unregister the disk I/O driver.
+9. Call the FatFs function :cpp:func:`f_mount` for the same drive number with NULL ``FATFS*`` argument to unmount the filesystem.
 
-10. Call :cpp:func:`esp_vfs_fat_unregister_path` with the path where the file system is mounted to remove FatFs from VFS, and free the ``FATFS`` structure allocated in Step 1.
+10. Call the FatFs function :cpp:func:`ff_diskio_register` with NULL ``ff_diskio_impl_t*`` argument and the same drive number to unregister the disk I/O driver.
+
+11. Call :cpp:func:`esp_vfs_fat_unregister_path` with the path where the file system is mounted to remove FatFs from VFS, and free the ``FATFS`` structure allocated in Step 1.
 
 The convenience functions :cpp:func:`esp_vfs_fat_sdmmc_mount`, :cpp:func:`esp_vfs_fat_sdspi_mount`, and :cpp:func:`esp_vfs_fat_sdcard_unmount` wrap the steps described above and also handle SD card initialization. These functions are described in the next section.
 

docs/zh_CN/api-reference/storage/fatfs.rst

@@ -26,21 +26,23 @@ FatFs 与 VFS 配合使用
 
 2. 调用 :cpp:func:`ff_diskio_register`，为步骤 1 中的驱动编号注册磁盘 I/O 驱动；
 
-3. 调用 FatFs 函数 ``f_mount``，随后调用 ``f_fdisk`` 或 ``f_mkfs``，并使用与传递到 :cpp:func:`esp_vfs_fat_register` 相同的驱动编号挂载文件系统。请参考 `FatFs 文档 <http://elm-chan.org/fsw/ff/doc/mount.html>`_，查看更多信息；
+3. 调用 FatFs 函数 :cpp:func:`f_mount`，随后调用 :cpp:func:`f_fdisk` 或 :cpp:func:`f_mkfs`，并使用与传递到 :cpp:func:`esp_vfs_fat_register` 相同的驱动编号挂载文件系统。请参考 `FatFs 文档 <http://elm-chan.org/fsw/ff/doc/mount.html>`_，查看更多信息；
 
 4. 调用 C 标准库和 POSIX API 对路径中带有步骤 1 中所述前缀的文件（例如，``"/sdcard/hello.txt"``）执行打开、读取、写入、擦除、复制等操作。文件系统默认使用 `8.3 文件名 <https://en.wikipedia.org/wiki/8.3_filename>`_ 格式 (SFN)。如需使用长文件名 (LFN)，启用 :ref:`CONFIG_FATFS_LONG_FILENAMES` 选项。请参考 `here <http://elm-chan.org/fsw/ff/doc/filename.html>`_，查看更多信息；
 
-5. 选择启用 :ref:`CONFIG_FATFS_USE_FASTSEEK` 选项，可以使用 POSIX lseek 实现快速执行。快速查找不适用于编辑模式下的文件，所以，使用快速查找时，应在只读模式下打开（或者关闭然后重新打开）文件；
+5. 可以启用 :ref:`CONFIG_FATFS_USE_FASTSEEK` 选项，可以使用 POSIX lseek 实现快速执行。快速查找不适用于编辑模式下的文件，所以，使用快速查找时，应在只读模式下打开（或者关闭然后重新打开）文件；
 
-6. 也可选择直接调用 FatFs 库函数，但需要使用没有 VFS 前缀的路径（例如，``"/hello.txt"``）；
+6. 可以启用 :ref:`CONFIG_FATFS_IMMEDIATE_FSYNC` 选项，在每次调用 :cpp:func:`vfs_fat_write`、:cpp:func:`vfs_fat_pwrite`、:cpp:func:`vfs_fat_link`、:cpp:func:`vfs_fat_truncate` 和 :cpp:func:`vfs_fat_ftruncate` 函数之后，自动调用 :cpp:func:`f_sync` 以同步最近的文件改动。该功能可提高文件系统中文件的一致性和文件大小报告的准确性，但是由于需要频繁进行磁盘操作，性能将会受到影响；
 
-7. 关闭所有打开的文件；
+7. 可以直接调用 FatFs 库函数，但需要使用没有 VFS 前缀的路径，如 ``"/hello.txt"``；
 
-8. 调用 FatFs 函数 ``f_mount`` 并使用 NULL ``FATFS*`` 参数，为与上述编号相同的驱动卸载文件系统；
+8. 关闭所有打开的文件；
 
-9. 调用 FatFs 函数 :cpp:func:`ff_diskio_register` 并使用 NULL ``ff_diskio_impl_t*`` 参数和相同的驱动编号，来释放注册的磁盘 I/O 驱动；
+9. 调用 FatFs 函数 :cpp:func:`f_mount` 并使用 NULL ``FATFS*`` 参数，为与上述编号相同的驱动卸载文件系统；
 
-10. 调用 :cpp:func:`esp_vfs_fat_unregister_path` 并使用文件系统挂载的路径将 FatFs 从 VFS 中移除，并释放步骤 1 中分配的 ``FATFS`` 结构。
+10. 调用 FatFs 函数 :cpp:func:`ff_diskio_register` 并使用 NULL ``ff_diskio_impl_t*`` 参数和相同的驱动编号，来释放注册的磁盘 I/O 驱动；
+
+11. 调用 :cpp:func:`esp_vfs_fat_unregister_path` 并使用文件系统挂载的路径将 FatFs 从 VFS 中移除，并释放步骤 1 中分配的 ``FATFS`` 结构。
 
 便捷函数 :cpp:func:`esp_vfs_fat_sdmmc_mount`、:cpp:func:`esp_vfs_fat_sdspi_mount` 和 :cpp:func:`esp_vfs_fat_sdcard_unmount` 对上述步骤进行了封装，并加入了对 SD 卡初始化的处理。我们将在下一章节详细介绍以上函数。