Always allocate linear memory using mmap (#3052)

With this approach we can omit using memset() for the newly allocated memory
therefore the physical pages are not being used unless touched by the program.

This also simplifies the implementation.

.github/workflows/compilation_on_android_ubuntu.yml

@@ -145,7 +145,6 @@ jobs:
             "-DWAMR_BUILD_SIMD=1",
             "-DWAMR_BUILD_TAIL_CALL=1",
             "-DWAMR_DISABLE_HW_BOUND_CHECK=1",
-            "-DWAMR_ENABLE_SHARED_MEMORY_MMAP=1",
           ]
         os: [ubuntu-22.04]
         platform: [android, linux]

build-scripts/config_common.cmake

@@ -248,12 +248,6 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1)
 else ()
   add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0)
 endif ()
-if (WAMR_ENABLE_SHARED_MEMORY_MMAP EQUAL 1)
-  add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=1)
-  message ("     Shared memory allocated using mmap enabled")
-else ()
-  add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=0)
-endif ()
 if (WAMR_BUILD_THREAD_MGR EQUAL 1)
   message ("     Thread manager enabled")
 endif ()

core/config.h

@@ -188,6 +188,10 @@
 #define WASM_ENABLE_APP_FRAMEWORK 0
 #endif
 
+#ifndef WASM_HAVE_MREMAP
+#define WASM_HAVE_MREMAP 0
+#endif
+
 /* Bulk memory operation */
 #ifndef WASM_ENABLE_BULK_MEMORY
 #define WASM_ENABLE_BULK_MEMORY 0
@@ -521,9 +525,4 @@
 #define WASM_ENABLE_QUICK_AOT_ENTRY 1
 #endif
 
-/* Disable mmap based shared memory by default */
-#ifndef WASM_ENABLE_SHARED_MEMORY_MMAP
-#define WASM_ENABLE_SHARED_MEMORY_MMAP 0
-#endif
-
 #endif /* end of _CONFIG_H_ */

core/iwasm/aot/aot_runtime.c

@@ -340,9 +340,6 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
 static void
 memories_deinstantiate(AOTModuleInstance *module_inst)
 {
-#ifdef WASM_LINEAR_MEMORY_MMAP
-    uint64 map_size;
-#endif
     uint32 i;
     AOTMemoryInstance *memory_inst;
 
@@ -364,23 +361,7 @@ memories_deinstantiate(AOTModuleInstance *module_inst)
             }
 
             if (memory_inst->memory_data) {
-#ifndef OS_ENABLE_HW_BOUND_CHECK
-#ifdef WASM_LINEAR_MEMORY_MMAP
-                if (shared_memory_is_shared(memory_inst)) {
-                    map_size = (uint64)memory_inst->num_bytes_per_page
-                               * memory_inst->max_page_count;
-                    wasm_munmap_linear_memory(memory_inst->memory_data,
-                                              map_size, map_size);
-                }
-                else
-#endif
-                    wasm_runtime_free(memory_inst->memory_data);
-#else
-                map_size = (uint64)memory_inst->num_bytes_per_page
-                           * memory_inst->cur_page_count;
-                wasm_munmap_linear_memory(memory_inst->memory_data, map_size,
-                                          8 * (uint64)BH_GB);
-#endif
+                wasm_deallocate_linear_memory(memory_inst);
             }
         }
     }
@@ -402,14 +383,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
     uint32 heap_offset = num_bytes_per_page * init_page_count;
     uint64 memory_data_size, max_memory_data_size;
     uint8 *p = NULL, *global_addr;
-#ifdef WASM_LINEAR_MEMORY_MMAP
-    uint8 *mapped_mem = NULL;
-    uint64 map_size;
-#endif
 
+    bool is_shared_memory = false;
 #if WASM_ENABLE_SHARED_MEMORY != 0
-    bool is_shared_memory = memory->memory_flags & 0x02 ? true : false;
-
+    is_shared_memory = memory->memory_flags & 0x02 ? true : false;
     /* Shared memory */
     if (is_shared_memory && parent != NULL) {
         AOTMemoryInstance *shared_memory_instance;
@@ -519,55 +496,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
                 module->aux_stack_size);
     LOG_VERBOSE("  heap offset: %u, heap size: %d\n", heap_offset, heap_size);
 
-    memory_data_size = (uint64)num_bytes_per_page * init_page_count;
     max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
-    bh_assert(memory_data_size <= UINT32_MAX);
     bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB);
     (void)max_memory_data_size;
 
-#ifndef OS_ENABLE_HW_BOUND_CHECK
-#if WASM_ENABLE_SHARED_MEMORY != 0
-    if (is_shared_memory) {
-#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0
-        map_size = max_memory_data_size;
-        if (max_memory_data_size > 0
-            && !(p = mapped_mem =
-                     wasm_mmap_linear_memory(map_size, &max_memory_data_size,
-                                             error_buf, error_buf_size))) {
-            return NULL;
-        }
-#else
-        /* Allocate maximum memory size when memory is shared */
-        if (max_memory_data_size > 0
-            && !(p = runtime_malloc(max_memory_data_size, error_buf,
-                                    error_buf_size))) {
-            return NULL;
-        }
-#endif
-    }
-    else
-#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */
-    {
-        /* Allocate initial memory size when memory is not shared */
-        if (memory_data_size > 0
-            && !(p = runtime_malloc(memory_data_size, error_buf,
-                                    error_buf_size))) {
-            return NULL;
-        }
-    }
-#else  /* else of OS_ENABLE_HW_BOUND_CHECK */
-    /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
-     *   ea = i + memarg.offset
-     * both i and memarg.offset are u32 in range 0 to 4G
-     * so the range of ea is 0 to 8G
-     */
-    map_size = 8 * (uint64)BH_GB;
-    if (!(p = mapped_mem = wasm_mmap_linear_memory(
-              map_size, &memory_data_size, error_buf, error_buf_size))) {
-        set_error_buf(error_buf, error_buf_size, "mmap memory failed");
+    if (wasm_allocate_linear_memory(&p, is_shared_memory, num_bytes_per_page,
+                                    init_page_count, max_page_count,
+                                    &memory_data_size)
+        != BHT_OK) {
+        set_error_buf(error_buf, error_buf_size,
+                      "allocate linear memory failed");
         return NULL;
     }
-#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
 
     memory_inst->module_type = Wasm_Module_AoT;
     memory_inst->num_bytes_per_page = num_bytes_per_page;
@@ -617,16 +557,8 @@ fail2:
     if (heap_size > 0)
         wasm_runtime_free(memory_inst->heap_handle);
 fail1:
-#ifdef WASM_LINEAR_MEMORY_MMAP
-    if (mapped_mem)
-        wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size);
-    else
-#endif
-    {
-        if (memory_inst->memory_data)
-            wasm_runtime_free(memory_inst->memory_data);
-    }
-    memory_inst->memory_data = NULL;
+    wasm_deallocate_linear_memory(memory_inst);
+
     return NULL;
 }
 

core/iwasm/common/wasm_memory.c

@@ -42,6 +42,14 @@ static void (*free_func)(void *ptr) = NULL;
 
 static unsigned int global_pool_size;
 
+static uint32
+align_as_and_cast(uint64 size, uint64 alignment)
+{
+    uint64 aligned_size = (size + alignment - 1) & ~(alignment - 1);
+
+    return aligned_size > UINT32_MAX ? UINT32_MAX : (uint32)aligned_size;
+}
+
 static bool
 wasm_memory_init_with_pool(void *mem, unsigned int bytes)
 {
@@ -629,157 +637,70 @@ wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory,
 #endif
 }
 
-#ifndef OS_ENABLE_HW_BOUND_CHECK
-bool
-wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
+static void
+wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size)
 {
-    WASMMemoryInstance *memory = wasm_get_default_memory(module);
-    uint8 *memory_data_old, *memory_data_new, *heap_data_old;
-    uint32 num_bytes_per_page, heap_size, total_size_old = 0;
-    uint32 cur_page_count, max_page_count, total_page_count;
-    uint64 total_size_new;
-    bool ret = true;
-    enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR;
-
-    if (!memory) {
-        ret = false;
-        goto return_func;
-    }
-
-    heap_data_old = memory->heap_data;
-    heap_size = (uint32)(memory->heap_data_end - memory->heap_data);
-
-    memory_data_old = memory->memory_data;
-    total_size_old = memory->memory_data_size;
+#ifdef BH_PLATFORM_WINDOWS
+    os_mem_decommit(mapped_mem, commit_size);
+#else
+    (void)commit_size;
+#endif
+    os_munmap(mapped_mem, map_size);
+}
 
-    num_bytes_per_page = memory->num_bytes_per_page;
-    cur_page_count = memory->cur_page_count;
-    max_page_count = memory->max_page_count;
-    total_page_count = inc_page_count + cur_page_count;
-    total_size_new = num_bytes_per_page * (uint64)total_page_count;
+static void *
+wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size,
+                          uint64 commit_size)
+{
+    void *new_mem;
 
-    if (inc_page_count <= 0)
-        /* No need to enlarge memory */
-        return true;
+    bh_assert(new_size > 0);
+    bh_assert(new_size > old_size);
 
-    if (total_page_count < cur_page_count) { /* integer overflow */
-        ret = false;
-        goto return_func;
+    if (mapped_mem) {
+        new_mem = os_mremap(mapped_mem, old_size, new_size);
     }
-
-    if (total_page_count > max_page_count) {
-        failure_reason = MAX_SIZE_REACHED;
-        ret = false;
-        goto return_func;
+    else {
+        new_mem = os_mmap(NULL, new_size, MMAP_PROT_NONE, MMAP_MAP_NONE,
+                          os_get_invalid_handle());
     }
-
-    bh_assert(total_size_new <= 4 * (uint64)BH_GB);
-    if (total_size_new > UINT32_MAX) {
-        /* Resize to 1 page with size 4G-1 */
-        num_bytes_per_page = UINT32_MAX;
-        total_page_count = max_page_count = 1;
-        total_size_new = UINT32_MAX;
+    if (!new_mem) {
+        return NULL;
     }
 
-#if WASM_ENABLE_SHARED_MEMORY != 0
-    if (shared_memory_is_shared(memory)) {
-        memory->num_bytes_per_page = num_bytes_per_page;
-        memory->cur_page_count = total_page_count;
-        memory->max_page_count = max_page_count;
-        SET_LINEAR_MEMORY_SIZE(memory, (uint32)total_size_new);
-        memory->memory_data_end = memory->memory_data + (uint32)total_size_new;
-
-        wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new);
-        return true;
+#ifdef BH_PLATFORM_WINDOWS
+    if (commit_size > 0
+        && !os_mem_commit(new_mem, commit_size,
+                          MMAP_PROT_READ | MMAP_PROT_WRITE)) {
+        os_munmap(new_mem, new_size);
+        return NULL;
     }
 #endif
 
-    if (heap_size > 0) {
-        if (mem_allocator_is_heap_corrupted(memory->heap_handle)) {
-            wasm_runtime_show_app_heap_corrupted_prompt();
-            ret = false;
-            goto return_func;
-        }
-    }
-
-    if (!(memory_data_new =
-              wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) {
-        if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) {
-            ret = false;
-            goto return_func;
-        }
-        if (memory_data_old) {
-            bh_memcpy_s(memory_data_new, (uint32)total_size_new,
-                        memory_data_old, total_size_old);
-            wasm_runtime_free(memory_data_old);
-        }
-    }
-
-    memset(memory_data_new + total_size_old, 0,
-           (uint32)total_size_new - total_size_old);
-
-    if (heap_size > 0) {
-        if (mem_allocator_migrate(memory->heap_handle,
-                                  (char *)heap_data_old
-                                      + (memory_data_new - memory_data_old),
-                                  heap_size)
-            != 0) {
-            /* Don't return here as memory->memory_data is obsolete and
-               must be updated to be correctly used later. */
-            ret = false;
-        }
+    if (os_mprotect(new_mem, commit_size, MMAP_PROT_READ | MMAP_PROT_WRITE)
+        != 0) {
+        wasm_munmap_linear_memory(new_mem, new_size, new_size);
+        return NULL;
     }
 
-    memory->heap_data = memory_data_new + (heap_data_old - memory_data_old);
-    memory->heap_data_end = memory->heap_data + heap_size;
-
-    memory->num_bytes_per_page = num_bytes_per_page;
-    memory->cur_page_count = total_page_count;
-    memory->max_page_count = max_page_count;
-    memory->memory_data_size = (uint32)total_size_new;
-
-    memory->memory_data = memory_data_new;
-    memory->memory_data_end = memory_data_new + (uint32)total_size_new;
-
-    wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new);
-
-#if defined(os_writegsbase)
-    /* write base addr of linear memory to GS segment register */
-    os_writegsbase(memory_data_new);
-#endif
-
-return_func:
-    if (!ret && enlarge_memory_error_cb) {
-        WASMExecEnv *exec_env = NULL;
-
-#if WASM_ENABLE_INTERP != 0
-        if (module->module_type == Wasm_Module_Bytecode)
-            exec_env =
-                ((WASMModuleInstanceExtra *)module->e)->common.cur_exec_env;
-#endif
-#if WASM_ENABLE_AOT != 0
-        if (module->module_type == Wasm_Module_AoT)
-            exec_env =
-                ((AOTModuleInstanceExtra *)module->e)->common.cur_exec_env;
-#endif
-
-        enlarge_memory_error_cb(inc_page_count, total_size_old, 0,
-                                failure_reason,
-                                (WASMModuleInstanceCommon *)module, exec_env,
-                                enlarge_memory_error_user_data);
-    }
+    return new_mem;
+}
 
-    return ret;
+static void *
+wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size)
+{
+    return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size);
 }
-#else
+
 bool
 wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
 {
     WASMMemoryInstance *memory = wasm_get_default_memory(module);
-    uint32 num_bytes_per_page, total_size_old = 0;
+    uint8 *memory_data_old, *memory_data_new, *heap_data_old;
+    uint32 num_bytes_per_page, heap_size, total_size_old = 0;
     uint32 cur_page_count, max_page_count, total_page_count;
     uint64 total_size_new;
-    bool ret = true;
+    bool ret = true, full_size_mmaped;
     enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR;
 
     if (!memory) {
@@ -787,10 +708,23 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
         goto return_func;
     }
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    full_size_mmaped = true;
+#elif WASM_ENABLE_SHARED_MEMORY != 0
+    full_size_mmaped = shared_memory_is_shared(memory);
+#else
+    full_size_mmaped = false;
+#endif
+
+    memory_data_old = memory->memory_data;
+    total_size_old = memory->memory_data_size;
+
+    heap_data_old = memory->heap_data;
+    heap_size = (uint32)(memory->heap_data_end - memory->heap_data);
+
     num_bytes_per_page = memory->num_bytes_per_page;
     cur_page_count = memory->cur_page_count;
     max_page_count = memory->max_page_count;
-    total_size_old = num_bytes_per_page * cur_page_count;
     total_page_count = inc_page_count + cur_page_count;
     total_size_new = num_bytes_per_page * (uint64)total_page_count;
 
@@ -817,29 +751,64 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
         total_size_new = UINT32_MAX;
     }
 
+    if (full_size_mmaped) {
 #ifdef BH_PLATFORM_WINDOWS
-    if (!os_mem_commit(memory->memory_data_end,
-                       (uint32)total_size_new - total_size_old,
-                       MMAP_PROT_READ | MMAP_PROT_WRITE)) {
-        ret = false;
-        goto return_func;
-    }
+        if (!os_mem_commit(memory->memory_data_end,
+                           (uint32)total_size_new - total_size_old,
+                           MMAP_PROT_READ | MMAP_PROT_WRITE)) {
+            ret = false;
+            goto return_func;
+        }
 #endif
 
-    if (os_mprotect(memory->memory_data_end,
-                    (uint32)total_size_new - total_size_old,
-                    MMAP_PROT_READ | MMAP_PROT_WRITE)
-        != 0) {
+        if (os_mprotect(memory->memory_data_end,
+                        (uint32)total_size_new - total_size_old,
+                        MMAP_PROT_READ | MMAP_PROT_WRITE)
+            != 0) {
 #ifdef BH_PLATFORM_WINDOWS
-        os_mem_decommit(memory->memory_data_end,
-                        (uint32)total_size_new - total_size_old);
+            os_mem_decommit(memory->memory_data_end,
+                            (uint32)total_size_new - total_size_old);
 #endif
-        ret = false;
-        goto return_func;
+            ret = false;
+            goto return_func;
+        }
     }
+    else {
+        if (heap_size > 0) {
+            if (mem_allocator_is_heap_corrupted(memory->heap_handle)) {
+                wasm_runtime_show_app_heap_corrupted_prompt();
+                ret = false;
+                goto return_func;
+            }
+        }
+
+        if (!(memory_data_new = wasm_mremap_linear_memory(
+                  memory_data_old, total_size_old, (uint32)total_size_new,
+                  (uint32)total_size_new))) {
+            ret = false;
+            goto return_func;
+        }
+
+        if (heap_size > 0) {
+            if (mem_allocator_migrate(memory->heap_handle,
+                                      (char *)heap_data_old
+                                          + (memory_data_new - memory_data_old),
+                                      heap_size)
+                != 0) {
+                /* Don't return here as memory->memory_data is obsolete and
+                must be updated to be correctly used later. */
+                ret = false;
+            }
+        }
 
-    /* The increased pages are filled with zero by the OS when os_mmap,
-       no need to memset it again here */
+        memory->heap_data = memory_data_new + (heap_data_old - memory_data_old);
+        memory->heap_data_end = memory->heap_data + heap_size;
+        memory->memory_data = memory_data_new;
+#if defined(os_writegsbase)
+        /* write base addr of linear memory to GS segment register */
+        os_writegsbase(memory_data_new);
+#endif
+    }
 
     memory->num_bytes_per_page = num_bytes_per_page;
     memory->cur_page_count = total_page_count;
@@ -872,7 +841,6 @@ return_func:
 
     return ret;
 }
-#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
 
 void
 wasm_runtime_set_enlarge_mem_error_callback(
@@ -899,3 +867,75 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
 
     return ret;
 }
+
+void
+wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst)
+{
+    uint64 map_size;
+
+    bh_assert(memory_inst);
+    bh_assert(memory_inst->memory_data);
+
+#ifndef OS_ENABLE_HW_BOUND_CHECK
+#if WASM_ENABLE_SHARED_MEMORY != 0
+    if (shared_memory_is_shared(memory_inst)) {
+        map_size = (uint64)memory_inst->num_bytes_per_page
+                   * memory_inst->max_page_count;
+    }
+    else
+#endif
+    {
+        map_size = (uint64)memory_inst->num_bytes_per_page
+                   * memory_inst->cur_page_count;
+    }
+#else
+    map_size = 8 * (uint64)BH_GB;
+#endif
+    wasm_munmap_linear_memory(memory_inst->memory_data,
+                              memory_inst->memory_data_size, map_size);
+    memory_inst->memory_data = NULL;
+}
+
+int
+wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
+                            uint64 num_bytes_per_page, uint64 init_page_count,
+                            uint64 max_page_count, uint64 *memory_data_size)
+{
+    uint64 map_size, page_size;
+
+    bh_assert(data);
+    bh_assert(memory_data_size);
+
+#ifndef OS_ENABLE_HW_BOUND_CHECK
+#if WASM_ENABLE_SHARED_MEMORY != 0
+    if (is_shared_memory) {
+        /* Allocate maximum memory size when memory is shared */
+        map_size = max_page_count * num_bytes_per_page;
+    }
+    else
+#endif
+    {
+        map_size = init_page_count * num_bytes_per_page;
+    }
+#else  /* else of OS_ENABLE_HW_BOUND_CHECK */
+    /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
+     *   ea = i + memarg.offset
+     * both i and memarg.offset are u32 in range 0 to 4G
+     * so the range of ea is 0 to 8G
+     */
+    map_size = 8 * (uint64)BH_GB;
+#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
+
+    page_size = os_getpagesize();
+    *memory_data_size = init_page_count * num_bytes_per_page;
+    bh_assert(*memory_data_size <= UINT32_MAX);
+    align_as_and_cast(*memory_data_size, page_size);
+
+    if (map_size > 0) {
+        if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) {
+            return BHT_ERROR;
+        }
+    }
+
+    return BHT_OK;
+}

core/iwasm/common/wasm_memory.h

@@ -42,6 +42,14 @@ void
 wasm_runtime_set_enlarge_mem_error_callback(
     const enlarge_memory_error_callback_t callback, void *user_data);
 
+void
+wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst);
+
+int
+wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
+                            uint64 num_bytes_per_page, uint64 init_page_count,
+                            uint64 max_page_count, uint64 *memory_data_size);
+
 #ifdef __cplusplus
 }
 #endif

core/iwasm/common/wasm_runtime_common.c

@@ -6269,67 +6269,3 @@ wasm_runtime_set_linux_perf(bool flag)
     enable_linux_perf = flag;
 }
 #endif
-
-#ifdef WASM_LINEAR_MEMORY_MMAP
-void
-wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size)
-{
-#ifdef BH_PLATFORM_WINDOWS
-    os_mem_decommit(mapped_mem, commit_size);
-#else
-    (void)commit_size;
-#endif
-    os_munmap(mapped_mem, map_size);
-}
-
-void *
-wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size,
-                        char *error_buf, uint32 error_buf_size)
-{
-    uint64 page_size = os_getpagesize();
-    void *mapped_mem = NULL;
-    uint64 memory_data_size;
-
-    bh_assert(io_memory_data_size);
-
-    memory_data_size =
-        (*io_memory_data_size + page_size - 1) & ~(page_size - 1);
-
-    if (memory_data_size > UINT32_MAX)
-        memory_data_size = UINT32_MAX;
-
-    if (!(mapped_mem = os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE,
-                               os_get_invalid_handle()))) {
-        set_error_buf(error_buf, error_buf_size, "mmap memory failed");
-        goto fail1;
-    }
-
-#ifdef BH_PLATFORM_WINDOWS
-    if (memory_data_size > 0
-        && !os_mem_commit(mapped_mem, memory_data_size,
-                          MMAP_PROT_READ | MMAP_PROT_WRITE)) {
-        set_error_buf(error_buf, error_buf_size, "commit memory failed");
-        os_munmap(mapped_mem, map_size);
-        goto fail1;
-    }
-#endif
-
-    if (os_mprotect(mapped_mem, memory_data_size,
-                    MMAP_PROT_READ | MMAP_PROT_WRITE)
-        != 0) {
-        set_error_buf(error_buf, error_buf_size, "mprotect memory failed");
-        goto fail2;
-    }
-
-    /* Newly allocated pages are filled with zero by the OS, we don't fill it
-     * again here */
-
-    *io_memory_data_size = memory_data_size;
-
-    return mapped_mem;
-fail2:
-    wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size);
-fail1:
-    return NULL;
-}
-#endif

core/iwasm/common/wasm_runtime_common.h

@@ -320,11 +320,6 @@ LOAD_I16(void *addr)
 #define SHARED_MEMORY_UNLOCK(memory) (void)0
 #endif
 
-#if defined(OS_ENABLE_HW_BOUND_CHECK) \
-    || (WASM_ENABLE_SHARED_MEMORY != 0 && WASM_ENABLE_SHARED_MEMORY_MMAP != 0)
-#define WASM_LINEAR_MEMORY_MMAP
-#endif
-
 typedef struct WASMModuleCommon {
     /* Module type, for module loaded from WASM bytecode binary,
        this field is Wasm_Module_Bytecode, and this structure should
@@ -421,16 +416,6 @@ typedef struct WASMRegisteredModule {
 } WASMRegisteredModule;
 #endif
 
-typedef struct WASMMemoryInstanceCommon {
-    uint32 module_type;
-
-    /* The following uint8[1] member is a dummy just to indicate
-       some module_type dependent members follow.
-       Typically it should be accessed by casting to the corresponding
-       actual module_type dependent structure, not via this member. */
-    uint8 memory_inst_data[1];
-} WASMMemoryInstanceCommon;
-
 typedef package_type_t PackageType;
 typedef wasm_section_t WASMSection, AOTSection;
 
@@ -1098,14 +1083,6 @@ wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst,
 void
 wasm_runtime_show_app_heap_corrupted_prompt();
 
-void
-wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size,
-                          uint64 map_size);
-
-void *
-wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size,
-                        char *error_buf, uint32 error_buf_size);
-
 #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
 void
 wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list);

core/iwasm/interpreter/wasm_runtime.c

@@ -115,9 +115,6 @@ static void
 memories_deinstantiate(WASMModuleInstance *module_inst,
                        WASMMemoryInstance **memories, uint32 count)
 {
-#ifdef WASM_LINEAR_MEMORY_MMAP
-    uint64 map_size;
-#endif
     uint32 i;
     if (memories) {
         for (i = 0; i < count; i++) {
@@ -144,23 +141,7 @@ memories_deinstantiate(WASMModuleInstance *module_inst,
                     memories[i]->heap_handle = NULL;
                 }
                 if (memories[i]->memory_data) {
-#ifndef OS_ENABLE_HW_BOUND_CHECK
-#ifdef WASM_LINEAR_MEMORY_MMAP
-                    if (shared_memory_is_shared(memories[i])) {
-                        map_size = (uint64)memories[i]->num_bytes_per_page
-                                   * memories[i]->max_page_count;
-                        wasm_munmap_linear_memory(memories[i]->memory_data,
-                                                  map_size, map_size);
-                    }
-                    else
-#endif
-                        wasm_runtime_free(memories[i]->memory_data);
-#else
-                    map_size = (uint64)memories[i]->num_bytes_per_page
-                               * memories[i]->cur_page_count;
-                    wasm_munmap_linear_memory(memories[i]->memory_data,
-                                              map_size, 8 * (uint64)BH_GB);
-#endif
+                    wasm_deallocate_linear_memory(memories[i]);
                 }
             }
         }
@@ -182,13 +163,10 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
     uint32 inc_page_count, aux_heap_base, global_idx;
     uint32 bytes_of_last_page, bytes_to_page_end;
     uint8 *global_addr;
-#ifdef WASM_LINEAR_MEMORY_MMAP
-    uint8 *mapped_mem = NULL;
-    uint64 map_size;
-#endif
 
+    bool is_shared_memory = false;
 #if WASM_ENABLE_SHARED_MEMORY != 0
-    bool is_shared_memory = flags & 0x02 ? true : false;
+    is_shared_memory = flags & 0x02 ? true : false;
 
     /* shared memory */
     if (is_shared_memory && parent != NULL) {
@@ -197,6 +175,10 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
         shared_memory_inc_reference(memory);
         return memory;
     }
+#else
+    (void)parent;
+    (void)memory_idx;
+    (void)flags;
 #endif /* end of WASM_ENABLE_SHARED_MEMORY */
 
     if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1
@@ -296,57 +278,20 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
                 num_bytes_per_page, init_page_count, max_page_count);
     LOG_VERBOSE("  heap offset: %u, heap size: %d\n", heap_offset, heap_size);
 
-    memory_data_size = (uint64)num_bytes_per_page * init_page_count;
     max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
-    bh_assert(memory_data_size <= UINT32_MAX);
     bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB);
     (void)max_memory_data_size;
 
     bh_assert(memory != NULL);
 
-#ifndef OS_ENABLE_HW_BOUND_CHECK
-#if WASM_ENABLE_SHARED_MEMORY != 0
-    if (is_shared_memory) {
-        /* Allocate maximum memory size when memory is shared */
-#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0
-        map_size = max_memory_data_size;
-        if (max_memory_data_size > 0
-            && !(memory->memory_data = mapped_mem =
-                     wasm_mmap_linear_memory(map_size, &max_memory_data_size,
-                                             error_buf, error_buf_size))) {
-            goto fail1;
-        }
-#else
-        if (max_memory_data_size > 0
-            && !(memory->memory_data = runtime_malloc(
-                     max_memory_data_size, error_buf, error_buf_size))) {
-            goto fail1;
-        }
-#endif
-    }
-    else
-#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */
-    {
-        /* Allocate initial memory size when memory is not shared */
-        if (memory_data_size > 0
-            && !(memory->memory_data = runtime_malloc(
-                     memory_data_size, error_buf, error_buf_size))) {
-            goto fail1;
-        }
-    }
-#else  /* else of OS_ENABLE_HW_BOUND_CHECK */
-    /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
-     *   ea = i + memarg.offset
-     * both i and memarg.offset are u32 in range 0 to 4G
-     * so the range of ea is 0 to 8G
-     */
-    map_size = 8 * (uint64)BH_GB;
-    if (!(memory->memory_data = mapped_mem = wasm_mmap_linear_memory(
-              map_size, &memory_data_size, error_buf, error_buf_size))) {
-        set_error_buf(error_buf, error_buf_size, "mmap memory failed");
-        goto fail1;
+    if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory,
+                                    num_bytes_per_page, init_page_count,
+                                    max_page_count, &memory_data_size)
+        != BHT_OK) {
+        set_error_buf(error_buf, error_buf_size,
+                      "allocate linear memory failed");
+        return NULL;
     }
-#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
 
     memory->module_type = Wasm_Module_Bytecode;
     memory->num_bytes_per_page = num_bytes_per_page;
@@ -364,13 +309,13 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
 
         if (!(memory->heap_handle = runtime_malloc(
                   (uint64)heap_struct_size, error_buf, error_buf_size))) {
-            goto fail2;
+            goto fail1;
         }
         if (!mem_allocator_create_with_struct_and_pool(
                 memory->heap_handle, heap_struct_size, memory->heap_data,
                 heap_size)) {
             set_error_buf(error_buf, error_buf_size, "init app heap failed");
-            goto fail3;
+            goto fail2;
         }
     }
 
@@ -388,20 +333,13 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
     LOG_VERBOSE("Memory instantiate success.");
     return memory;
 
-fail3:
+fail2:
     if (heap_size > 0)
         wasm_runtime_free(memory->heap_handle);
-fail2:
-#ifdef WASM_LINEAR_MEMORY_MMAP
-    if (mapped_mem)
-        wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size);
-    else
-#endif
-    {
-        if (memory->memory_data)
-            wasm_runtime_free(memory->memory_data);
-    }
 fail1:
+    if (memory->memory_data)
+        wasm_deallocate_linear_memory(memory);
+
     return NULL;
 }
 
@@ -633,7 +571,7 @@ fail:
  * Destroy function instances.
  */
 static void
-functions_deinstantiate(WASMFunctionInstance *functions, uint32 count)
+functions_deinstantiate(WASMFunctionInstance *functions)
 {
     if (functions) {
         wasm_runtime_free(functions);
@@ -1561,6 +1499,7 @@ set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode,
 
 #if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
       && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */
+    (void)first_time_set;
     module_inst->e->running_mode = running_mode;
 
     if (running_mode == Mode_Interp) {
@@ -2342,12 +2281,10 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
     }
 
     tables_deinstantiate(module_inst);
-    functions_deinstantiate(module_inst->e->functions,
-                            module_inst->e->function_count);
+    functions_deinstantiate(module_inst->e->functions);
 #if WASM_ENABLE_TAGS != 0
     tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs);
 #endif
-
     globals_deinstantiate(module_inst->e->globals);
     export_functions_deinstantiate(module_inst->export_functions);
 #if WASM_ENABLE_TAGS != 0

core/shared/platform/common/memory/mremap.c

@@ -0,0 +1,12 @@
+/*
+ * Copyright (C) 2024 Amazon Inc.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "bh_memutils.h"
+
+void *
+os_mremap(void *old_addr, size_t old_size, size_t new_size)
+{
+    return bh_memory_remap_slow(old_addr, old_size, new_size);
+}

core/shared/platform/common/memory/platform_api_memory.cmake

@@ -0,0 +1,4 @@
+# Copyright (C) 2024 Amazon Inc.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+file (GLOB_RECURSE PLATFORM_COMMON_MEMORY_SOURCE ${CMAKE_CURRENT_LIST_DIR}/*.c)

core/shared/platform/common/posix/platform_api_posix.cmake

@@ -16,4 +16,19 @@ else()
     set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
 endif()
 
+# This is to support old CMake version. Newer version of CMake could use
+# list APPEND/POP_BACK methods.
+include(CheckSymbolExists)
+set (CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE ${CMAKE_REQUIRED_DEFINITIONS})
+check_symbol_exists (mremap "sys/mman.h" MREMAP_EXISTS)
+list (REMOVE_AT CMAKE_REQUIRED_DEFINITIONS 0)
+
+if(MREMAP_EXISTS)
+    add_definitions (-DWASM_HAVE_MREMAP=1)
+else()
+    add_definitions (-DWASM_HAVE_MREMAP=0)
+    include (${CMAKE_CURRENT_LIST_DIR}/../memory/platform_api_memory.cmake)
+    set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
+endif()
+
 set (PLATFORM_COMMON_POSIX_SOURCE ${source_all} )

core/shared/platform/common/posix/posix_memmap.c

@@ -3,6 +3,12 @@
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  */
 
+#if !defined(_GNU_SOURCE) && WASM_HAVE_MREMAP != 0
+/* Enable mremap */
+#define _GNU_SOURCE
+#include "bh_memutils.h"
+#endif
+
 #include "platform_api_vmcore.h"
 
 #if defined(__APPLE__) || defined(__MACH__)
@@ -236,6 +242,23 @@ os_munmap(void *addr, size_t size)
     }
 }
 
+#if WASM_HAVE_MREMAP != 0
+void *
+os_mremap(void *old_addr, size_t old_size, size_t new_size)
+{
+    void *ptr = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE);
+
+    if (ptr == MAP_FAILED) {
+#if BH_ENABLE_TRACE_MMAP != 0
+        os_printf("mremap failed: %d\n", errno);
+#endif
+        return bh_memory_remap_slow(old_addr, old_size, new_size);
+    }
+
+    return ptr;
+}
+#endif
+
 int
 os_mprotect(void *addr, size_t size, int prot)
 {

core/shared/platform/darwin/shared_platform.cmake

@@ -12,6 +12,9 @@ include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake)
 
 file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c)
 
+include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
+set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
+
 set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE})
 
 file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h)

core/shared/platform/include/platform_api_vmcore.h

@@ -132,7 +132,7 @@ enum {
     MMAP_MAP_32BIT = 1,
     /* Don't interpret addr as a hint: place the mapping at exactly
        that address. */
-    MMAP_MAP_FIXED = 2
+    MMAP_MAP_FIXED = 2,
 };
 
 void *
@@ -142,6 +142,11 @@ os_munmap(void *addr, size_t size);
 int
 os_mprotect(void *addr, size_t size, int prot);
 
+/* Doesn't guarantee that protection flags will be preserved.
+   os_mprotect() must be called after remapping. */
+void *
+os_mremap(void *old_addr, size_t old_size, size_t new_size);
+
 #if (WASM_MEM_DUAL_BUS_MIRROR != 0)
 void *
 os_get_dbus_mirror(void *ibus);

core/shared/platform/linux-sgx/platform_internal.h

@@ -79,6 +79,8 @@ os_get_invalid_handle()
     return -1;
 }
 
+#define os_getpagesize getpagesize
+
 #ifdef __cplusplus
 }
 #endif

core/shared/platform/linux-sgx/shared_platform.cmake

@@ -37,6 +37,9 @@ else()
   set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
 endif()
 
+include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
+set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
+
 file (GLOB source_all_untrusted ${PLATFORM_SHARED_DIR}/untrusted/*.c)
 
 set (PLATFORM_SHARED_SOURCE ${source_all})

core/shared/platform/nuttx/platform_internal.h

@@ -44,6 +44,8 @@ typedef pthread_t korp_thread;
 typedef pthread_rwlock_t korp_rwlock;
 typedef sem_t korp_sem;
 
+#define os_getpagesize getpagesize
+
 #define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
 
 #define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB)

core/shared/platform/nuttx/shared_platform.cmake

@@ -16,5 +16,6 @@ if (WAMR_BUILD_LIBC_WASI EQUAL 1)
   set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
 endif ()
 
-set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE})
+include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
+set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE} ${PLATFORM_COMMON_MEMORY_SOURCE})
 

core/shared/platform/windows/shared_platform.cmake

@@ -20,6 +20,9 @@ else()
     set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
 endif()
 
+include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
+set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
+
 set (PLATFORM_SHARED_SOURCE ${source_all})
 
 file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h)

core/shared/utils/bh_memutils.c

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2024 Amazon Inc.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "bh_memutils.h"
+
+void *
+bh_memory_remap_slow(void *old_addr, size_t old_size, size_t new_size)
+{
+    void *new_memory =
+        os_mmap(NULL, new_size, MMAP_PROT_WRITE | MMAP_PROT_READ, 0, -1);
+    if (!new_memory) {
+        return NULL;
+    }
+    /*
+     * bh_memcpy_s can't be used as it doesn't support values bigger than
+     * UINT32_MAX
+     */
+    memcpy(new_memory, old_addr, new_size < old_size ? new_size : old_size);
+    os_munmap(old_addr, old_size);
+
+    return new_memory;
+}

core/shared/utils/bh_memutils.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 Amazon Inc.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef _BH_MEMUTILS_H
+#define _BH_MEMUTILS_H
+
+#include "bh_platform.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Remaps memory by mapping a new region, copying data from the old
+ * region and umapping the old region.
+ *
+ * Unless the behavior is desired, in most cases os_mremap should be used
+ * as it's at worst equally slow as this function, and on some platforms
+ * (e.g. posix with mremap) os_mremap will perform better.
+ *
+ * @param old_addr an old address.
+ * @param old_size a size of the old address.
+ * @param new_size a size of the new memory region.
+ * @return a pointer to the new memory region.
+ */
+void *
+bh_memory_remap_slow(void *old_addr, size_t old_size, size_t new_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* end of _BH_MEMUTILS_H */

doc/memory_usage.md

@@ -51,14 +51,9 @@ They are in the number of WASM pages, each of which is of 65536 bytes.
 The `max` is optional for non-shared memory. When omitted, it effectivily
 means unlimited.
 
-If `OS_ENABLE_HW_BOUND_CHECK` is enabled, the memory is allocated via
-`os_mmap` and `os_mem_commit`/`os_mprotect`.
-Otherwise, it's allocated from the global heap.
+The linear memory is allocated via `os_mmap` and `os_mem_commit`/`os_mprotect`.
 
-If the memory is shared and `OS_ENABLE_HW_BOUND_CHECK` is not enabled,
-the `max` size of memory is allocated on instantiation.
-
-Otherwise, the `min` size of memory is allocated on instantiation.
+The `min` size of memory is allocated on instantiation.
 It can later grow up to the `max` size via the `memory.grow` instruction.
 
 Libc heap

product-mini/platforms/linux-sgx/CMakeLists.txt

@@ -176,3 +176,10 @@ else()
         OUTPUT_VARIABLE cmdOutput
     )
 endif()
+
+if (WAMR_BUILD_SPEC_TEST EQUAL 1)
+    execute_process(
+        COMMAND bash -c "sed -i -E 's/<ReservedMemMaxSize>0x1000000<\\/ReservedMemMaxSize>/<ReservedMemMaxSize>0x8000000<\\/ReservedMemMaxSize>/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.config.xml"
+        OUTPUT_VARIABLE cmdOutput
+    )
+endif()

product-mini/platforms/nuttx/wamr.mk

@@ -391,6 +391,7 @@ CSRCS += nuttx_platform.c \
          posix_thread.c \
          posix_time.c \
          posix_sleep.c \
+         mremap.c \
          mem_alloc.c \
          ems_kfc.c \
          ems_alloc.c \
@@ -401,6 +402,7 @@ CSRCS += nuttx_platform.c \
          bh_hashmap.c \
          bh_list.c \
          bh_log.c \
+         bh_memutils.c \
          bh_queue.c \
          bh_vector.c \
          bh_read_file.c \
@@ -416,6 +418,7 @@ CSRCS += nuttx_platform.c \
 ASRCS += $(INVOKE_NATIVE)
 
 VPATH += $(SHARED_ROOT)/platform/nuttx
+VPATH += $(SHARED_ROOT)/platform/common/memory
 VPATH += $(SHARED_ROOT)/platform/common/posix
 VPATH += $(SHARED_ROOT)/platform/common/libc-util
 VPATH += $(SHARED_ROOT)/mem-alloc