Fix linear memory page count issues (#1380)

Fix issue reported in #1289 and #1371.
Enable to set the max page count to 65536.

core/iwasm/aot/aot_runtime.c

@@ -412,18 +412,27 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
     if (init_page_count == max_page_count && init_page_count == 1) {
         /* If only one page and at most one page, we just append
            the app heap to the end of linear memory, enlarge the
-           num_bytes_per_page, and don't change the page count*/
+           num_bytes_per_page, and don't change the page count */
         heap_offset = num_bytes_per_page;
         num_bytes_per_page += heap_size;
         if (num_bytes_per_page < heap_size) {
             set_error_buf(error_buf, error_buf_size,
-                          "memory size must be at most 65536 pages (4GiB)");
+                          "failed to insert app heap into linear memory, "
+                          "try using `--heap_size=0` option");
             return NULL;
         }
     }
     else if (heap_size > 0) {
-        if (module->aux_heap_base_global_index != (uint32)-1
-            && module->aux_heap_base < num_bytes_per_page * init_page_count) {
+        if (init_page_count == max_page_count && init_page_count == 0) {
+            /* If the memory data size is always 0, we resize it to
+               one page for app heap */
+            num_bytes_per_page = heap_size;
+            heap_offset = 0;
+            inc_page_count = 1;
+        }
+        else if (module->aux_heap_base_global_index != (uint32)-1
+                 && module->aux_heap_base
+                        < num_bytes_per_page * init_page_count) {
             /* Insert app heap before __heap_base */
             aux_heap_base = module->aux_heap_base;
             bytes_of_last_page = aux_heap_base % num_bytes_per_page;
@@ -464,13 +473,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
         }
         init_page_count += inc_page_count;
         max_page_count += inc_page_count;
-        if (init_page_count > 65536) {
+        if (init_page_count > DEFAULT_MAX_PAGES) {
             set_error_buf(error_buf, error_buf_size,
-                          "memory size must be at most 65536 pages (4GiB)");
+                          "failed to insert app heap into linear memory, "
+                          "try using `--heap_size=0` option");
             return NULL;
         }
-        if (max_page_count > 65536)
-            max_page_count = 65536;
+        else if (init_page_count == DEFAULT_MAX_PAGES) {
+            num_bytes_per_page = UINT32_MAX;
+            init_page_count = max_page_count = 1;
+        }
+        if (max_page_count > DEFAULT_MAX_PAGES)
+            max_page_count = DEFAULT_MAX_PAGES;
     }
 
     LOG_VERBOSE("Memory instantiate:");
@@ -487,6 +501,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
         total_size = (uint64)num_bytes_per_page * max_page_count;
     }
 #endif
+    bh_assert(total_size <= UINT32_MAX);
 
 #ifndef OS_ENABLE_HW_BOUND_CHECK
     /* Allocate memory */
@@ -502,9 +517,8 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
      * both i and memarg.offset are u32 in range 0 to 4G
      * so the range of ea is 0 to 8G
      */
-    if (total_size >= UINT32_MAX
-        || !(p = mapped_mem =
-                 os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) {
+    if (!(p = mapped_mem =
+              os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) {
         set_error_buf(error_buf, error_buf_size, "mmap memory failed");
         return NULL;
     }
@@ -529,15 +543,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
      * again here */
 #endif /* end of OS_ENABLE_HW_BOUND_CHECK */
 
+    if (total_size > UINT32_MAX)
+        total_size = UINT32_MAX;
+
     memory_inst->module_type = Wasm_Module_AoT;
     memory_inst->num_bytes_per_page = num_bytes_per_page;
     memory_inst->cur_page_count = init_page_count;
     memory_inst->max_page_count = max_page_count;
+    memory_inst->memory_data_size = (uint32)total_size;
 
     /* Init memory info */
     memory_inst->memory_data.ptr = p;
     memory_inst->memory_data_end.ptr = p + (uint32)total_size;
-    memory_inst->memory_data_size = (uint32)total_size;
 
     /* Initialize heap info */
     memory_inst->heap_data.ptr = p + heap_offset;
@@ -561,21 +578,19 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
     }
 
     if (total_size > 0) {
-        if (sizeof(uintptr_t) == sizeof(uint64)) {
-            memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
-            memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
-            memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
-            memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
-            memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
-        }
-        else {
-            memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
-            memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
-            memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
-            memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
-            memory_inst->mem_bound_check_16bytes.u32[0] =
-                (uint32)total_size - 16;
-        }
+#if UINTPTR_MAX == UINT64_MAX
+        memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
+        memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
+        memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
+        memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
+        memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
+#else
+        memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
+        memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
+        memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
+        memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
+        memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16;
+#endif
     }
 
 #if WASM_ENABLE_SHARED_MEMORY != 0
@@ -2032,26 +2047,29 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr,
 bool
 aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
 {
-    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
-    uint32 num_bytes_per_page, cur_page_count, max_page_count;
-    uint32 total_page_count, total_size_old, heap_size;
-    uint64 total_size;
-    uint8 *memory_data_old, *heap_data_old, *memory_data, *heap_data;
+    AOTMemoryInstance *memory = aot_get_default_memory(module_inst);
+    uint8 *memory_data_old, *memory_data_new, *heap_data_old;
+    uint32 num_bytes_per_page, heap_size, total_size_old;
+    uint32 cur_page_count, max_page_count, total_page_count;
+    uint64 total_size_new;
     bool ret = true;
 
-    if (!memory_inst)
+    if (!memory)
         return false;
 
-    num_bytes_per_page = memory_inst->num_bytes_per_page;
-    cur_page_count = memory_inst->cur_page_count;
-    max_page_count = memory_inst->max_page_count;
-    total_page_count = cur_page_count + inc_page_count;
-    total_size_old = memory_inst->memory_data_size;
-    total_size = (uint64)num_bytes_per_page * total_page_count;
-    heap_size = (uint32)((uint8 *)memory_inst->heap_data_end.ptr
-                         - (uint8 *)memory_inst->heap_data.ptr);
-    memory_data_old = (uint8 *)memory_inst->memory_data.ptr;
-    heap_data_old = (uint8 *)memory_inst->heap_data.ptr;
+    heap_data_old = (uint8 *)memory->heap_data.ptr;
+    heap_size = (uint32)((uint8 *)memory->heap_data_end.ptr
+                         - (uint8 *)memory->heap_data.ptr);
+
+    memory_data_old = (uint8 *)memory->memory_data.ptr;
+    total_size_old =
+        (uint32)((uint8 *)memory->memory_data_end.ptr - memory_data_old);
+
+    num_bytes_per_page = memory->num_bytes_per_page;
+    cur_page_count = memory->cur_page_count;
+    max_page_count = memory->max_page_count;
+    total_page_count = inc_page_count + cur_page_count;
+    total_size_new = num_bytes_per_page * (uint64)total_page_count;
 
     if (inc_page_count <= 0)
         /* No need to enlarge memory */
@@ -2062,94 +2080,103 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
         return false;
     }
 
-    if (total_size >= UINT32_MAX) {
-        return false;
+    bh_assert(total_size_new <= 4 * (uint64)BH_GB);
+    if (total_size_new > UINT32_MAX) {
+        /* Resize to 1 page with size 4G-1 */
+        num_bytes_per_page = UINT32_MAX;
+        total_page_count = max_page_count = 1;
+        total_size_new = UINT32_MAX;
     }
 
 #if WASM_ENABLE_SHARED_MEMORY != 0
-    if (memory_inst->is_shared) {
-        /* For shared memory, we have reserved the maximum spaces during
-            instantiate, only change the cur_page_count here */
-        memory_inst->cur_page_count = total_page_count;
+    if (memory->is_shared) {
+        memory->num_bytes_per_page = UINT32_MAX;
+        memory->cur_page_count = total_page_count;
+        memory->max_page_count = max_page_count;
+        memory->memory_data_size = (uint32)total_size_new;
         return true;
     }
 #endif
 
     if (heap_size > 0) {
-        if (mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) {
+        if (mem_allocator_is_heap_corrupted(memory->heap_handle.ptr)) {
             wasm_runtime_show_app_heap_corrupted_prompt();
             return false;
         }
     }
 
-    if (!(memory_data =
-              wasm_runtime_realloc(memory_data_old, (uint32)total_size))) {
-        if (!(memory_data = wasm_runtime_malloc((uint32)total_size))) {
+    if (!(memory_data_new =
+              wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) {
+        if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) {
             return false;
         }
         if (memory_data_old) {
-            bh_memcpy_s(memory_data, (uint32)total_size, memory_data_old,
-                        total_size_old);
+            bh_memcpy_s(memory_data_new, (uint32)total_size_new,
+                        memory_data_old, total_size_old);
             wasm_runtime_free(memory_data_old);
         }
     }
 
-    memset(memory_data + total_size_old, 0,
-           (uint32)total_size - total_size_old);
-
-    memory_inst->cur_page_count = total_page_count;
-    memory_inst->memory_data_size = (uint32)total_size;
-    memory_inst->memory_data.ptr = memory_data;
-    memory_inst->memory_data_end.ptr = memory_data + total_size;
+    memset(memory_data_new + total_size_old, 0,
+           (uint32)total_size_new - total_size_old);
 
     if (heap_size > 0) {
-        if (mem_allocator_migrate(memory_inst->heap_handle.ptr,
+        if (mem_allocator_migrate(memory->heap_handle.ptr,
                                   (char *)heap_data_old
-                                      + (memory_data - memory_data_old),
-                                  heap_size)) {
+                                      + (memory_data_new - memory_data_old),
+                                  heap_size)
+            != 0) {
             /* Don't return here as memory->memory_data is obsolete and
                must be updated to be correctly used later. */
             ret = false;
         }
     }
 
-    heap_data = memory_data + (heap_data_old - memory_data_old);
-    memory_inst->heap_data.ptr = heap_data;
-    memory_inst->heap_data_end.ptr = heap_data + heap_size;
+    memory->heap_data.ptr = memory_data_new + (heap_data_old - memory_data_old);
+    memory->heap_data_end.ptr = (uint8 *)memory->heap_data.ptr + heap_size;
+
+    memory->num_bytes_per_page = num_bytes_per_page;
+    memory->cur_page_count = total_page_count;
+    memory->max_page_count = max_page_count;
+
+    memory->memory_data.ptr = memory_data_new;
+    memory->memory_data_end.ptr = memory_data_new + (uint32)total_size_new;
+    memory->memory_data_size = (uint32)total_size_new;
+
+#if UINTPTR_MAX == UINT64_MAX
+    memory->mem_bound_check_1byte.u64 = total_size_new - 1;
+    memory->mem_bound_check_2bytes.u64 = total_size_new - 2;
+    memory->mem_bound_check_4bytes.u64 = total_size_new - 4;
+    memory->mem_bound_check_8bytes.u64 = total_size_new - 8;
+    memory->mem_bound_check_16bytes.u64 = total_size_new - 16;
+#else
+    memory->mem_bound_check_1byte.u32[0] = (uint32)total_size_new - 1;
+    memory->mem_bound_check_2bytes.u32[0] = (uint32)total_size_new - 2;
+    memory->mem_bound_check_4bytes.u32[0] = (uint32)total_size_new - 4;
+    memory->mem_bound_check_8bytes.u32[0] = (uint32)total_size_new - 8;
+    memory->mem_bound_check_16bytes.u32[0] = (uint32)total_size_new - 16;
+#endif
 
-    if (sizeof(uintptr_t) == sizeof(uint64)) {
-        memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
-        memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
-        memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
-        memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
-        memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
-    }
-    else {
-        memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
-        memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
-        memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
-        memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
-        memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16;
-    }
     return ret;
 }
 #else /* else of OS_ENABLE_HW_BOUND_CHECK */
 bool
 aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
 {
-    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
-    uint32 num_bytes_per_page, cur_page_count, max_page_count;
-    uint32 total_page_count;
-    uint64 total_size;
+    AOTMemoryInstance *memory = aot_get_default_memory(module_inst);
+    uint32 num_bytes_per_page, total_size_old;
+    uint32 cur_page_count, max_page_count, total_page_count;
+    uint64 total_size_new;
 
-    if (!memory_inst)
+    if (!memory)
         return false;
 
-    num_bytes_per_page = memory_inst->num_bytes_per_page;
-    cur_page_count = memory_inst->cur_page_count;
-    max_page_count = memory_inst->max_page_count;
-    total_page_count = cur_page_count + inc_page_count;
-    total_size = (uint64)num_bytes_per_page * total_page_count;
+    num_bytes_per_page = memory->num_bytes_per_page;
+    cur_page_count = memory->cur_page_count;
+    max_page_count = memory->max_page_count;
+    total_size_old = num_bytes_per_page * cur_page_count;
+    total_page_count = inc_page_count + cur_page_count;
+    total_size_new = num_bytes_per_page * (uint64)total_page_count;
 
     if (inc_page_count <= 0)
         /* No need to enlarge memory */
@@ -2160,21 +2187,29 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
         return false;
     }
 
+    bh_assert(total_size_new <= 4 * (uint64)BH_GB);
+    if (total_size_new > UINT32_MAX) {
+        /* Resize to 1 page with size 4G-1 */
+        num_bytes_per_page = UINT32_MAX;
+        total_page_count = max_page_count = 1;
+        total_size_new = UINT32_MAX;
+    }
+
 #ifdef BH_PLATFORM_WINDOWS
-    if (!os_mem_commit(memory_inst->memory_data_end.ptr,
-                       num_bytes_per_page * inc_page_count,
+    if (!os_mem_commit(memory->memory_data_end.ptr,
+                       (uint32)total_size_new - total_size_old,
                        MMAP_PROT_READ | MMAP_PROT_WRITE)) {
         return false;
     }
 #endif
 
-    if (os_mprotect(memory_inst->memory_data_end.ptr,
-                    num_bytes_per_page * inc_page_count,
+    if (os_mprotect(memory->memory_data_end.ptr,
+                    (uint32)total_size_new - total_size_old,
                     MMAP_PROT_READ | MMAP_PROT_WRITE)
         != 0) {
 #ifdef BH_PLATFORM_WINDOWS
-        os_mem_decommit(memory_inst->memory_data_end.ptr,
-                        num_bytes_per_page * inc_page_count);
+        os_mem_decommit(memory->memory_data_end.ptr,
+                        (uint32)total_size_new - total_size_old);
 #endif
         return false;
     }
@@ -2182,25 +2217,19 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
     /* The increased pages are filled with zero by the OS when os_mmap,
        no need to memset it again here */
 
-    memory_inst->cur_page_count = total_page_count;
-    memory_inst->memory_data_size = (uint32)total_size;
-    memory_inst->memory_data_end.ptr =
-        (uint8 *)memory_inst->memory_data.ptr + (uint32)total_size;
+    memory->num_bytes_per_page = num_bytes_per_page;
+    memory->cur_page_count = total_page_count;
+    memory->max_page_count = max_page_count;
+    memory->memory_data_size = (uint32)total_size_new;
+    memory->memory_data_end.ptr =
+        (uint8 *)memory->memory_data.ptr + (uint32)total_size_new;
+
+    memory->mem_bound_check_1byte.u64 = total_size_new - 1;
+    memory->mem_bound_check_2bytes.u64 = total_size_new - 2;
+    memory->mem_bound_check_4bytes.u64 = total_size_new - 4;
+    memory->mem_bound_check_8bytes.u64 = total_size_new - 8;
+    memory->mem_bound_check_16bytes.u64 = total_size_new - 16;
 
-    if (sizeof(uintptr_t) == sizeof(uint64)) {
-        memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
-        memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
-        memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
-        memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
-        memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
-    }
-    else {
-        memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
-        memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
-        memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
-        memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
-        memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16;
-    }
     return true;
 }
 #endif /* end of OS_ENABLE_HW_BOUND_CHECK */

core/iwasm/interpreter/wasm.h

@@ -29,6 +29,7 @@ extern "C" {
 #define VALUE_TYPE_ANY 0x42
 
 #define DEFAULT_NUM_BYTES_PER_PAGE 65536
+#define DEFAULT_MAX_PAGES 65536
 
 #define NULL_REF (0xFFFFFFFF)
 

core/iwasm/interpreter/wasm_loader.c

@@ -1266,7 +1266,7 @@ fail:
 static bool
 check_memory_init_size(uint32 init_size, char *error_buf, uint32 error_buf_size)
 {
-    if (init_size > 65536) {
+    if (init_size > DEFAULT_MAX_PAGES) {
         set_error_buf(error_buf, error_buf_size,
                       "memory size must be at most 65536 pages (4GiB)");
         return false;
@@ -1284,7 +1284,7 @@ check_memory_max_size(uint32 init_size, uint32 max_size, char *error_buf,
         return false;
     }
 
-    if (max_size > 65536) {
+    if (max_size > DEFAULT_MAX_PAGES) {
         set_error_buf(error_buf, error_buf_size,
                       "memory size must be at most 65536 pages (4GiB)");
         return false;
@@ -1299,12 +1299,12 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
                    char *error_buf, uint32 error_buf_size)
 {
     const uint8 *p = *p_buf, *p_end = buf_end;
-    uint32 pool_size = wasm_runtime_memory_pool_size();
 #if WASM_ENABLE_APP_FRAMEWORK != 0
+    uint32 pool_size = wasm_runtime_memory_pool_size();
     uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
                             / DEFAULT_NUM_BYTES_PER_PAGE;
 #else
-    uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE;
+    uint32 max_page_count = DEFAULT_MAX_PAGES;
 #endif /* WASM_ENABLE_APP_FRAMEWORK */
     uint32 declare_max_page_count_flag = 0;
     uint32 declare_init_page_count = 0;
@@ -1529,12 +1529,12 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
             char *error_buf, uint32 error_buf_size)
 {
     const uint8 *p = *p_buf, *p_end = buf_end, *p_org;
-    uint32 pool_size = wasm_runtime_memory_pool_size();
 #if WASM_ENABLE_APP_FRAMEWORK != 0
+    uint32 pool_size = wasm_runtime_memory_pool_size();
     uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
                             / DEFAULT_NUM_BYTES_PER_PAGE;
 #else
-    uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE;
+    uint32 max_page_count = DEFAULT_MAX_PAGES;
 #endif
 
     p_org = p;
@@ -3317,17 +3317,30 @@ load_from_sections(WASMModule *module, WASMSection *sections,
 #if WASM_ENABLE_MULTI_MODULE == 0
         if (module->import_memory_count) {
             memory_import = &module->import_memories[0].u.memory;
-            /* Memory init page count cannot be larger than 65536, we don't
-               check integer overflow again. */
-            memory_import->num_bytes_per_page *= memory_import->init_page_count;
-            memory_import->init_page_count = memory_import->max_page_count = 1;
+            if (memory_import->init_page_count < DEFAULT_MAX_PAGES)
+                memory_import->num_bytes_per_page *=
+                    memory_import->init_page_count;
+            else
+                memory_import->num_bytes_per_page = UINT32_MAX;
+
+            if (memory_import->init_page_count > 0)
+                memory_import->init_page_count = memory_import->max_page_count =
+                    1;
+            else
+                memory_import->init_page_count = memory_import->max_page_count =
+                    0;
         }
         if (module->memory_count) {
-            /* Memory init page count cannot be larger than 65536, we don't
-               check integer overflow again. */
             memory = &module->memories[0];
-            memory->num_bytes_per_page *= memory->init_page_count;
-            memory->init_page_count = memory->max_page_count = 1;
+            if (memory->init_page_count < DEFAULT_MAX_PAGES)
+                memory->num_bytes_per_page *= memory->init_page_count;
+            else
+                memory->num_bytes_per_page = UINT32_MAX;
+
+            if (memory->init_page_count > 0)
+                memory->init_page_count = memory->max_page_count = 1;
+            else
+                memory->init_page_count = memory->max_page_count = 0;
         }
 #endif
     }

core/iwasm/interpreter/wasm_mini_loader.c

@@ -534,12 +534,12 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
                    char *error_buf, uint32 error_buf_size)
 {
     const uint8 *p = *p_buf, *p_end = buf_end;
-    uint32 pool_size = wasm_runtime_memory_pool_size();
 #if WASM_ENABLE_APP_FRAMEWORK != 0
+    uint32 pool_size = wasm_runtime_memory_pool_size();
     uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
                             / DEFAULT_NUM_BYTES_PER_PAGE;
 #else
-    uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE;
+    uint32 max_page_count = DEFAULT_MAX_PAGES;
 #endif /* WASM_ENABLE_APP_FRAMEWORK */
     uint32 declare_max_page_count_flag = 0;
     uint32 declare_init_page_count = 0;
@@ -650,12 +650,12 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory,
             char *error_buf, uint32 error_buf_size)
 {
     const uint8 *p = *p_buf, *p_end = buf_end, *p_org;
-    uint32 pool_size = wasm_runtime_memory_pool_size();
 #if WASM_ENABLE_APP_FRAMEWORK != 0
+    uint32 pool_size = wasm_runtime_memory_pool_size();
     uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT
                             / DEFAULT_NUM_BYTES_PER_PAGE;
 #else
-    uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE;
+    uint32 max_page_count = DEFAULT_MAX_PAGES;
 #endif
 
     p_org = p;
@@ -2153,18 +2153,31 @@ load_from_sections(WASMModule *module, WASMSection *sections,
 
         if (module->import_memory_count) {
             memory_import = &module->import_memories[0].u.memory;
-            /* Memory init page count cannot be larger than 65536, we don't
-               check integer overflow again. */
-            memory_import->num_bytes_per_page *= memory_import->init_page_count;
-            memory_import->init_page_count = memory_import->max_page_count = 1;
+            if (memory_import->init_page_count < DEFAULT_MAX_PAGES)
+                memory_import->num_bytes_per_page *=
+                    memory_import->init_page_count;
+            else
+                memory_import->num_bytes_per_page = UINT32_MAX;
+
+            if (memory_import->init_page_count > 0)
+                memory_import->init_page_count = memory_import->max_page_count =
+                    1;
+            else
+                memory_import->init_page_count = memory_import->max_page_count =
+                    0;
         }
 
         if (module->memory_count) {
-            /* Memory init page count cannot be larger than 65536, we don't
-               check integer overflow again. */
             memory = &module->memories[0];
-            memory->num_bytes_per_page *= memory->init_page_count;
-            memory->init_page_count = memory->max_page_count = 1;
+            if (memory->init_page_count < DEFAULT_MAX_PAGES)
+                memory->num_bytes_per_page *= memory->init_page_count;
+            else
+                memory->num_bytes_per_page = UINT32_MAX;
+
+            if (memory->init_page_count > 0)
+                memory->init_page_count = memory->max_page_count = 1;
+            else
+                memory->init_page_count = memory->max_page_count = 0;
         }
     }
 

core/iwasm/interpreter/wasm_runtime.c

@@ -206,18 +206,27 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
     if (init_page_count == max_page_count && init_page_count == 1) {
         /* If only one page and at most one page, we just append
            the app heap to the end of linear memory, enlarge the
-           num_bytes_per_page, and don't change the page count*/
+           num_bytes_per_page, and don't change the page count */
         heap_offset = num_bytes_per_page;
         num_bytes_per_page += heap_size;
         if (num_bytes_per_page < heap_size) {
             set_error_buf(error_buf, error_buf_size,
-                          "memory size must be at most 65536 pages (4GiB)");
+                          "failed to insert app heap into linear memory, "
+                          "try using `--heap_size=0` option");
             return NULL;
         }
     }
     else if (heap_size > 0) {
-        if (module->aux_heap_base_global_index != (uint32)-1
-            && module->aux_heap_base < num_bytes_per_page * init_page_count) {
+        if (init_page_count == max_page_count && init_page_count == 0) {
+            /* If the memory data size is always 0, we resize it to
+               one page for app heap */
+            num_bytes_per_page = heap_size;
+            heap_offset = 0;
+            inc_page_count = 1;
+        }
+        else if (module->aux_heap_base_global_index != (uint32)-1
+                 && module->aux_heap_base
+                        < num_bytes_per_page * init_page_count) {
             /* Insert app heap before __heap_base */
             aux_heap_base = module->aux_heap_base;
             bytes_of_last_page = aux_heap_base % num_bytes_per_page;
@@ -259,13 +268,18 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
         }
         init_page_count += inc_page_count;
         max_page_count += inc_page_count;
-        if (init_page_count > 65536) {
+        if (init_page_count > DEFAULT_MAX_PAGES) {
             set_error_buf(error_buf, error_buf_size,
-                          "memory size must be at most 65536 pages (4GiB)");
+                          "failed to insert app heap into linear memory, "
+                          "try using `--heap_size=0` option");
             return NULL;
         }
-        if (max_page_count > 65536)
-            max_page_count = 65536;
+        else if (init_page_count == DEFAULT_MAX_PAGES) {
+            num_bytes_per_page = UINT32_MAX;
+            init_page_count = max_page_count = 1;
+        }
+        if (max_page_count > DEFAULT_MAX_PAGES)
+            max_page_count = DEFAULT_MAX_PAGES;
     }
 
     LOG_VERBOSE("Memory instantiate:");
@@ -280,6 +294,7 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
         memory_data_size = (uint64)num_bytes_per_page * max_page_count;
     }
 #endif
+    bh_assert(memory_data_size <= 4 * (uint64)BH_GB);
 
     /* Allocate memory space, addr data and global data */
     if (!(memory = runtime_malloc((uint64)sizeof(WASMMemoryInstance), error_buf,
@@ -301,9 +316,8 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
      * both i and memarg.offset are u32 in range 0 to 4G
      * so the range of ea is 0 to 8G
      */
-    if (memory_data_size >= UINT32_MAX
-        || !(memory->memory_data = mapped_mem =
-                 os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) {
+    if (!(memory->memory_data = mapped_mem =
+              os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) {
         set_error_buf(error_buf, error_buf_size, "mmap memory failed");
         goto fail1;
     }
@@ -327,10 +341,14 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
      * again here */
 #endif /* end of OS_ENABLE_HW_BOUND_CHECK */
 
+    if (memory_data_size > UINT32_MAX)
+        memory_data_size = (uint32)memory_data_size;
+
     memory->module_type = Wasm_Module_Bytecode;
     memory->num_bytes_per_page = num_bytes_per_page;
     memory->cur_page_count = init_page_count;
     memory->max_page_count = max_page_count;
+    memory->memory_data_size = (uint32)memory_data_size;
 
     memory->heap_data = memory->memory_data + heap_offset;
     memory->heap_data_end = memory->heap_data + heap_size;
@@ -2419,39 +2437,49 @@ bool
 wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
 {
     WASMMemoryInstance *memory = module->default_memory;
-    uint8 *new_memory_data, *memory_data, *heap_data_old;
-    uint32 heap_size, total_size_old, total_page_count;
-    uint64 total_size;
+    uint8 *memory_data_old, *memory_data_new, *heap_data_old;
+    uint32 num_bytes_per_page, heap_size, total_size_old;
+    uint32 cur_page_count, max_page_count, total_page_count;
+    uint64 total_size_new;
     bool ret = true;
 
     if (!memory)
         return false;
 
-    memory_data = memory->memory_data;
-    heap_size = (uint32)(memory->heap_data_end - memory->heap_data);
-    total_size_old = (uint32)(memory->memory_data_end - memory_data);
-    total_page_count = inc_page_count + memory->cur_page_count;
-    total_size = memory->num_bytes_per_page * (uint64)total_page_count;
     heap_data_old = memory->heap_data;
+    heap_size = (uint32)(memory->heap_data_end - memory->heap_data);
+
+    memory_data_old = memory->memory_data;
+    total_size_old = memory->memory_data_size;
+
+    num_bytes_per_page = memory->num_bytes_per_page;
+    cur_page_count = memory->cur_page_count;
+    max_page_count = memory->max_page_count;
+    total_page_count = inc_page_count + cur_page_count;
+    total_size_new = num_bytes_per_page * (uint64)total_page_count;
 
     if (inc_page_count <= 0)
         /* No need to enlarge memory */
         return true;
 
-    if (total_page_count < memory->cur_page_count /* integer overflow */
-        || total_page_count > memory->max_page_count) {
+    if (total_page_count < cur_page_count /* integer overflow */
+        || total_page_count > max_page_count) {
         return false;
     }
 
-    if (total_size >= UINT32_MAX) {
-        return false;
+    bh_assert(total_size_new <= 4 * (uint64)BH_GB);
+    if (total_size_new > UINT32_MAX) {
+        /* Resize to 1 page with size 4G-1 */
+        num_bytes_per_page = UINT32_MAX;
+        total_page_count = max_page_count = 1;
+        total_size_new = UINT32_MAX;
     }
 
 #if WASM_ENABLE_SHARED_MEMORY != 0
     if (memory->is_shared) {
-        /* For shared memory, we have reserved the maximum spaces during
-            instantiate, only change the cur_page_count here */
+        memory->num_bytes_per_page = UINT32_MAX;
         memory->cur_page_count = total_page_count;
+        memory->max_page_count = max_page_count;
         return true;
     }
 #endif
@@ -2463,25 +2491,25 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
         }
     }
 
-    if (!(new_memory_data =
-              wasm_runtime_realloc(memory_data, (uint32)total_size))) {
-        if (!(new_memory_data = wasm_runtime_malloc((uint32)total_size))) {
+    if (!(memory_data_new =
+              wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) {
+        if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) {
             return false;
         }
-        if (memory_data) {
-            bh_memcpy_s(new_memory_data, (uint32)total_size, memory_data,
-                        total_size_old);
-            wasm_runtime_free(memory_data);
+        if (memory_data_old) {
+            bh_memcpy_s(memory_data_new, (uint32)total_size_new,
+                        memory_data_old, total_size_old);
+            wasm_runtime_free(memory_data_old);
         }
     }
 
-    memset(new_memory_data + total_size_old, 0,
-           (uint32)total_size - total_size_old);
+    memset(memory_data_new + total_size_old, 0,
+           (uint32)total_size_new - total_size_old);
 
     if (heap_size > 0) {
         if (mem_allocator_migrate(memory->heap_handle,
                                   (char *)heap_data_old
-                                      + (new_memory_data - memory_data),
+                                      + (memory_data_new - memory_data_old),
                                   heap_size)
             != 0) {
             /* Don't return here as memory->memory_data is obsolete and
@@ -2490,26 +2518,30 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
         }
     }
 
-    memory->memory_data = new_memory_data;
-    memory->cur_page_count = total_page_count;
-    memory->heap_data = new_memory_data + (heap_data_old - memory_data);
+    memory->heap_data = memory_data_new + (heap_data_old - memory_data_old);
     memory->heap_data_end = memory->heap_data + heap_size;
-    memory->memory_data_end =
-        memory->memory_data + memory->num_bytes_per_page * total_page_count;
+
+    memory->num_bytes_per_page = num_bytes_per_page;
+    memory->cur_page_count = total_page_count;
+    memory->max_page_count = max_page_count;
+    memory->memory_data_size = (uint32)total_size_new;
+
+    memory->memory_data = memory_data_new;
+    memory->memory_data_end = memory_data_new + (uint32)total_size_new;
 
 #if WASM_ENABLE_FAST_JIT != 0
 #if UINTPTR_MAX == UINT64_MAX
-    memory->mem_bound_check_1byte = total_size - 1;
-    memory->mem_bound_check_2bytes = total_size - 2;
-    memory->mem_bound_check_4bytes = total_size - 4;
-    memory->mem_bound_check_8bytes = total_size - 8;
-    memory->mem_bound_check_16bytes = total_size - 16;
+    memory->mem_bound_check_1byte = total_size_new - 1;
+    memory->mem_bound_check_2bytes = total_size_new - 2;
+    memory->mem_bound_check_4bytes = total_size_new - 4;
+    memory->mem_bound_check_8bytes = total_size_new - 8;
+    memory->mem_bound_check_16bytes = total_size_new - 16;
 #else
-    memory->mem_bound_check_1byte = (uint32)total_size - 1;
-    memory->mem_bound_check_2bytes = (uint32)total_size - 2;
-    memory->mem_bound_check_4bytes = (uint32)total_size - 4;
-    memory->mem_bound_check_8bytes = (uint32)total_size - 8;
-    memory->mem_bound_check_16bytes = (uint32)total_size - 16;
+    memory->mem_bound_check_1byte = (uint32)total_size_new - 1;
+    memory->mem_bound_check_2bytes = (uint32)total_size_new - 2;
+    memory->mem_bound_check_4bytes = (uint32)total_size_new - 4;
+    memory->mem_bound_check_8bytes = (uint32)total_size_new - 8;
+    memory->mem_bound_check_16bytes = (uint32)total_size_new - 16;
 #endif
 #endif
 
@@ -2520,39 +2552,52 @@ bool
 wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
 {
     WASMMemoryInstance *memory = module->default_memory;
-    uint32 num_bytes_per_page, total_page_count;
+    uint32 num_bytes_per_page, total_size_old;
+    uint32 cur_page_count, max_page_count, total_page_count;
+    uint64 total_size_new;
 
     if (!memory)
         return false;
 
-    total_page_count = inc_page_count + memory->cur_page_count;
+    num_bytes_per_page = memory->num_bytes_per_page;
+    cur_page_count = memory->cur_page_count;
+    max_page_count = memory->max_page_count;
+    total_size_old = num_bytes_per_page * cur_page_count;
+    total_page_count = inc_page_count + cur_page_count;
+    total_size_new = num_bytes_per_page * (uint64)total_page_count;
 
     if (inc_page_count <= 0)
         /* No need to enlarge memory */
         return true;
 
-    if (total_page_count < memory->cur_page_count /* integer overflow */
-        || total_page_count > memory->max_page_count) {
+    if (total_page_count < cur_page_count /* integer overflow */
+        || total_page_count > max_page_count) {
         return false;
     }
 
-    num_bytes_per_page = memory->num_bytes_per_page;
+    bh_assert(total_size_new <= 4 * (uint64)BH_GB);
+    if (total_size_new > UINT32_MAX) {
+        /* Resize to 1 page with size 4G-1 */
+        num_bytes_per_page = UINT32_MAX;
+        total_page_count = max_page_count = 1;
+        total_size_new = UINT32_MAX;
+    }
 
 #ifdef BH_PLATFORM_WINDOWS
     if (!os_mem_commit(memory->memory_data_end,
-                       num_bytes_per_page * inc_page_count,
+                       (uint32)total_size_new - total_size_old,
                        MMAP_PROT_READ | MMAP_PROT_WRITE)) {
         return false;
     }
 #endif
 
     if (os_mprotect(memory->memory_data_end,
-                    num_bytes_per_page * inc_page_count,
+                    (uint32)total_size_new - total_size_old,
                     MMAP_PROT_READ | MMAP_PROT_WRITE)
         != 0) {
 #ifdef BH_PLATFORM_WINDOWS
         os_mem_decommit(memory->memory_data_end,
-                        num_bytes_per_page * inc_page_count);
+                        (uint32)total_size_new - total_size_old);
 #endif
         return false;
     }
@@ -2560,9 +2605,20 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
     /* The increased pages are filled with zero by the OS when os_mmap,
        no need to memset it again here */
 
+    memory->num_bytes_per_page = num_bytes_per_page;
     memory->cur_page_count = total_page_count;
-    memory->memory_data_end =
-        memory->memory_data + num_bytes_per_page * total_page_count;
+    memory->max_page_count = max_page_count;
+    memory->memory_data_size = (uint32)total_size_new;
+    memory->memory_data_end = memory->memory_data + (uint32)total_size_new;
+
+#if WASM_ENABLE_FAST_JIT != 0
+    memory->mem_bound_check_1byte = total_size_new - 1;
+    memory->mem_bound_check_2bytes = total_size_new - 2;
+    memory->mem_bound_check_4bytes = total_size_new - 4;
+    memory->mem_bound_check_8bytes = total_size_new - 8;
+    memory->mem_bound_check_16bytes = total_size_new - 16;
+#endif
+
     return true;
 }
 #endif /* end of OS_ENABLE_HW_BOUND_CHECK */

core/iwasm/interpreter/wasm_runtime.h

@@ -26,12 +26,25 @@ struct WASMMemoryInstance {
     uint32 module_type;
     /* Shared memory flag */
     bool is_shared;
+
     /* Number bytes per page */
     uint32 num_bytes_per_page;
     /* Current page count */
     uint32 cur_page_count;
     /* Maximum page count */
     uint32 max_page_count;
+    /* Memory data size */
+    uint32 memory_data_size;
+
+    /**
+     * Memory data begin address, Note:
+     *   the app-heap might be inserted in to the linear memory,
+     *   when memory is re-allocated, the heap data and memory data
+     *   must be copied to new memory also
+     */
+    uint8 *memory_data;
+    /* Memory data end address */
+    uint8 *memory_data_end;
 
     /* Heap data base address */
     uint8 *heap_data;
@@ -45,14 +58,6 @@ struct WASMMemoryInstance {
     korp_mutex mem_lock;
 #endif
 
-    /* Memory data end address */
-    uint8 *memory_data_end;
-
-    /* Memory data begin address, the layout is: memory data + heap data
-       Note: when memory is re-allocated, the heap data and memory data
-             must be copied to new memory also. */
-    uint8 *memory_data;
-
 #if WASM_ENABLE_FAST_JIT != 0
 #if UINTPTR_MAX == UINT64_MAX
     uint64 mem_bound_check_1byte;