wasm-micro-runtime/core/iwasm/aot/aot_runtime.c

/*
 * Copyright (C) 2019 Intel Corporation. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 */

#include "aot_runtime.h"
#include "bh_log.h"
#include "mem_alloc.h"
#include "../common/wasm_runtime_common.h"
#if WASM_ENABLE_SHARED_MEMORY != 0
#include "../common/wasm_shared_memory.h"
#endif
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif

/*
 * Note: These offsets need to match the values hardcoded in
 * AoT compilation code: aot_create_func_context, check_suspend_flags.
 */

bh_static_assert(offsetof(WASMExecEnv, module_inst) == 2 * sizeof(uintptr_t));
bh_static_assert(offsetof(WASMExecEnv, argv_buf) == 3 * sizeof(uintptr_t));
bh_static_assert(offsetof(WASMExecEnv, native_stack_boundary)
                 == 4 * sizeof(uintptr_t));
bh_static_assert(offsetof(WASMExecEnv, suspend_flags) == 5 * sizeof(uintptr_t));
bh_static_assert(offsetof(WASMExecEnv, aux_stack_boundary)
                 == 6 * sizeof(uintptr_t));
bh_static_assert(offsetof(WASMExecEnv, aux_stack_bottom)
                 == 7 * sizeof(uintptr_t));
bh_static_assert(offsetof(WASMExecEnv, native_symbol) == 8 * sizeof(uintptr_t));

static void
set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
{
    if (error_buf != NULL) {
        snprintf(error_buf, error_buf_size, "AOT module instantiate failed: %s",
                 string);
    }
}

static void
set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...)
{
    va_list args;
    char buf[128];

    if (error_buf != NULL) {
        va_start(args, format);
        vsnprintf(buf, sizeof(buf), format, args);
        va_end(args);
        snprintf(error_buf, error_buf_size, "AOT module instantiate failed: %s",
                 buf);
    }
}

static void *
runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
{
    void *mem;

    if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) {
        set_error_buf(error_buf, error_buf_size, "allocate memory failed");
        return NULL;
    }

    memset(mem, 0, (uint32)size);
    return mem;
}

static bool
check_global_init_expr(const AOTModule *module, uint32 global_index,
                       char *error_buf, uint32 error_buf_size)
{
    if (global_index >= module->import_global_count + module->global_count) {
        set_error_buf_v(error_buf, error_buf_size, "unknown global %d",
                        global_index);
        return false;
    }

    /**
     * Currently, constant expressions occurring as initializers of
     * globals are further constrained in that contained global.get
     * instructions are only allowed to refer to imported globals.
     *
     * And initializer expression cannot reference a mutable global.
     */
    if (global_index >= module->import_global_count
        || module->import_globals->is_mutable) {
        set_error_buf(error_buf, error_buf_size,
                      "constant expression required");
        return false;
    }

    return true;
}

static void
init_global_data(uint8 *global_data, uint8 type, WASMValue *initial_value)
{
    switch (type) {
        case VALUE_TYPE_I32:
        case VALUE_TYPE_F32:
#if WASM_ENABLE_REF_TYPES != 0
        case VALUE_TYPE_FUNCREF:
        case VALUE_TYPE_EXTERNREF:
#endif
            *(int32 *)global_data = initial_value->i32;
            break;
        case VALUE_TYPE_I64:
        case VALUE_TYPE_F64:
            bh_memcpy_s(global_data, sizeof(int64), &initial_value->i64,
                        sizeof(int64));
            break;
#if WASM_ENABLE_SIMD != 0
        case VALUE_TYPE_V128:
            bh_memcpy_s(global_data, sizeof(V128), &initial_value->i64,
                        sizeof(V128));
            break;
#endif
        default:
            bh_assert(0);
    }
}

static bool
global_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
                   char *error_buf, uint32 error_buf_size)
{
    uint32 i;
    InitializerExpression *init_expr;
    uint8 *p = (uint8 *)module_inst->global_data.ptr;
    AOTImportGlobal *import_global = module->import_globals;
    AOTGlobal *global = module->globals;

    /* Initialize import global data */
    for (i = 0; i < module->import_global_count; i++, import_global++) {
        bh_assert(import_global->data_offset
                  == (uint32)(p - (uint8 *)module_inst->global_data.ptr));
        init_global_data(p, import_global->type,
                         &import_global->global_data_linked);
        p += import_global->size;
    }

    /* Initialize defined global data */
    for (i = 0; i < module->global_count; i++, global++) {
        bh_assert(global->data_offset
                  == (uint32)(p - (uint8 *)module_inst->global_data.ptr));
        init_expr = &global->init_expr;
        switch (init_expr->init_expr_type) {
            case INIT_EXPR_TYPE_GET_GLOBAL:
            {
                if (!check_global_init_expr(module, init_expr->u.global_index,
                                            error_buf, error_buf_size)) {
                    return false;
                }
                init_global_data(
                    p, global->type,
                    &module->import_globals[init_expr->u.global_index]
                         .global_data_linked);
                break;
            }
#if WASM_ENABLE_REF_TYPES != 0
            case INIT_EXPR_TYPE_REFNULL_CONST:
            {
                *(uint32 *)p = NULL_REF;
                break;
            }
#endif
            default:
            {
                init_global_data(p, global->type, &init_expr->u);
                break;
            }
        }
        p += global->size;
    }

    bh_assert(module_inst->global_data_size
              == (uint32)(p - (uint8 *)module_inst->global_data.ptr));
    return true;
}

AOTTableInstance *
aot_next_tbl_inst(const AOTTableInstance *tbl_inst)
{
    uint32 offset = offsetof(AOTTableInstance, data);
    offset += tbl_inst->max_size * sizeof(uint32);
    return (AOTTableInstance *)((uint8 *)tbl_inst + offset);
}

static inline AOTTableInstance *
aot_get_table_inst(const AOTModuleInstance *module_inst, uint32 tbl_idx)
{
    uint32 i = 0;
    AOTTableInstance *tbl_inst = (AOTTableInstance *)module_inst->tables.ptr;

    while (i != tbl_idx) {
        tbl_inst = aot_next_tbl_inst(tbl_inst);
        ++i;
    }

    return tbl_inst;
}

static bool
table_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
                  char *error_buf, uint32 error_buf_size)
{
    uint32 i, global_index, global_data_offset, base_offset, length;
    AOTTableInitData *table_seg;
    AOTTableInstance *tbl_inst = (AOTTableInstance *)module_inst->tables.ptr;

    /*
     * treat import table like a local one until we enable module linking
     * in AOT mode
     */
    for (i = 0; i != module_inst->table_count; ++i) {
        if (i < module->import_table_count) {
            AOTImportTable *import_table = module->import_tables + i;
            tbl_inst->cur_size = import_table->table_init_size;
            tbl_inst->max_size = aot_get_imp_tbl_data_slots(import_table);
        }
        else {
            AOTTable *table = module->tables + (i - module->import_table_count);
            tbl_inst->cur_size = table->table_init_size;
            tbl_inst->max_size = aot_get_tbl_data_slots(table);
        }

        tbl_inst = aot_next_tbl_inst(tbl_inst);
    }

    /* fill table with element segment content */
    for (i = 0; i < module->table_init_data_count; i++) {
        table_seg = module->table_init_data_list[i];

#if WASM_ENABLE_REF_TYPES != 0
        if (!wasm_elem_is_active(table_seg->mode))
            continue;
#endif

        bh_assert(table_seg->table_index < module_inst->table_count);

        tbl_inst = aot_get_table_inst(module_inst, table_seg->table_index);
        bh_assert(tbl_inst);

        bh_assert(
            table_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST
            || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL
#if WASM_ENABLE_REF_TYPES != 0
            || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST
            || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST
#endif
        );

        /* Resolve table data base offset */
        if (table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
            global_index = table_seg->offset.u.global_index;

            if (!check_global_init_expr(module, global_index, error_buf,
                                        error_buf_size)) {
                return false;
            }

            if (global_index < module->import_global_count)
                global_data_offset =
                    module->import_globals[global_index].data_offset;
            else
                global_data_offset =
                    module->globals[global_index - module->import_global_count]
                        .data_offset;

            base_offset = *(uint32 *)((uint8 *)module_inst->global_data.ptr
                                      + global_data_offset);
        }
        else
            base_offset = (uint32)table_seg->offset.u.i32;

        /* Copy table data */
        /* base_offset only since length might negative */
        if (base_offset > tbl_inst->cur_size) {
#if WASM_ENABLE_REF_TYPES != 0
            set_error_buf(error_buf, error_buf_size,
                          "out of bounds table access");
#else
            set_error_buf(error_buf, error_buf_size,
                          "elements segment does not fit");
#endif
            return false;
        }

        /* base_offset + length(could be zero) */
        length = table_seg->func_index_count;
        if (base_offset + length > tbl_inst->cur_size) {
#if WASM_ENABLE_REF_TYPES != 0
            set_error_buf(error_buf, error_buf_size,
                          "out of bounds table access");
#else
            set_error_buf(error_buf, error_buf_size,
                          "elements segment does not fit");
#endif
            return false;
        }

        /**
         * Check function index in the current module inst for now.
         * will check the linked table inst owner in future
         */
        bh_memcpy_s((uint32 *)tbl_inst->data + base_offset,
                    (tbl_inst->max_size - base_offset) * sizeof(uint32),
                    table_seg->func_indexes, length * sizeof(uint32));
    }

    return true;
}

static void
memories_deinstantiate(AOTModuleInstance *module_inst)
{
    uint32 i;
    AOTMemoryInstance *memory_inst;

    for (i = 0; i < module_inst->memory_count; i++) {
        memory_inst = ((AOTMemoryInstance **)module_inst->memories.ptr)[i];
        if (memory_inst) {
#if WASM_ENABLE_SHARED_MEMORY != 0
            if (memory_inst->is_shared) {
                int32 ref_count = shared_memory_dec_reference(
                    (WASMModuleCommon *)module_inst->aot_module.ptr);
                bh_assert(ref_count >= 0);

                /* if the reference count is not zero,
                    don't free the memory */
                if (ref_count > 0)
                    continue;
            }
#endif
            if (memory_inst->heap_handle.ptr) {
                mem_allocator_destroy(memory_inst->heap_handle.ptr);
                wasm_runtime_free(memory_inst->heap_handle.ptr);
            }

            if (memory_inst->memory_data.ptr) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
                wasm_runtime_free(memory_inst->memory_data.ptr);
#else
#ifdef BH_PLATFORM_WINDOWS
                os_mem_decommit(memory_inst->memory_data.ptr,
                                memory_inst->num_bytes_per_page
                                    * memory_inst->cur_page_count);
#endif
                os_munmap((uint8 *)memory_inst->memory_data.ptr,
                          8 * (uint64)BH_GB);
#endif
            }
        }
    }
    wasm_runtime_free(module_inst->memories.ptr);
}

static AOTMemoryInstance *
memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
                   AOTMemoryInstance *memory_inst, AOTMemory *memory,
                   uint32 heap_size, char *error_buf, uint32 error_buf_size)
{
    void *heap_handle;
    uint32 num_bytes_per_page = memory->num_bytes_per_page;
    uint32 init_page_count = memory->mem_init_page_count;
    uint32 max_page_count = memory->mem_max_page_count;
    uint32 inc_page_count, aux_heap_base, global_idx;
    uint32 bytes_of_last_page, bytes_to_page_end;
    uint32 heap_offset = num_bytes_per_page * init_page_count;
    uint64 total_size;
    uint8 *p = NULL, *global_addr;
#ifdef OS_ENABLE_HW_BOUND_CHECK
    uint8 *mapped_mem;
    uint64 map_size = 8 * (uint64)BH_GB;
    uint64 page_size = os_getpagesize();
#endif

#if WASM_ENABLE_SHARED_MEMORY != 0
    bool is_shared_memory = memory->memory_flags & 0x02 ? true : false;

    /* Shared memory */
    if (is_shared_memory) {
        AOTMemoryInstance *shared_memory_instance;
        WASMSharedMemNode *node =
            wasm_module_get_shared_memory((WASMModuleCommon *)module);
        /* If the memory of this module has been instantiated,
            return the memory instance directly */
        if (node) {
            uint32 ref_count;
            ref_count = shared_memory_inc_reference((WASMModuleCommon *)module);
            bh_assert(ref_count > 0);
            shared_memory_instance =
                (AOTMemoryInstance *)shared_memory_get_memory_inst(node);
            bh_assert(shared_memory_instance);

            (void)ref_count;
            return shared_memory_instance;
        }
    }
#endif

    if (heap_size > 0 && module->malloc_func_index != (uint32)-1
        && module->free_func_index != (uint32)-1) {
        /* Disable app heap, use malloc/free function exported
           by wasm app to allocate/free memory instead */
        heap_size = 0;
    }

    if (init_page_count == max_page_count && init_page_count == 1) {
        /* If only one page and at most one page, we just append
           the app heap to the end of linear memory, enlarge the
           num_bytes_per_page, and don't change the page count*/
        heap_offset = num_bytes_per_page;
        num_bytes_per_page += heap_size;
        if (num_bytes_per_page < heap_size) {
            set_error_buf(error_buf, error_buf_size,
                          "memory size must be at most 65536 pages (4GiB)");
            return NULL;
        }
    }
    else if (heap_size > 0) {
        if (module->aux_heap_base_global_index != (uint32)-1
            && module->aux_heap_base < num_bytes_per_page * init_page_count) {
            /* Insert app heap before __heap_base */
            aux_heap_base = module->aux_heap_base;
            bytes_of_last_page = aux_heap_base % num_bytes_per_page;
            if (bytes_of_last_page == 0)
                bytes_of_last_page = num_bytes_per_page;
            bytes_to_page_end = num_bytes_per_page - bytes_of_last_page;
            inc_page_count =
                (heap_size - bytes_to_page_end + num_bytes_per_page - 1)
                / num_bytes_per_page;
            heap_offset = aux_heap_base;
            aux_heap_base += heap_size;

            bytes_of_last_page = aux_heap_base % num_bytes_per_page;
            if (bytes_of_last_page == 0)
                bytes_of_last_page = num_bytes_per_page;
            bytes_to_page_end = num_bytes_per_page - bytes_of_last_page;
            if (bytes_to_page_end < 1 * BH_KB) {
                aux_heap_base += 1 * BH_KB;
                inc_page_count++;
            }

            /* Adjust __heap_base global value */
            global_idx = module->aux_heap_base_global_index
                         - module->import_global_count;
            global_addr = (uint8 *)module_inst->global_data.ptr
                          + module->globals[global_idx].data_offset;
            *(uint32 *)global_addr = aux_heap_base;
            LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base);
        }
        else {
            /* Insert app heap before new page */
            inc_page_count =
                (heap_size + num_bytes_per_page - 1) / num_bytes_per_page;
            heap_offset = num_bytes_per_page * init_page_count;
            heap_size = num_bytes_per_page * inc_page_count;
            if (heap_size > 0)
                heap_size -= 1 * BH_KB;
        }
        init_page_count += inc_page_count;
        max_page_count += inc_page_count;
        if (init_page_count > 65536) {
            set_error_buf(error_buf, error_buf_size,
                          "memory size must be at most 65536 pages (4GiB)");
            return NULL;
        }
        if (max_page_count > 65536)
            max_page_count = 65536;
    }

    LOG_VERBOSE("Memory instantiate:");
    LOG_VERBOSE("  page bytes: %u, init pages: %u, max pages: %u",
                num_bytes_per_page, init_page_count, max_page_count);
    LOG_VERBOSE("  data offset: %u, stack size: %d", module->aux_data_end,
                module->aux_stack_size);
    LOG_VERBOSE("  heap offset: %u, heap size: %d\n", heap_offset, heap_size);

    total_size = (uint64)num_bytes_per_page * init_page_count;
#if WASM_ENABLE_SHARED_MEMORY != 0
    if (is_shared_memory) {
        /* Allocate max page for shared memory */
        total_size = (uint64)num_bytes_per_page * max_page_count;
    }
#endif

#ifndef OS_ENABLE_HW_BOUND_CHECK
    /* Allocate memory */
    if (total_size > 0
        && !(p = runtime_malloc(total_size, error_buf, error_buf_size))) {
        return NULL;
    }
#else
    total_size = (total_size + page_size - 1) & ~(page_size - 1);

    /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
     *   ea = i + memarg.offset
     * both i and memarg.offset are u32 in range 0 to 4G
     * so the range of ea is 0 to 8G
     */
    if (total_size >= UINT32_MAX
        || !(p = mapped_mem =
                 os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) {
        set_error_buf(error_buf, error_buf_size, "mmap memory failed");
        return NULL;
    }

#ifdef BH_PLATFORM_WINDOWS
    if (!os_mem_commit(p, total_size, MMAP_PROT_READ | MMAP_PROT_WRITE)) {
        set_error_buf(error_buf, error_buf_size, "commit memory failed");
        os_munmap(mapped_mem, map_size);
        return NULL;
    }
#endif

    if (os_mprotect(p, total_size, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) {
        set_error_buf(error_buf, error_buf_size, "mprotect memory failed");
#ifdef BH_PLATFORM_WINDOWS
        os_mem_decommit(p, total_size);
#endif
        os_munmap(mapped_mem, map_size);
        return NULL;
    }
    /* Newly allocated pages are filled with zero by the OS, we don't fill it
     * again here */
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */

    memory_inst->module_type = Wasm_Module_AoT;
    memory_inst->num_bytes_per_page = num_bytes_per_page;
    memory_inst->cur_page_count = init_page_count;
    memory_inst->max_page_count = max_page_count;

    /* Init memory info */
    memory_inst->memory_data.ptr = p;
    memory_inst->memory_data_end.ptr = p + (uint32)total_size;
    memory_inst->memory_data_size = (uint32)total_size;

    /* Initialize heap info */
    memory_inst->heap_data.ptr = p + heap_offset;
    memory_inst->heap_data_end.ptr = p + heap_offset + heap_size;
    if (heap_size > 0) {
        uint32 heap_struct_size = mem_allocator_get_heap_struct_size();

        if (!(heap_handle = runtime_malloc((uint64)heap_struct_size, error_buf,
                                           error_buf_size))) {
            goto fail1;
        }

        memory_inst->heap_handle.ptr = heap_handle;

        if (!mem_allocator_create_with_struct_and_pool(
                heap_handle, heap_struct_size, memory_inst->heap_data.ptr,
                heap_size)) {
            set_error_buf(error_buf, error_buf_size, "init app heap failed");
            goto fail2;
        }
    }

    if (total_size > 0) {
        if (sizeof(uintptr_t) == sizeof(uint64)) {
            memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
            memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
            memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
            memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
            memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
        }
        else {
            memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
            memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
            memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
            memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
            memory_inst->mem_bound_check_16bytes.u32[0] =
                (uint32)total_size - 16;
        }
    }

#if WASM_ENABLE_SHARED_MEMORY != 0
    if (is_shared_memory) {
        memory_inst->is_shared = true;
        if (!shared_memory_set_memory_inst(
                (WASMModuleCommon *)module,
                (WASMMemoryInstanceCommon *)memory_inst)) {
            set_error_buf(error_buf, error_buf_size, "allocate memory failed");
            goto fail3;
        }
    }
#endif

    return memory_inst;

#if WASM_ENABLE_SHARED_MEMORY != 0
fail3:
    if (heap_size > 0)
        mem_allocator_destroy(memory_inst->heap_handle.ptr);
#endif
fail2:
    if (heap_size > 0)
        wasm_runtime_free(memory_inst->heap_handle.ptr);
fail1:
#ifndef OS_ENABLE_HW_BOUND_CHECK
    if (memory_inst->memory_data.ptr)
        wasm_runtime_free(memory_inst->memory_data.ptr);
#else
#ifdef BH_PLATFORM_WINDOWS
    if (memory_inst->memory_data.ptr)
        os_mem_decommit(p, total_size);
#endif
    os_munmap(mapped_mem, map_size);
#endif
    memory_inst->memory_data.ptr = NULL;
    return NULL;
}

static AOTMemoryInstance *
aot_get_default_memory(AOTModuleInstance *module_inst)
{
    if (module_inst->memories.ptr)
        return ((AOTMemoryInstance **)module_inst->memories.ptr)[0];
    else
        return NULL;
}

static bool
memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
                     uint32 heap_size, char *error_buf, uint32 error_buf_size)
{
    uint32 global_index, global_data_offset, base_offset, length;
    uint32 i, memory_count = module->memory_count;
    AOTMemoryInstance *memories, *memory_inst;
    AOTMemInitData *data_seg;
    uint64 total_size;

    module_inst->memory_count = memory_count;
    total_size = sizeof(AOTPointer) * (uint64)memory_count;
    if (!(module_inst->memories.ptr =
              runtime_malloc(total_size, error_buf, error_buf_size))) {
        return false;
    }

    memories = module_inst->global_table_data.memory_instances;
    for (i = 0; i < memory_count; i++, memories++) {
        memory_inst = memory_instantiate(module_inst, module, memories,
                                         &module->memories[i], heap_size,
                                         error_buf, error_buf_size);
        if (!memory_inst) {
            return false;
        }

        ((AOTMemoryInstance **)module_inst->memories.ptr)[i] = memory_inst;
    }

    /* Get default memory instance */
    memory_inst = aot_get_default_memory(module_inst);
    if (!memory_inst) {
        /* Ignore setting memory init data if no memory inst is created */
        return true;
    }

    for (i = 0; i < module->mem_init_data_count; i++) {
        data_seg = module->mem_init_data_list[i];
#if WASM_ENABLE_BULK_MEMORY != 0
        if (data_seg->is_passive)
            continue;
#endif

        bh_assert(data_seg->offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST
                  || data_seg->offset.init_expr_type
                         == INIT_EXPR_TYPE_GET_GLOBAL);

        /* Resolve memory data base offset */
        if (data_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
            global_index = data_seg->offset.u.global_index;

            if (!check_global_init_expr(module, global_index, error_buf,
                                        error_buf_size)) {
                return false;
            }

            if (global_index < module->import_global_count)
                global_data_offset =
                    module->import_globals[global_index].data_offset;
            else
                global_data_offset =
                    module->globals[global_index - module->import_global_count]
                        .data_offset;

            base_offset = *(uint32 *)((uint8 *)module_inst->global_data.ptr
                                      + global_data_offset);
        }
        else {
            base_offset = (uint32)data_seg->offset.u.i32;
        }

        /* Copy memory data */
        bh_assert(memory_inst->memory_data.ptr
                  || memory_inst->memory_data_size == 0);

        /* Check memory data */
        /* check offset since length might negative */
        if (base_offset > memory_inst->memory_data_size) {
            LOG_DEBUG("base_offset(%d) > memory_data_size(%d)", base_offset,
                      memory_inst->memory_data_size);
#if WASM_ENABLE_REF_TYPES != 0
            set_error_buf(error_buf, error_buf_size,
                          "out of bounds memory access");
#else
            set_error_buf(error_buf, error_buf_size,
                          "data segment does not fit");
#endif
            return false;
        }

        /* check offset + length(could be zero) */
        length = data_seg->byte_count;
        if (base_offset + length > memory_inst->memory_data_size) {
            LOG_DEBUG("base_offset(%d) + length(%d) > memory_data_size(%d)",
                      base_offset, length, memory_inst->memory_data_size);
#if WASM_ENABLE_REF_TYPES != 0
            set_error_buf(error_buf, error_buf_size,
                          "out of bounds memory access");
#else
            set_error_buf(error_buf, error_buf_size,
                          "data segment does not fit");
#endif
            return false;
        }

        if (memory_inst->memory_data.ptr) {
            bh_memcpy_s((uint8 *)memory_inst->memory_data.ptr + base_offset,
                        memory_inst->memory_data_size - base_offset,
                        data_seg->bytes, length);
        }
    }

    return true;
}

static bool
init_func_ptrs(AOTModuleInstance *module_inst, AOTModule *module,
               char *error_buf, uint32 error_buf_size)
{
    uint32 i;
    void **func_ptrs;
    uint64 total_size = ((uint64)module->import_func_count + module->func_count)
                        * sizeof(void *);

    if (module->import_func_count + module->func_count == 0)
        return true;

    /* Allocate memory */
    if (!(module_inst->func_ptrs.ptr =
              runtime_malloc(total_size, error_buf, error_buf_size))) {
        return false;
    }

    /* Set import function pointers */
    func_ptrs = (void **)module_inst->func_ptrs.ptr;
    for (i = 0; i < module->import_func_count; i++, func_ptrs++) {
        *func_ptrs = (void *)module->import_funcs[i].func_ptr_linked;
        if (!*func_ptrs) {
            const char *module_name = module->import_funcs[i].module_name;
            const char *field_name = module->import_funcs[i].func_name;
            LOG_WARNING("warning: failed to link import function (%s, %s)",
                        module_name, field_name);
        }
    }

    /* Set defined function pointers */
    bh_memcpy_s(func_ptrs, sizeof(void *) * module->func_count,
                module->func_ptrs, sizeof(void *) * module->func_count);
    return true;
}

static bool
init_func_type_indexes(AOTModuleInstance *module_inst, AOTModule *module,
                       char *error_buf, uint32 error_buf_size)
{
    uint32 i;
    uint32 *func_type_index;
    uint64 total_size = ((uint64)module->import_func_count + module->func_count)
                        * sizeof(uint32);

    if (module->import_func_count + module->func_count == 0)
        return true;

    /* Allocate memory */
    if (!(module_inst->func_type_indexes.ptr =
              runtime_malloc(total_size, error_buf, error_buf_size))) {
        return false;
    }

    /* Set import function type indexes */
    func_type_index = (uint32 *)module_inst->func_type_indexes.ptr;
    for (i = 0; i < module->import_func_count; i++, func_type_index++)
        *func_type_index = module->import_funcs[i].func_type_index;

    bh_memcpy_s(func_type_index, sizeof(uint32) * module->func_count,
                module->func_type_indexes, sizeof(uint32) * module->func_count);
    return true;
}

static bool
create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module,
                    char *error_buf, uint32 error_buf_size)
{
    AOTExport *exports = module->exports;
    AOTFunctionInstance *export_func;
    uint64 size;
    uint32 i, func_index, ftype_index;

    if (module_inst->export_func_count > 0) {
        /* Allocate memory */
        size = sizeof(AOTFunctionInstance)
               * (uint64)module_inst->export_func_count;
        if (!(module_inst->export_funcs.ptr = export_func =
                  runtime_malloc(size, error_buf, error_buf_size))) {
            return false;
        }

        for (i = 0; i < module->export_count; i++) {
            if (exports[i].kind == EXPORT_KIND_FUNC) {
                export_func->func_name = exports[i].name;
                export_func->func_index = exports[i].index;
                if (export_func->func_index < module->import_func_count) {
                    export_func->is_import_func = true;
                    export_func->u.func_import =
                        &module->import_funcs[export_func->func_index];
                }
                else {
                    export_func->is_import_func = false;
                    func_index =
                        export_func->func_index - module->import_func_count;
                    ftype_index = module->func_type_indexes[func_index];
                    export_func->u.func.func_type =
                        module->func_types[ftype_index];
                    export_func->u.func.func_ptr =
                        module->func_ptrs[func_index];
                }
                export_func++;
            }
        }
    }

    return true;
}

static bool
create_exports(AOTModuleInstance *module_inst, AOTModule *module,
               char *error_buf, uint32 error_buf_size)
{
    AOTExport *exports = module->exports;
    uint32 i;

    for (i = 0; i < module->export_count; i++) {
        switch (exports[i].kind) {
            case EXPORT_KIND_FUNC:
                module_inst->export_func_count++;
                break;
            case EXPORT_KIND_GLOBAL:
                module_inst->export_global_count++;
                break;
            case EXPORT_KIND_TABLE:
                module_inst->export_tab_count++;
                break;
            case EXPORT_KIND_MEMORY:
                module_inst->export_mem_count++;
                break;
            default:
                return false;
        }
    }

    return create_export_funcs(module_inst, module, error_buf, error_buf_size);
}

static bool
clear_wasi_proc_exit_exception(AOTModuleInstance *module_inst)
{
#if WASM_ENABLE_LIBC_WASI != 0
    const char *exception = aot_get_exception(module_inst);
    if (exception && !strcmp(exception, "Exception: wasi proc exit")) {
        /* The "wasi proc exit" exception is thrown by native lib to
           let wasm app exit, which is a normal behavior, we clear
           the exception here. */
        aot_set_exception(module_inst, NULL);
        return true;
    }
    return false;
#else
    return false;
#endif
}

static bool
execute_post_inst_function(AOTModuleInstance *module_inst)
{
    AOTFunctionInstance *post_inst_func =
        aot_lookup_function(module_inst, "__post_instantiate", "()");

    if (!post_inst_func)
        /* Not found */
        return true;

    return aot_create_exec_env_and_call_function(module_inst, post_inst_func, 0,
                                                 NULL);
}

static bool
execute_start_function(AOTModuleInstance *module_inst)
{
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;
    WASMExecEnv *exec_env;
    typedef void (*F)(WASMExecEnv *);
    union {
        F f;
        void *v;
    } u;

    if (!module->start_function)
        return true;

    if (!(exec_env =
              wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst,
                                   module_inst->default_wasm_stack_size))) {
        aot_set_exception(module_inst, "allocate memory failed");
        return false;
    }

    u.v = module->start_function;
    u.f(exec_env);

    wasm_exec_env_destroy(exec_env);
    (void)clear_wasi_proc_exit_exception(module_inst);
    return !aot_get_exception(module_inst);
}

#if WASM_ENABLE_BULK_MEMORY != 0
static bool
execute_memory_init_function(AOTModuleInstance *module_inst)
{
    AOTFunctionInstance *memory_init_func =
        aot_lookup_function(module_inst, "__wasm_call_ctors", "()");

    if (!memory_init_func)
        /* Not found */
        return true;

    return aot_create_exec_env_and_call_function(module_inst, memory_init_func,
                                                 0, NULL);
}
#endif

AOTModuleInstance *
aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size,
                uint32 heap_size, char *error_buf, uint32 error_buf_size)
{
    AOTModuleInstance *module_inst;
    const uint32 module_inst_struct_size =
        offsetof(AOTModuleInstance, global_table_data.bytes);
    const uint64 module_inst_mem_inst_size =
        (uint64)module->memory_count * sizeof(AOTMemoryInstance);
    uint64 total_size, table_size = 0;
    uint8 *p;
    uint32 i;

    /* Check heap size */
    heap_size = align_uint(heap_size, 8);
    if (heap_size > APP_HEAP_SIZE_MAX)
        heap_size = APP_HEAP_SIZE_MAX;

    total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size
                 + module->global_data_size;

    /*
     * calculate size of table data
     */
    for (i = 0; i != module->import_table_count; ++i) {
        table_size += offsetof(AOTTableInstance, data);
        table_size +=
            (uint64)sizeof(uint32)
            * (uint64)aot_get_imp_tbl_data_slots(module->import_tables + i);
    }

    for (i = 0; i != module->table_count; ++i) {
        table_size += offsetof(AOTTableInstance, data);
        table_size += (uint64)sizeof(uint32)
                      * (uint64)aot_get_tbl_data_slots(module->tables + i);
    }
    total_size += table_size;

    /* Allocate module instance, global data, table data and heap data */
    if (!(module_inst =
              runtime_malloc(total_size, error_buf, error_buf_size))) {
        return NULL;
    }

    module_inst->module_type = Wasm_Module_AoT;
    module_inst->aot_module.ptr = module;

    /* Initialize global info */
    p = (uint8 *)module_inst + module_inst_struct_size
        + module_inst_mem_inst_size;
    module_inst->global_data.ptr = p;
    module_inst->global_data_size = module->global_data_size;
    if (!global_instantiate(module_inst, module, error_buf, error_buf_size))
        goto fail;

    /* Initialize table info */
    p += module->global_data_size;
    module_inst->tables.ptr = p;
    module_inst->table_count = module->table_count + module->import_table_count;
    /* Set all elements to -1 to mark them as uninitialized elements */
    memset(module_inst->tables.ptr, 0xff, (uint32)table_size);
    if (!table_instantiate(module_inst, module, error_buf, error_buf_size))
        goto fail;

    /* Initialize memory space */
    if (!memories_instantiate(module_inst, module, heap_size, error_buf,
                              error_buf_size))
        goto fail;

    /* Initialize function pointers */
    if (!init_func_ptrs(module_inst, module, error_buf, error_buf_size))
        goto fail;

    /* Initialize function type indexes */
    if (!init_func_type_indexes(module_inst, module, error_buf, error_buf_size))
        goto fail;

    if (!create_exports(module_inst, module, error_buf, error_buf_size))
        goto fail;

#if WASM_ENABLE_LIBC_WASI != 0
    if (!is_sub_inst) {
        if (!wasm_runtime_init_wasi(
                (WASMModuleInstanceCommon *)module_inst,
                module->wasi_args.dir_list, module->wasi_args.dir_count,
                module->wasi_args.map_dir_list, module->wasi_args.map_dir_count,
                module->wasi_args.env, module->wasi_args.env_count,
                module->wasi_args.addr_pool, module->wasi_args.addr_count,
                module->wasi_args.argv, module->wasi_args.argc,
                module->wasi_args.stdio[0], module->wasi_args.stdio[1],
                module->wasi_args.stdio[2], error_buf, error_buf_size))
            goto fail;
    }
#endif

    /* Initialize the thread related data */
    if (stack_size == 0)
        stack_size = DEFAULT_WASM_STACK_SIZE;
#if WASM_ENABLE_SPEC_TEST != 0
    if (stack_size < 48 * 1024)
        stack_size = 48 * 1024;
#endif
    module_inst->default_wasm_stack_size = stack_size;

#if WASM_ENABLE_PERF_PROFILING != 0
    total_size = (uint64)sizeof(AOTFuncPerfProfInfo)
                 * (module->import_func_count + module->func_count);
    if (!(module_inst->func_perf_profilings.ptr =
              runtime_malloc(total_size, error_buf, error_buf_size))) {
        goto fail;
    }
#endif

#if WASM_ENABLE_DUMP_CALL_STACK != 0
    if (!(module_inst->frames.ptr =
              runtime_malloc(sizeof(Vector), error_buf, error_buf_size))) {
        goto fail;
    }
#endif

    /* Execute __post_instantiate function and start function*/
    if (!execute_post_inst_function(module_inst)
        || !execute_start_function(module_inst)) {
        set_error_buf(error_buf, error_buf_size, module_inst->cur_exception);
        goto fail;
    }

#if WASM_ENABLE_BULK_MEMORY != 0
#if WASM_ENABLE_LIBC_WASI != 0
    if (!module->import_wasi_api) {
#endif
        /* Only execute the memory init function for main instance because
            the data segments will be dropped once initialized.
        */
        if (!is_sub_inst) {
            if (!execute_memory_init_function(module_inst)) {
                set_error_buf(error_buf, error_buf_size,
                              module_inst->cur_exception);
                goto fail;
            }
        }
#if WASM_ENABLE_LIBC_WASI != 0
    }
#endif
#endif

#if WASM_ENABLE_MEMORY_TRACING != 0
    wasm_runtime_dump_module_inst_mem_consumption(
        (WASMModuleInstanceCommon *)module_inst);
#endif

    return module_inst;

fail:
    aot_deinstantiate(module_inst, is_sub_inst);
    return NULL;
}

bool
aot_create_exec_env_singleton(AOTModuleInstance *module_inst)
{
    WASMExecEnv *exec_env =
        wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst,
                             module_inst->default_wasm_stack_size);
    if (exec_env)
        module_inst->exec_env_singleton.ptr = exec_env;

    return exec_env ? true : false;
}

void
aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst)
{
#if WASM_ENABLE_LIBC_WASI != 0
    /* Destroy wasi resource before freeing app heap, since some fields of
       wasi contex are allocated from app heap, and if app heap is freed,
       these fields will be set to NULL, we cannot free their internal data
       which may allocated from global heap. */
    /* Only destroy wasi ctx in the main module instance */
    if (!is_sub_inst)
        wasm_runtime_destroy_wasi((WASMModuleInstanceCommon *)module_inst);
#endif

#if WASM_ENABLE_PERF_PROFILING != 0
    if (module_inst->func_perf_profilings.ptr)
        wasm_runtime_free(module_inst->func_perf_profilings.ptr);
#endif

#if WASM_ENABLE_DUMP_CALL_STACK != 0
    if (module_inst->frames.ptr) {
        bh_vector_destroy(module_inst->frames.ptr);
        wasm_runtime_free(module_inst->frames.ptr);
        module_inst->frames.ptr = NULL;
    }
#endif

    if (module_inst->memories.ptr)
        memories_deinstantiate(module_inst);

    if (module_inst->export_funcs.ptr)
        wasm_runtime_free(module_inst->export_funcs.ptr);

    if (module_inst->func_ptrs.ptr)
        wasm_runtime_free(module_inst->func_ptrs.ptr);

    if (module_inst->func_type_indexes.ptr)
        wasm_runtime_free(module_inst->func_type_indexes.ptr);

    if (module_inst->exec_env_singleton.ptr)
        wasm_exec_env_destroy(
            (WASMExecEnv *)module_inst->exec_env_singleton.ptr);

    wasm_runtime_free(module_inst);
}

AOTFunctionInstance *
aot_lookup_function(const AOTModuleInstance *module_inst, const char *name,
                    const char *signature)
{
    uint32 i;
    AOTFunctionInstance *export_funcs =
        (AOTFunctionInstance *)module_inst->export_funcs.ptr;

    for (i = 0; i < module_inst->export_func_count; i++)
        if (!strcmp(export_funcs[i].func_name, name))
            return &export_funcs[i];
    (void)signature;
    return NULL;
}

#ifdef OS_ENABLE_HW_BOUND_CHECK

#ifndef BH_PLATFORM_WINDOWS
void
aot_signal_handler(WASMSignalInfo *sig_info)
{
    WASMExecEnv *exec_env_tls = sig_info->exec_env_tls;
    void *sig_addr = sig_info->sig_addr;
    AOTModuleInstance *module_inst;
    AOTMemoryInstance *memory_inst;
    WASMJmpBuf *jmpbuf_node;
    uint8 *mapped_mem_start_addr = NULL;
    uint8 *mapped_mem_end_addr = NULL;
    uint8 *stack_min_addr;
    uint32 page_size;
    uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;

    /* Check whether current thread is running aot function */
    if (exec_env_tls && exec_env_tls->handle == os_self_thread()
        && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) {
        /* Get mapped mem info of current instance */
        module_inst = (AOTModuleInstance *)exec_env_tls->module_inst;
        /* Get the default memory instance */
        memory_inst = aot_get_default_memory(module_inst);
        if (memory_inst) {
            mapped_mem_start_addr = (uint8 *)memory_inst->memory_data.ptr;
            mapped_mem_end_addr =
                (uint8 *)memory_inst->memory_data.ptr + 8 * (uint64)BH_GB;
        }

        /* Get stack info of current thread */
        page_size = os_getpagesize();
        stack_min_addr = os_thread_get_stack_boundary();

        if (memory_inst
            && (mapped_mem_start_addr <= (uint8 *)sig_addr
                && (uint8 *)sig_addr < mapped_mem_end_addr)) {
            /* The address which causes segmentation fault is inside
               the memory instance's guard regions */
            aot_set_exception_with_id(module_inst,
                                      EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS);
            os_longjmp(jmpbuf_node->jmpbuf, 1);
        }
        else if (stack_min_addr - page_size <= (uint8 *)sig_addr
                 && (uint8 *)sig_addr
                        < stack_min_addr + page_size * guard_page_count) {
            /* The address which causes segmentation fault is inside
               native thread's guard page */
            aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW);
            os_longjmp(jmpbuf_node->jmpbuf, 1);
        }
    }
}
#else  /* else of BH_PLATFORM_WINDOWS */
LONG
aot_exception_handler(WASMSignalInfo *sig_info)
{
    WASMExecEnv *exec_env_tls = sig_info->exec_env_tls;
    EXCEPTION_POINTERS *exce_info = sig_info->exce_info;
    PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord;
    uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1];
    AOTModuleInstance *module_inst;
    AOTMemoryInstance *memory_inst;
    WASMJmpBuf *jmpbuf_node;
    uint8 *mapped_mem_start_addr = NULL;
    uint8 *mapped_mem_end_addr = NULL;
    uint32 page_size = os_getpagesize();

    if (exec_env_tls && exec_env_tls->handle == os_self_thread()
        && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) {
        module_inst = (AOTModuleInstance *)exec_env_tls->module_inst;
        if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
            /* Get the default memory instance */
            memory_inst = aot_get_default_memory(module_inst);
            if (memory_inst) {
                mapped_mem_start_addr = (uint8 *)memory_inst->memory_data.ptr;
                mapped_mem_end_addr =
                    (uint8 *)memory_inst->memory_data.ptr + 8 * (uint64)BH_GB;
                if (mapped_mem_start_addr <= (uint8 *)sig_addr
                    && (uint8 *)sig_addr < mapped_mem_end_addr) {
                    /* The address which causes segmentation fault is inside
                       aot instance's guard regions.
                       Set exception and let the aot func continue to run, when
                       the aot func returns, the caller will check whether the
                       exception is thrown and return to runtime. */
                    aot_set_exception_with_id(module_inst,
                                              EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS);
                    /* Skip current instruction */
                    exce_info->ContextRecord->Rip++;
                    return EXCEPTION_CONTINUE_EXECUTION;
                }
            }
        }
        else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) {
            /* Set stack overflow exception and let the aot func continue
               to run, when the aot func returns, the caller will check
               whether the exception is thrown and return to runtime, and
               the damaged stack will be recovered by _resetstkoflw(). */
            aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW);
            return EXCEPTION_CONTINUE_EXECUTION;
        }
    }

    os_printf("Unhandled exception thrown:  exception code: 0x%lx, "
              "exception address: %p, exception information: %p\n",
              ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress,
              sig_addr);
    return EXCEPTION_CONTINUE_SEARCH;
}
#endif /* end of BH_PLATFORM_WINDOWS */

static bool
invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr,
                                  const WASMType *func_type,
                                  const char *signature, void *attachment,
                                  uint32 *argv, uint32 argc, uint32 *argv_ret)
{
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
    WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop;
    uint32 page_size = os_getpagesize();
    uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
    uint16 param_count = func_type->param_count;
    uint16 result_count = func_type->result_count;
    const uint8 *types = func_type->types;
#ifdef BH_PLATFORM_WINDOWS
    const char *exce;
    int result;
#endif
    bool ret;

    /* Check native stack overflow firstly to ensure we have enough
       native stack to run the following codes before actually calling
       the aot function in invokeNative function. */
    if ((uint8 *)&module_inst < exec_env->native_stack_boundary
                                    + page_size * (guard_page_count + 1)) {
        aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW);
        return false;
    }

    if (exec_env_tls && (exec_env_tls != exec_env)) {
        aot_set_exception(module_inst, "invalid exec env");
        return false;
    }

    if (!os_thread_signal_inited()) {
        aot_set_exception(module_inst, "thread signal env not inited");
        return false;
    }

    wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node);

    wasm_runtime_set_exec_env_tls(exec_env);
    if (os_setjmp(jmpbuf_node.jmpbuf) == 0) {
        /* Quick call with func_ptr if the function signature is simple */
        if (!signature && param_count == 1 && types[0] == VALUE_TYPE_I32) {
            if (result_count == 0) {
                void (*NativeFunc)(WASMExecEnv *, uint32) =
                    (void (*)(WASMExecEnv *, uint32))func_ptr;
                NativeFunc(exec_env, argv[0]);
                ret = aot_get_exception(module_inst) ? false : true;
            }
            else if (result_count == 1
                     && types[param_count] == VALUE_TYPE_I32) {
                uint32 (*NativeFunc)(WASMExecEnv *, uint32) =
                    (uint32(*)(WASMExecEnv *, uint32))func_ptr;
                argv_ret[0] = NativeFunc(exec_env, argv[0]);
                ret = aot_get_exception(module_inst) ? false : true;
            }
            else {
                ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type,
                                                 signature, attachment, argv,
                                                 argc, argv_ret);
            }
        }
        else {
            ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type,
                                             signature, attachment, argv, argc,
                                             argv_ret);
        }
#ifdef BH_PLATFORM_WINDOWS
        if ((exce = aot_get_exception(module_inst))
            && strstr(exce, "native stack overflow")) {
            /* After a stack overflow, the stack was left
               in a damaged state, let the CRT repair it */
            result = _resetstkoflw();
            bh_assert(result != 0);
        }
#endif
    }
    else {
        /* Exception has been set in signal handler before calling longjmp */
        ret = false;
    }

    jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env);
    bh_assert(&jmpbuf_node == jmpbuf_node_pop);
    if (!exec_env->jmpbuf_stack_top) {
        wasm_runtime_set_exec_env_tls(NULL);
    }
    if (!ret) {
        os_sigreturn();
        os_signal_unmask();
    }
    (void)jmpbuf_node_pop;
    return ret;
}

#define invoke_native_internal invoke_native_with_hw_bound_check
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
#define invoke_native_internal wasm_runtime_invoke_native
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */

bool
aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function,
                  unsigned argc, uint32 argv[])
{
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    AOTFuncType *func_type = function->u.func.func_type;
    uint32 result_count = func_type->result_count;
    uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
    bool ret;

    if (argc < func_type->param_cell_num) {
        char buf[108];
        snprintf(buf, sizeof(buf),
                 "invalid argument count %u, must be no smaller than %u", argc,
                 func_type->param_cell_num);
        aot_set_exception(module_inst, buf);
        return false;
    }
    argc = func_type->param_cell_num;

#if WASM_ENABLE_LAZY_JIT != 0
    if (!function->u.func.func_ptr) {
        AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr;
        if (!(function->u.func.func_ptr =
                  aot_lookup_orcjit_func(aot_module->comp_ctx->orc_lazyjit,
                                         module_inst, function->func_index))) {
            return false;
        }
    }
#endif

    /* set thread handle and stack boundary */
    wasm_exec_env_set_thread_info(exec_env);

    if (ext_ret_count > 0) {
        uint32 cell_num = 0, i;
        uint8 *ext_ret_types = func_type->types + func_type->param_count + 1;
        uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL;
        uint32 *argv_ret = argv;
        uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count);
        uint64 size;

        /* Allocate memory all arguments */
        size =
            sizeof(uint32) * (uint64)argc /* original arguments */
            + sizeof(void *)
                  * (uint64)ext_ret_count /* extra result values' addr */
            + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */
        if (size > sizeof(argv1_buf)
            && !(argv1 = runtime_malloc(size, module_inst->cur_exception,
                                        sizeof(module_inst->cur_exception)))) {
            aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY);
            return false;
        }

        /* Copy original arguments */
        bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc);

        /* Get the extra result value's address */
        ext_rets =
            argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;

        /* Append each extra result value's address to original arguments */
        for (i = 0; i < ext_ret_count; i++) {
            *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) =
                (uintptr_t)(ext_rets + cell_num);
            cell_num += wasm_value_type_cell_num(ext_ret_types[i]);
        }

#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
        if (!aot_alloc_frame(exec_env, function->func_index)) {
            wasm_runtime_free(argv1);
            return false;
        }
#endif

        ret = invoke_native_internal(exec_env, function->u.func.func_ptr,
                                     func_type, NULL, NULL, argv1, argc, argv);

        if (!ret || aot_get_exception(module_inst)) {
            if (argv1 != argv1_buf)
                wasm_runtime_free(argv1);

            if (clear_wasi_proc_exit_exception(module_inst))
                ret = true;
            else
                ret = false;
        }

#if WASM_ENABLE_DUMP_CALL_STACK != 0
        if (!ret) {
            if (aot_create_call_stack(exec_env)) {
                aot_dump_call_stack(exec_env, true, NULL, 0);
            }
        }
#endif

#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
        aot_free_frame(exec_env);
#endif
        if (!ret)
            return ret;

        /* Get extra result values */
        switch (func_type->types[func_type->param_count]) {
            case VALUE_TYPE_I32:
            case VALUE_TYPE_F32:
#if WASM_ENABLE_REF_TYPES != 0
            case VALUE_TYPE_FUNCREF:
            case VALUE_TYPE_EXTERNREF:
#endif
                argv_ret++;
                break;
            case VALUE_TYPE_I64:
            case VALUE_TYPE_F64:
                argv_ret += 2;
                break;
#if WASM_ENABLE_SIMD != 0
            case VALUE_TYPE_V128:
                argv_ret += 4;
                break;
#endif
            default:
                bh_assert(0);
                break;
        }
        ext_rets =
            argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
        bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets,
                    sizeof(uint32) * cell_num);
        if (argv1 != argv1_buf)
            wasm_runtime_free(argv1);

        return true;
    }
    else {
#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
        if (!aot_alloc_frame(exec_env, function->func_index)) {
            return false;
        }
#endif

        ret = invoke_native_internal(exec_env, function->u.func.func_ptr,
                                     func_type, NULL, NULL, argv, argc, argv);

        if (clear_wasi_proc_exit_exception(module_inst))
            ret = true;

#if WASM_ENABLE_DUMP_CALL_STACK != 0
        if (aot_get_exception(module_inst)) {
            if (aot_create_call_stack(exec_env)) {
                aot_dump_call_stack(exec_env, true, NULL, 0);
            }
        }
#endif

#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
        aot_free_frame(exec_env);
#endif

        return ret && !aot_get_exception(module_inst) ? true : false;
    }
}

bool
aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst,
                                      AOTFunctionInstance *func, unsigned argc,
                                      uint32 argv[])
{
    WASMExecEnv *exec_env = NULL, *existing_exec_env = NULL;
    bool ret;

#if defined(OS_ENABLE_HW_BOUND_CHECK)
    existing_exec_env = exec_env = wasm_runtime_get_exec_env_tls();
#elif WASM_ENABLE_THREAD_MGR != 0
    existing_exec_env = exec_env =
        wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst);
#endif

    if (!existing_exec_env) {
        if (!(exec_env =
                  wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst,
                                       module_inst->default_wasm_stack_size))) {
            aot_set_exception(module_inst, "allocate memory failed");
            return false;
        }
    }

    ret = aot_call_function(exec_env, func, argc, argv);

    /* don't destroy the exec_env if it isn't created in this function */
    if (!existing_exec_env)
        wasm_exec_env_destroy(exec_env);

    return ret;
}

void
aot_set_exception(AOTModuleInstance *module_inst, const char *exception)
{
    if (exception)
        snprintf(module_inst->cur_exception, sizeof(module_inst->cur_exception),
                 "Exception: %s", exception);
    else
        module_inst->cur_exception[0] = '\0';
}

void
aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id)
{
    switch (id) {
        case EXCE_UNREACHABLE:
            aot_set_exception(module_inst, "unreachable");
            break;
        case EXCE_OUT_OF_MEMORY:
            aot_set_exception(module_inst, "allocate memory failed");
            break;
        case EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS:
            aot_set_exception(module_inst, "out of bounds memory access");
            break;
        case EXCE_INTEGER_OVERFLOW:
            aot_set_exception(module_inst, "integer overflow");
            break;
        case EXCE_INTEGER_DIVIDE_BY_ZERO:
            aot_set_exception(module_inst, "integer divide by zero");
            break;
        case EXCE_INVALID_CONVERSION_TO_INTEGER:
            aot_set_exception(module_inst, "invalid conversion to integer");
            break;
        case EXCE_INVALID_FUNCTION_TYPE_INDEX:
            aot_set_exception(module_inst, "indirect call type mismatch");
            break;
        case EXCE_INVALID_FUNCTION_INDEX:
            aot_set_exception(module_inst, "invalid function index");
            break;
        case EXCE_UNDEFINED_ELEMENT:
            aot_set_exception(module_inst, "undefined element");
            break;
        case EXCE_UNINITIALIZED_ELEMENT:
            aot_set_exception(module_inst, "uninitialized element");
            break;
        case EXCE_CALL_UNLINKED_IMPORT_FUNC:
            aot_set_exception(module_inst,
                              "failed to call unlinked import function");
            break;
        case EXCE_NATIVE_STACK_OVERFLOW:
            aot_set_exception(module_inst, "native stack overflow");
            break;
        case EXCE_UNALIGNED_ATOMIC:
            aot_set_exception(module_inst, "unaligned atomic");
            break;
        case EXCE_AUX_STACK_OVERFLOW:
            aot_set_exception(module_inst, "wasm auxiliary stack overflow");
            break;
        case EXCE_AUX_STACK_UNDERFLOW:
            aot_set_exception(module_inst, "wasm auxiliary stack underflow");
            break;
        case EXCE_OUT_OF_BOUNDS_TABLE_ACCESS:
            aot_set_exception(module_inst, "out of bounds table access");
            break;
        default:
            break;
    }
}

const char *
aot_get_exception(AOTModuleInstance *module_inst)
{
    if (module_inst->cur_exception[0] == '\0')
        return NULL;
    else
        return module_inst->cur_exception;
}

void
aot_clear_exception(AOTModuleInstance *module_inst)
{
    module_inst->cur_exception[0] = '\0';
}

static bool
execute_malloc_function(AOTModuleInstance *module_inst,
                        AOTFunctionInstance *malloc_func,
                        AOTFunctionInstance *retain_func, uint32 size,
                        uint32 *p_result)
{
#ifdef OS_ENABLE_HW_BOUND_CHECK
    WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
#endif
    uint32 argv[2], argc;
    bool ret;

    argv[0] = size;
    argc = 1;
    if (retain_func) {
        argv[1] = 0;
        argc = 2;
    }

#ifdef OS_ENABLE_HW_BOUND_CHECK
    if (exec_env_tls != NULL) {
        bh_assert(exec_env_tls->module_inst
                  == (WASMModuleInstanceCommon *)module_inst);
        ret = aot_call_function(exec_env_tls, malloc_func, argc, argv);

        if (retain_func && ret) {
            ret = aot_call_function(exec_env_tls, retain_func, 1, argv);
        }
    }
    else
#endif
    {
        ret = aot_create_exec_env_and_call_function(module_inst, malloc_func,
                                                    argc, argv);

        if (retain_func && ret) {
            ret = aot_create_exec_env_and_call_function(module_inst,
                                                        retain_func, 1, argv);
        }
    }

    if (ret)
        *p_result = argv[0];
    return ret;
}

static bool
execute_free_function(AOTModuleInstance *module_inst,
                      AOTFunctionInstance *free_func, uint32 offset)
{
#ifdef OS_ENABLE_HW_BOUND_CHECK
    WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
#endif
    uint32 argv[2];

    argv[0] = offset;
#ifdef OS_ENABLE_HW_BOUND_CHECK
    if (exec_env_tls != NULL) {
        bh_assert(exec_env_tls->module_inst
                  == (WASMModuleInstanceCommon *)module_inst);
        return aot_call_function(exec_env_tls, free_func, 1, argv);
    }
    else
#endif
    {
        return aot_create_exec_env_and_call_function(module_inst, free_func, 1,
                                                     argv);
    }
}

uint32
aot_module_malloc(AOTModuleInstance *module_inst, uint32 size,
                  void **p_native_addr)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;
    uint8 *addr = NULL;
    uint32 offset = 0;

    if (!memory_inst) {
        aot_set_exception(module_inst, "uninitialized memory");
        return 0;
    }

    if (memory_inst->heap_handle.ptr) {
        addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size);
    }
    else if (module->malloc_func_index != (uint32)-1
             && module->free_func_index != (uint32)-1) {
        AOTFunctionInstance *malloc_func, *retain_func = NULL;
        char *malloc_func_name;
        char *malloc_func_sig;

        if (module->retain_func_index != (uint32)-1) {
            malloc_func_name = "__new";
            malloc_func_sig = "(ii)i";
            retain_func = aot_lookup_function(module_inst, "__retain", "(i)i");
            if (!retain_func)
                retain_func = aot_lookup_function(module_inst, "__pin", "(i)i");
            bh_assert(retain_func);
        }
        else {
            malloc_func_name = "malloc";
            malloc_func_sig = "(i)i";
        }
        malloc_func =
            aot_lookup_function(module_inst, malloc_func_name, malloc_func_sig);

        if (!malloc_func
            || !execute_malloc_function(module_inst, malloc_func, retain_func,
                                        size, &offset)) {
            return 0;
        }
        addr = offset ? (uint8 *)memory_inst->memory_data.ptr + offset : NULL;
    }

    if (!addr) {
        if (memory_inst->heap_handle.ptr
            && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) {
            wasm_runtime_show_app_heap_corrupted_prompt();
            aot_set_exception(module_inst, "app heap corrupted");
        }
        else {
            LOG_WARNING("warning: allocate %u bytes memory failed", size);
        }
        return 0;
    }
    if (p_native_addr)
        *p_native_addr = addr;
    return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr);
}

uint32
aot_module_realloc(AOTModuleInstance *module_inst, uint32 ptr, uint32 size,
                   void **p_native_addr)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint8 *addr = NULL;

    if (!memory_inst) {
        aot_set_exception(module_inst, "uninitialized memory");
        return 0;
    }

    if (memory_inst->heap_handle.ptr) {
        addr = mem_allocator_realloc(
            memory_inst->heap_handle.ptr,
            ptr ? (uint8 *)memory_inst->memory_data.ptr + ptr : NULL, size);
    }

    /* Only support realloc in WAMR's app heap */

    if (!addr) {
        if (memory_inst->heap_handle.ptr
            && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) {
            aot_set_exception(module_inst, "app heap corrupted");
        }
        else {
            aot_set_exception(module_inst, "out of memory");
        }
        return 0;
    }

    if (p_native_addr)
        *p_native_addr = addr;
    return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr);
}

void
aot_module_free(AOTModuleInstance *module_inst, uint32 ptr)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;

    if (!memory_inst) {
        return;
    }

    if (ptr) {
        uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + ptr;
        if (memory_inst->heap_handle.ptr
            && (uint8 *)memory_inst->heap_data.ptr < addr
            && addr < (uint8 *)memory_inst->heap_data_end.ptr) {
            mem_allocator_free(memory_inst->heap_handle.ptr, addr);
        }
        else if (module->malloc_func_index != (uint32)-1
                 && module->free_func_index != (uint32)-1
                 && (uint8 *)memory_inst->memory_data.ptr <= addr
                 && addr < (uint8 *)memory_inst->memory_data_end.ptr) {
            AOTFunctionInstance *free_func;
            char *free_func_name;

            if (module->retain_func_index != (uint32)-1) {
                free_func_name = "__release";
            }
            else {
                free_func_name = "free";
            }
            free_func =
                aot_lookup_function(module_inst, free_func_name, "(i)i");
            if (!free_func && module->retain_func_index != (uint32)-1)
                free_func = aot_lookup_function(module_inst, "__unpin", "(i)i");

            if (free_func)
                execute_free_function(module_inst, free_func, ptr);
        }
    }
}

uint32
aot_module_dup_data(AOTModuleInstance *module_inst, const char *src,
                    uint32 size)
{
    char *buffer;
    uint32 buffer_offset =
        aot_module_malloc(module_inst, size, (void **)&buffer);

    if (buffer_offset != 0) {
        buffer = aot_addr_app_to_native(module_inst, buffer_offset);
        bh_memcpy_s(buffer, size, src, size);
    }
    return buffer_offset;
}

bool
aot_validate_app_addr(AOTModuleInstance *module_inst, uint32 app_offset,
                      uint32 size)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);

    if (!memory_inst) {
        goto fail;
    }

    /* integer overflow check */
    if (app_offset > UINT32_MAX - size) {
        goto fail;
    }

    if (app_offset + size <= memory_inst->memory_data_size) {
        return true;
    }
fail:
    aot_set_exception(module_inst, "out of bounds memory access");
    return false;
}

bool
aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr,
                         uint32 size)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint8 *addr = (uint8 *)native_ptr;

    if (!memory_inst) {
        goto fail;
    }

    /* integer overflow check */
    if ((uintptr_t)addr > UINTPTR_MAX - size) {
        goto fail;
    }

    if ((uint8 *)memory_inst->memory_data.ptr <= addr
        && addr + size <= (uint8 *)memory_inst->memory_data_end.ptr)
        return true;
fail:
    aot_set_exception(module_inst, "out of bounds memory access");
    return false;
}

void *
aot_addr_app_to_native(AOTModuleInstance *module_inst, uint32 app_offset)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint8 *addr;

    if (!memory_inst) {
        return NULL;
    }

    addr = (uint8 *)memory_inst->memory_data.ptr + app_offset;

    if ((uint8 *)memory_inst->memory_data.ptr <= addr
        && addr < (uint8 *)memory_inst->memory_data_end.ptr)
        return addr;
    return NULL;
}

uint32
aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint8 *addr = (uint8 *)native_ptr;

    if (!memory_inst) {
        return 0;
    }

    if ((uint8 *)memory_inst->memory_data.ptr <= addr
        && addr < (uint8 *)memory_inst->memory_data_end.ptr)
        return (uint32)(addr - (uint8 *)memory_inst->memory_data.ptr);
    return 0;
}

bool
aot_get_app_addr_range(AOTModuleInstance *module_inst, uint32 app_offset,
                       uint32 *p_app_start_offset, uint32 *p_app_end_offset)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint32 memory_data_size;

    if (!memory_inst) {
        return false;
    }

    memory_data_size = memory_inst->memory_data_size;

    if (app_offset < memory_data_size) {
        if (p_app_start_offset)
            *p_app_start_offset = 0;
        if (p_app_end_offset)
            *p_app_end_offset = memory_data_size;
        return true;
    }
    return false;
}

bool
aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr,
                          uint8 **p_native_start_addr,
                          uint8 **p_native_end_addr)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint8 *addr = (uint8 *)native_ptr;

    if (!memory_inst) {
        return false;
    }

    if ((uint8 *)memory_inst->memory_data.ptr <= addr
        && addr < (uint8 *)memory_inst->memory_data_end.ptr) {
        if (p_native_start_addr)
            *p_native_start_addr = (uint8 *)memory_inst->memory_data.ptr;
        if (p_native_end_addr)
            *p_native_end_addr = (uint8 *)memory_inst->memory_data_end.ptr;
        return true;
    }
    return false;
}

#ifndef OS_ENABLE_HW_BOUND_CHECK
bool
aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint32 num_bytes_per_page, cur_page_count, max_page_count;
    uint32 total_page_count, total_size_old, heap_size;
    uint64 total_size;
    uint8 *memory_data_old, *heap_data_old, *memory_data, *heap_data;
    bool ret = true;

    if (!memory_inst)
        return false;

    num_bytes_per_page = memory_inst->num_bytes_per_page;
    cur_page_count = memory_inst->cur_page_count;
    max_page_count = memory_inst->max_page_count;
    total_page_count = cur_page_count + inc_page_count;
    total_size_old = memory_inst->memory_data_size;
    total_size = (uint64)num_bytes_per_page * total_page_count;
    heap_size = (uint32)((uint8 *)memory_inst->heap_data_end.ptr
                         - (uint8 *)memory_inst->heap_data.ptr);
    memory_data_old = (uint8 *)memory_inst->memory_data.ptr;
    heap_data_old = (uint8 *)memory_inst->heap_data.ptr;

    if (inc_page_count <= 0)
        /* No need to enlarge memory */
        return true;

    if (total_page_count < cur_page_count /* integer overflow */
        || total_page_count > max_page_count) {
        return false;
    }

    if (total_size >= UINT32_MAX) {
        return false;
    }

#if WASM_ENABLE_SHARED_MEMORY != 0
    if (memory_inst->is_shared) {
        /* For shared memory, we have reserved the maximum spaces during
            instantiate, only change the cur_page_count here */
        memory_inst->cur_page_count = total_page_count;
        return true;
    }
#endif

    if (heap_size > 0) {
        if (mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) {
            wasm_runtime_show_app_heap_corrupted_prompt();
            return false;
        }
    }

    if (!(memory_data =
              wasm_runtime_realloc(memory_data_old, (uint32)total_size))) {
        if (!(memory_data = wasm_runtime_malloc((uint32)total_size))) {
            return false;
        }
        if (memory_data_old) {
            bh_memcpy_s(memory_data, (uint32)total_size, memory_data_old,
                        total_size_old);
            wasm_runtime_free(memory_data_old);
        }
    }

    memset(memory_data + total_size_old, 0,
           (uint32)total_size - total_size_old);

    memory_inst->cur_page_count = total_page_count;
    memory_inst->memory_data_size = (uint32)total_size;
    memory_inst->memory_data.ptr = memory_data;
    memory_inst->memory_data_end.ptr = memory_data + total_size;

    if (heap_size > 0) {
        if (mem_allocator_migrate(memory_inst->heap_handle.ptr,
                                  (char *)heap_data_old
                                      + (memory_data - memory_data_old),
                                  heap_size)) {
            /* Don't return here as memory->memory_data is obsolete and
               must be updated to be correctly used later. */
            ret = false;
        }
    }

    heap_data = memory_data + (heap_data_old - memory_data_old);
    memory_inst->heap_data.ptr = heap_data;
    memory_inst->heap_data_end.ptr = heap_data + heap_size;

    if (sizeof(uintptr_t) == sizeof(uint64)) {
        memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
        memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
        memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
        memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
        memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
    }
    else {
        memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
        memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
        memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
        memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
        memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16;
    }
    return ret;
}
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
bool
aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint32 num_bytes_per_page, cur_page_count, max_page_count;
    uint32 total_page_count;
    uint64 total_size;

    if (!memory_inst)
        return false;

    num_bytes_per_page = memory_inst->num_bytes_per_page;
    cur_page_count = memory_inst->cur_page_count;
    max_page_count = memory_inst->max_page_count;
    total_page_count = cur_page_count + inc_page_count;
    total_size = (uint64)num_bytes_per_page * total_page_count;

    if (inc_page_count <= 0)
        /* No need to enlarge memory */
        return true;

    if (total_page_count < cur_page_count /* integer overflow */
        || total_page_count > max_page_count) {
        return false;
    }

#ifdef BH_PLATFORM_WINDOWS
    if (!os_mem_commit(memory_inst->memory_data_end.ptr,
                       num_bytes_per_page * inc_page_count,
                       MMAP_PROT_READ | MMAP_PROT_WRITE)) {
        return false;
    }
#endif

    if (os_mprotect(memory_inst->memory_data_end.ptr,
                    num_bytes_per_page * inc_page_count,
                    MMAP_PROT_READ | MMAP_PROT_WRITE)
        != 0) {
#ifdef BH_PLATFORM_WINDOWS
        os_mem_decommit(memory_inst->memory_data_end.ptr,
                        num_bytes_per_page * inc_page_count);
#endif
        return false;
    }

    /* The increased pages are filled with zero by the OS when os_mmap,
       no need to memset it again here */

    memory_inst->cur_page_count = total_page_count;
    memory_inst->memory_data_size = (uint32)total_size;
    memory_inst->memory_data_end.ptr =
        (uint8 *)memory_inst->memory_data.ptr + (uint32)total_size;

    if (sizeof(uintptr_t) == sizeof(uint64)) {
        memory_inst->mem_bound_check_1byte.u64 = total_size - 1;
        memory_inst->mem_bound_check_2bytes.u64 = total_size - 2;
        memory_inst->mem_bound_check_4bytes.u64 = total_size - 4;
        memory_inst->mem_bound_check_8bytes.u64 = total_size - 8;
        memory_inst->mem_bound_check_16bytes.u64 = total_size - 16;
    }
    else {
        memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1;
        memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2;
        memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4;
        memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8;
        memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16;
    }
    return true;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */

bool
aot_is_wasm_type_equal(AOTModuleInstance *module_inst, uint32 type1_idx,
                       uint32 type2_idx)
{
    WASMType *type1, *type2;
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;

    if (type1_idx >= module->func_type_count
        || type2_idx >= module->func_type_count) {
        aot_set_exception(module_inst, "type index out of bounds");
        return false;
    }

    if (type1_idx == type2_idx)
        return true;

    type1 = module->func_types[type1_idx];
    type2 = module->func_types[type2_idx];

    return wasm_type_equal(type1, type2);
}

bool
aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
                  uint32 *argv)
{
    AOTModuleInstance *module_inst =
        (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env);
    AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr;
    uint32 *func_type_indexes = (uint32 *)module_inst->func_type_indexes.ptr;
    uint32 func_type_idx = func_type_indexes[func_idx];
    AOTFuncType *func_type = aot_module->func_types[func_type_idx];
    void **func_ptrs = (void **)module_inst->func_ptrs.ptr;
    void *func_ptr = func_ptrs[func_idx];
    AOTImportFunc *import_func;
    const char *signature;
    void *attachment;
    char buf[96];

    bh_assert(func_idx < aot_module->import_func_count);

    import_func = aot_module->import_funcs + func_idx;
    if (!func_ptr) {
        snprintf(buf, sizeof(buf),
                 "failed to call unlinked import function (%s, %s)",
                 import_func->module_name, import_func->func_name);
        aot_set_exception(module_inst, buf);
        return false;
    }

    attachment = import_func->attachment;
    if (import_func->call_conv_wasm_c_api) {
        return wasm_runtime_invoke_c_api_native(
            (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc,
            argv, import_func->wasm_c_api_with_env, attachment);
    }
    else if (!import_func->call_conv_raw) {
        signature = import_func->signature;
        return wasm_runtime_invoke_native(exec_env, func_ptr, func_type,
                                          signature, attachment, argv, argc,
                                          argv);
    }
    else {
        signature = import_func->signature;
        return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
                                              signature, attachment, argv, argc,
                                              argv);
    }
}

bool
aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
                  uint32 argc, uint32 *argv)
{
    AOTModuleInstance *module_inst =
        (AOTModuleInstance *)wasm_runtime_get_module_inst(exec_env);
    AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr;
    uint32 *func_type_indexes = (uint32 *)module_inst->func_type_indexes.ptr;
    AOTTableInstance *tbl_inst;
    AOTFuncType *func_type;
    void **func_ptrs = (void **)module_inst->func_ptrs.ptr, *func_ptr;
    uint32 func_type_idx, func_idx, ext_ret_count;
    AOTImportFunc *import_func;
    const char *signature = NULL;
    void *attachment = NULL;
    char buf[96];
    bool ret;

    /* this function is called from native code, so exec_env->handle and
       exec_env->native_stack_boundary must have been set, we don't set
       it again */

    if ((uint8 *)&module_inst < exec_env->native_stack_boundary) {
        aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW);
        return false;
    }

    tbl_inst = aot_get_table_inst(module_inst, tbl_idx);
    bh_assert(tbl_inst);

    if (table_elem_idx >= tbl_inst->cur_size) {
        aot_set_exception_with_id(module_inst, EXCE_UNDEFINED_ELEMENT);
        return false;
    }

    func_idx = ((uint32 *)tbl_inst->data)[table_elem_idx];
    if (func_idx == (uint32)-1) {
        aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT);
        return false;
    }

    func_type_idx = func_type_indexes[func_idx];
    func_type = aot_module->func_types[func_type_idx];

#if WASM_ENABLE_LAZY_JIT != 0
    if (func_idx >= aot_module->import_func_count && !func_ptrs[func_idx]) {
        if (!(func_ptr = aot_lookup_orcjit_func(
                  aot_module->comp_ctx->orc_lazyjit, module_inst, func_idx))) {
            return false;
        }
    }
#endif

    if (!(func_ptr = func_ptrs[func_idx])) {
        bh_assert(func_idx < aot_module->import_func_count);
        import_func = aot_module->import_funcs + func_idx;
        snprintf(buf, sizeof(buf),
                 "failed to call unlinked import function (%s, %s)",
                 import_func->module_name, import_func->func_name);
        aot_set_exception(module_inst, buf);
        return false;
    }

    if (func_idx < aot_module->import_func_count) {
        /* Call native function */
        import_func = aot_module->import_funcs + func_idx;
        signature = import_func->signature;
        if (import_func->call_conv_raw) {
            attachment = import_func->attachment;
            return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
                                                  signature, attachment, argv,
                                                  argc, argv);
        }
    }

    ext_ret_count =
        func_type->result_count > 1 ? func_type->result_count - 1 : 0;
    if (ext_ret_count > 0) {
        uint32 argv1_buf[32], *argv1 = argv1_buf;
        uint32 *ext_rets = NULL, *argv_ret = argv;
        uint32 cell_num = 0, i;
        uint8 *ext_ret_types = func_type->types + func_type->param_count + 1;
        uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count);
        uint64 size;

        /* Allocate memory all arguments */
        size =
            sizeof(uint32) * (uint64)argc /* original arguments */
            + sizeof(void *)
                  * (uint64)ext_ret_count /* extra result values' addr */
            + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */
        if (size > sizeof(argv1_buf)
            && !(argv1 = runtime_malloc(size, module_inst->cur_exception,
                                        sizeof(module_inst->cur_exception)))) {
            aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY);
            return false;
        }

        /* Copy original arguments */
        bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc);

        /* Get the extra result value's address */
        ext_rets =
            argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;

        /* Append each extra result value's address to original arguments */
        for (i = 0; i < ext_ret_count; i++) {
            *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) =
                (uintptr_t)(ext_rets + cell_num);
            cell_num += wasm_value_type_cell_num(ext_ret_types[i]);
        }

        ret = invoke_native_internal(exec_env, func_ptr, func_type, signature,
                                     attachment, argv1, argc, argv);
        if (!ret || aot_get_exception(module_inst)) {
            if (argv1 != argv1_buf)
                wasm_runtime_free(argv1);
            if (clear_wasi_proc_exit_exception(module_inst))
                return true;
            return false;
        }

        /* Get extra result values */
        switch (func_type->types[func_type->param_count]) {
            case VALUE_TYPE_I32:
            case VALUE_TYPE_F32:
#if WASM_ENABLE_REF_TYPES != 0
            case VALUE_TYPE_FUNCREF:
            case VALUE_TYPE_EXTERNREF:
#endif
                argv_ret++;
                break;
            case VALUE_TYPE_I64:
            case VALUE_TYPE_F64:
                argv_ret += 2;
                break;
#if WASM_ENABLE_SIMD != 0
            case VALUE_TYPE_V128:
                argv_ret += 4;
                break;
#endif
            default:
                bh_assert(0);
                break;
        }
        ext_rets =
            argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
        bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets,
                    sizeof(uint32) * cell_num);

        if (argv1 != argv1_buf)
            wasm_runtime_free(argv1);

        return true;
    }
    else {
        ret = invoke_native_internal(exec_env, func_ptr, func_type, signature,
                                     attachment, argv, argc, argv);
        if (clear_wasi_proc_exit_exception(module_inst))
            return true;
        return ret;
    }
}

/**
 * Check whether the app address and the buf is inside the linear memory,
 * and convert the app address into native address
 */
bool
aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str,
                               uint32 app_buf_addr, uint32 app_buf_size,
                               void **p_native_addr)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    uint8 *native_addr;

    if (!memory_inst) {
        goto fail;
    }

    native_addr = (uint8 *)memory_inst->memory_data.ptr + app_buf_addr;

    /* No need to check the app_offset and buf_size if memory access
       boundary check with hardware trap is enabled */
#ifndef OS_ENABLE_HW_BOUND_CHECK
    if (app_buf_addr >= memory_inst->memory_data_size) {
        goto fail;
    }

    if (!is_str) {
        if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) {
            goto fail;
        }
    }
    else {
        const char *str, *str_end;

        /* The whole string must be in the linear memory */
        str = (const char *)native_addr;
        str_end = (const char *)memory_inst->memory_data_end.ptr;
        while (str < str_end && *str != '\0')
            str++;
        if (str == str_end)
            goto fail;
    }
#endif

    *p_native_addr = (void *)native_addr;
    return true;
fail:
    aot_set_exception(module_inst, "out of bounds memory access");
    return false;
}

void *
aot_memmove(void *dest, const void *src, size_t n)
{
    return memmove(dest, src, n);
}

void *
aot_memset(void *s, int c, size_t n)
{
    return memset(s, c, n);
}

#if WASM_ENABLE_BULK_MEMORY != 0
bool
aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset,
                uint32 len, uint32 dst)
{
    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
    AOTModule *aot_module;
    uint8 *data = NULL;
    uint8 *maddr;
    uint64 seg_len = 0;

    aot_module = (AOTModule *)module_inst->aot_module.ptr;
    if (aot_module->is_jit_mode) {
#if WASM_ENABLE_JIT != 0
        seg_len =
            aot_module->wasm_module->data_segments[seg_index]->data_length;
        data = aot_module->wasm_module->data_segments[seg_index]->data;
#endif
    }
    else {
        seg_len = aot_module->mem_init_data_list[seg_index]->byte_count;
        data = aot_module->mem_init_data_list[seg_index]->bytes;
    }

    if (!aot_validate_app_addr(module_inst, dst, len))
        return false;

    if ((uint64)offset + (uint64)len > seg_len) {
        aot_set_exception(module_inst, "out of bounds memory access");
        return false;
    }

    maddr = aot_addr_app_to_native(module_inst, dst);

    bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len);
    return true;
}

bool
aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index)
{
    AOTModule *aot_module = (AOTModule *)(module_inst->aot_module.ptr);

    if (aot_module->is_jit_mode) {
#if WASM_ENABLE_JIT != 0
        aot_module->wasm_module->data_segments[seg_index]->data_length = 0;
        /* Currently we can't free the dropped data segment
            as they are stored in wasm bytecode */
#endif
    }
    else {
        aot_module->mem_init_data_list[seg_index]->byte_count = 0;
        /* Currently we can't free the dropped data segment
            as the mem_init_data_count is a continuous array */
    }
    return true;
}
#endif /* WASM_ENABLE_BULK_MEMORY */

#if WASM_ENABLE_THREAD_MGR != 0
bool
aot_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size)
{
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;

    uint32 stack_top_idx = module->aux_stack_top_global_index;
    uint32 data_end = module->aux_data_end;
    uint32 stack_bottom = module->aux_stack_bottom;
    bool is_stack_before_data = stack_bottom < data_end ? true : false;

    /* Check the aux stack space, currently we don't allocate space in heap */
    if ((is_stack_before_data && (size > start_offset))
        || ((!is_stack_before_data) && (start_offset - data_end < size)))
        return false;

    if (stack_top_idx != (uint32)-1) {
        /* The aux stack top is a wasm global,
            set the initial value for the global */
        uint32 global_offset = module->globals[stack_top_idx].data_offset;
        uint8 *global_addr =
            (uint8 *)module_inst->global_data.ptr + global_offset;
        *(int32 *)global_addr = start_offset;

        /* The aux stack boundary is a constant value,
            set the value to exec_env */
        exec_env->aux_stack_boundary.boundary = start_offset - size;
        exec_env->aux_stack_bottom.bottom = start_offset;
        return true;
    }

    return false;
}

bool
aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size)
{
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;

    /* The aux stack information is resolved in loader
        and store in module */
    uint32 stack_bottom = module->aux_stack_bottom;
    uint32 total_aux_stack_size = module->aux_stack_size;

    if (stack_bottom != 0 && total_aux_stack_size != 0) {
        if (start_offset)
            *start_offset = stack_bottom;
        if (size)
            *size = total_aux_stack_size;
        return true;
    }
    return false;
}
#endif

#if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0)
static void
const_string_node_size_cb(void *key, void *value, void *p_const_string_size)
{
    uint32 const_string_size = 0;
    const_string_size += bh_hash_map_get_elem_struct_size();
    const_string_size += strlen((const char *)value) + 1;
    *(uint32 *)p_const_string_size += const_string_size;
}

void
aot_get_module_mem_consumption(const AOTModule *module,
                               WASMModuleMemConsumption *mem_conspn)
{
    uint32 i, size;

    memset(mem_conspn, 0, sizeof(*mem_conspn));

    mem_conspn->module_struct_size = sizeof(AOTModule);

    mem_conspn->types_size = sizeof(AOTFuncType *) * module->func_type_count;
    for (i = 0; i < module->func_type_count; i++) {
        AOTFuncType *type = module->func_types[i];
        size = offsetof(AOTFuncType, types)
               + sizeof(uint8) * (type->param_count + type->result_count);
        mem_conspn->types_size += size;
    }

    mem_conspn->imports_size =
        sizeof(AOTImportMemory) * module->import_memory_count
        + sizeof(AOTImportTable) * module->import_table_count
        + sizeof(AOTImportGlobal) * module->import_global_count
        + sizeof(AOTImportFunc) * module->import_func_count;

    /* func_ptrs and func_type_indexes */
    mem_conspn->functions_size =
        (sizeof(void *) + sizeof(uint32)) * module->func_count;

    mem_conspn->tables_size = sizeof(AOTTable) * module->table_count;

    mem_conspn->memories_size = sizeof(AOTMemory) * module->memory_count;
    mem_conspn->globals_size = sizeof(AOTGlobal) * module->global_count;
    mem_conspn->exports_size = sizeof(AOTExport) * module->export_count;

    mem_conspn->table_segs_size =
        sizeof(AOTTableInitData *) * module->table_init_data_count;
    for (i = 0; i < module->table_init_data_count; i++) {
        AOTTableInitData *init_data = module->table_init_data_list[i];
        size = offsetof(AOTTableInitData, func_indexes)
               + sizeof(uint32) * init_data->func_index_count;
        mem_conspn->table_segs_size += size;
    }

    mem_conspn->data_segs_size =
        sizeof(AOTMemInitData *) * module->mem_init_data_count;
    for (i = 0; i < module->mem_init_data_count; i++) {
        mem_conspn->data_segs_size += sizeof(AOTMemInitData);
    }

    if (module->const_str_set) {
        uint32 const_string_size = 0;

        mem_conspn->const_strs_size =
            bh_hash_map_get_struct_size(module->const_str_set);

        bh_hash_map_traverse(module->const_str_set, const_string_node_size_cb,
                             (void *)&const_string_size);
        mem_conspn->const_strs_size += const_string_size;
    }

    /* code size + literal size + object data section size */
    mem_conspn->aot_code_size =
        module->code_size + module->literal_size
        + sizeof(AOTObjectDataSection) * module->data_section_count;
    for (i = 0; i < module->data_section_count; i++) {
        AOTObjectDataSection *obj_data = module->data_sections + i;
        mem_conspn->aot_code_size += sizeof(uint8) * obj_data->size;
    }

    mem_conspn->total_size += mem_conspn->module_struct_size;
    mem_conspn->total_size += mem_conspn->types_size;
    mem_conspn->total_size += mem_conspn->imports_size;
    mem_conspn->total_size += mem_conspn->functions_size;
    mem_conspn->total_size += mem_conspn->tables_size;
    mem_conspn->total_size += mem_conspn->memories_size;
    mem_conspn->total_size += mem_conspn->globals_size;
    mem_conspn->total_size += mem_conspn->exports_size;
    mem_conspn->total_size += mem_conspn->table_segs_size;
    mem_conspn->total_size += mem_conspn->data_segs_size;
    mem_conspn->total_size += mem_conspn->const_strs_size;
    mem_conspn->total_size += mem_conspn->aot_code_size;
}

void
aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst,
                                    WASMModuleInstMemConsumption *mem_conspn)
{
    AOTTableInstance *tbl_inst;
    uint32 i;

    memset(mem_conspn, 0, sizeof(*mem_conspn));

    mem_conspn->module_inst_struct_size = sizeof(AOTModuleInstance);

    mem_conspn->memories_size =
        sizeof(AOTPointer) * module_inst->memory_count
        + sizeof(AOTMemoryInstance) * module_inst->memory_count;
    for (i = 0; i < module_inst->memory_count; i++) {
        AOTMemoryInstance *mem_inst =
            ((AOTMemoryInstance **)module_inst->memories.ptr)[i];
        mem_conspn->memories_size +=
            mem_inst->num_bytes_per_page * mem_inst->cur_page_count;
        mem_conspn->app_heap_size = (uint8 *)mem_inst->heap_data_end.ptr
                                    - (uint8 *)mem_inst->heap_data.ptr;
        /* size of app heap structure */
        mem_conspn->memories_size += mem_allocator_get_heap_struct_size();
    }

    tbl_inst = module_inst->tables.ptr;
    for (i = 0; i < module_inst->table_count; i++) {
        mem_conspn->tables_size += offsetof(AOTTableInstance, data);
        mem_conspn->tables_size += sizeof(uint32) * tbl_inst->max_size;
        tbl_inst = aot_next_tbl_inst(tbl_inst);
    }

    /* func_ptrs and func_type_indexes */
    mem_conspn->functions_size =
        (sizeof(void *) + sizeof(uint32))
        * (((AOTModule *)module_inst->aot_module.ptr)->import_func_count
           + ((AOTModule *)module_inst->aot_module.ptr)->func_count);

    mem_conspn->globals_size = module_inst->global_data_size;

    mem_conspn->exports_size =
        sizeof(AOTFunctionInstance) * (uint64)module_inst->export_func_count;

    mem_conspn->total_size += mem_conspn->module_inst_struct_size;
    mem_conspn->total_size += mem_conspn->memories_size;
    mem_conspn->total_size += mem_conspn->functions_size;
    mem_conspn->total_size += mem_conspn->tables_size;
    mem_conspn->total_size += mem_conspn->globals_size;
    mem_conspn->total_size += mem_conspn->exports_size;
}
#endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \
                 || (WASM_ENABLE_MEMORY_TRACING != 0) */

#if WASM_ENABLE_REF_TYPES != 0
void
aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx)
{
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;
    AOTTableInitData *tbl_seg = module->table_init_data_list[tbl_seg_idx];
    tbl_seg->is_dropped = true;
}

void
aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx,
               uint32 tbl_seg_idx, uint32 length, uint32 src_offset,
               uint32 dst_offset)
{
    AOTTableInstance *tbl_inst;
    AOTTableInitData *tbl_seg;
    const AOTModule *module = module_inst->aot_module.ptr;

    tbl_inst = aot_get_table_inst(module_inst, tbl_idx);
    bh_assert(tbl_inst);

    tbl_seg = module->table_init_data_list[tbl_seg_idx];
    bh_assert(tbl_seg);

    if (!length) {
        return;
    }

    if (length + src_offset > tbl_seg->func_index_count
        || dst_offset + length > tbl_inst->cur_size) {
        aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
        return;
    }

    if (tbl_seg->is_dropped) {
        aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
        return;
    }

    if (!wasm_elem_is_passive(tbl_seg->mode)) {
        aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
        return;
    }

    bh_memcpy_s((uint8 *)tbl_inst + offsetof(AOTTableInstance, data)
                    + dst_offset * sizeof(uint32),
                (tbl_inst->cur_size - dst_offset) * sizeof(uint32),
                tbl_seg->func_indexes + src_offset, length * sizeof(uint32));
}

void
aot_table_copy(AOTModuleInstance *module_inst, uint32 src_tbl_idx,
               uint32 dst_tbl_idx, uint32 length, uint32 src_offset,
               uint32 dst_offset)
{
    AOTTableInstance *src_tbl_inst, *dst_tbl_inst;

    src_tbl_inst = aot_get_table_inst(module_inst, src_tbl_idx);
    bh_assert(src_tbl_inst);

    dst_tbl_inst = aot_get_table_inst(module_inst, dst_tbl_idx);
    bh_assert(dst_tbl_inst);

    if ((uint64)dst_offset + length > dst_tbl_inst->cur_size
        || (uint64)src_offset + length > src_tbl_inst->cur_size) {
        aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
        return;
    }

    /* if src_offset >= dst_offset, copy from front to back */
    /* if src_offset < dst_offset, copy from back to front */
    /* merge all together */
    bh_memmove_s((uint8 *)(dst_tbl_inst) + offsetof(AOTTableInstance, data)
                     + dst_offset * sizeof(uint32),
                 (dst_tbl_inst->cur_size - dst_offset) * sizeof(uint32),
                 (uint8 *)(src_tbl_inst) + offsetof(AOTTableInstance, data)
                     + src_offset * sizeof(uint32),
                 length * sizeof(uint32));
}

void
aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length,
               uint32 val, uint32 data_offset)
{
    AOTTableInstance *tbl_inst;

    tbl_inst = aot_get_table_inst(module_inst, tbl_idx);
    bh_assert(tbl_inst);

    if (data_offset + length > tbl_inst->cur_size) {
        aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
        return;
    }

    for (; length != 0; data_offset++, length--) {
        tbl_inst->data[data_offset] = val;
    }
}

uint32
aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx,
               uint32 inc_entries, uint32 init_val)
{
    uint32 entry_count, i, orig_tbl_sz;
    AOTTableInstance *tbl_inst;

    tbl_inst = aot_get_table_inst(module_inst, tbl_idx);
    if (!tbl_inst) {
        return (uint32)-1;
    }

    orig_tbl_sz = tbl_inst->cur_size;

    if (!inc_entries) {
        return orig_tbl_sz;
    }

    if (tbl_inst->cur_size > UINT32_MAX - inc_entries) {
        return (uint32)-1;
    }

    entry_count = tbl_inst->cur_size + inc_entries;
    if (entry_count > tbl_inst->max_size) {
        return (uint32)-1;
    }

    /* fill in */
    for (i = 0; i < inc_entries; ++i) {
        tbl_inst->data[tbl_inst->cur_size + i] = init_val;
    }

    tbl_inst->cur_size = entry_count;
    return orig_tbl_sz;
}
#endif /* WASM_ENABLE_REF_TYPES != 0 */

#if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0)
#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
static const char *
lookup_func_name(const char **func_names, uint32 *func_indexes,
                 uint32 func_index_count, uint32 func_index)
{
    int64 low = 0, mid;
    int64 high = func_index_count - 1;

    while (low <= high) {
        mid = (low + high) / 2;
        if (func_index == func_indexes[mid]) {
            return func_names[mid];
        }
        else if (func_index < func_indexes[mid])
            high = mid - 1;
        else
            low = mid + 1;
    }

    return NULL;
}
#endif /* WASM_ENABLE_CUSTOM_NAME_SECTION != 0 */

static const char *
get_func_name_from_index(const AOTModuleInstance *module_inst,
                         uint32 func_index)
{
    const char *func_name = NULL;
    AOTModule *module = module_inst->aot_module.ptr;

#if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
    if ((func_name =
             lookup_func_name(module->aux_func_names, module->aux_func_indexes,
                              module->aux_func_name_count, func_index))) {
        return func_name;
    }
#endif

    if (func_index < module->import_func_count) {
        func_name = module->import_funcs[func_index].func_name;
    }
    else {
        uint32 i;

        for (i = 0; i < module->export_count; i++) {
            AOTExport export = module->exports[i];
            if (export.index == func_index && export.kind == EXPORT_KIND_FUNC) {
                func_name = export.name;
                break;
            }
        }
    }

    return func_name;
}

bool
aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
{
    AOTFrame *frame =
        wasm_exec_env_alloc_wasm_frame(exec_env, sizeof(AOTFrame));
#if WASM_ENABLE_PERF_PROFILING != 0
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    AOTFuncPerfProfInfo *func_perf_prof =
        (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings.ptr
        + func_index;
#endif

    if (!frame) {
        aot_set_exception((AOTModuleInstance *)exec_env->module_inst,
                          "auxiliary call stack overflow");
        return false;
    }

#if WASM_ENABLE_PERF_PROFILING != 0
    frame->time_started = os_time_get_boot_microsecond();
    frame->func_perf_prof_info = func_perf_prof;
#endif

    frame->prev_frame = (AOTFrame *)exec_env->cur_frame;
    exec_env->cur_frame = (struct WASMInterpFrame *)frame;

    frame->func_index = func_index;
    return true;
}

void
aot_free_frame(WASMExecEnv *exec_env)
{
    AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame;
    AOTFrame *prev_frame = cur_frame->prev_frame;

#if WASM_ENABLE_PERF_PROFILING != 0
    cur_frame->func_perf_prof_info->total_exec_time +=
        os_time_get_boot_microsecond() - cur_frame->time_started;
    cur_frame->func_perf_prof_info->total_exec_cnt++;
#endif

    wasm_exec_env_free_wasm_frame(exec_env, cur_frame);
    exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame;
}
#endif /* end of (WASM_ENABLE_DUMP_CALL_STACK != 0) \
                 || (WASM_ENABLE_PERF_PROFILING != 0) */

#if WASM_ENABLE_DUMP_CALL_STACK != 0
bool
aot_create_call_stack(struct WASMExecEnv *exec_env)
{
    AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame,
             *first_frame = cur_frame;
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    uint32 n = 0;

    while (cur_frame) {
        cur_frame = cur_frame->prev_frame;
        n++;
    }

    /* release previous stack frames and create new ones */
    if (!bh_vector_destroy(module_inst->frames.ptr)
        || !bh_vector_init(module_inst->frames.ptr, n, sizeof(WASMCApiFrame),
                           false)) {
        return false;
    }

    cur_frame = first_frame;
    while (cur_frame) {
        WASMCApiFrame frame = { 0 };
        frame.instance = module_inst;
        frame.module_offset = 0;
        frame.func_index = cur_frame->func_index;
        frame.func_offset = 0;
        frame.func_name_wp =
            get_func_name_from_index(module_inst, cur_frame->func_index);

        if (!bh_vector_append(module_inst->frames.ptr, &frame)) {
            bh_vector_destroy(module_inst->frames.ptr);
            return false;
        }

        cur_frame = cur_frame->prev_frame;
    }

    return true;
}

#define PRINT_OR_DUMP()                                                   \
    do {                                                                  \
        total_len +=                                                      \
            wasm_runtime_dump_line_buf_impl(line_buf, print, &buf, &len); \
        if ((!print) && buf && (len == 0)) {                              \
            return total_len;                                             \
        }                                                                 \
    } while (0)

uint32
aot_dump_call_stack(WASMExecEnv *exec_env, bool print, char *buf, uint32 len)
{
    AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
    uint32 n = 0, total_len = 0, total_frames;
    /* reserve 256 bytes for line buffer, any line longer than 256 bytes
     * will be truncated */
    char line_buf[256];

    if (!module_inst->frames.ptr) {
        return 0;
    }

    total_frames = bh_vector_size(module_inst->frames.ptr);
    if (total_frames == 0) {
        return 0;
    }

    snprintf(line_buf, sizeof(line_buf), "\n");
    PRINT_OR_DUMP();

    while (n < total_frames) {
        WASMCApiFrame frame = { 0 };
        uint32 line_length, i;

        if (!bh_vector_get(module_inst->frames.ptr, n, &frame)) {
            return 0;
        }

        /* function name not exported, print number instead */
        if (frame.func_name_wp == NULL) {
            line_length = snprintf(line_buf, sizeof(line_buf), "#%02d $f%d\n",
                                   n, frame.func_index);
        }
        else {
            line_length = snprintf(line_buf, sizeof(line_buf), "#%02d %s\n", n,
                                   frame.func_name_wp);
        }

        if (line_length >= sizeof(line_buf)) {
            uint32 line_buffer_len = sizeof(line_buf);
            /* If line too long, ensure the last character is '\n' */
            for (i = line_buffer_len - 5; i < line_buffer_len - 2; i++) {
                line_buf[i] = '.';
            }
            line_buf[line_buffer_len - 2] = '\n';
        }

        PRINT_OR_DUMP();

        n++;
    }
    snprintf(line_buf, sizeof(line_buf), "\n");
    PRINT_OR_DUMP();

    return total_len + 1;
}
#endif /* end of WASM_ENABLE_DUMP_CALL_STACK */

#if WASM_ENABLE_PERF_PROFILING != 0
void
aot_dump_perf_profiling(const AOTModuleInstance *module_inst)
{
    AOTFuncPerfProfInfo *perf_prof =
        (AOTFuncPerfProfInfo *)module_inst->func_perf_profilings.ptr;
    AOTModule *module = (AOTModule *)module_inst->aot_module.ptr;
    uint32 total_func_count = module->import_func_count + module->func_count, i;
    const char *func_name;

    os_printf("Performance profiler data:\n");
    for (i = 0; i < total_func_count; i++, perf_prof++) {
        func_name = get_func_name_from_index(module_inst, i);

        if (func_name)
            os_printf("  func %s, execution time: %.3f ms, execution count: %d "
                      "times\n",
                      func_name, perf_prof->total_exec_time / 1000.0f,
                      perf_prof->total_exec_cnt);
        else
            os_printf("  func %d, execution time: %.3f ms, execution count: %d "
                      "times\n",
                      i, perf_prof->total_exec_time / 1000.0f,
                      perf_prof->total_exec_cnt);
    }
}
#endif /* end of WASM_ENABLE_PERF_PROFILING */