Refine aot call_indirect opcode translation (#492)

Re-implement aot call_indirect opcode translation: when calling non-import function, translate it by LLVM call IR to call the function in AOTed code, so as to avoid calling runtime aot_call_indirect API which is much slower. For import function, keep calling aot_call_indirect API due to the possible pointer/string argument conversion.

And add prompt info while app heap is corrupted, change emit_leb to emit_uint32 inter fast-interp to refine footprint.

Signed-off-by: Wenyong Huang <wenyong.huang@intel.com>

core/iwasm/aot/aot_runtime.c

@@ -1381,7 +1381,17 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size,
     }
 
     if (!addr) {
-        aot_set_exception(module_inst, "out of memory");
+        if (memory_inst->heap_handle.ptr
+            && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) {
+            LOG_ERROR("Error: app heap is corrupted, if the wasm file "
+                      "is compiled by wasi-sdk-12.0 or larger version, "
+                      "please add -Wl,--export=malloc -Wl,--export=free "
+                      " to export malloc and free functions.");
+            aot_set_exception(module_inst, "app heap corrupted");
+        }
+        else {
+            aot_set_exception(module_inst, "out of memory");
+        }
         return 0;
     }
     if (p_native_addr)
@@ -1804,7 +1814,6 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
 
 bool
 aot_call_indirect(WASMExecEnv *exec_env,
-                  bool check_func_type, uint32 func_type_idx,
                   uint32 table_elem_idx,
                   uint32 argc, uint32 *argv)
 {
@@ -1816,8 +1825,7 @@ aot_call_indirect(WASMExecEnv *exec_env,
     AOTFuncType *func_type;
     void **func_ptrs = (void**)module_inst->func_ptrs.ptr, *func_ptr;
     uint32 table_size = module_inst->table_size;
-    uint32 func_idx, func_type_idx1;
-    uint32 ext_ret_count;
+    uint32 func_type_idx, func_idx, ext_ret_count;
     AOTImportFunc *import_func;
     const char *signature = NULL;
     void *attachment = NULL;
@@ -1844,15 +1852,8 @@ aot_call_indirect(WASMExecEnv *exec_env,
         return false;
     }
 
-    func_type_idx1 = func_type_indexes[func_idx];
-    if (check_func_type
-        && !aot_is_wasm_type_equal(module_inst, func_type_idx,
-                                   func_type_idx1)) {
-        aot_set_exception_with_id(module_inst,
-                                  EXCE_INVALID_FUNCTION_TYPE_INDEX);
-        return false;
-    }
-    func_type = aot_module->func_types[func_type_idx1];
+    func_type_idx = func_type_indexes[func_idx];
+    func_type = aot_module->func_types[func_type_idx];
 
     if (!(func_ptr = func_ptrs[func_idx])) {
         bh_assert(func_idx < aot_module->import_func_count);

core/iwasm/aot/aot_runtime.h

@@ -521,7 +521,6 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
 
 bool
 aot_call_indirect(WASMExecEnv *exec_env,
-                  bool check_func_type, uint32 func_type_idx,
                   uint32 table_elem_idx,
                   uint32 argc, uint32 *argv);
 

core/iwasm/common/wasm_runtime_common.c

@@ -3327,14 +3327,11 @@ wasm_runtime_call_indirect(WASMExecEnv *exec_env,
 
 #if WASM_ENABLE_INTERP != 0
     if (exec_env->module_inst->module_type == Wasm_Module_Bytecode)
-        return wasm_call_indirect(exec_env,
-                                  element_indices,
-                                  argc, argv);
+        return wasm_call_indirect(exec_env, element_indices, argc, argv);
 #endif
 #if WASM_ENABLE_AOT != 0
     if (exec_env->module_inst->module_type == Wasm_Module_AoT)
-        return aot_call_indirect(exec_env, false, 0,
-                                 element_indices, argc, argv);
+        return aot_call_indirect(exec_env, element_indices, argc, argv);
 #endif
     return false;
 }

core/iwasm/compilation/aot_emit_function.c

@@ -347,7 +347,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                                                  + ext_ret_count);
     if (total_size >= UINT32_MAX
         || !(param_values = wasm_runtime_malloc((uint32)total_size))) {
-        aot_set_last_error("Allocate memory failed.");
+        aot_set_last_error("allocate memory failed.");
         return false;
     }
 
@@ -408,7 +408,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
         total_size = sizeof(LLVMTypeRef) * (uint64)(param_count + 1);
         if (total_size >= UINT32_MAX
             || !(param_types = wasm_runtime_malloc((uint32)total_size))) {
-            aot_set_last_error("Allocate memory failed.");
+            aot_set_last_error("allocate memory failed.");
             goto fail;
         }
 
@@ -510,22 +510,12 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     char buf[32], *func_name = "aot_call_indirect";
     uint32 i, cell_num = 0, ret_cell_num, argv_cell_num;
 
-#if WASM_ENABLE_THREAD_MGR != 0
-    /* Insert suspend check point */
-    if (comp_ctx->enable_thread_mgr) {
-        if (!check_suspend_flags(comp_ctx, func_ctx))
-            return false;
-    }
-#endif
-
     /* prepare function type of aot_call_indirect */
     func_param_types[0] = comp_ctx->exec_env_type;  /* exec_env */
-    func_param_types[1] = INT8_TYPE;                /* check_func_type */
-    func_param_types[2] = I32_TYPE;                 /* func_type_idx */
-    func_param_types[3] = I32_TYPE;                 /* table_elem_idx */
-    func_param_types[4] = I32_TYPE;                 /* argc */
-    func_param_types[5] = INT32_PTR_TYPE;           /* argv */
-    if (!(func_type = LLVMFunctionType(INT8_TYPE, func_param_types, 6, false))) {
+    func_param_types[1] = I32_TYPE;                 /* table_elem_idx */
+    func_param_types[2] = I32_TYPE;                 /* argc */
+    func_param_types[3] = INT32_PTR_TYPE;           /* argv */
+    if (!(func_type = LLVMFunctionType(INT8_TYPE, func_param_types, 4, false))) {
         aot_set_last_error("llvm add function type failed.");
         return false;
     }
@@ -588,20 +578,18 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     }
 
     func_param_values[0] = func_ctx->exec_env;
-    func_param_values[1] = I8_CONST(true);
-    func_param_values[2] = func_type_idx;
-    func_param_values[3] = table_elem_idx;
-    func_param_values[4] = I32_CONST(param_cell_num);
-    func_param_values[5] = func_ctx->argv_buf;
+    func_param_values[1] = table_elem_idx;
+    func_param_values[2] = I32_CONST(param_cell_num);
+    func_param_values[3] = func_ctx->argv_buf;
 
-    if (!func_param_values[1] || !func_param_values[4]) {
+    if (!func_param_values[2]) {
         aot_set_last_error("llvm create const failed.");
         return false;
     }
 
     /* call aot_call_indirect() function */
     if (!(res = LLVMBuildCall(comp_ctx->builder, func,
-                              func_param_values, 6, "res"))) {
+                              func_param_values, 4, "res"))) {
         aot_set_last_error("llvm build call failed.");
         return false;
     }
@@ -642,89 +630,453 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                              uint32 type_idx)
 {
     AOTFuncType *func_type;
-    LLVMValueRef elem_idx, ftype_idx;
-    LLVMValueRef *param_values = NULL, *value_rets = NULL, res = NULL;
-    LLVMTypeRef *param_types = NULL;
-    int32 i, param_count, result_count;
-    uint32 param_cell_num;
+    LLVMValueRef elem_idx, table_elem, func_idx;
+    LLVMValueRef ftype_idx_ptr, ftype_idx, ftype_idx_const;
+    LLVMValueRef cmp_elem_idx, cmp_func_idx, cmp_ftype_idx;
+    LLVMValueRef func, func_ptr, table_size_const;
+    LLVMValueRef ext_ret_offset, ext_ret_ptr, ext_ret, res;
+    LLVMValueRef *param_values = NULL, *value_rets = NULL;
+    LLVMValueRef *result_phis = NULL, value_ret, import_func_count;
+    LLVMTypeRef *param_types = NULL, ret_type;
+    LLVMTypeRef llvm_func_type, llvm_func_ptr_type;
+    LLVMTypeRef ext_ret_ptr_type;
+    LLVMBasicBlockRef check_elem_idx_succ, check_ftype_idx_succ;
+    LLVMBasicBlockRef check_func_idx_succ, block_return, block_curr;
+    LLVMBasicBlockRef block_call_import, block_call_non_import;
+    uint32 total_param_count, func_param_count, func_result_count;
+    uint32 table_init_size = 0, ext_cell_num, param_cell_num, i, j;
+    uint8 wasm_ret_type, *wasm_ret_types;
     uint64 total_size;
-    uint8 *wasm_ret_types = NULL;
-    bool ret = false;
+    char buf[32];
+    bool ret;
 
     /* Check function type index */
     if (type_idx >= comp_ctx->comp_data->func_type_count) {
-        aot_set_last_error("type index is overflow");
+        aot_set_last_error("function type index out of range");
         return false;
     }
 
-    ftype_idx = I32_CONST(type_idx);
-    CHECK_LLVM_CONST(ftype_idx);
+    /* Find the equivalent function type whose type index is the smallest:
+       the callee function's type index is also converted to the smallest
+       one in wasm loader, so we can just check whether the two type indexes
+       are equal (the type index of call_indirect opcode and callee func),
+       we don't need to check whether the whole function types are equal,
+       including param types and result types. */
+    type_idx = wasm_get_smallest_type_idx(comp_ctx->comp_data->func_types,
+                                          comp_ctx->comp_data->func_type_count,
+                                          type_idx);
+    ftype_idx_const = I32_CONST(type_idx);
+    CHECK_LLVM_CONST(ftype_idx_const);
 
     func_type = comp_ctx->comp_data->func_types[type_idx];
-
-    param_cell_num = func_type->param_cell_num;
-    result_count = func_type->result_count;
-    wasm_ret_types = func_type->types + func_type->param_count;
+    func_param_count = func_type->param_count;
+    func_result_count = func_type->result_count;
 
     POP_I32(elem_idx);
 
+    if (comp_ctx->comp_data->import_table_count > 0) {
+        table_init_size = comp_ctx->comp_data->import_tables[0]
+                          .table_init_size;
+    }
+    else if (comp_ctx->comp_data->table_count > 0) {
+        table_init_size = comp_ctx->comp_data->tables[0]
+                          .table_init_size;
+    }
+    else {
+        aot_set_last_error("table index out of range");
+        goto fail;
+    }
+    table_size_const = I32_CONST(table_init_size);
+    CHECK_LLVM_CONST(table_size_const);
+
+    /* Check if (uint32)elem index >= table size */
+    if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE,
+                                       elem_idx, table_size_const,
+                                       "cmp_elem_idx"))) {
+        aot_set_last_error("llvm build icmp failed.");
+        goto fail;
+    }
+
+    /* Throw exception if elem index >= table size */
+    if (!(check_elem_idx_succ =
+                LLVMAppendBasicBlockInContext(comp_ctx->context,
+                                              func_ctx->func,
+                                              "check_elem_idx_succ"))) {
+        aot_set_last_error("llvm add basic block failed.");
+        goto fail;
+    }
+
+    LLVMMoveBasicBlockAfter(check_elem_idx_succ,
+                            LLVMGetInsertBlock(comp_ctx->builder));
+
+    if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNDEFINED_ELEMENT,
+                             true, cmp_elem_idx, check_elem_idx_succ)))
+        goto fail;
+
+    /* Load function index */
+    if (!(table_elem = LLVMBuildInBoundsGEP(comp_ctx->builder,
+                                            func_ctx->table_base,
+                                            &elem_idx, 1, "table_elem"))) {
+        aot_set_last_error("llvm build add failed.");
+        goto fail;
+    }
+
+    if (!(func_idx = LLVMBuildLoad(comp_ctx->builder,
+                                   table_elem, "func_idx"))) {
+        aot_set_last_error("llvm build load failed.");
+        goto fail;
+    }
+
+    /* Check if func_idx == -1 */
+    if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
+                                       func_idx, I32_NEG_ONE,
+                                       "cmp_func_idx"))) {
+        aot_set_last_error("llvm build icmp failed.");
+        goto fail;
+    }
+
+    /* Throw exception if func_idx == -1 */
+    if (!(check_func_idx_succ =
+                LLVMAppendBasicBlockInContext(comp_ctx->context,
+                                              func_ctx->func,
+                                              "check_func_idx_succ"))) {
+        aot_set_last_error("llvm add basic block failed.");
+        goto fail;
+    }
+
+    LLVMMoveBasicBlockAfter(check_func_idx_succ,
+                            LLVMGetInsertBlock(comp_ctx->builder));
+
+    if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNINITIALIZED_ELEMENT,
+                             true, cmp_func_idx, check_func_idx_succ)))
+        goto fail;
+
+    /* Load function type index */
+    if (!(ftype_idx_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder,
+                                               func_ctx->func_type_indexes,
+                                               &func_idx, 1,
+                                               "ftype_idx_ptr"))) {
+        aot_set_last_error("llvm build inbounds gep failed.");
+        goto fail;
+    }
+
+    if (!(ftype_idx = LLVMBuildLoad(comp_ctx->builder, ftype_idx_ptr,
+                                    "ftype_idx"))) {
+        aot_set_last_error("llvm build load failed.");
+        goto fail;
+    }
+
+    /* Check if function type index not equal */
+    if (!(cmp_ftype_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE,
+                                        ftype_idx, ftype_idx_const,
+                                        "cmp_ftype_idx"))) {
+        aot_set_last_error("llvm build icmp failed.");
+        goto fail;
+    }
+
+    /* Throw exception if ftype_idx != ftype_idx_const */
+    if (!(check_ftype_idx_succ =
+                LLVMAppendBasicBlockInContext(comp_ctx->context,
+                                              func_ctx->func,
+                                              "check_ftype_idx_succ"))) {
+        aot_set_last_error("llvm add basic block failed.");
+        goto fail;
+    }
+
+    LLVMMoveBasicBlockAfter(check_ftype_idx_succ,
+                            LLVMGetInsertBlock(comp_ctx->builder));
+
+    if (!(aot_emit_exception(comp_ctx, func_ctx,
+                             EXCE_INVALID_FUNCTION_TYPE_INDEX,
+                             true, cmp_ftype_idx, check_ftype_idx_succ)))
+        goto fail;
+
     /* Initialize parameter types of the LLVM function */
-    param_count = (int32)func_type->param_count;
-    total_size = sizeof(LLVMTypeRef) * (uint64)param_count;
+    total_param_count = 1 + func_param_count;
+
+    /* Extra function results' addresses (except the first one) are
+     * appended to aot function parameters. */
+    if (func_result_count > 1)
+      total_param_count += func_result_count - 1;
+
+    total_size = sizeof(LLVMTypeRef) * (uint64)total_param_count;
     if (total_size >= UINT32_MAX
         || !(param_types = wasm_runtime_malloc((uint32)total_size))) {
-        aot_set_last_error("Allocate memory failed.");
+        aot_set_last_error("allocate memory failed.");
         goto fail;
     }
 
-    for (i = 0; i < param_count; i++)
-        param_types[i] = TO_LLVM_TYPE(func_type->types[i]);
+    /* Prepare param types */
+    j = 0;
+    param_types[j++] = comp_ctx->exec_env_type;
+    for (i = 0; i < func_param_count; i++)
+        param_types[j++] = TO_LLVM_TYPE(func_type->types[i]);
+
+    for (i = 1; i < func_result_count; i++, j++) {
+        param_types[j] =
+            TO_LLVM_TYPE(func_type->types[func_param_count + i]);
+        if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) {
+            aot_set_last_error("llvm get pointer type failed.");
+            goto fail;
+        }
+    }
+
+    /* Resolve return type of the LLVM function */
+    if (func_result_count) {
+        wasm_ret_type = func_type->types[func_param_count];
+        ret_type = TO_LLVM_TYPE(wasm_ret_type);
+    }
+    else {
+        wasm_ret_type = VALUE_TYPE_VOID;
+        ret_type = VOID_TYPE;
+    }
 
     /* Allocate memory for parameters */
-    total_size = sizeof(LLVMValueRef) * (uint64)param_count;
+    total_size = sizeof(LLVMValueRef) * (uint64)total_param_count;
     if (total_size >= UINT32_MAX
         || !(param_values = wasm_runtime_malloc((uint32)total_size))) {
-        aot_set_last_error("Allocate memory failed.");
+        aot_set_last_error("allocate memory failed.");
         goto fail;
     }
 
+    /* First parameter is exec env */
+    j = 0;
+    param_values[j++] = func_ctx->exec_env;
+
     /* Pop parameters from stack */
-    for (i = param_count - 1; i >= 0; i--)
-        POP(param_values[i], func_type->types[i]);
+    for (i = func_param_count - 1; (int32)i >= 0; i--)
+        POP(param_values[i + j], func_type->types[i]);
+
+    /* Prepare extra parameters */
+    ext_cell_num = 0;
+    for (i = 1; i < func_result_count; i++) {
+        ext_ret_offset = I32_CONST(ext_cell_num);
+        CHECK_LLVM_CONST(ext_ret_offset);
+
+        snprintf(buf, sizeof(buf), "ext_ret%d_ptr", i - 1);
+        if (!(ext_ret_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder,
+                                                 func_ctx->argv_buf,
+                                                 &ext_ret_offset, 1, buf))) {
+            aot_set_last_error("llvm build GEP failed.");
+            goto fail;
+        }
+
+        ext_ret_ptr_type = param_types[func_param_count + i];
+        snprintf(buf, sizeof(buf), "ext_ret%d_ptr_cast", i - 1);
+        if (!(ext_ret_ptr = LLVMBuildBitCast(comp_ctx->builder,
+                                             ext_ret_ptr, ext_ret_ptr_type,
+                                             buf))) {
+            aot_set_last_error("llvm build bit cast failed.");
+            goto fail;
+        }
+
+        param_values[func_param_count + i] = ext_ret_ptr;
+        ext_cell_num += wasm_value_type_cell_num(
+                func_type->types[func_param_count + i]);
+    }
+
+    if (ext_cell_num > 64) {
+        aot_set_last_error("prepare call-indirect arguments failed: "
+                           "maximum 64 extra cell number supported.");
+        goto fail;
+    }
+
+#if WASM_ENABLE_THREAD_MGR != 0
+    /* Insert suspend check point */
+    if (comp_ctx->enable_thread_mgr) {
+        if (!check_suspend_flags(comp_ctx, func_ctx))
+            goto fail;
+    }
+#endif
+
+    /* Add basic blocks */
+    block_call_import =
+        LLVMAppendBasicBlockInContext(comp_ctx->context, func_ctx->func,
+                                      "call_import");
+    block_call_non_import =
+        LLVMAppendBasicBlockInContext(comp_ctx->context, func_ctx->func,
+                                      "call_non_import");
+    block_return =
+        LLVMAppendBasicBlockInContext(comp_ctx->context, func_ctx->func,
+                                      "func_return");
+    if (!block_call_import || !block_call_non_import || !block_return) {
+        aot_set_last_error("llvm add basic block failed.");
+        goto fail;
+    }
+
+    LLVMMoveBasicBlockAfter(block_call_import,
+                            LLVMGetInsertBlock(comp_ctx->builder));
+    LLVMMoveBasicBlockAfter(block_call_non_import, block_call_import);
+    LLVMMoveBasicBlockAfter(block_return, block_call_non_import);
+
+    import_func_count = I32_CONST(comp_ctx->comp_data->import_func_count);
+    CHECK_LLVM_CONST(import_func_count);
+
+    /* Check if func_idx < import_func_count */
+    if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntULT,
+                                       func_idx, import_func_count,
+                                       "cmp_func_idx"))) {
+        aot_set_last_error("llvm build icmp failed.");
+        goto fail;
+    }
+
+    /* If func_idx < import_func_count, jump to call import block,
+       else jump to call non-import block */
+    if (!LLVMBuildCondBr(comp_ctx->builder, cmp_func_idx,
+                         block_call_import, block_call_non_import)) {
+        aot_set_last_error("llvm build cond br failed.");
+        goto fail;
+    }
+
+    /* Add result phis for return block */
+    LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return);
+
+    if (func_result_count > 0) {
+        total_size = sizeof(LLVMValueRef) * (uint64)func_result_count;
+        if (total_size >= UINT32_MAX
+            || !(result_phis = wasm_runtime_malloc((uint32)total_size))) {
+            aot_set_last_error("allocate memory failed.");
+            goto fail;
+        }
+        memset(result_phis, 0, (uint32)total_size);
+        for (i = 0; i < func_result_count; i++) {
+            LLVMTypeRef tmp_type =
+                TO_LLVM_TYPE(func_type->types[func_param_count + i]);
+            if (!(result_phis[i] = LLVMBuildPhi(comp_ctx->builder,
+                                                tmp_type, "phi"))) {
+                aot_set_last_error("llvm build phi failed.");
+                goto fail;
+            }
+        }
+    }
+
+    /* Translate call import block */
+    LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_import);
 
     /* Allocate memory for result values */
-    total_size = sizeof(LLVMValueRef) * (uint64)result_count;
+    total_size = sizeof(LLVMValueRef) * (uint64)func_result_count;
     if (total_size >= UINT32_MAX
         || !(value_rets = wasm_runtime_malloc((uint32)total_size))) {
-        aot_set_last_error("Allocate memory failed.");
+        aot_set_last_error("allocate memory failed.");
         goto fail;
     }
     memset(value_rets, 0, total_size);
 
+    param_cell_num = func_type->param_cell_num;
+    wasm_ret_types = func_type->types + func_type->param_count;
+
     if (!call_aot_call_indirect_func(comp_ctx, func_ctx,
                                      func_type, ftype_idx, elem_idx,
-                                     param_types, param_values,
-                                     param_count, param_cell_num,
-                                     result_count, wasm_ret_types,
+                                     param_types + 1, param_values + 1,
+                                     func_param_count, param_cell_num,
+                                     func_result_count, wasm_ret_types,
                                      value_rets, &res))
         goto fail;
 
-    for (i = 0; i < func_type->result_count; i++)
-        PUSH(value_rets[i], func_type->types[func_type->param_count + i]);
-
-    /* Check whether there was exception thrown when executing the function */
+    /* Check whether exception was thrown when executing the function */
     if (!check_call_return(comp_ctx, func_ctx, res))
         goto fail;
 
+    block_curr = LLVMGetInsertBlock(comp_ctx->builder);
+    for (i = 0; i < func_result_count; i++) {
+        LLVMAddIncoming(result_phis[i], &value_rets[i], &block_curr, 1);
+    }
+
+    if (!LLVMBuildBr(comp_ctx->builder, block_return)) {
+        aot_set_last_error("llvm build br failed.");
+        goto fail;
+    }
+
+    /* Translate call non-import block */
+    LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_non_import);
+
+    if (comp_ctx->enable_bound_check
+        && !check_stack_boundary(comp_ctx, func_ctx,
+                                 param_cell_num + ext_cell_num + 1
+                                 /* Reserve some local variables */
+                                 + 16))
+        goto fail;
+
+    /* Load function pointer */
+    if (!(func_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->func_ptrs,
+                                          &func_idx, 1, "func_ptr_tmp"))) {
+        aot_set_last_error("llvm build inbounds gep failed.");
+        goto fail;
+    }
+
+    if (!(func_ptr = LLVMBuildLoad(comp_ctx->builder, func_ptr, "func_ptr"))) {
+        aot_set_last_error("llvm build load failed.");
+        goto fail;
+    }
+
+    if (!(llvm_func_type = LLVMFunctionType(ret_type, param_types,
+                                            total_param_count, false))
+        || !(llvm_func_ptr_type = LLVMPointerType(llvm_func_type, 0))) {
+        aot_set_last_error("llvm add function type failed.");
+        goto fail;
+    }
+
+    if (!(func = LLVMBuildBitCast(comp_ctx->builder,
+                                  func_ptr, llvm_func_ptr_type,
+                                  "indirect_func"))) {
+        aot_set_last_error("llvm build bit cast failed.");
+        goto fail;
+    }
+
+    if (!(value_ret = LLVMBuildCall(comp_ctx->builder, func,
+                                    param_values, total_param_count,
+                                    func_result_count > 0
+                                    ? "ret" : ""))) {
+        aot_set_last_error("llvm build call failed.");
+        goto fail;
+    }
+
+    /* Check whether exception was thrown when executing the function */
+    if (!check_exception_thrown(comp_ctx, func_ctx))
+        goto fail;
+
+    if (func_result_count > 0) {
+        block_curr = LLVMGetInsertBlock(comp_ctx->builder);
+
+        /* Push the first result to stack */
+        LLVMAddIncoming(result_phis[0], &value_ret, &block_curr, 1);
+
+        /* Load extra result from its address and push to stack */
+        for (i = 1; i < func_result_count; i++) {
+            snprintf(buf, sizeof(buf), "ext_ret%d", i - 1);
+            if (!(ext_ret = LLVMBuildLoad(comp_ctx->builder,
+                                          param_values[func_param_count + i],
+                                          buf))) {
+                aot_set_last_error("llvm build load failed.");
+                goto fail;
+            }
+            LLVMAddIncoming(result_phis[i], &ext_ret, &block_curr, 1);
+        }
+    }
+
+    if (!LLVMBuildBr(comp_ctx->builder, block_return)) {
+        aot_set_last_error("llvm build br failed.");
+        goto fail;
+    }
+
+    /* Translate function return block */
+    LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return);
+
+    for (i = 0; i < func_result_count; i++) {
+        PUSH(result_phis[i], func_type->types[func_param_count + i]);
+    }
+
     ret = true;
 
 fail:
-    if (value_rets)
-        wasm_runtime_free(value_rets);
     if (param_values)
         wasm_runtime_free(param_values);
     if (param_types)
         wasm_runtime_free(param_types);
+    if (value_rets)
+        wasm_runtime_free(value_rets);
+    if (result_phis)
+        wasm_runtime_free(result_phis);
     return ret;
 }
 

core/iwasm/compilation/aot_llvm.c

@@ -465,9 +465,13 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
 static bool
 create_table_base(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
 {
+    AOTCompData *comp_data = comp_ctx->comp_data;
+    uint64 module_inst_mem_inst_size =
+        (uint64)comp_data->memory_count * sizeof(AOTMemoryInstance);
     LLVMValueRef offset;
 
     offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data.bytes)
+                       + module_inst_mem_inst_size
                        + comp_ctx->comp_data->global_data_size);
     func_ctx->table_base = LLVMBuildInBoundsGEP(comp_ctx->builder,
                                                 func_ctx->aot_inst,
@@ -544,6 +548,43 @@ create_func_type_indexes(AOTCompContext *comp_ctx,
     return true;
 }
 
+static bool
+create_func_ptrs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+    LLVMValueRef offset;
+
+    offset = I32_CONST(offsetof(AOTModuleInstance, func_ptrs));
+    func_ctx->func_ptrs = LLVMBuildInBoundsGEP(comp_ctx->builder,
+                                                func_ctx->aot_inst,
+                                                &offset, 1, "func_ptrs_offset");
+    if (!func_ctx->func_ptrs) {
+        aot_set_last_error("llvm build in bounds gep failed.");
+        return false;
+    }
+    func_ctx->func_ptrs = LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs,
+                                           comp_ctx->exec_env_type, "func_ptrs_tmp");
+    if (!func_ctx->func_ptrs) {
+        aot_set_last_error("llvm build bit cast failed.");
+        return false;
+    }
+
+    func_ctx->func_ptrs = LLVMBuildLoad(comp_ctx->builder, func_ctx->func_ptrs,
+                                        "func_ptrs_ptr");
+    if (!func_ctx->func_ptrs) {
+        aot_set_last_error("llvm build load failed.");
+        return false;
+    }
+
+    func_ctx->func_ptrs = LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs,
+                                           comp_ctx->exec_env_type, "func_ptrs");
+    if (!func_ctx->func_ptrs) {
+        aot_set_last_error("llvm build bit cast failed.");
+        return false;
+    }
+
+    return true;
+}
+
 /**
  * Create function compiler context
  */
@@ -746,6 +787,10 @@ aot_create_func_context(AOTCompData *comp_data, AOTCompContext *comp_ctx,
     if (!create_func_type_indexes(comp_ctx, func_ctx))
         goto fail;
 
+    /* Load function pointers */
+    if (!create_func_ptrs(comp_ctx, func_ctx))
+        goto fail;
+
     return func_ctx;
 
 fail:

core/iwasm/compilation/aot_llvm.h

@@ -122,6 +122,7 @@ typedef struct AOTFuncContext {
   LLVMValueRef argv_buf;
   LLVMValueRef native_stack_bound;
   LLVMValueRef last_alloca;
+  LLVMValueRef func_ptrs;
 
   AOTMemInfo *mem_info;
 
@@ -130,7 +131,6 @@ typedef struct AOTFuncContext {
   bool mem_space_unchanged;
   AOTCheckedAddrList checked_addr_list;
 
-  LLVMBasicBlockRef *exception_blocks;
   LLVMBasicBlockRef got_exception_block;
   LLVMBasicBlockRef func_return_block;
   LLVMValueRef exception_id_phi;

core/iwasm/interpreter/wasm.h

@@ -517,6 +517,19 @@ wasm_type_equal(const WASMType *type1, const WASMType *type2)
         ? true : false;
 }
 
+inline static uint32
+wasm_get_smallest_type_idx(WASMType **types, uint32 type_count,
+                           uint32 cur_type_idx)
+{
+    uint32 i;
+
+    for (i = 0; i < cur_type_idx; i++) {
+        if (wasm_type_equal(types[cur_type_idx], types[i]))
+            return i;
+    }
+    return cur_type_idx;
+}
+
 static inline uint32
 block_type_get_param_types(BlockType *block_type,
                            uint8 **p_param_types)

core/iwasm/interpreter/wasm_interp_fast.c

@@ -380,48 +380,11 @@ popcount64(uint64 u)
     return ret;
 }
 
-static uint64
-read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign)
-{
-    uint64 result = 0;
-    uint32 shift = 0;
-    uint32 bcnt = 0;
-    uint64 byte;
-
-    while (true) {
-        byte = buf[*p_offset];
-        *p_offset += 1;
-        result |= ((byte & 0x7f) << shift);
-        shift += 7;
-        if ((byte & 0x80) == 0) {
-            break;
-        }
-        bcnt += 1;
-    }
-    if (sign && (shift < maxbits) && (byte & 0x40)) {
-        /* Sign extend */
-        result |= - ((uint64)1 << shift);
-    }
-    return result;
-}
-
-#define read_leb_uint32(p, p_end, res) do {     \
-  uint8 _val = *p;                              \
-  if (!(_val & 0x80)) {                         \
-    res = _val;                                 \
-    p++;                                        \
-    break;                                      \
-  }                                             \
-  uint32 _off = 0;                              \
-  res = (uint32)read_leb(p, &_off, 32, false);  \
-  p += _off;                                    \
-} while (0)
-
 #define read_uint32(p) (p += sizeof(uint32), *(uint32 *)(p - sizeof(uint32)))
 
 #define GET_LOCAL_INDEX_TYPE_AND_OFFSET() do {                      \
     uint32 param_count = cur_func->param_count;                     \
-    read_leb_uint32(frame_ip, frame_ip_end, local_idx);             \
+    local_idx = read_uint32(frame_ip);                              \
     bh_assert(local_idx < param_count + cur_func->local_count);     \
     local_offset = cur_func->local_offsets[local_idx];              \
     if (local_idx < param_count)                                    \

core/iwasm/interpreter/wasm_loader.c

@@ -791,6 +791,12 @@ load_function_import(const WASMModule *parent_module, WASMModule *sub_module,
         return false;
     }
 
+#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0)
+    declare_type_index = wasm_get_smallest_type_idx(
+            parent_module->types, parent_module->type_count,
+            declare_type_index);
+#endif
+
     declare_func_type = parent_module->types[declare_type_index];
 
     if (wasm_runtime_is_host_module(sub_module_name)) {
@@ -1724,6 +1730,11 @@ load_function_section(const uint8 *buf, const uint8 *buf_end,
                 return false;
             }
 
+#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0)
+            type_index = wasm_get_smallest_type_idx(
+                    module->types, module->type_count, type_index);
+#endif
+
             read_leb_uint32(p_code, buf_code_end, code_size);
             if (code_size == 0
                 || p_code + code_size > buf_code_end) {
@@ -4312,10 +4323,6 @@ fail:
     LOG_OP("%d\t", value);                                          \
   } while (0)
 
-#define emit_leb() do {                                             \
-    wasm_loader_emit_leb(loader_ctx, p_org, p);                     \
-  } while (0)
-
 static bool
 wasm_loader_ctx_reinit(WASMLoaderContext *ctx)
 {
@@ -4402,21 +4409,6 @@ wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size)
         ctx->code_compiled_size -= size;
 }
 
-static void
-wasm_loader_emit_leb(WASMLoaderContext *ctx, uint8* start, uint8* end)
-{
-    if (ctx->p_code_compiled) {
-        bh_memcpy_s(ctx->p_code_compiled,
-                    ctx->p_code_compiled_end - ctx->p_code_compiled,
-                    start, end - start);
-        ctx->p_code_compiled += (end - start);
-    }
-    else {
-        ctx->code_compiled_size += (end - start);
-    }
-
-}
-
 static bool
 preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode,
                           uint32 local_index, uint32 local_type, bool *preserved,
@@ -6497,8 +6489,7 @@ handle_op_block_and_loop:
                     }
                 }
                 else {   /* local index larger than 255, reserve leb */
-                    p_org ++;
-                    emit_leb();
+                    emit_uint32(loader_ctx, local_idx);
                     POP_OFFSET_TYPE(local_type);
                 }
 #else
@@ -6554,11 +6545,10 @@ handle_op_block_and_loop:
                     }
                 }
                 else {  /* local index larger than 255, reserve leb */
-                    p_org ++;
-                    emit_leb();
+                    emit_uint32(loader_ctx, local_idx);
                 }
                 emit_operand(loader_ctx, *(loader_ctx->frame_offset -
-                        wasm_value_type_cell_num(local_type)));
+                                           wasm_value_type_cell_num(local_type)));
 #else
 #if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0)
                 if (local_offset < 0x80) {

core/iwasm/interpreter/wasm_mini_loader.c

@@ -3062,10 +3062,6 @@ wasm_loader_pop_frame_csp(WASMLoaderContext *ctx,
     LOG_OP("%d\t", value);                                          \
   } while (0)
 
-#define emit_leb() do {                                             \
-    wasm_loader_emit_leb(loader_ctx, p_org, p);                     \
-  } while (0)
-
 static bool
 wasm_loader_ctx_reinit(WASMLoaderContext *ctx)
 {
@@ -3152,21 +3148,6 @@ wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size)
         ctx->code_compiled_size -= size;
 }
 
-static void
-wasm_loader_emit_leb(WASMLoaderContext *ctx, uint8* start, uint8* end)
-{
-    if (ctx->p_code_compiled) {
-        bh_memcpy_s(ctx->p_code_compiled,
-                    ctx->p_code_compiled_end - ctx->p_code_compiled,
-                    start, end - start);
-        ctx->p_code_compiled += (end - start);
-    }
-    else {
-        ctx->code_compiled_size += (end - start);
-    }
-
-}
-
 static bool
 preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode,
                           uint32 local_index, uint32 local_type, bool *preserved,
@@ -4970,8 +4951,7 @@ handle_op_block_and_loop:
                     }
                 }
                 else {   /* local index larger than 255, reserve leb */
-                    p_org ++;
-                    emit_leb();
+                    emit_uint32(loader_ctx, local_idx);
                     POP_OFFSET_TYPE(local_type);
                 }
 #else
@@ -5027,11 +5007,10 @@ handle_op_block_and_loop:
                     }
                 }
                 else {  /* local index larger than 255, reserve leb */
-                    p_org ++;
-                    emit_leb();
+                    emit_uint32(loader_ctx, local_idx);
                 }
                 emit_operand(loader_ctx, *(loader_ctx->frame_offset -
-                        wasm_value_type_cell_num(local_type)));
+                                           wasm_value_type_cell_num(local_type)));
 #else
 #if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0)
                 if (local_offset < 0x80) {

core/iwasm/interpreter/wasm_runtime.c

@@ -1688,7 +1688,17 @@ wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size,
     }
 
     if (!addr) {
-        wasm_set_exception(module_inst, "out of memory");
+        if (memory->heap_handle
+            && mem_allocator_is_heap_corrupted(memory->heap_handle)) {
+            LOG_ERROR("Error: app heap is corrupted, if the wasm file "
+                      "is compiled by wasi-sdk-12.0 or larger version, "
+                      "please add -Wl,--export=malloc -Wl,--export=free "
+                      " to export malloc and free functions.");
+            wasm_set_exception(module_inst, "app heap corrupted");
+        }
+        else {
+            wasm_set_exception(module_inst, "out of memory");
+        }
         return 0;
     }
     if (p_native_addr)

core/shared/mem-alloc/ems/ems_gc.h

@@ -109,12 +109,14 @@ gc_migrate(gc_handle_t handle,
            char *pool_buf_new, gc_size_t pool_buf_size);
 
 /**
- * Destroy lock of heap
+ * Check whether the heap is corrupted
  *
- * @param handle the heap handle
+ * @param handle handle of the heap
+ *
+ * @return true if success, false otherwise
  */
-void
-gc_destroy_lock(gc_handle_t handle);
+bool
+gc_is_heap_corrupted(gc_handle_t handle);
 
 /**
  * Get Heap Stats

core/shared/mem-alloc/ems/ems_kfc.c

@@ -204,11 +204,12 @@ gc_migrate(gc_handle_t handle,
     return 0;
 }
 
-void
-gc_destroy_lock(gc_handle_t handle)
+bool
+gc_is_heap_corrupted(gc_handle_t handle)
 {
-    gc_heap_t *heap = (gc_heap_t *) handle;
-    os_mutex_destroy(&heap->lock);
+    gc_heap_t *heap = (gc_heap_t *)handle;
+
+    return heap->is_heap_corrupted ? true : false;
 }
 
 #if BH_ENABLE_GC_VERIFY != 0

core/shared/mem-alloc/mem_alloc.c

@@ -63,10 +63,10 @@ mem_allocator_migrate(mem_allocator_t allocator,
                       pool_buf_new, pool_buf_size);
 }
 
-void
-mem_allocator_destroy_lock(mem_allocator_t allocator)
+bool
+mem_allocator_is_heap_corrupted(mem_allocator_t allocator)
 {
-    gc_destroy_lock((gc_handle_t) allocator);
+    return gc_is_heap_corrupted((gc_handle_t) allocator);
 }
 
 #else /* else of DEFAULT_MEM_ALLOCATOR */
@@ -181,19 +181,5 @@ mem_allocator_migrate(mem_allocator_t allocator,
                         (mem_allocator_tlsf *) allocator_old);
 }
 
-int
-mem_allocator_init_lock(mem_allocator_t allocator)
-{
-    mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator;
-    return os_mutex_init(&allocator_tlsf->lock);
-}
-
-void
-mem_allocator_destroy_lock(mem_allocator_t allocator)
-{
-    mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator;
-    os_mutex_destroy(&allocator_tlsf->lock);
-}
-
 #endif /* end of DEFAULT_MEM_ALLOCATOR */
 

core/shared/mem-alloc/mem_alloc.h

@@ -42,8 +42,8 @@ int
 mem_allocator_migrate(mem_allocator_t allocator,
                       char *pool_buf_new, uint32 pool_buf_size);
 
-void
-mem_allocator_destroy_lock(mem_allocator_t allocator);
+bool
+mem_allocator_is_heap_corrupted(mem_allocator_t allocator);
 
 #ifdef __cplusplus
 }