Refine Fast JIT call indirect and call native process (#1620)

Translate call_indirect opcode by calling wasm functions with Fast JIT IRs instead of
calling jit_call_indirect runtime API, so as to improve the performance.

Translate call native function process with Fast JIT IRs to validate each pointer argument
and convert it into native address, and then call the native function directly instead
of calling jit_invoke_native runtime API, so as to improve the performance.

core/iwasm/compilation/aot_emit_table.c

@@ -15,18 +15,10 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx,
     AOTImportTable *imp_tbls = comp_ctx->comp_data->import_tables;
     AOTTable *tbls = comp_ctx->comp_data->tables;
 
-    if (comp_ctx->is_jit_mode) {
-        offset = offsetof(WASMModuleInstance, global_table_data.bytes)
-                 + (uint64)comp_ctx->comp_data->memory_count
-                       * sizeof(AOTMemoryInstance)
-                 + comp_ctx->comp_data->global_data_size;
-    }
-    else {
-        offset = offsetof(AOTModuleInstance, global_table_data.bytes)
-                 + (uint64)comp_ctx->comp_data->memory_count
-                       * sizeof(AOTMemoryInstance)
-                 + comp_ctx->comp_data->global_data_size;
-    }
+    offset =
+        offsetof(AOTModuleInstance, global_table_data.bytes)
+        + (uint64)comp_ctx->comp_data->memory_count * sizeof(AOTMemoryInstance)
+        + comp_ctx->comp_data->global_data_size;
 
     while (i < tbl_idx && i < comp_ctx->comp_data->import_table_count) {
         offset += offsetof(AOTTableInstance, elems);

core/iwasm/compilation/aot_emit_variable.c

@@ -120,16 +120,9 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     LLVMValueRef offset, global_ptr, global, res;
     LLVMTypeRef ptr_type = NULL;
 
-    if (comp_ctx->is_jit_mode) {
-        global_base_offset =
-            offsetof(WASMModuleInstance, global_table_data.bytes)
-            + sizeof(WASMMemoryInstance) * comp_ctx->comp_data->memory_count;
-    }
-    else {
-        global_base_offset =
-            offsetof(AOTModuleInstance, global_table_data.bytes)
-            + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count;
-    }
+    global_base_offset =
+        offsetof(AOTModuleInstance, global_table_data.bytes)
+        + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count;
 
     bh_assert(global_idx < import_global_count + comp_data->global_count);
 

core/iwasm/fast-jit/fe/jit_emit_function.c

@@ -9,6 +9,10 @@
 #include "../jit_codegen.h"
 #include "../../interpreter/wasm_runtime.h"
 
+static bool
+emit_callnative(JitCompContext *cc, JitReg native_func_reg, JitReg res,
+                JitReg *params, uint32 param_count);
+
 /* Prepare parameters for the function to call */
 static bool
 pre_call(JitCompContext *cc, const WASMType *func_type)
@@ -62,7 +66,8 @@ fail:
 
 /* Push results */
 static bool
-post_return(JitCompContext *cc, const WASMType *func_type, JitReg first_res)
+post_return(JitCompContext *cc, const WASMType *func_type, JitReg first_res,
+            bool update_committed_sp)
 {
     uint32 i, n;
     JitReg value;
@@ -132,14 +137,82 @@ post_return(JitCompContext *cc, const WASMType *func_type, JitReg first_res)
         }
     }
 
-    /* Update the committed_sp as the callee has updated the frame sp */
-    cc->jit_frame->committed_sp = cc->jit_frame->sp;
+    if (update_committed_sp)
+        /* Update the committed_sp as the callee has updated the frame sp */
+        cc->jit_frame->committed_sp = cc->jit_frame->sp;
 
     return true;
 fail:
     return false;
 }
 
+static bool
+pre_load(JitCompContext *cc, JitReg *argvs, const WASMType *func_type)
+{
+    JitReg value;
+    uint32 i;
+
+    /* Prepare parameters for the function to call */
+    for (i = 0; i < func_type->param_count; i++) {
+        switch (func_type->types[func_type->param_count - 1 - i]) {
+            case VALUE_TYPE_I32:
+#if WASM_ENABLE_REF_TYPES != 0
+            case VALUE_TYPE_EXTERNREF:
+            case VALUE_TYPE_FUNCREF:
+#endif
+                POP_I32(value);
+                argvs[func_type->param_count - 1 - i] = value;
+                break;
+            case VALUE_TYPE_I64:
+                POP_I64(value);
+                argvs[func_type->param_count - 1 - i] = value;
+                break;
+            case VALUE_TYPE_F32:
+                POP_F32(value);
+                argvs[func_type->param_count - 1 - i] = value;
+                break;
+            case VALUE_TYPE_F64:
+                POP_F64(value);
+                argvs[func_type->param_count - 1 - i] = value;
+                break;
+            default:
+                bh_assert(0);
+                goto fail;
+        }
+    }
+
+    gen_commit_sp_ip(cc->jit_frame);
+
+    return true;
+fail:
+    return false;
+}
+
+static JitReg
+create_first_res_reg(JitCompContext *cc, const WASMType *func_type)
+{
+    if (func_type->result_count) {
+        switch (func_type->types[func_type->param_count]) {
+            case VALUE_TYPE_I32:
+#if WASM_ENABLE_REF_TYPES != 0
+            case VALUE_TYPE_EXTERNREF:
+            case VALUE_TYPE_FUNCREF:
+#endif
+                return jit_cc_new_reg_I32(cc);
+            case VALUE_TYPE_I64:
+                return jit_cc_new_reg_I64(cc);
+            case VALUE_TYPE_F32:
+                return jit_cc_new_reg_F32(cc);
+            case VALUE_TYPE_F64:
+                return jit_cc_new_reg_F64(cc);
+            default:
+                bh_assert(0);
+                return 0;
+        }
+    }
+    return 0;
+}
+
 bool
 jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
 {
@@ -148,89 +221,197 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
     WASMFunction *func;
     WASMType *func_type;
     JitFrame *jit_frame = cc->jit_frame;
-    JitReg native_ret;
     JitReg fast_jit_func_ptrs, jitted_code = 0;
-    uint32 jitted_func_idx;
-
-    if (func_idx >= wasm_module->import_function_count) {
-        fast_jit_func_ptrs = get_fast_jit_func_ptrs_reg(jit_frame);
-        jitted_code = jit_cc_new_reg_ptr(cc);
-        /* jitted_code = func_ptrs[func_idx - import_function_count] */
-        jitted_func_idx = func_idx - wasm_module->import_function_count;
-        GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs,
-                 NEW_CONST(I32, (uint32)sizeof(void *) * jitted_func_idx));
-    }
+    JitReg native_func, *argvs = NULL, *argvs1 = NULL, func_params[5];
+    JitReg native_addr_ptr, module_inst_reg, ret, res;
+    uint32 jitted_func_idx, i;
+    uint64 total_size;
+    const char *signature = NULL;
+    /* Whether the argument is a pointer/str argument and
+       need to call jit_check_app_addr_and_convert */
+    bool is_pointer_arg;
+    bool return_value = false;
 
     if (func_idx < wasm_module->import_function_count) {
+        /* The function to call is an import function */
         func_import = &wasm_module->import_functions[func_idx].u.function;
         func_type = func_import->func_type;
-    }
-    else {
-        func = wasm_module
-                   ->functions[func_idx - wasm_module->import_function_count];
-        func_type = func->func_type;
-    }
 
-    if (!pre_call(cc, func_type)) {
-        goto fail;
-    }
+        /* Call fast_jit_invoke_native in some cases */
+        if (!func_import->func_ptr_linked /* import func hasn't been linked */
+            || func_import->call_conv_wasm_c_api /* linked by wasm_c_api */
+            || func_import->call_conv_raw /* registered as raw mode */
+            || func_type->param_count >= 5 /* registered as normal mode, but
+                                              jit_emit_callnative only supports
+                                              maximum 6 registers now
+                                              (include exec_nev) */) {
+            JitReg arg_regs[3];
+
+            if (!pre_call(cc, func_type)) {
+                goto fail;
+            }
 
-    if (func_idx < wasm_module->import_function_count) {
-        JitReg arg_regs[3];
+            /* Call fast_jit_invoke_native */
+            ret = jit_cc_new_reg_I32(cc);
+            arg_regs[0] = cc->exec_env_reg;
+            arg_regs[1] = NEW_CONST(I32, func_idx);
+            arg_regs[2] = cc->fp_reg;
+            if (!jit_emit_callnative(cc, fast_jit_invoke_native, ret, arg_regs,
+                                     3)) {
+                goto fail;
+            }
+
+            /* Convert the return value from bool to uint32 */
+            GEN_INSN(AND, ret, ret, NEW_CONST(I32, 0xFF));
+
+            /* Check whether there is exception thrown */
+            GEN_INSN(CMP, cc->cmp_reg, ret, NEW_CONST(I32, 0));
+            if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ,
+                                    cc->cmp_reg, NULL)) {
+                goto fail;
+            }
+
+            if (!post_return(cc, func_type, 0, true)) {
+                goto fail;
+            }
+
+            return true;
+        }
+
+        /* Import function was registered as normal mode, and its argument count
+           is no more than 5, we directly call it */
+
+        signature = func_import->signature;
+        bh_assert(signature);
+
+        /* Allocate memory for argvs*/
+        total_size = sizeof(JitReg) * (uint64)(func_type->param_count);
+        if (total_size > 0) {
+            if (total_size >= UINT32_MAX
+                || !(argvs = jit_malloc((uint32)total_size))) {
+                goto fail;
+            }
+        }
+
+        /* Pop function params from stack and store them into argvs */
+        if (!pre_load(cc, argvs, func_type)) {
+            goto fail;
+        }
+
+        ret = jit_cc_new_reg_I32(cc);
+        func_params[0] = module_inst_reg = get_module_inst_reg(jit_frame);
+        func_params[4] = native_addr_ptr = jit_cc_new_reg_ptr(cc);
+        GEN_INSN(ADD, native_addr_ptr, cc->exec_env_reg,
+                 NEW_CONST(PTR, offsetof(WASMExecEnv, jit_cache)));
+
+        /* Traverse each pointer/str argument, call
+           jit_check_app_addr_and_convert to check whether it is
+           in the range of linear memory and and convert it from
+           app offset into native address */
+        for (i = 0; i < func_type->param_count; i++) {
+
+            is_pointer_arg = false;
+
+            if (signature[i + 1] == '*') {
+                /* param is a pointer */
+                is_pointer_arg = true;
+                func_params[1] = NEW_CONST(I32, false); /* is_str = false */
+                func_params[2] = argvs[i];
+                if (signature[i + 2] == '~') {
+                    /* pointer with length followed */
+                    func_params[3] = argvs[i + 1];
+                }
+                else {
+                    /* pointer with length followed */
+                    func_params[3] = NEW_CONST(I32, 1);
+                }
+            }
+            else if (signature[i + 1] == '$') {
+                /* param is a string */
+                is_pointer_arg = true;
+                func_params[1] = NEW_CONST(I32, true); /* is_str = true */
+                func_params[2] = argvs[i];
+                func_params[3] = NEW_CONST(I32, 1);
+            }
+
+            if (is_pointer_arg) {
+                if (!jit_emit_callnative(cc, jit_check_app_addr_and_convert,
+                                         ret, func_params, 5)) {
+                    goto fail;
+                }
+
+                /* Convert the return value from bool to uint32 */
+                GEN_INSN(AND, ret, ret, NEW_CONST(I32, 0xFF));
+                /* Check whether there is exception thrown */
+                GEN_INSN(CMP, cc->cmp_reg, ret, NEW_CONST(I32, 0));
+                if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ,
+                                        cc->cmp_reg, NULL)) {
+                    return false;
+                }
 
-        native_ret = jit_cc_new_reg_I32(cc);
-        arg_regs[0] = cc->exec_env_reg;
-        arg_regs[1] = NEW_CONST(I32, func_idx);
-        arg_regs[2] = cc->fp_reg;
+                /* Load native addr from pointer of native addr,
+                   or exec_env->jit_cache */
+                argvs[i] = jit_cc_new_reg_ptr(cc);
+                GEN_INSN(LDPTR, argvs[i], native_addr_ptr, NEW_CONST(I32, 0));
+            }
+        }
+
+        res = create_first_res_reg(cc, func_type);
 
-        if (!jit_emit_callnative(cc, fast_jit_invoke_native, native_ret,
-                                 arg_regs, 3)) {
-            return false;
+        /* Prepare arguments of the native function */
+        if (!(argvs1 =
+                  jit_calloc(sizeof(JitReg) * (func_type->param_count + 1)))) {
+            goto fail;
+        }
+        argvs1[0] = cc->exec_env_reg;
+        for (i = 0; i < func_type->param_count; i++) {
+            argvs1[i + 1] = argvs[i];
         }
-        /* Convert bool to uint32 */
-        GEN_INSN(AND, native_ret, native_ret, NEW_CONST(I32, 0xFF));
+
+        /* Call the native function */
+        native_func = NEW_CONST(PTR, (uintptr_t)func_import->func_ptr_linked);
+        if (!emit_callnative(cc, native_func, res, argvs1,
+                             func_type->param_count + 1)) {
+            jit_free(argvs1);
+            goto fail;
+        }
+        jit_free(argvs1);
 
         /* Check whether there is exception thrown */
-        GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0));
-        if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BEQ,
+        GEN_INSN(LDI8, ret, module_inst_reg,
+                 NEW_CONST(I32, offsetof(WASMModuleInstance, cur_exception)));
+        GEN_INSN(CMP, cc->cmp_reg, ret, NEW_CONST(I32, 0));
+        if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BNE,
                                 cc->cmp_reg, NULL)) {
-            return false;
+            goto fail;
         }
 
-        if (!post_return(cc, func_type, 0)) {
+        if (!post_return(cc, func_type, res, false)) {
             goto fail;
         }
     }
     else {
-        JitReg res = 0;
+        /* The function to call is a bytecode function */
+        func = wasm_module
+                   ->functions[func_idx - wasm_module->import_function_count];
+        func_type = func->func_type;
 
-        if (func_type->result_count > 0) {
-            switch (func_type->types[func_type->param_count]) {
-                case VALUE_TYPE_I32:
-#if WASM_ENABLE_REF_TYPES != 0
-                case VALUE_TYPE_EXTERNREF:
-                case VALUE_TYPE_FUNCREF:
-#endif
-                    res = jit_cc_new_reg_I32(cc);
-                    break;
-                case VALUE_TYPE_I64:
-                    res = jit_cc_new_reg_I64(cc);
-                    break;
-                case VALUE_TYPE_F32:
-                    res = jit_cc_new_reg_F32(cc);
-                    break;
-                case VALUE_TYPE_F64:
-                    res = jit_cc_new_reg_F64(cc);
-                    break;
-                default:
-                    bh_assert(0);
-                    goto fail;
-            }
+        /* jitted_code = func_ptrs[func_idx - import_function_count] */
+        fast_jit_func_ptrs = get_fast_jit_func_ptrs_reg(jit_frame);
+        jitted_code = jit_cc_new_reg_ptr(cc);
+        jitted_func_idx = func_idx - wasm_module->import_function_count;
+        GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs,
+                 NEW_CONST(I32, (uint32)sizeof(void *) * jitted_func_idx));
+
+        if (!pre_call(cc, func_type)) {
+            goto fail;
         }
 
+        res = create_first_res_reg(cc, func_type);
+
         GEN_INSN(CALLBC, res, 0, jitted_code);
 
-        if (!post_return(cc, func_type, res)) {
+        if (!post_return(cc, func_type, res, true)) {
             goto fail;
         }
     }
@@ -243,9 +424,14 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
 
     /* Ignore tail call currently */
     (void)tail_call;
-    return true;
+
+    return_value = true;
+
 fail:
-    return false;
+    if (argvs)
+        jit_free(argvs);
+
+    return return_value;
 }
 
 static JitReg
@@ -264,81 +450,123 @@ pack_argv(JitCompContext *cc)
     return argv;
 }
 
-static bool
-unpack_argv(JitCompContext *cc, const WASMType *func_type, JitReg argv)
-{
-    uint32 i, offset_by_cell = 0;
-    JitReg value;
-
-    /* push results in argv to stack */
-    for (i = 0; i < func_type->result_count; i++) {
-        switch (func_type->types[func_type->param_count + i]) {
-            case VALUE_TYPE_I32:
-#if WASM_ENABLE_REF_TYPES != 0
-            case VALUE_TYPE_EXTERNREF:
-            case VALUE_TYPE_FUNCREF:
-#endif
-            {
-                value = jit_cc_new_reg_I32(cc);
-                GEN_INSN(LDI32, value, argv, NEW_CONST(I32, offset_by_cell));
-                PUSH_I32(value);
-                offset_by_cell += 4;
-                break;
-            }
-            case VALUE_TYPE_I64:
-            {
-                value = jit_cc_new_reg_I64(cc);
-                GEN_INSN(LDI64, value, argv, NEW_CONST(I32, offset_by_cell));
-                PUSH_I64(value);
-                offset_by_cell += 8;
-                break;
-            }
-            case VALUE_TYPE_F32:
-            {
-                value = jit_cc_new_reg_F32(cc);
-                GEN_INSN(LDF32, value, argv, NEW_CONST(I32, offset_by_cell));
-                PUSH_F32(value);
-                offset_by_cell += 4;
-                break;
-            }
-            case VALUE_TYPE_F64:
-            {
-                value = jit_cc_new_reg_F64(cc);
-                GEN_INSN(LDF64, value, argv, NEW_CONST(I32, offset_by_cell));
-                PUSH_F64(value);
-                offset_by_cell += 8;
-                break;
-            }
-            default:
-            {
-                bh_assert(0);
-                goto fail;
-            }
-        }
-    }
-
-    /* Update the committed_sp as the callee has updated the frame sp */
-    cc->jit_frame->committed_sp = cc->jit_frame->sp;
-
-    return true;
-fail:
-    return false;
-}
-
 bool
 jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
                              uint32 tbl_idx)
 {
+    WASMModule *wasm_module = cc->cur_wasm_module;
+    JitBasicBlock *block_import, *block_nonimport, *func_return;
     JitReg elem_idx, native_ret, argv, arg_regs[6];
+    JitFrame *jit_frame = cc->jit_frame;
+    JitReg tbl_size, offset, offset_i32;
+    JitReg func_import, func_idx, tbl_data, func_count;
+    JitReg func_type_indexes, func_type_idx, fast_jit_func_ptrs;
+    JitReg offset1_i32, offset1, func_type_idx1, res;
+    JitReg import_func_ptrs, jitted_code_idx, jitted_code;
     WASMType *func_type;
+    uint32 n;
 
     POP_I32(elem_idx);
 
-    func_type = cc->cur_wasm_module->types[type_idx];
+    /* check elem_idx */
+    tbl_size = get_table_cur_size_reg(jit_frame, tbl_idx);
+
+    GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_size);
+    if (!jit_emit_exception(cc, EXCE_UNDEFINED_ELEMENT, JIT_OP_BGEU,
+                            cc->cmp_reg, NULL))
+        goto fail;
+
+    /* check func_idx */
+    if (UINTPTR_MAX == UINT64_MAX) {
+        offset_i32 = jit_cc_new_reg_I32(cc);
+        offset = jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHL, offset_i32, elem_idx, NEW_CONST(I32, 2));
+        GEN_INSN(I32TOI64, offset, offset_i32);
+    }
+    else {
+        offset = jit_cc_new_reg_I32(cc);
+        GEN_INSN(SHL, offset, elem_idx, NEW_CONST(I32, 2));
+    }
+    func_idx = jit_cc_new_reg_I32(cc);
+    tbl_data = get_table_data_reg(jit_frame, tbl_idx);
+    GEN_INSN(LDI32, func_idx, tbl_data, offset);
+
+    GEN_INSN(CMP, cc->cmp_reg, func_idx, NEW_CONST(I32, -1));
+    if (!jit_emit_exception(cc, EXCE_UNINITIALIZED_ELEMENT, JIT_OP_BEQ,
+                            cc->cmp_reg, NULL))
+        goto fail;
+
+    func_count = NEW_CONST(I32, wasm_module->import_function_count
+                                    + wasm_module->function_count);
+    GEN_INSN(CMP, cc->cmp_reg, func_idx, func_count);
+    if (!jit_emit_exception(cc, EXCE_INVALID_FUNCTION_INDEX, JIT_OP_BGTU,
+                            cc->cmp_reg, NULL))
+        goto fail;
+
+    /* check func_type */
+    /* get func_type_idx from func_type_indexes */
+    if (UINTPTR_MAX == UINT64_MAX) {
+        offset1_i32 = jit_cc_new_reg_I32(cc);
+        offset1 = jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHL, offset1_i32, func_idx, NEW_CONST(I32, 2));
+        GEN_INSN(I32TOI64, offset1, offset1_i32);
+    }
+    else {
+        offset1 = jit_cc_new_reg_I32(cc);
+        GEN_INSN(SHL, offset1, func_idx, NEW_CONST(I32, 2));
+    }
+
+    func_type_indexes = get_func_type_indexes_reg(jit_frame);
+    func_type_idx = jit_cc_new_reg_I32(cc);
+    GEN_INSN(LDI32, func_type_idx, func_type_indexes, offset1);
+
+    type_idx = wasm_get_smallest_type_idx(wasm_module->types,
+                                          wasm_module->type_count, type_idx);
+    func_type_idx1 = NEW_CONST(I32, type_idx);
+    GEN_INSN(CMP, cc->cmp_reg, func_type_idx, func_type_idx1);
+    if (!jit_emit_exception(cc, EXCE_INVALID_FUNCTION_TYPE_INDEX, JIT_OP_BNE,
+                            cc->cmp_reg, NULL))
+        goto fail;
+
+    /* pop function arguments and store it to out area of callee stack frame */
+    func_type = wasm_module->types[type_idx];
     if (!pre_call(cc, func_type)) {
         goto fail;
     }
 
+    /* store elem_idx and func_idx to exec_env->jit_cache */
+    GEN_INSN(STI32, elem_idx, cc->exec_env_reg,
+             NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache)));
+    GEN_INSN(STI32, func_idx, cc->exec_env_reg,
+             NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4));
+
+    block_import = jit_cc_new_basic_block(cc, 0);
+    block_nonimport = jit_cc_new_basic_block(cc, 0);
+    func_return = jit_cc_new_basic_block(cc, 0);
+    if (!block_import || !block_nonimport || !func_return) {
+        goto fail;
+    }
+
+    /* Commit register values to locals and stacks */
+    gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp);
+    /* Clear frame values */
+    clear_values(jit_frame);
+
+    /* jump to block_import or block_nonimport */
+    GEN_INSN(CMP, cc->cmp_reg, func_idx,
+             NEW_CONST(I32, cc->cur_wasm_module->import_function_count));
+    GEN_INSN(BLTU, cc->cmp_reg, jit_basic_block_label(block_import),
+             jit_basic_block_label(block_nonimport));
+
+    /* block_import */
+    cc->cur_basic_block = block_import;
+
+    elem_idx = jit_cc_new_reg_I32(cc);
+    GEN_INSN(LDI32, elem_idx, cc->exec_env_reg,
+             NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache)));
+    GEN_INSN(LDI32, func_idx, cc->exec_env_reg,
+             NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4));
+
     argv = pack_argv(cc);
     if (!argv) {
         goto fail;
@@ -351,10 +579,25 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
     arg_regs[4] = NEW_CONST(I32, func_type->param_cell_num);
     arg_regs[5] = argv;
 
+    import_func_ptrs = get_import_func_ptrs_reg(jit_frame);
+    func_import = jit_cc_new_reg_ptr(cc);
+    if (UINTPTR_MAX == UINT64_MAX) {
+        JitReg func_import_offset = jit_cc_new_reg_I32(cc);
+        JitReg func_import_offset_i64 = jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHL, func_import_offset, func_idx, NEW_CONST(I32, 3));
+        GEN_INSN(I32TOI64, func_import_offset_i64, func_import_offset);
+        GEN_INSN(LDPTR, func_import, import_func_ptrs, func_import_offset_i64);
+    }
+    else {
+        JitReg func_import_offset = jit_cc_new_reg_I32(cc);
+        GEN_INSN(SHL, func_import_offset, func_idx, NEW_CONST(I32, 2));
+        GEN_INSN(LDPTR, func_import, import_func_ptrs, func_import_offset);
+    }
     if (!jit_emit_callnative(cc, fast_jit_call_indirect, native_ret, arg_regs,
                              6)) {
-        return false;
+        goto fail;
     }
+
     /* Convert bool to uint32 */
     GEN_INSN(AND, native_ret, native_ret, NEW_CONST(I32, 0xFF));
 
@@ -365,7 +608,137 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
         return false;
     }
 
-    if (!unpack_argv(cc, func_type, argv)) {
+    /* Store res into current frame, so that post_return in
+        block func_return can get the value */
+    n = cc->jit_frame->sp - cc->jit_frame->lp;
+    if (func_type->result_count > 0) {
+        switch (func_type->types[func_type->param_count]) {
+            case VALUE_TYPE_I32:
+#if WASM_ENABLE_REF_TYPES != 0
+            case VALUE_TYPE_EXTERNREF:
+            case VALUE_TYPE_FUNCREF:
+#endif
+                res = jit_cc_new_reg_I32(cc);
+                GEN_INSN(LDI32, res, argv, NEW_CONST(I32, 0));
+                GEN_INSN(STI32, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            case VALUE_TYPE_I64:
+                res = jit_cc_new_reg_I32(cc);
+                GEN_INSN(LDI64, res, argv, NEW_CONST(I32, 0));
+                GEN_INSN(STI64, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            case VALUE_TYPE_F32:
+                res = jit_cc_new_reg_I32(cc);
+                GEN_INSN(LDF32, res, argv, NEW_CONST(I32, 0));
+                GEN_INSN(STF32, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            case VALUE_TYPE_F64:
+                res = jit_cc_new_reg_I32(cc);
+                GEN_INSN(LDF64, res, argv, NEW_CONST(I32, 0));
+                GEN_INSN(STF64, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            default:
+                bh_assert(0);
+                goto fail;
+        }
+    }
+
+    gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp);
+    clear_values(jit_frame);
+    GEN_INSN(JMP, jit_basic_block_label(func_return));
+
+    /* basic_block non_import */
+    cc->cur_basic_block = block_nonimport;
+
+    GEN_INSN(LDI32, func_idx, cc->exec_env_reg,
+             NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4));
+
+    /* get jitted_code */
+    fast_jit_func_ptrs = get_fast_jit_func_ptrs_reg(jit_frame);
+    jitted_code_idx = jit_cc_new_reg_I32(cc);
+    jitted_code = jit_cc_new_reg_ptr(cc);
+    GEN_INSN(SUB, jitted_code_idx, func_idx,
+             NEW_CONST(I32, cc->cur_wasm_module->import_function_count));
+    if (UINTPTR_MAX == UINT64_MAX) {
+        JitReg jitted_code_offset = jit_cc_new_reg_I32(cc);
+        JitReg jitted_code_offset_64 = jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHL, jitted_code_offset, jitted_code_idx, NEW_CONST(I32, 3));
+        GEN_INSN(I32TOI64, jitted_code_offset_64, jitted_code_offset);
+        GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs, jitted_code_offset_64);
+    }
+    else {
+        JitReg jitted_code_offset = jit_cc_new_reg_I32(cc);
+        GEN_INSN(SHL, jitted_code_offset, jitted_code_idx, NEW_CONST(I32, 2));
+        GEN_INSN(LDPTR, jitted_code, fast_jit_func_ptrs, jitted_code_offset);
+    }
+
+    res = 0;
+    if (func_type->result_count > 0) {
+        switch (func_type->types[func_type->param_count]) {
+            case VALUE_TYPE_I32:
+#if WASM_ENABLE_REF_TYPES != 0
+            case VALUE_TYPE_EXTERNREF:
+            case VALUE_TYPE_FUNCREF:
+#endif
+                res = jit_cc_new_reg_I32(cc);
+                break;
+            case VALUE_TYPE_I64:
+                res = jit_cc_new_reg_I64(cc);
+                break;
+            case VALUE_TYPE_F32:
+                res = jit_cc_new_reg_F32(cc);
+                break;
+            case VALUE_TYPE_F64:
+                res = jit_cc_new_reg_F64(cc);
+                break;
+            default:
+                bh_assert(0);
+                goto fail;
+        }
+    }
+    GEN_INSN(CALLBC, res, 0, jitted_code);
+    /* Store res into current frame, so that post_return in
+        block func_return can get the value */
+    n = cc->jit_frame->sp - cc->jit_frame->lp;
+    if (func_type->result_count > 0) {
+        switch (func_type->types[func_type->param_count]) {
+            case VALUE_TYPE_I32:
+#if WASM_ENABLE_REF_TYPES != 0
+            case VALUE_TYPE_EXTERNREF:
+            case VALUE_TYPE_FUNCREF:
+#endif
+                GEN_INSN(STI32, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            case VALUE_TYPE_I64:
+                GEN_INSN(STI64, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            case VALUE_TYPE_F32:
+                GEN_INSN(STF32, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            case VALUE_TYPE_F64:
+                GEN_INSN(STF64, res, cc->fp_reg,
+                         NEW_CONST(I32, offset_of_local(n)));
+                break;
+            default:
+                bh_assert(0);
+                goto fail;
+        }
+    }
+    /* commit and clear jit frame, then jump to block func_ret */
+    gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp);
+    clear_values(jit_frame);
+    GEN_INSN(JMP, jit_basic_block_label(func_return));
+
+    /* translate block func_return */
+    cc->cur_basic_block = func_return;
+    if (!post_return(cc, func_type, 0, true)) {
         goto fail;
     }
 
@@ -418,9 +791,9 @@ fail:
 #endif
 
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
-bool
-jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res,
-                    JitReg *params, uint32 param_count)
+static bool
+emit_callnative(JitCompContext *cc, JitReg native_func_reg, JitReg res,
+                JitReg *params, uint32 param_count)
 {
     JitInsn *insn;
     char *i64_arg_names[] = { "rdi", "rsi", "rdx", "rcx", "r8", "r9" };
@@ -483,8 +856,7 @@ jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res,
         }
     }
 
-    insn = GEN_INSN(CALLNATIVE, res_hreg,
-                    NEW_CONST(PTR, (uintptr_t)native_func), param_count);
+    insn = GEN_INSN(CALLNATIVE, res_hreg, native_func_reg, param_count);
     if (!insn) {
         return false;
     }
@@ -515,17 +887,16 @@ jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res,
     return true;
 }
 #else
-bool
-jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res,
-                    JitReg *params, uint32 param_count)
+static bool
+emit_callnative(JitCompContext *cc, JitRef native_func_reg, JitReg res,
+                JitReg *params, uint32 param_count)
 {
     JitInsn *insn;
     uint32 i;
 
     bh_assert(param_count <= 6);
 
-    insn = GEN_INSN(CALLNATIVE, res, NEW_CONST(PTR, (uintptr_t)native_func),
-                    param_count);
+    insn = GEN_INSN(CALLNATIVE, res, native_func_reg, param_count);
     if (!insn)
         return false;
 
@@ -535,3 +906,11 @@ jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res,
     return true;
 }
 #endif
+
+bool
+jit_emit_callnative(JitCompContext *cc, void *native_func, JitReg res,
+                    JitReg *params, uint32 param_count)
+{
+    return emit_callnative(cc, NEW_CONST(PTR, (uintptr_t)native_func), res,
+                           params, param_count);
+}

core/iwasm/fast-jit/jit_frontend.c

@@ -793,7 +793,7 @@ init_func_translation(JitCompContext *cc)
     cc->spill_cache_offset = wasm_interp_interp_frame_size(total_cell_num);
     /* Set spill cache size according to max local cell num, max stack cell
        num and virtual fixed register num */
-    cc->spill_cache_size = (max_locals + max_stacks) * 4 + sizeof(void *) * 4;
+    cc->spill_cache_size = (max_locals + max_stacks) * 4 + sizeof(void *) * 5;
     cc->total_frame_size = cc->spill_cache_offset + cc->spill_cache_size;
     cc->jitted_return_address_offset =
         offsetof(WASMInterpFrame, jitted_return_addr);