Refine call native function from AOT code (#1015)

When calling native function from AOT code, current implementation is to return
back to runtime to call aot_invoke_native, which calls wasm_runtime_invoke_native
and the latter calls assembly code. We did it before as there may be pointer and
string arguments to check and convert if the native function's registered signature
has character '*' and '$'.
As the built-in native function's signatures can be gotten in compilation time, we
check the pointer/string arguments and convert them into native address in AOT
code, and then invoke the native function directly, so as to improve performance.

core/iwasm/aot/aot_reloc.h

@@ -113,6 +113,7 @@ typedef struct {
     REG_SYM(aot_call_indirect),           \
     REG_SYM(aot_enlarge_memory),          \
     REG_SYM(aot_set_exception),           \
+    REG_SYM(aot_check_app_addr_and_convert),\
     { "memset", (void*)aot_memset },      \
     { "memmove", (void*)aot_memmove },    \
     { "memcpy", (void*)aot_memmove },     \

core/iwasm/aot/aot_runtime.c

@@ -2431,6 +2431,56 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
     }
 }
 
+/**
+ * Check whether the app address and the buf is inside the linear memory,
+ * and convert the app address into native address
+ */
+bool
+aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str,
+                               uint32 app_buf_addr, uint32 app_buf_size,
+                               void **p_native_addr)
+{
+    AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst);
+    uint8 *native_addr;
+
+    if (!memory_inst) {
+        goto fail;
+    }
+
+    native_addr = (uint8 *)memory_inst->memory_data.ptr + app_buf_addr;
+
+    /* No need to check the app_offset and buf_size if memory access
+       boundary check with hardware trap is enabled */
+#ifndef OS_ENABLE_HW_BOUND_CHECK
+    if (app_buf_addr >= memory_inst->memory_data_size) {
+        goto fail;
+    }
+
+    if (!is_str) {
+        if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) {
+            goto fail;
+        }
+    }
+    else {
+        const char *str, *str_end;
+
+        /* The whole string must be in the linear memory */
+        str = (const char *)native_addr;
+        str_end = (const char *)memory_inst->memory_data_end.ptr;
+        while (str < str_end && *str != '\0')
+            str++;
+        if (str == str_end)
+            goto fail;
+    }
+#endif
+
+    *p_native_addr = (void *)native_addr;
+    return true;
+fail:
+    aot_set_exception(module_inst, "out of bounds memory access");
+    return false;
+}
+
 void *
 aot_memmove(void *dest, const void *src, size_t n)
 {

core/iwasm/aot/aot_runtime.h

@@ -647,6 +647,11 @@ bool
 aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
                   uint32 argc, uint32 *argv);
 
+bool
+aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str,
+                               uint32 app_buf_addr, uint32 app_buf_size,
+                               void **p_native_addr);
+
 uint32
 aot_get_plt_table_size();
 

core/iwasm/compilation/aot_emit_function.c

@@ -518,6 +518,111 @@ check_stack_boundary(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     return true;
 }
 
+/**
+ * Check whether the app address and its buffer are inside the linear memory,
+ * if no, throw exception
+ */
+static bool
+check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
+                           bool is_str_arg, LLVMValueRef app_addr,
+                           LLVMValueRef buf_size,
+                           LLVMValueRef *p_native_addr_converted)
+{
+    LLVMTypeRef func_type, func_ptr_type, func_param_types[5];
+    LLVMValueRef func, func_param_values[5], res, native_addr_ptr;
+    char *func_name = "aot_check_app_addr_and_convert";
+
+    /* prepare function type of aot_check_app_addr_and_convert */
+    func_param_types[0] = comp_ctx->aot_inst_type; /* module_inst */
+    func_param_types[1] = INT8_TYPE;               /* is_str_arg */
+    func_param_types[2] = I32_TYPE;                /* app_offset */
+    func_param_types[3] = I32_TYPE;                /* buf_size */
+    func_param_types[4] =
+        comp_ctx->basic_types.int8_pptr_type; /* p_native_addr */
+    if (!(func_type =
+              LLVMFunctionType(INT8_TYPE, func_param_types, 5, false))) {
+        aot_set_last_error("llvm add function type failed.");
+        return false;
+    }
+
+    /* prepare function pointer */
+    if (comp_ctx->is_jit_mode) {
+        if (!(func_ptr_type = LLVMPointerType(func_type, 0))) {
+            aot_set_last_error("create LLVM function type failed.");
+            return false;
+        }
+
+        /* JIT mode, call the function directly */
+        if (!(func =
+                  I64_CONST((uint64)(uintptr_t)aot_check_app_addr_and_convert))
+            || !(func = LLVMConstIntToPtr(func, func_ptr_type))) {
+            aot_set_last_error("create LLVM value failed.");
+            return false;
+        }
+    }
+    else if (comp_ctx->is_indirect_mode) {
+        int32 func_index;
+        if (!(func_ptr_type = LLVMPointerType(func_type, 0))) {
+            aot_set_last_error("create LLVM function type failed.");
+            return false;
+        }
+        func_index = aot_get_native_symbol_index(comp_ctx, func_name);
+        if (func_index < 0) {
+            return false;
+        }
+        if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol,
+                                             func_ptr_type, func_index))) {
+            return false;
+        }
+    }
+    else {
+        if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name))
+            && !(func =
+                     LLVMAddFunction(func_ctx->module, func_name, func_type))) {
+            aot_set_last_error("add LLVM function failed.");
+            return false;
+        }
+    }
+
+    if (!(native_addr_ptr = LLVMBuildBitCast(
+              comp_ctx->builder, func_ctx->argv_buf,
+              comp_ctx->basic_types.int8_pptr_type, "p_native_addr"))) {
+        aot_set_last_error("llvm build bit cast failed.");
+        return false;
+    }
+
+    func_param_values[0] = func_ctx->aot_inst;
+    func_param_values[1] = I8_CONST(is_str_arg);
+    func_param_values[2] = app_addr;
+    func_param_values[3] = buf_size;
+    func_param_values[4] = native_addr_ptr;
+
+    if (!func_param_values[1]) {
+        aot_set_last_error("llvm create const failed.");
+        return false;
+    }
+
+    /* call aot_check_app_addr_and_convert() function */
+    if (!(res = LLVMBuildCall(comp_ctx->builder, func, func_param_values, 5,
+                              "res"))) {
+        aot_set_last_error("llvm build call failed.");
+        return false;
+    }
+
+    /* Check whether exception was thrown when executing the function */
+    if (!check_call_return(comp_ctx, func_ctx, res)) {
+        return false;
+    }
+
+    if (!(*p_native_addr_converted = LLVMBuildLoad(
+              comp_ctx->builder, native_addr_ptr, "native_addr"))) {
+        aot_set_last_error("llvm build load failed.");
+        return false;
+    }
+
+    return true;
+}
+
 bool
 aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                     uint32 func_idx, bool tail_call)
@@ -539,6 +644,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     uint32 callee_cell_num;
     uint8 wasm_ret_type;
     uint8 *ext_ret_types = NULL;
+    const char *signature = NULL;
     bool ret = false;
     char buf[32];
 
@@ -557,11 +663,14 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     }
 
     /* Get function type */
-    if (func_idx < import_func_count)
+    if (func_idx < import_func_count) {
         func_type = import_funcs[func_idx].func_type;
-    else
+        signature = import_funcs[func_idx].signature;
+    }
+    else {
         func_type =
             func_ctxes[func_idx - import_func_count]->aot_func->func_type;
+    }
 
     /* Get param cell number */
     param_cell_num = func_type->param_cell_num;
@@ -659,8 +768,41 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
         j = 0;
         param_types[j++] = comp_ctx->exec_env_type;
 
-        for (i = 0; i < param_count; i++)
-            param_types[j++] = TO_LLVM_TYPE(func_type->types[i]);
+        for (i = 0; i < param_count; i++, j++) {
+            param_types[j] = TO_LLVM_TYPE(func_type->types[i]);
+
+            /* If the signature can be gotten, e.g. the signature of the builtin
+               native libraries, just check the app offset and buf size, and
+               then convert app offset to native addr and call the native func
+               directly, no need to call aot_invoke_native to call it */
+            if (signature) {
+                LLVMValueRef native_addr, native_addr_size;
+                if (signature[i + 1] == '*' || signature[i + 1] == '$') {
+                    param_types[j] = INT8_PTR_TYPE;
+                }
+                if (signature[i + 1] == '*') {
+                    if (signature[i + 2] == '~')
+                        native_addr_size = param_values[i + 2];
+                    else
+                        native_addr_size = I32_ONE;
+                    if (!check_app_addr_and_convert(
+                            comp_ctx, func_ctx, false, param_values[j],
+                            native_addr_size, &native_addr)) {
+                        goto fail;
+                    }
+                    param_values[j] = native_addr;
+                }
+                else if (signature[i + 1] == '$') {
+                    native_addr_size = I32_ZERO;
+                    if (!check_app_addr_and_convert(
+                            comp_ctx, func_ctx, true, param_values[j],
+                            native_addr_size, &native_addr)) {
+                        goto fail;
+                    }
+                    param_values[j] = native_addr;
+                }
+            }
+        }
 
         if (func_type->result_count) {
             wasm_ret_type = func_type->types[func_type->param_count];
@@ -671,17 +813,68 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
             ret_type = VOID_TYPE;
         }
 
-        /* call aot_invoke_native() */
-        if (!call_aot_invoke_native_func(
-                comp_ctx, func_ctx, import_func_idx, func_type, param_types + 1,
-                param_values + 1, param_count, param_cell_num, ret_type,
-                wasm_ret_type, &value_ret, &res))
-            goto fail;
+        if (!signature) {
+            /* call aot_invoke_native() */
+            if (!call_aot_invoke_native_func(
+                    comp_ctx, func_ctx, import_func_idx, func_type,
+                    param_types + 1, param_values + 1, param_count,
+                    param_cell_num, ret_type, wasm_ret_type, &value_ret, &res))
+                goto fail;
+            /* Check whether there was exception thrown when executing
+               the function */
+            if (!check_call_return(comp_ctx, func_ctx, res))
+                goto fail;
+        }
+        else { /* call native func directly */
+            LLVMTypeRef native_func_type, func_ptr_type;
+            LLVMValueRef func_ptr;
 
-        /* Check whether there was exception thrown when executing
-           the function */
-        if (!check_call_return(comp_ctx, func_ctx, res))
-            goto fail;
+            if (!(native_func_type = LLVMFunctionType(
+                      ret_type, param_types, param_count + 1, false))) {
+                aot_set_last_error("llvm add function type failed.");
+                goto fail;
+            }
+
+            if (!(func_ptr_type = LLVMPointerType(native_func_type, 0))) {
+                aot_set_last_error("create LLVM function type failed.");
+                goto fail;
+            }
+
+            /* Load function pointer */
+            if (!(func_ptr = LLVMBuildInBoundsGEP(
+                      comp_ctx->builder, func_ctx->func_ptrs, &import_func_idx,
+                      1, "native_func_ptr_tmp"))) {
+                aot_set_last_error("llvm build inbounds gep failed.");
+                goto fail;
+            }
+
+            if (!(func_ptr = LLVMBuildLoad(comp_ctx->builder, func_ptr,
+                                           "native_func_ptr"))) {
+                aot_set_last_error("llvm build load failed.");
+                goto fail;
+            }
+
+            if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr,
+                                          func_ptr_type, "native_func"))) {
+                aot_set_last_error("llvm bit cast failed.");
+                goto fail;
+            }
+
+            /* Call the function */
+            if (!(value_ret = LLVMBuildCall(
+                      comp_ctx->builder, func, param_values,
+                      (uint32)param_count + 1 + ext_ret_count,
+                      (func_type->result_count > 0 ? "call" : "")))) {
+                aot_set_last_error("LLVM build call failed.");
+                goto fail;
+            }
+
+            /* Check whether there was exception thrown when executing
+               the function */
+            if (!check_exception_thrown(comp_ctx, func_ctx)) {
+                goto fail;
+            }
+        }
     }
     else {
         bool recursive_call =

wamr-compiler/CMakeLists.txt

@@ -181,6 +181,8 @@ include (${SHARED_DIR}/utils/shared_utils.cmake)
 include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake)
 include (${IWASM_DIR}/libraries/thread-mgr/thread_mgr.cmake)
 include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake)
+include (${IWASM_DIR}/libraries/libc-wasi/libc_wasi.cmake)
+include (${IWASM_DIR}/libraries/lib-pthread/lib_pthread.cmake)
 include (${IWASM_DIR}/common/iwasm_common.cmake)
 include (${IWASM_DIR}/interpreter/iwasm_interp.cmake)
 include (${IWASM_DIR}/aot/iwasm_aot.cmake)
@@ -239,6 +241,8 @@ add_library (vmlib
              ${UNCOMMON_SHARED_SOURCE}
              ${THREAD_MGR_SOURCE}
              ${LIBC_BUILTIN_SOURCE}
+             ${LIBC_WASI_SOURCE}
+             ${LIB_PTHREAD_SOURCE}
              ${IWASM_COMMON_SOURCE}
              ${IWASM_INTERP_SOURCE}
              ${IWASM_AOT_SOURCE})