Refine AOT exception check when function return (#1752)

Refine AOT exception check in the caller when returning from callee function,
remove the exception check instructions when hw bound check is enabled to
improve the performance: create guard page to trigger signal handler when
exception occurs.

core/iwasm/aot/aot_runtime.c

@@ -1500,7 +1500,11 @@ aot_set_exception(AOTModuleInstance *module_inst, const char *exception)
 void
 aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id)
 {
-    wasm_set_exception_with_id(module_inst, id);
+    if (id != EXCE_ALREADY_THROWN)
+        wasm_set_exception_with_id(module_inst, id);
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    wasm_runtime_access_exce_check_guard_page();
+#endif
 }
 
 const char *
@@ -1755,6 +1759,7 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
     const char *signature;
     void *attachment;
     char buf[96];
+    bool ret = false;
 
     bh_assert(func_idx < aot_module->import_func_count);
 
@@ -1764,27 +1769,34 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
                  "failed to call unlinked import function (%s, %s)",
                  import_func->module_name, import_func->func_name);
         aot_set_exception(module_inst, buf);
-        return false;
+        goto fail;
     }
 
     attachment = import_func->attachment;
     if (import_func->call_conv_wasm_c_api) {
-        return wasm_runtime_invoke_c_api_native(
+        ret = wasm_runtime_invoke_c_api_native(
             (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc,
             argv, import_func->wasm_c_api_with_env, attachment);
     }
     else if (!import_func->call_conv_raw) {
         signature = import_func->signature;
-        return wasm_runtime_invoke_native(exec_env, func_ptr, func_type,
-                                          signature, attachment, argv, argc,
-                                          argv);
+        ret =
+            wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature,
+                                       attachment, argv, argc, argv);
     }
     else {
         signature = import_func->signature;
-        return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
-                                              signature, attachment, argv, argc,
-                                              argv);
+        ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
+                                             signature, attachment, argv, argc,
+                                             argv);
     }
+
+fail:
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+    return ret;
 }
 
 bool
@@ -1811,7 +1823,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
 
     if ((uint8 *)&module_inst < exec_env->native_stack_boundary) {
         aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW);
-        return false;
+        goto fail;
     }
 
     tbl_inst = module_inst->tables[tbl_idx];
@@ -1819,13 +1831,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
 
     if (table_elem_idx >= tbl_inst->cur_size) {
         aot_set_exception_with_id(module_inst, EXCE_UNDEFINED_ELEMENT);
-        return false;
+        goto fail;
     }
 
     func_idx = tbl_inst->elems[table_elem_idx];
     if (func_idx == NULL_REF) {
         aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT);
-        return false;
+        goto fail;
     }
 
     func_type_idx = func_type_indexes[func_idx];
@@ -1843,7 +1855,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
                  "failed to call unlinked import function (%s, %s)",
                  import_func->module_name, import_func->func_name);
         aot_set_exception(module_inst, buf);
-        return false;
+        goto fail;
     }
 
     if (func_idx < aot_module->import_func_count) {
@@ -1852,9 +1864,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
         signature = import_func->signature;
         if (import_func->call_conv_raw) {
             attachment = import_func->attachment;
-            return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
-                                                  signature, attachment, argv,
-                                                  argc, argv);
+            ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
+                                                 signature, attachment, argv,
+                                                 argc, argv);
+            if (!ret)
+                goto fail;
+
+            return true;
         }
     }
 
@@ -1878,7 +1894,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
             && !(argv1 = runtime_malloc(size, module_inst->cur_exception,
                                         sizeof(module_inst->cur_exception)))) {
             aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY);
-            return false;
+            goto fail;
         }
 
         /* Copy original arguments */
@@ -1897,12 +1913,10 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
 
         ret = invoke_native_internal(exec_env, func_ptr, func_type, signature,
                                      attachment, argv1, argc, argv);
-        if (!ret || aot_get_exception(module_inst)) {
+        if (!ret) {
             if (argv1 != argv1_buf)
                 wasm_runtime_free(argv1);
-            if (clear_wasi_proc_exit_exception(module_inst))
-                return true;
-            return false;
+            goto fail;
         }
 
         /* Get extra result values */
@@ -1941,10 +1955,20 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
     else {
         ret = invoke_native_internal(exec_env, func_ptr, func_type, signature,
                                      attachment, argv, argc, argv);
-        if (clear_wasi_proc_exit_exception(module_inst))
-            return true;
-        return ret;
+        if (!ret)
+            goto fail;
+
+        return true;
     }
+
+fail:
+    if (clear_wasi_proc_exit_exception(module_inst))
+        return true;
+
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    wasm_runtime_access_exce_check_guard_page();
+#endif
+    return false;
 }
 
 bool
@@ -1952,8 +1976,17 @@ aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str,
                                uint32 app_buf_addr, uint32 app_buf_size,
                                void **p_native_addr)
 {
-    return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr,
-                                           app_buf_size, p_native_addr);
+    bool ret;
+
+    ret = wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr,
+                                          app_buf_size, p_native_addr);
+
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+
+    return ret;
 }
 
 void *

core/iwasm/common/wasm_exec_env.c

@@ -56,6 +56,12 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
 #endif
 #endif
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!(exec_env->exce_check_guard_page =
+              os_mmap(NULL, os_getpagesize(), MMAP_PROT_NONE, MMAP_MAP_NONE)))
+        goto fail5;
+#endif
+
     exec_env->module_inst = module_inst;
     exec_env->wasm_stack_size = stack_size;
     exec_env->wasm_stack.s.top_boundary =
@@ -76,6 +82,12 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
 
     return exec_env;
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+fail5:
+#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0
+    wasm_cluster_destroy_exenv_status(exec_env->current_status);
+#endif
+#endif
 #if WASM_ENABLE_THREAD_MGR != 0
 #if WASM_ENABLE_DEBUG_INTERP != 0
 fail4:
@@ -96,6 +108,9 @@ fail1:
 void
 wasm_exec_env_destroy_internal(WASMExecEnv *exec_env)
 {
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    os_munmap(exec_env->exce_check_guard_page, os_getpagesize());
+#endif
 #if WASM_ENABLE_THREAD_MGR != 0
     os_mutex_destroy(&exec_env->wait_lock);
     os_cond_destroy(&exec_env->wait_cond);

core/iwasm/common/wasm_exec_env.h

@@ -137,6 +137,8 @@ typedef struct WASMExecEnv {
 
 #ifdef OS_ENABLE_HW_BOUND_CHECK
     WASMJmpBuf *jmpbuf_stack_top;
+    /* One guard page for the exception check */
+    uint8 *exce_check_guard_page;
 #endif
 
 #if WASM_ENABLE_MEMORY_PROFILING != 0
@@ -199,7 +201,8 @@ wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size)
        the outs area contains const cells, its size may be larger than current
        frame size, we should check again before putting the function arguments
        into the outs area. */
-    if (addr + size * 2 > exec_env->wasm_stack.s.top_boundary) {
+    if (size * 2
+        > (uint32)(uintptr_t)(exec_env->wasm_stack.s.top_boundary - addr)) {
         /* WASM stack overflow. */
         return NULL;
     }

core/iwasm/common/wasm_runtime_common.c

@@ -185,6 +185,12 @@ runtime_signal_handler(void *sig_addr)
             os_longjmp(jmpbuf_node->jmpbuf, 1);
         }
 #endif
+        else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr
+                 && (uint8 *)sig_addr
+                        < exec_env_tls->exce_check_guard_page + page_size) {
+            bh_assert(wasm_get_exception(module_inst));
+            os_longjmp(jmpbuf_node->jmpbuf, 1);
+        }
     }
 }
 #else
@@ -1435,6 +1441,17 @@ wasm_runtime_get_user_data(WASMExecEnv *exec_env)
     return exec_env->user_data;
 }
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+void
+wasm_runtime_access_exce_check_guard_page()
+{
+    if (exec_env_tls && exec_env_tls->handle == os_self_thread()) {
+        uint32 page_size = os_getpagesize();
+        memset(exec_env_tls->exce_check_guard_page, 0, page_size);
+    }
+}
+#endif
+
 WASMType *
 wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function,
                                uint32 module_type)

core/iwasm/common/wasm_runtime_common.h

@@ -554,6 +554,12 @@ wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data);
 WASM_RUNTIME_API_EXTERN void *
 wasm_runtime_get_user_data(WASMExecEnv *exec_env);
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+/* Access exception check guard page to trigger the signal handler */
+void
+wasm_runtime_access_exce_check_guard_page();
+#endif
+
 /* See wasm_export.h for description */
 WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_call_wasm(WASMExecEnv *exec_env,

core/iwasm/compilation/aot_emit_function.c

@@ -35,7 +35,14 @@ create_func_return_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
         /* Create return IR */
         LLVMPositionBuilderAtEnd(comp_ctx->builder,
                                  func_ctx->func_return_block);
-        if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) {
+        if (!comp_ctx->enable_bound_check) {
+            if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ALREADY_THROWN,
+                                    false, NULL, NULL)) {
+                return false;
+            }
+        }
+        else if (!aot_build_zero_function_ret(comp_ctx, func_ctx,
+                                              aot_func_type)) {
             return false;
         }
     }
@@ -494,7 +501,8 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     }
 
     /* Check whether exception was thrown when executing the function */
-    if (!check_call_return(comp_ctx, func_ctx, res)) {
+    if (comp_ctx->enable_bound_check
+        && !check_call_return(comp_ctx, func_ctx, res)) {
         return false;
     }
 
@@ -707,7 +715,8 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                 goto fail;
             /* Check whether there was exception thrown when executing
                the function */
-            if (!check_call_return(comp_ctx, func_ctx, res))
+            if (comp_ctx->enable_bound_check
+                && !check_call_return(comp_ctx, func_ctx, res))
                 goto fail;
         }
         else { /* call native func directly */
@@ -823,7 +832,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
 
         /* Check whether there was exception thrown when executing
            the function */
-        if (!tail_call && !recursive_call
+        if (!tail_call && !recursive_call && comp_ctx->enable_bound_check
             && !check_exception_thrown(comp_ctx, func_ctx))
             goto fail;
     }
@@ -1395,7 +1404,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
         goto fail;
 
     /* Check whether exception was thrown when executing the function */
-    if (!check_call_return(comp_ctx, func_ctx, res))
+    if (comp_ctx->enable_bound_check
+        && !check_call_return(comp_ctx, func_ctx, res))
         goto fail;
 
     block_curr = LLVMGetInsertBlock(comp_ctx->builder);
@@ -1454,7 +1464,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     }
 
     /* Check whether exception was thrown when executing the function */
-    if (!check_exception_thrown(comp_ctx, func_ctx))
+    if (comp_ctx->enable_bound_check
+        && !check_exception_thrown(comp_ctx, func_ctx))
         goto fail;
 
     if (func_result_count > 0) {

core/iwasm/compilation/aot_llvm_extra.cpp

@@ -344,11 +344,12 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module)
 
         if (!disable_llvm_lto) {
             /* Apply LTO for AOT mode */
-#if LLVM_VERSION_MAJOR < 14
-            MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL));
-#else
-            MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL));
-#endif
+            if (comp_ctx->comp_data->func_count >= 10)
+                /* Adds the pre-link optimizations if the func count
+                   is large enough */
+                MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL));
+            else
+                MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL));
         }
         else {
             MPM.addPass(PB.buildPerModuleDefaultPipeline(OL));

core/iwasm/interpreter/wasm_runtime.c

@@ -2781,7 +2781,11 @@ wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf,
 void
 jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id)
 {
-    wasm_set_exception_with_id(module_inst, id);
+    if (id != EXCE_ALREADY_THROWN)
+        wasm_set_exception_with_id(module_inst, id);
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    wasm_runtime_access_exce_check_guard_page();
+#endif
 }
 
 bool
@@ -2789,8 +2793,15 @@ jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
                                uint32 app_buf_addr, uint32 app_buf_size,
                                void **p_native_addr)
 {
-    return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr,
-                                           app_buf_size, p_native_addr);
+    bool ret = wasm_check_app_addr_and_convert(
+        module_inst, is_str, app_buf_addr, app_buf_size, p_native_addr);
+
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+
+    return ret;
 }
 #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
           || WASM_ENABLE_WAMR_COMPILER != 0 */
@@ -2811,12 +2822,20 @@ bool
 llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
                        uint32 argc, uint32 *argv)
 {
+    bool ret;
+
 #if WASM_ENABLE_JIT != 0
     if (Wasm_Module_AoT == exec_env->module_inst->module_type) {
         return aot_call_indirect(exec_env, tbl_idx, elem_idx, argc, argv);
     }
 #endif
-    return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
+
+    ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+    return ret;
 }
 
 bool
@@ -2833,6 +2852,7 @@ llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
     const char *signature;
     void *attachment;
     char buf[96];
+    bool ret = false;
 
 #if WASM_ENABLE_JIT != 0
     if (Wasm_Module_AoT == exec_env->module_inst->module_type) {
@@ -2855,27 +2875,34 @@ llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
                  "failed to call unlinked import function (%s, %s)",
                  import_func->module_name, import_func->field_name);
         wasm_set_exception(module_inst, buf);
-        return false;
+        goto fail;
     }
 
     attachment = import_func->attachment;
     if (import_func->call_conv_wasm_c_api) {
-        return wasm_runtime_invoke_c_api_native(
+        ret = wasm_runtime_invoke_c_api_native(
             (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc,
             argv, import_func->wasm_c_api_with_env, attachment);
     }
     else if (!import_func->call_conv_raw) {
         signature = import_func->signature;
-        return wasm_runtime_invoke_native(exec_env, func_ptr, func_type,
-                                          signature, attachment, argv, argc,
-                                          argv);
+        ret =
+            wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature,
+                                       attachment, argv, argc, argv);
     }
     else {
         signature = import_func->signature;
-        return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
-                                              signature, attachment, argv, argc,
-                                              argv);
+        ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
+                                             signature, attachment, argv, argc,
+                                             argv);
     }
+
+fail:
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+    return ret;
 }
 
 #if WASM_ENABLE_BULK_MEMORY != 0