Enable LLVM link time optimization (LTO) for AOT (#831)

Enable LLVM link time optimization for AOT and enable it by default,
and provide "wamrc --disable-llvm-lto" option to disable it.
And modify the spec test script, disable it by default when testing
spec cases.

Signed-off-by: Wenyong Huang <wenyong.huang@intel.com>

core/iwasm/aot/aot_reloc.h

@@ -110,8 +110,9 @@ typedef struct {
     REG_SYM(aot_call_indirect),           \
     REG_SYM(aot_enlarge_memory),          \
     REG_SYM(aot_set_exception),           \
-    {"memset", (void*)aot_memset},        \
-    {"memmove", (void*)aot_memmove},      \
+    { "memset", (void*)aot_memset },      \
+    { "memmove", (void*)aot_memmove },    \
+    { "memcpy", (void*)aot_memmove },     \
     REG_SYM(fmin),                        \
     REG_SYM(fminf),                       \
     REG_SYM(fmax),                        \

core/iwasm/compilation/aot_compiler.c

@@ -2548,6 +2548,7 @@ aot_compile_wasm(AOTCompContext *comp_ctx)
 
     bh_print_time("Begin to run function optimization passes");
 
+    /* Run function pass manager */
     if (comp_ctx->optimize) {
         LLVMInitializeFunctionPassManager(comp_ctx->pass_mgr);
         for (i = 0; i < comp_ctx->func_ctx_count; i++)
@@ -2555,6 +2556,36 @@ aot_compile_wasm(AOTCompContext *comp_ctx)
                                        comp_ctx->func_ctxes[i]->func);
     }
 
+    /* Run common pass manager */
+    if (comp_ctx->optimize && !comp_ctx->is_jit_mode
+        && !comp_ctx->disable_llvm_lto) {
+        LLVMPassManagerRef common_pass_mgr = NULL;
+        LLVMPassManagerBuilderRef pass_mgr_builder = NULL;
+
+        if (!(common_pass_mgr = LLVMCreatePassManager())) {
+            aot_set_last_error("create pass manager failed");
+            return false;
+        }
+
+        if (!(pass_mgr_builder = LLVMPassManagerBuilderCreate())) {
+            aot_set_last_error("create pass manager builder failed");
+            LLVMDisposePassManager(common_pass_mgr);
+            return false;
+        }
+
+        LLVMPassManagerBuilderSetOptLevel(pass_mgr_builder,
+                                          comp_ctx->opt_level);
+        LLVMPassManagerBuilderPopulateModulePassManager(pass_mgr_builder,
+                                                        common_pass_mgr);
+        LLVMPassManagerBuilderPopulateLTOPassManager(
+            pass_mgr_builder, common_pass_mgr, true, true);
+
+        LLVMRunPassManager(common_pass_mgr, comp_ctx->module);
+
+        LLVMDisposePassManager(common_pass_mgr);
+        LLVMPassManagerBuilderDispose(pass_mgr_builder);
+    }
+
     return true;
 }
 

core/iwasm/compilation/aot_llvm.c

@@ -1482,6 +1482,12 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
     if (option->disable_llvm_intrinsics)
         comp_ctx->disable_llvm_intrinsics = true;
 
+    if (option->disable_llvm_lto)
+        comp_ctx->disable_llvm_lto = true;
+
+    comp_ctx->opt_level = option->opt_level;
+    comp_ctx->size_level = option->size_level;
+
     if (option->is_jit_mode) {
         char *triple_jit = NULL;
 

core/iwasm/compilation/aot_llvm.h

@@ -17,6 +17,7 @@
 #include "llvm-c/Transforms/Utils.h"
 #include "llvm-c/Transforms/Scalar.h"
 #include "llvm-c/Transforms/Vectorize.h"
+#include "llvm-c/Transforms/PassManagerBuilder.h"
 
 #if WASM_ENABLE_LAZY_JIT != 0
 #include "aot_llvm_lazyjit.h"
@@ -304,9 +305,15 @@ typedef struct AOTCompContext {
     /* Disable LLVM built-in intrinsics */
     bool disable_llvm_intrinsics;
 
+    /* Disable LLVM link time optimization */
+    bool disable_llvm_lto;
+
     /* Whether optimize the JITed code */
     bool optimize;
 
+    uint32 opt_level;
+    uint32 size_level;
+
     /* LLVM pass manager to optimize the JITed code */
     LLVMPassManagerRef pass_mgr;
 
@@ -352,6 +359,7 @@ typedef struct AOTCompOption {
     bool enable_aux_stack_check;
     bool enable_aux_stack_frame;
     bool disable_llvm_intrinsics;
+    bool disable_llvm_lto;
     uint32 opt_level;
     uint32 size_level;
     uint32 output_format;

core/iwasm/compilation/aot_llvm_extra.cpp

@@ -33,6 +33,9 @@ WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
 extern "C" bool
 aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str);
 
+extern "C" void
+aot_func_disable_tce(LLVMValueRef func);
+
 LLVMBool
 WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
                                  LLVMModuleRef M,
@@ -143,3 +146,14 @@ aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str)
     return true;
 #endif /* WASM_ENABLE_SIMD */
 }
+
+void
+aot_func_disable_tce(LLVMValueRef func)
+{
+    Function *F = unwrap<Function>(func);
+    auto Attrs = F->getAttributes();
+
+    Attrs = Attrs.addAttribute(F->getContext(), AttributeList::FunctionIndex,
+                               "disable-tail-calls", "true");
+    F->setAttributes(Attrs);
+}

core/iwasm/include/aot_export.h

@@ -54,6 +54,7 @@ typedef struct AOTCompOption {
     bool enable_aux_stack_check;
     bool enable_aux_stack_frame;
     bool disable_llvm_intrinsics;
+    bool disable_llvm_lto;
     uint32_t opt_level;
     uint32_t size_level;
     uint32_t output_format;

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -946,6 +946,11 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r):
         cmd.append("--enable-ref-types")
         cmd.append("--enable-bulk-memory")
 
+    # disable llvm link time optimization as it might convert
+    # code of tail call into code of dead loop, and stack overflow
+    # exception isn't thrown in several cases
+    cmd.append("--disable-llvm-lto")
+
     cmd += ["-o", aot_tempfile, wasm_tempfile]
 
     log("Running: %s" % " ".join(cmd))

wamr-compiler/main.c

@@ -62,6 +62,7 @@ print_help()
     printf("  --enable-perf-profiling   Enable function performance profiling\n");
     printf("  --enable-indirect-mode    Enalbe call function through symbol table but not direct call\n");
     printf("  --disable-llvm-intrinsics Disable the LLVM built-in intrinsics\n");
+    printf("  --disable-llvm-lto        Disable the LLVM link time optimization\n");
     printf("  -v=n                      Set log verbose level (0 to 5, default is 2), larger with more log\n");
     printf("Examples: wamrc -o test.aot test.wasm\n");
     printf("          wamrc --target=i386 -o test.aot test.wasm\n");
@@ -198,6 +199,9 @@ main(int argc, char *argv[])
         else if (!strcmp(argv[0], "--disable-llvm-intrinsics")) {
             option.disable_llvm_intrinsics = true;
         }
+        else if (!strcmp(argv[0], "--disable-llvm-lto")) {
+            option.disable_llvm_lto = true;
+        }
         else
             return print_help();
     }