Support integrate 3rd-party toolchains into wamrc (#1237)

Support integrating 3rd-party toolchain llc compiler or asm compiler
into wamrc by setting environment variable WAMRC_LLC_COMPILER
or WAMRC_ASM_COMPILER, wamrc will use these tools to generate
object file from LLVM IR firstly, and then refactor the object file into
aot file.

core/iwasm/compilation/aot.h

@@ -15,7 +15,9 @@
 extern "C" {
 #endif
 
+#ifndef AOT_FUNC_PREFIX
 #define AOT_FUNC_PREFIX "aot_func#"
+#endif
 
 typedef InitializerExpression AOTInitExpr;
 typedef WASMType AOTFuncType;

core/iwasm/compilation/aot_compiler.c

@@ -2886,6 +2886,36 @@ aot_compile_wasm(AOTCompContext *comp_ctx)
     return true;
 }
 
+#if !(defined(_WIN32) || defined(_WIN32_))
+char *
+aot_generate_tempfile_name(const char *prefix, const char *extension,
+                           char *buffer, uint32 len)
+{
+    int fd, name_len;
+
+    name_len = snprintf(buffer, len, "%s-XXXXXX", prefix);
+
+    if ((fd = mkstemp(buffer)) <= 0) {
+        aot_set_last_error("make temp file failed.");
+        return NULL;
+    }
+
+    /* close and remove temp file */
+    close(fd);
+    unlink(buffer);
+
+    /* Check if buffer length is enough */
+    /* name_len + '.' + extension + '\0' */
+    if (name_len + 1 + strlen(extension) + 1 > len) {
+        aot_set_last_error("temp file name too long.");
+        return NULL;
+    }
+
+    snprintf(buffer + name_len, len - name_len, ".%s", extension);
+    return buffer;
+}
+#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */
+
 #if WASM_ENABLE_LAZY_JIT == 0
 bool
 aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name)
@@ -2915,6 +2945,83 @@ aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name)
 
     bh_print_time("Begin to emit object file");
 
+#if !(defined(_WIN32) || defined(_WIN32_))
+    if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) {
+        char cmd[1024];
+        int ret;
+
+        if (comp_ctx->external_llc_compiler) {
+            char bc_file_name[64];
+
+            if (!aot_generate_tempfile_name("wamrc-bc", "bc", bc_file_name,
+                                            sizeof(bc_file_name))) {
+                return false;
+            }
+
+            if (LLVMWriteBitcodeToFile(comp_ctx->module, bc_file_name) != 0) {
+                aot_set_last_error("emit llvm bitcode file failed.");
+                return false;
+            }
+
+            snprintf(cmd, sizeof(cmd), "%s %s -o %s %s",
+                     comp_ctx->external_llc_compiler,
+                     comp_ctx->llc_compiler_flags ? comp_ctx->llc_compiler_flags
+                                                  : "-O3 -c",
+                     file_name, bc_file_name);
+            LOG_VERBOSE("invoking external LLC compiler:\n\t%s", cmd);
+
+            ret = system(cmd);
+            /* remove temp bitcode file */
+            unlink(bc_file_name);
+
+            if (ret != 0) {
+                aot_set_last_error("failed to compile LLVM bitcode to obj file "
+                                   "with external LLC compiler.");
+                return false;
+            }
+        }
+        else if (comp_ctx->external_asm_compiler) {
+            char asm_file_name[64];
+
+            if (!aot_generate_tempfile_name("wamrc-asm", "s", asm_file_name,
+                                            sizeof(asm_file_name))) {
+                return false;
+            }
+
+            if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine,
+                                            comp_ctx->module, asm_file_name,
+                                            LLVMAssemblyFile, &err)
+                != 0) {
+                if (err) {
+                    LLVMDisposeMessage(err);
+                    err = NULL;
+                }
+                aot_set_last_error("emit elf to assembly file failed.");
+                return false;
+            }
+
+            snprintf(cmd, sizeof(cmd), "%s %s -o %s %s",
+                     comp_ctx->external_asm_compiler,
+                     comp_ctx->asm_compiler_flags ? comp_ctx->asm_compiler_flags
+                                                  : "-O3 -c",
+                     file_name, asm_file_name);
+            LOG_VERBOSE("invoking external ASM compiler:\n\t%s", cmd);
+
+            ret = system(cmd);
+            /* remove temp assembly file */
+            unlink(asm_file_name);
+
+            if (ret != 0) {
+                aot_set_last_error("failed to compile Assembly file to obj "
+                                   "file with external ASM compiler.");
+                return false;
+            }
+        }
+
+        return true;
+    }
+#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */
+
     if (!strncmp(LLVMGetTargetName(target), "arc", 3))
         /* Emit to assmelby file instead for arc target
            as it cannot emit to object file */

core/iwasm/compilation/aot_compiler.h

@@ -371,6 +371,10 @@ aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data,
 bool
 aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name);
 
+char *
+aot_generate_tempfile_name(const char *prefix, const char *extension,
+                           char *buffer, uint32 len);
+
 #ifdef __cplusplus
 } /* end of extern "C" */
 #endif

core/iwasm/compilation/aot_emit_aot_file.c

@@ -2692,8 +2692,41 @@ aot_obj_data_create(AOTCompContext *comp_ctx)
     memset(obj_data, 0, sizeof(AOTObjectData));
 
     bh_print_time("Begin to emit object file");
+    if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) {
+#if defined(_WIN32) || defined(_WIN32_)
+        aot_set_last_error("external toolchain not supported on Windows");
+        goto fail;
+#else
+        /* Generate a temp file name */
+        int ret;
+        char obj_file_name[64];
+
+        if (!aot_generate_tempfile_name("wamrc-obj", "o", obj_file_name,
+                                        sizeof(obj_file_name))) {
+            goto fail;
+        }
+
+        if (!aot_emit_object_file(comp_ctx, obj_file_name)) {
+            goto fail;
+        }
+
+        /* create memory buffer from object file */
+        ret = LLVMCreateMemoryBufferWithContentsOfFile(
+            obj_file_name, &obj_data->mem_buf, &err);
+        /* remove temp object file */
+        unlink(obj_file_name);
 
-    if (!strncmp(LLVMGetTargetName(target), "arc", 3)) {
+        if (ret != 0) {
+            if (err) {
+                LLVMDisposeMessage(err);
+                err = NULL;
+            }
+            aot_set_last_error("create mem buffer with file failed.");
+            goto fail;
+        }
+#endif /* end of defined(_WIN32) || defined(_WIN32_) */
+    }
+    else if (!strncmp(LLVMGetTargetName(target), "arc", 3)) {
 #if defined(_WIN32) || defined(_WIN32_)
         aot_set_last_error("emit object file on Windows is unsupported.");
         goto fail;

core/iwasm/compilation/aot_llvm.c

@@ -1660,6 +1660,51 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
         opt_level = option->opt_level;
         size_level = option->size_level;
 
+        /* verify external llc compiler */
+        comp_ctx->external_llc_compiler = getenv("WAMRC_LLC_COMPILER");
+        if (comp_ctx->external_llc_compiler) {
+#if defined(_WIN32) || defined(_WIN32_)
+            comp_ctx->external_llc_compiler = NULL;
+            LOG_WARNING("External LLC compiler not supported on Windows.");
+#else
+            if (access(comp_ctx->external_llc_compiler, X_OK) != 0) {
+                LOG_WARNING("WAMRC_LLC_COMPILER [%s] not found, fallback to "
+                            "default pipeline",
+                            comp_ctx->external_llc_compiler);
+                comp_ctx->external_llc_compiler = NULL;
+            }
+            else {
+                comp_ctx->llc_compiler_flags = getenv("WAMRC_LLC_FLAGS");
+                LOG_VERBOSE("Using external LLC compiler [%s]",
+                            comp_ctx->external_llc_compiler);
+            }
+#endif
+        }
+
+        /* verify external asm compiler */
+        if (!comp_ctx->external_llc_compiler) {
+            comp_ctx->external_asm_compiler = getenv("WAMRC_ASM_COMPILER");
+            if (comp_ctx->external_asm_compiler) {
+#if defined(_WIN32) || defined(_WIN32_)
+                comp_ctx->external_asm_compiler = NULL;
+                LOG_WARNING("External ASM compiler not supported on Windows.");
+#else
+                if (access(comp_ctx->external_asm_compiler, X_OK) != 0) {
+                    LOG_WARNING(
+                        "WAMRC_ASM_COMPILER [%s] not found, fallback to "
+                        "default pipeline",
+                        comp_ctx->external_asm_compiler);
+                    comp_ctx->external_asm_compiler = NULL;
+                }
+                else {
+                    comp_ctx->asm_compiler_flags = getenv("WAMRC_ASM_FLAGS");
+                    LOG_VERBOSE("Using external ASM compiler [%s]",
+                                comp_ctx->external_asm_compiler);
+                }
+#endif
+            }
+        }
+
         if (arch) {
             /* Add default sub-arch if not specified */
             if (!strcmp(arch, "arm"))

core/iwasm/compilation/aot_llvm.h

@@ -14,6 +14,7 @@
 #include "llvm-c/Object.h"
 #include "llvm-c/ExecutionEngine.h"
 #include "llvm-c/Analysis.h"
+#include "llvm-c/BitWriter.h"
 #include "llvm-c/Transforms/Utils.h"
 #include "llvm-c/Transforms/Scalar.h"
 #include "llvm-c/Transforms/Vectorize.h"
@@ -350,6 +351,20 @@ typedef struct AOTCompContext {
     uint32 func_ctx_count;
     char **custom_sections_wp;
     uint32 custom_sections_count;
+
+    /* 3rd-party toolchains */
+    /* External llc compiler, if specified, wamrc will emit the llvm-ir file and
+     * invoke the llc compiler to generate object file.
+     * This can be used when we want to benefit from the optimization of other
+     * LLVM based toolchains */
+    const char *external_llc_compiler;
+    const char *llc_compiler_flags;
+    /* External asm compiler, if specified, wamrc will emit the text-based
+     * assembly file (.s) and invoke the llc compiler to generate object file.
+     * This will be useful when the upstream LLVM doesn't support to emit object
+     * file for some architecture (such as arc) */
+    const char *external_asm_compiler;
+    const char *asm_compiler_flags;
 } AOTCompContext;
 
 enum {

doc/build_wasm_app.md

@@ -330,6 +330,36 @@ Examples: wamrc -o test.aot test.wasm
           wamrc --target=i386 --format=object -o test.o test.wasm
 ```
 
+## AoT compilation with 3rd-party toolchains
+
+`wamrc` uses LLVM to compile wasm bytecode to AoT file, this works for most of the architectures, but there may be circumstances where you want to use 3rd-party toolchains to take over some steps of the compilation pipeline, e.g.
+
+1. The upstream LLVM doesn't support generating object file for your CPU architecture (such as ARC), then we may need some other assembler to do such things.
+2. You may get some other LLVM-based toolchains which may have better optimizations for the specific target, then you may want your toolchain to take over all optimization steps.
+
+`wamrc` provides two environment variables to achieve these:
+- `WAMRC_LLC_COMPILER`
+
+  When specified, `wamrc` will emit the optimized LLVM-IR (.bc) to a file, and invoke `$WAMRC_LLC_COMPILER` with ` -c -O3 ` to generate the object file.
+
+  Optionally, you can use environment variable `WAMRC_LLC_FLAGS` to overwrite the default flags.
+
+- `WAMRC_ASM_COMPILER`
+
+  When specified, `wamrc` will emit the text based assembly file (.s), and invoke `$WAMRC_ASM_COMPILER` with ` -c -O3 ` to generate the object file.
+
+  Optionally, you can use environment variable `WAMRC_ASM_FLAGS` to overwrite the default flags.
+
+### Usage example
+``` bash
+WAMRC_LLC_COMPILER=<path/to/your/compiler/driver> ./wamrc -o test.aot test.wasm
+```
+
+> Note: `wamrc` will verify whether the specified file exists and executable. If verification failed, `wamrc` will report a warning and fallback to normal pipeline. Since the verification is based on file, you **must specify the absolute path to the binary** even if it's in `$PATH`
+
+> Note: `WAMRC_LLC_COMPILER` has higher priority than `WAMRC_ASM_COMPILER`, if `WAMRC_LLC_COMPILER` is set and verified, then `WAMRC_ASM_COMPILER` will be ignored.
+
+> Note: the `LLC` and `ASM` in the env name just means this compiler will be used to compile the `LLVM IR file`/`assembly file` to object file, usually passing the compiler driver is the simplest way. (e.g. for LLVM toolchain, you don't need to pass `/usr/bin/llc`, using `/usr/bin/clang` is OK)
 
 Run WASM app in WAMR mini product build
 =======================================

wamr-compiler/CMakeLists.txt

@@ -41,6 +41,10 @@ if (WAMR_BUILD_LLVM_LEGACY_PM EQUAL 1)
   add_definitions(-DWASM_ENABLE_LLVM_LEGACY_PM=1)
 endif()
 
+if (DEFINED WAMR_BUILD_AOT_FUNC_PREFIX)
+  add_definitions(-DAOT_FUNC_PREFIX="${WAMR_BUILD_AOT_FUNC_PREFIX}")
+endif ()
+
 # Set WAMR_BUILD_TARGET, currently values supported:
 # "X86_64", "AMD_64", "X86_32", "ARM_32", "MIPS_32", "XTENSA_32"
 if (NOT WAMR_BUILD_TARGET)