Implement atomic and memset/memmove/memcpy intrinsic for riscv (#841)

- Lower aotmic instruction to non-atomic form on some platforms
- Lower memset/memmove/memcpy for XIP
- Disable rtti in cmake

core/iwasm/aot/aot_intrinsic.c

@@ -53,7 +53,8 @@ static const aot_intrinsic g_intrinsic_mapping[] = {
     { "f64_convert_i64_u", "aot_intrinsic_u64_to_f64", AOT_INTRINSIC_FLAG_U64_TO_F64 },
     { "f32_convert_i64_s", "aot_intrinsic_i64_to_f32", AOT_INTRINSIC_FLAG_I64_TO_F32 },
     { "f32_convert_i64_u", "aot_intrinsic_u64_to_f32", AOT_INTRINSIC_FLAG_U64_TO_F32 },
-    { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_I32_TO_F64 },
+    { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_F64_TO_U32 },
+    { "i32_trunc_f64_s", "aot_intrinsic_f64_to_i32", AOT_INTRINSIC_FLAG_F64_TO_I32 },
     { "f32_demote_f64", "aot_intrinsic_f64_to_f32", AOT_INTRINSIC_FLAG_F64_TO_F32 },
     { "f64_promote_f32", "aot_intrinsic_f32_to_f64", AOT_INTRINSIC_FLAG_F32_TO_F64 },
     { "f32_cmp", "aot_intrinsic_f32_cmp", AOT_INTRINSIC_FLAG_F32_CMP },

core/iwasm/aot/aot_reloc.h

@@ -99,6 +99,7 @@ typedef struct {
     REG_SYM(aot_intrinsic_i64_to_f64),    \
     REG_SYM(aot_intrinsic_u64_to_f64),    \
     REG_SYM(aot_intrinsic_f64_to_f32),    \
+    REG_SYM(aot_intrinsic_f64_to_i32),    \
     REG_SYM(aot_intrinsic_f64_to_u32),    \
     REG_SYM(aot_intrinsic_f32_to_f64),    \
     REG_SYM(aot_intrinsic_f32_cmp),       \

core/iwasm/compilation/aot_compiler.c

@@ -2538,6 +2538,39 @@ fail:
     return false;
 }
 
+/* Check whether the target supports hardware atomic instructions */
+static bool
+aot_require_lower_atomic_pass(AOTCompContext *comp_ctx)
+{
+    bool ret = false;
+    if (!strncmp(comp_ctx->target_arch, "riscv", 5)) {
+        char *feature =
+            LLVMGetTargetMachineFeatureString(comp_ctx->target_machine);
+
+        if (feature) {
+            if (!strstr(feature, "+a")) {
+                ret = true;
+            }
+            LLVMDisposeMessage(feature);
+        }
+    }
+    return ret;
+}
+
+/* Check whether the target needs to expand switch to if/else */
+static bool
+aot_require_lower_switch_pass(AOTCompContext *comp_ctx)
+{
+    bool ret = false;
+
+    /* IR switch/case will cause .rodata relocation on riscv */
+    if (!strncmp(comp_ctx->target_arch, "riscv", 5)) {
+        ret = true;
+    }
+
+    return ret;
+}
+
 bool
 aot_compile_wasm(AOTCompContext *comp_ctx)
 {
@@ -2625,6 +2658,27 @@ aot_compile_wasm(AOTCompContext *comp_ctx)
         LLVMPassManagerBuilderDispose(pass_mgr_builder);
     }
 
+    if (comp_ctx->optimize && comp_ctx->is_indirect_mode) {
+        LLVMPassManagerRef common_pass_mgr = NULL;
+
+        if (!(common_pass_mgr = LLVMCreatePassManager())) {
+            aot_set_last_error("create pass manager failed");
+            return false;
+        }
+
+        aot_add_expand_memory_op_pass(common_pass_mgr);
+
+        if (aot_require_lower_atomic_pass(comp_ctx))
+            LLVMAddLowerAtomicPass(common_pass_mgr);
+
+        if (aot_require_lower_switch_pass(comp_ctx))
+            LLVMAddLowerSwitchPass(common_pass_mgr);
+
+        LLVMRunPassManager(common_pass_mgr, comp_ctx->module);
+
+        LLVMDisposePassManager(common_pass_mgr);
+    }
+
     return true;
 }
 

core/iwasm/compilation/aot_emit_memory.c

@@ -6,6 +6,7 @@
 #include "aot_emit_memory.h"
 #include "aot_emit_exception.h"
 #include "../aot/aot_runtime.h"
+#include "aot_intrinsic.h"
 
 #define BUILD_ICMP(op, left, right, res, name)                                \
     do {                                                                      \
@@ -951,13 +952,66 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
     if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len)))
         return false;
 
-    /* TODO: lookup func ptr of "memmove" to call for XIP mode */
+    if (comp_ctx->is_indirect_mode) {
+        LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type;
+        LLVMValueRef func, params[3];
+        int32 func_idx;
 
-    if (!(res = LLVMBuildMemMove(comp_ctx->builder, dst_addr, 1, src_addr, 1,
-                                 len))) {
-        aot_set_last_error("llvm build memmove failed.");
-        return false;
+        if (!(dst_addr =
+                  LLVMBuildBitCast(comp_ctx->builder, dst_addr, INT32_PTR_TYPE,
+                                   "memmove dst addr cast type"))) {
+            aot_set_last_error("llvm cast memmove dst addr type failed.");
+            return false;
+        }
+
+        if (!(src_addr =
+                  LLVMBuildBitCast(comp_ctx->builder, src_addr, INT32_PTR_TYPE,
+                                   "memmove src addr cast type"))) {
+            aot_set_last_error("llvm cast memmove src addr type failed.");
+            return false;
+        }
+
+        param_types[0] = INT32_PTR_TYPE;
+        param_types[1] = INT32_PTR_TYPE;
+        param_types[2] = I32_TYPE;
+        ret_type = INT32_PTR_TYPE;
+
+        if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) {
+            aot_set_last_error("create LLVM function type failed.");
+            return false;
+        }
+
+        if (!(func_ptr_type = LLVMPointerType(func_type, 0))) {
+            aot_set_last_error("create LLVM function pointer type failed.");
+            return false;
+        }
+
+        func_idx = aot_get_native_symbol_index(comp_ctx, "memmove");
+        if (func_idx < 0) {
+            return false;
+        }
+        if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol,
+                                             func_ptr_type, func_idx))) {
+            return false;
+        }
+
+        params[0] = dst_addr;
+        params[1] = src_addr;
+        params[2] = len;
+        if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3,
+                                  "call memmove"))) {
+            aot_set_last_error("llvm build memmove failed.");
+            return false;
+        }
     }
+    else {
+        if (!(res = LLVMBuildMemMove(comp_ctx->builder, dst_addr, 1, src_addr,
+                                     1, len))) {
+            aot_set_last_error("llvm build memmove failed.");
+            return false;
+        }
+    }
+
     return true;
 fail:
     return false;
@@ -981,11 +1035,57 @@ aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
         return false;
     }
 
-    /* TODO: lookup func ptr of "memset" to call for XIP mode */
+    if (comp_ctx->is_indirect_mode) {
+        LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type;
+        LLVMValueRef func, params[3];
+        int32 func_idx;
 
-    if (!(res = LLVMBuildMemSet(comp_ctx->builder, dst_addr, val, len, 1))) {
-        aot_set_last_error("llvm build memset failed.");
-        return false;
+        if (!(dst_addr =
+                  LLVMBuildBitCast(comp_ctx->builder, dst_addr, INT32_PTR_TYPE,
+                                   "memset dst addr cast type"))) {
+            aot_set_last_error("llvm cast memset dst addr type failed.");
+            return false;
+        }
+
+        param_types[0] = INT32_PTR_TYPE;
+        param_types[1] = INT8_TYPE;
+        param_types[2] = I32_TYPE;
+        ret_type = INT32_PTR_TYPE;
+
+        if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) {
+            aot_set_last_error("create LLVM function type failed.");
+            return false;
+        }
+
+        if (!(func_ptr_type = LLVMPointerType(func_type, 0))) {
+            aot_set_last_error("create LLVM function pointer type failed.");
+            return false;
+        }
+
+        func_idx = aot_get_native_symbol_index(comp_ctx, "memset");
+        if (func_idx < 0) {
+            return false;
+        }
+        if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol,
+                                             func_ptr_type, func_idx))) {
+            return false;
+        }
+
+        params[0] = dst_addr;
+        params[1] = val;
+        params[2] = len;
+        if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3,
+                                  "call memset"))) {
+            aot_set_last_error("llvm build memset failed.");
+            return false;
+        }
+    }
+    else {
+        if (!(res =
+                  LLVMBuildMemSet(comp_ctx->builder, dst_addr, val, len, 1))) {
+            aot_set_last_error("llvm build memset failed.");
+            return false;
+        }
     }
     return true;
 fail:
@@ -1127,16 +1227,20 @@ aot_compile_op_atomic_cmpxchg(AOTCompContext *comp_ctx,
     }
 
     if (op_type == VALUE_TYPE_I32) {
-        if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE,
-                                     "result_i32"))) {
-            goto fail;
+        if (LLVMTypeOf(result) != I32_TYPE) {
+            if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE,
+                                         "result_i32"))) {
+                goto fail;
+            }
         }
         PUSH_I32(result);
     }
     else {
-        if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I64_TYPE,
-                                     "result_i64"))) {
-            goto fail;
+        if (LLVMTypeOf(result) != I64_TYPE) {
+            if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I64_TYPE,
+                                         "result_i64"))) {
+                goto fail;
+            }
         }
         PUSH_I64(result);
     }

core/iwasm/compilation/aot_llvm.h

@@ -445,6 +445,9 @@ aot_get_func_from_table(const AOTCompContext *comp_ctx, LLVMValueRef base,
 bool
 aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str);
 
+void
+aot_add_expand_memory_op_pass(LLVMPassManagerRef pass);
+
 #if WASM_ENABLE_LAZY_JIT != 0
 void
 aot_handle_llvm_errmsg(char *error_buf, uint32 error_buf_size,

core/iwasm/compilation/aot_llvm_extra.cpp

@@ -6,20 +6,29 @@
 #include <llvm/ADT/SmallVector.h>
 #include <llvm/ADT/Twine.h>
 #include <llvm/ADT/Triple.h>
+#include <llvm/Analysis/TargetTransformInfo.h>
+#include <llvm/CodeGen/TargetPassConfig.h>
 #include <llvm/ExecutionEngine/ExecutionEngine.h>
 #include <llvm/MC/MCSubtargetInfo.h>
 #include <llvm/Support/TargetSelect.h>
 #include <llvm/Target/TargetMachine.h>
 #include <llvm-c/Core.h>
 #include <llvm-c/ExecutionEngine.h>
+#include <llvm-c/Initialization.h>
 #include <llvm/ExecutionEngine/GenericValue.h>
 #include <llvm/ExecutionEngine/JITEventListener.h>
 #include <llvm/ExecutionEngine/RTDyldMemoryManager.h>
 #include <llvm/IR/DerivedTypes.h>
 #include <llvm/IR/Module.h>
+#include <llvm/IR/Instructions.h>
+#include <llvm/IR/IntrinsicInst.h>
+#include <llvm/IR/LegacyPassManager.h>
+#include <llvm/Support/CommandLine.h>
 #include <llvm/Support/ErrorHandling.h>
 #include <llvm/Target/CodeGenCWrappers.h>
+#include <llvm/Target/TargetMachine.h>
 #include <llvm/Target/TargetOptions.h>
+#include <llvm/Transforms/Utils/LowerMemIntrinsics.h>
 #include <cstring>
 
 using namespace llvm;
@@ -33,6 +42,9 @@ WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
 extern "C" bool
 aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str);
 
+extern "C" void
+aot_add_expand_memory_op_pass(LLVMPassManagerRef pass);
+
 extern "C" void
 aot_func_disable_tce(LLVMValueRef func);
 
@@ -106,6 +118,109 @@ WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT,
     return 1;
 }
 
+class ExpandMemoryOpPass : public llvm::ModulePass
+{
+  public:
+    static char ID;
+
+    ExpandMemoryOpPass()
+      : ModulePass(ID)
+    {}
+
+    bool runOnModule(Module &M) override;
+
+    bool expandMemIntrinsicUses(Function &F);
+    StringRef getPassName() const override
+    {
+        return "Expand memory operation intrinsics";
+    }
+
+    void getAnalysisUsage(AnalysisUsage &AU) const override
+    {
+        AU.addRequired<TargetTransformInfoWrapperPass>();
+    }
+};
+
+char ExpandMemoryOpPass::ID = 0;
+
+bool
+ExpandMemoryOpPass::expandMemIntrinsicUses(Function &F)
+{
+    Intrinsic::ID ID = F.getIntrinsicID();
+    bool Changed = false;
+
+    for (auto I = F.user_begin(), E = F.user_end(); I != E;) {
+        Instruction *Inst = cast<Instruction>(*I);
+        ++I;
+
+        switch (ID) {
+            case Intrinsic::memcpy:
+            {
+                auto *Memcpy = cast<MemCpyInst>(Inst);
+                Function *ParentFunc = Memcpy->getParent()->getParent();
+                const TargetTransformInfo &TTI =
+                    getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
+                        *ParentFunc);
+                expandMemCpyAsLoop(Memcpy, TTI);
+                Changed = true;
+                Memcpy->eraseFromParent();
+                break;
+            }
+            case Intrinsic::memmove:
+            {
+                auto *Memmove = cast<MemMoveInst>(Inst);
+                expandMemMoveAsLoop(Memmove);
+                Changed = true;
+                Memmove->eraseFromParent();
+                break;
+            }
+            case Intrinsic::memset:
+            {
+                auto *Memset = cast<MemSetInst>(Inst);
+                expandMemSetAsLoop(Memset);
+                Changed = true;
+                Memset->eraseFromParent();
+                break;
+            }
+            default:
+                break;
+        }
+    }
+
+    return Changed;
+}
+
+bool
+ExpandMemoryOpPass::runOnModule(Module &M)
+{
+    bool Changed = false;
+
+    for (Function &F : M) {
+        if (!F.isDeclaration())
+            continue;
+
+        switch (F.getIntrinsicID()) {
+            case Intrinsic::memcpy:
+            case Intrinsic::memmove:
+            case Intrinsic::memset:
+                if (expandMemIntrinsicUses(F))
+                    Changed = true;
+                break;
+
+            default:
+                break;
+        }
+    }
+
+    return Changed;
+}
+
+void
+aot_add_expand_memory_op_pass(LLVMPassManagerRef pass)
+{
+    unwrap(pass)->add(new ExpandMemoryOpPass());
+}
+
 bool
 aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str)
 {

core/iwasm/compilation/iwasm_compl.cmake

@@ -16,3 +16,11 @@ endif()
 
 set (IWASM_COMPL_SOURCE ${source_all})
 
+# Disalbe rtti to works with LLVM
+
+if (MSVC)
+  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
+else()
+  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
+endif()
+