wasm-micro-runtime/core/iwasm/compilation/aot_emit_function.c

/*
 * Copyright (C) 2019 Intel Corporation. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 */

#include "aot_emit_function.h"
#include "aot_emit_exception.h"
#include "aot_emit_control.h"
#include "../aot/aot_runtime.h"

/* Check whether there was exception thrown, if yes, return directly */
static bool
check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
    AOTFuncType *aot_func_type = func_ctx->aot_func->func_type;
    LLVMBasicBlockRef block_curr, check_exce_succ;
    LLVMValueRef value, cmp;

    /* Load the first byte of aot_module_inst->cur_exception, and check
       whether it is '\0'. If yes, no exception was thrown. */
    if (!(value = LLVMBuildLoad(comp_ctx->builder, func_ctx->cur_exception,
                                 "exce_value"))
        || !(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
                                  value, I8_ZERO, "cmp"))) {
        aot_set_last_error("llvm build icmp failed.");
        return false;
    }

    /* Add check exection success block */
    if (!(check_exce_succ = LLVMAppendBasicBlockInContext(comp_ctx->context,
                                                          func_ctx->func,
                                                          "check_exce_succ"))) {
        aot_set_last_error("llvm add basic block failed.");
        return false;
    }

    block_curr = LLVMGetInsertBlock(comp_ctx->builder);
    LLVMMoveBasicBlockAfter(check_exce_succ, block_curr);

    /* Create function return block if it isn't created */
    if (!func_ctx->func_return_block) {
        if (!(func_ctx->func_return_block =
                LLVMAppendBasicBlockInContext(comp_ctx->context,
                                              func_ctx->func,
                                              "func_ret"))) {
            aot_set_last_error("llvm add basic block failed.");
            return false;
        }

        /* Create return IR */
        LLVMPositionBuilderAtEnd(comp_ctx->builder, func_ctx->func_return_block);
        if (aot_func_type->result_count) {
            switch (aot_func_type->types[aot_func_type->param_count]) {
                case VALUE_TYPE_I32:
                    LLVMBuildRet(comp_ctx->builder, I32_ZERO);
                    break;
                case VALUE_TYPE_I64:
                    LLVMBuildRet(comp_ctx->builder, I64_ZERO);
                    break;
                case VALUE_TYPE_F32:
                    LLVMBuildRet(comp_ctx->builder, F32_ZERO);
                    break;
                case VALUE_TYPE_F64:
                    LLVMBuildRet(comp_ctx->builder, F64_ZERO);
                    break;
            }
        }
        else {
            LLVMBuildRetVoid(comp_ctx->builder);
        }
    }

    LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
    /* Create condition br */
    if (!LLVMBuildCondBr(comp_ctx->builder, cmp,
                         check_exce_succ, func_ctx->func_return_block)) {
        aot_set_last_error("llvm build cond br failed.");
        return false;
    }

    LLVMPositionBuilderAtEnd(comp_ctx->builder, check_exce_succ);
    return true;
}

bool
aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                    uint32 func_idx, uint8 **p_frame_ip)
{
    uint32 import_func_count = comp_ctx->comp_data->import_func_count;
    AOTImportFunc *import_funcs = comp_ctx->comp_data->import_funcs;
    uint32 func_count = comp_ctx->func_ctx_count;
    AOTFuncContext **func_ctxes = comp_ctx->func_ctxes;
    AOTFuncType *func_type;
    LLVMTypeRef *param_types = NULL, ret_type, f_type, f_ptr_type;
    LLVMValueRef *param_values = NULL, value_ret, func, value, cmp;
    LLVMBasicBlockRef check_func_ptr_succ;
    int32 i, j = 0, param_count;
    void *func_ptr;
    uint64 total_size;
    bool ret = false;

    /* Check function index */
    if (func_idx >= import_func_count + func_count) {
        aot_set_last_error("Function index out of range.");
        return false;
    }

    /* Get function type */
    if (func_idx < import_func_count)
        func_type = import_funcs[func_idx].func_type;
    else
        func_type = func_ctxes[func_idx - import_func_count]->
                                                aot_func->func_type;

    /* Allocate memory for parameters */
    param_count = (int32)func_type->param_count;
    total_size = sizeof(LLVMValueRef) * (uint64)(param_count + 1);
    if (total_size >= UINT32_MAX
        || !(param_values = wasm_malloc((uint32)total_size))) {
        aot_set_last_error("Allocate memory failed.");
        return false;
    }

    /* First parameter is exec env */
    param_values[j++] = func_ctx->exec_env;

    /* Pop parameters from stack */
    for (i = param_count - 1; i >= 0; i--)
        POP(param_values[i + j], func_type->types[i]);

    if (func_idx < import_func_count) {
        /* Get function pointer linked */
        func_ptr = import_funcs[func_idx].func_ptr_linked;

        /* Initialize parameter types of the LLVM function */
        total_size = sizeof(LLVMTypeRef) * (uint64)(param_count + 1);
        if (total_size >= UINT32_MAX
            || !(param_types = wasm_malloc((uint32)total_size))) {
            aot_set_last_error("Allocate memory failed.");
            goto fail;
        }

        j = 0;
        param_types[j++] = comp_ctx->exec_env_type;

        for (i = 0; i < param_count; i++)
            param_types[j++] = TO_LLVM_TYPE(func_type->types[i]);

        /* Resolve return type of the LLVM function */
        if (func_type->result_count)
            ret_type = TO_LLVM_TYPE(func_type->types[func_type->param_count]);
        else
            ret_type = VOID_TYPE;

        /* Resolve function prototype */
        if (!(f_type = LLVMFunctionType(ret_type, param_types,
                                        (uint32)param_count + 1, false))
            || !(f_ptr_type = LLVMPointerType(f_type, 0))) {
            aot_set_last_error("create LLVM function type failed.");
            goto fail;
        }

        if (comp_ctx->is_jit_mode) {
            if (!func_ptr) {
                /* The import function isn't linked, throw exception
                   when calling it. */
                if (!aot_emit_exception(comp_ctx, func_ctx,
                                        EXCE_CALL_UNLINKED_IMPORT_FUNC,
                                        false, NULL, NULL))
                    goto fail;
                ret = aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
                goto fail;
            }

            /* JIT mode, call the linked function directly */
            if (!(value = I64_CONST((uint64)(uintptr_t)func_ptr))
                || !(func = LLVMConstIntToPtr(value, f_ptr_type))) {
                aot_set_last_error("create LLVM value failed.");
                goto fail;
            }
        }
        else {
            /* Load function pointer */
            if (!(value = I32_CONST(func_idx))
                || !(func_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder,
                                                     func_ctx->func_ptrs,
                                                     &value, 1, "func_ptr"))) {
                aot_set_last_error("llvm build inbounds gep failed.");
                goto fail;
            }

            if (!(func = LLVMBuildLoad(comp_ctx->builder, func_ptr, "func_tmp"))) {
                aot_set_last_error("llvm build load failed.");
                goto fail;
            }

            /* Check whether import function is NULL */
            if (!(cmp = LLVMBuildIsNull(comp_ctx->builder, func, "is_func_null"))) {
                aot_set_last_error("llvm build icmp failed.");
                goto fail;
            }

            /* Throw exception if import function is NULL */
            if (!(check_func_ptr_succ =
                        LLVMAppendBasicBlockInContext(comp_ctx->context,
                                                      func_ctx->func,
                                                      "check_func_ptr_succ"))) {
                aot_set_last_error("llvm add basic block failed.");
                goto fail;
            }

            LLVMMoveBasicBlockAfter(check_func_ptr_succ,
                                    LLVMGetInsertBlock(comp_ctx->builder));

            if (!(aot_emit_exception(comp_ctx, func_ctx,
                                     EXCE_CALL_UNLINKED_IMPORT_FUNC,
                                     true, cmp, check_func_ptr_succ)))
                goto fail;

            if (!(func = LLVMBuildBitCast(comp_ctx->builder, func,
                                          f_ptr_type, "func"))) {
                aot_set_last_error("create LLVM value failed.");
                goto fail;
            }
        }
    }
    else {
        func = func_ctxes[func_idx - import_func_count]->func;
    }

    /* Call the function */
    if (!(value_ret = LLVMBuildCall(comp_ctx->builder, func,
                                    param_values, (uint32)param_count + 1,
                                    (func_type->result_count > 0
                                     ? "call" : "")))) {
        aot_set_last_error("LLVM build call failed.");
        goto fail;
    }

    /* Set calling convention for the call with the func's calling convention */
    LLVMSetInstructionCallConv(value_ret, LLVMGetFunctionCallConv(func));

    if (func_type->result_count > 0)
        PUSH(value_ret, func_type->types[func_type->param_count]);

    /* Check whether there was exception thrown when executing the function */
    if (!check_exception_thrown(comp_ctx, func_ctx))
        goto fail;

    ret = true;
fail:
    if (param_types)
        wasm_free(param_types);
    if (param_values)
        wasm_free(param_values);
  return ret;
}

bool
aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                             uint32 type_idx)
{
    AOTFuncType *func_type;
    LLVMValueRef elem_idx, table_elem, func_idx, ftype_idx_ptr, ftype_idx;
    LLVMValueRef cmp_elem_idx, cmp_func_idx, is_ftype_match, is_ftype_mismatch;
    LLVMValueRef func, func_ptr, func_const, table_size_const, cmp_func_ptr;
    LLVMValueRef *param_values = NULL, param_values_tmp[3], value_ret;
    LLVMTypeRef *param_types = NULL, param_types_tmp[3], ret_type,
                f_type, f_ptr_type;
    LLVMBasicBlockRef check_elem_idx_succ, check_ftype_idx_succ;
    LLVMBasicBlockRef check_func_idx_succ, check_func_ptr_succ;
    int32 i, j = 0, param_count;
    uint64 total_size;
    bool ret;
    char *func_name = "aot_is_wasm_type_equal";

    /* Check function type index */
    if (type_idx >= comp_ctx->comp_data->func_type_count) {
        aot_set_last_error("type index is overflow");
        return false;
    }

    func_type = comp_ctx->comp_data->func_types[type_idx];

    POP_I32(elem_idx);

    table_size_const = I32_CONST(comp_ctx->comp_data->table_size);
    CHECK_LLVM_CONST(table_size_const);

    /* Check if (uint32)elem index >= table size */
    if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE,
                                       elem_idx, table_size_const,
                                       "cmp_elem_idx"))) {
        aot_set_last_error("llvm build icmp failed.");
        goto fail;
    }

    /* Throw exception if elem index >= table size */
    if (!(check_elem_idx_succ =
                LLVMAppendBasicBlockInContext(comp_ctx->context,
                                              func_ctx->func,
                                              "check_elem_idx_succ"))) {
        aot_set_last_error("llvm add basic block failed.");
        goto fail;
    }

    LLVMMoveBasicBlockAfter(check_elem_idx_succ,
                            LLVMGetInsertBlock(comp_ctx->builder));

    if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNDEFINED_ELEMENT,
                             true, cmp_elem_idx, check_elem_idx_succ)))
        goto fail;

    /* Load function index */
    if (!(table_elem = LLVMBuildInBoundsGEP(comp_ctx->builder,
                                            func_ctx->table_base,
                                            &elem_idx, 1, "table_elem"))) {
        aot_set_last_error("llvm build add failed.");
        goto fail;
    }

    if (!(func_idx = LLVMBuildLoad(comp_ctx->builder, table_elem, "func_idx"))) {
        aot_set_last_error("llvm build load failed.");
        goto fail;
    }

    /* Check if func_idx == -1 */
    if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
                                       func_idx, I32_NEG_ONE,
                                       "cmp_func_idx"))) {
        aot_set_last_error("llvm build icmp failed.");
        goto fail;
    }

    /* Throw exception if func_idx == -1 */
    if (!(check_func_idx_succ =
                LLVMAppendBasicBlockInContext(comp_ctx->context,
                                              func_ctx->func,
                                              "check_func_idx_succ"))) {
        aot_set_last_error("llvm add basic block failed.");
        goto fail;
    }

    LLVMMoveBasicBlockAfter(check_func_idx_succ,
                            LLVMGetInsertBlock(comp_ctx->builder));

    if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNINITIALIZED_ELEMENT,
                             true, cmp_func_idx, check_func_idx_succ)))
        goto fail;

    /* Load function type index */
    if (!(ftype_idx_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder,
                                               func_ctx->func_type_indexes,
                                               &func_idx, 1,
                                               "ftype_idx_ptr"))) {
        aot_set_last_error("llvm build inbounds gep failed.");
        goto fail;
    }

    if (!(ftype_idx = LLVMBuildLoad(comp_ctx->builder, ftype_idx_ptr,
                                    "ftype_idx"))) {
        aot_set_last_error("llvm build load failed.");
        goto fail;
    }

    /* Call aot_is_type_equal() to check whether function type match */
    param_types_tmp[0] = INT8_PTR_TYPE;
    param_types_tmp[1] = I32_TYPE;
    param_types_tmp[2] = I32_TYPE;
    ret_type = INT8_TYPE;

    /* Create function type */
    if (!(f_type = LLVMFunctionType(ret_type, param_types_tmp,
                                    3, false))) {
        aot_set_last_error("create LLVM function type failed.");
        goto fail;
    }

    if (comp_ctx->is_jit_mode) {
        /* Create function type */
        if (!(f_ptr_type = LLVMPointerType(f_type, 0))) {
            aot_set_last_error("create LLVM function type failed.");
            goto fail;
        }
        /* Create LLVM function with const function pointer */
        if (!(func_const =
                    I64_CONST((uint64)(uintptr_t)aot_is_wasm_type_equal))
                || !(func = LLVMConstIntToPtr(func_const, f_ptr_type))) {
            aot_set_last_error("create LLVM value failed.");
            goto fail;
        }
    }
    else {
        /* Create LLVM function with external function pointer */
        if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name))
            && !(func = LLVMAddFunction(comp_ctx->module, func_name, f_type))) {
            aot_set_last_error("add LLVM function failed.");
            goto fail;
        }
    }

    /* Call the aot_is_type_equal() function */
    param_values_tmp[0] = func_ctx->aot_inst;
    param_values_tmp[1] = I32_CONST(type_idx);
    param_values_tmp[2] = ftype_idx;

    CHECK_LLVM_CONST(param_values_tmp[1]);

    if (!(is_ftype_match = LLVMBuildCall(comp_ctx->builder, func,
                                         param_values_tmp, 3,
                                         "is_ftype_match"))) {
        aot_set_last_error("llvm build icmp failed.");
        goto fail;
    }

    if (!(is_ftype_mismatch = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
                                            is_ftype_match, I8_ZERO,
                                            "is_ftype_mismatch"))) {
        aot_set_last_error("llvm build icmp failed.");
        goto fail;
    }

    if (!(check_ftype_idx_succ =
                LLVMAppendBasicBlockInContext(comp_ctx->context,
                                              func_ctx->func,
                                              "check_ftype_idx_success"))) {
        aot_set_last_error("llvm add basic block failed.");
        goto fail;
    }

    LLVMMoveBasicBlockAfter(check_ftype_idx_succ,
                            LLVMGetInsertBlock(comp_ctx->builder));

    if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INVALID_FUNCTION_TYPE_INDEX,
                             true, is_ftype_mismatch, check_ftype_idx_succ)))
        goto fail;

    /* Load function pointer */
    if (!(func_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->func_ptrs,
                                          &func_idx, 1, "func_ptr"))) {
        aot_set_last_error("llvm build inbounds gep failed.");
        goto fail;
    }

    if (!(func = LLVMBuildLoad(comp_ctx->builder, func_ptr, "func_tmp"))) {
        aot_set_last_error("llvm build load failed.");
        goto fail;
    }

    /* Check whether import function is NULL */
    if (!(cmp_func_ptr = LLVMBuildIsNull(comp_ctx->builder, func, "is_func_null"))) {
        aot_set_last_error("llvm build is null failed.");
        goto fail;
    }

    /* Throw exception if import function is NULL */
    if (!(check_func_ptr_succ =
                LLVMAppendBasicBlockInContext(comp_ctx->context,
                                              func_ctx->func,
                                              "check_func_ptr_succ"))) {
        aot_set_last_error("llvm add basic block failed.");
        goto fail;
    }

    LLVMMoveBasicBlockAfter(check_func_ptr_succ,
                            LLVMGetInsertBlock(comp_ctx->builder));

    if (!(aot_emit_exception(comp_ctx, func_ctx,
                             EXCE_CALL_UNLINKED_IMPORT_FUNC,
                             true, cmp_func_ptr, check_func_ptr_succ)))
        goto fail;

    /* Initialize parameter types of the LLVM function */
    param_count = (int32)func_type->param_count;
    total_size = sizeof(LLVMTypeRef) * (uint64)(param_count + 1);
    if (total_size >= UINT32_MAX
        || !(param_types = wasm_malloc((uint32)total_size))) {
        aot_set_last_error("Allocate memory failed.");
        goto fail;
    }

    j = 0;
    param_types[j++] = comp_ctx->exec_env_type;
    for (i = 0; i < param_count; i++)
        param_types[j++] = TO_LLVM_TYPE(func_type->types[i]);

    /* Resolve return type of the LLVM function */
    if (func_type->result_count)
        ret_type = TO_LLVM_TYPE(func_type->types[func_type->param_count]);
    else
        ret_type = VOID_TYPE;

    /* Resolve function prototype */
    if (!(f_type = LLVMFunctionType(ret_type, param_types,
                                    (uint32)param_count + 1, false))
        || !(f_ptr_type = LLVMPointerType(f_type, 0))) {
        aot_set_last_error("create LLVM function type failed.");
        goto fail;
    }

    if (!(func = LLVMBuildBitCast(comp_ctx->builder, func,
                                  f_ptr_type, "func"))) {
        aot_set_last_error("create LLVM value failed.");
        goto fail;
    }

    /* Allocate memory for parameters */
    total_size = sizeof(LLVMValueRef) * (uint64)(param_count + 1);
    if (total_size >= UINT32_MAX
        || !(param_values = wasm_malloc((uint32)total_size))) {
        aot_set_last_error("Allocate memory failed.");
        goto fail;
    }

    /* First parameter is exec env */
    j = 0;
    param_values[j++] = func_ctx->exec_env;

    /* Pop parameters from stack */
    for (i = param_count - 1; i >= 0; i--)
        POP(param_values[i + j], func_type->types[i]);

    /* Call the function */
    if (!(value_ret = LLVMBuildCall(comp_ctx->builder, func,
                                    param_values, (uint32)param_count + 1,
                                    (func_type->result_count > 0
                                     ? "call_indirect" : "")))) {
        aot_set_last_error("LLVM build call failed.");
        goto fail;
    }

    if (func_type->result_count > 0)
        PUSH(value_ret, func_type->types[func_type->param_count]);

    /* Check whether there was exception thrown when executing the function */
    if (!check_exception_thrown(comp_ctx, func_ctx))
        goto fail;

    ret = true;

fail:
    if (param_values)
        wasm_free(param_values);
    if (param_types)
        wasm_free(param_types);
    return ret;
}