Enable running mode control for runtime and module instance (#1923)

Enable setting running mode when executing a wasm bytecode file
- Four running modes are supported: interpreter, fast-jit, llvm-jit and multi-tier-jit
- Add APIs to set/get the default running mode of the runtime
- Add APIs to set/get the running mode of a wasm module instance
- Add running mode options for iwasm command line tool

And add size/opt level options for LLVM JIT

core/iwasm/common/wasm_runtime_common.c

@@ -128,6 +128,12 @@ runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst,
 static JitCompOptions jit_options = { 0 };
 #endif
 
+#if WASM_ENABLE_JIT != 0
+static LLVMJITOptions llvm_jit_options = { 3, 3 };
+#endif
+
+static RunningMode runtime_running_mode = Mode_Default;
+
 #ifdef OS_ENABLE_HW_BOUND_CHECK
 /* The exec_env of thread local storage, set before calling function
    and used in signal handler, as we cannot get it from the argument
@@ -514,6 +520,20 @@ wasm_runtime_destroy()
     wasm_runtime_memory_destroy();
 }
 
+RunningMode
+wasm_runtime_get_default_running_mode(void)
+{
+    return runtime_running_mode;
+}
+
+#if WASM_ENABLE_JIT != 0
+LLVMJITOptions
+wasm_runtime_get_llvm_jit_options(void)
+{
+    return llvm_jit_options;
+}
+#endif
+
 bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args)
 {
@@ -521,10 +541,20 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args)
                                   &init_args->mem_alloc_option))
         return false;
 
+    if (!wasm_runtime_set_default_running_mode(init_args->running_mode)) {
+        wasm_runtime_memory_destroy();
+        return false;
+    }
+
 #if WASM_ENABLE_FAST_JIT != 0
     jit_options.code_cache_size = init_args->fast_jit_code_cache_size;
 #endif
 
+#if WASM_ENABLE_JIT != 0
+    llvm_jit_options.size_level = init_args->llvm_jit_size_level;
+    llvm_jit_options.opt_level = init_args->llvm_jit_opt_level;
+#endif
+
     if (!wasm_runtime_env_init()) {
         wasm_runtime_memory_destroy();
         return false;
@@ -554,6 +584,47 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args)
     return true;
 }
 
+bool
+wasm_runtime_is_running_mode_supported(RunningMode running_mode)
+{
+    if (running_mode == Mode_Default) {
+        return true;
+    }
+    else if (running_mode == Mode_Interp) {
+#if WASM_ENABLE_INTERP != 0
+        return true;
+#endif
+    }
+    else if (running_mode == Mode_Fast_JIT) {
+#if WASM_ENABLE_FAST_JIT != 0
+        return true;
+#endif
+    }
+    else if (running_mode == Mode_LLVM_JIT) {
+#if WASM_ENABLE_JIT != 0
+        return true;
+#endif
+    }
+    else if (running_mode == Mode_Multi_Tier_JIT) {
+#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
+    && WASM_ENABLE_LAZY_JIT != 0
+        return true;
+#endif
+    }
+
+    return false;
+}
+
+bool
+wasm_runtime_set_default_running_mode(RunningMode running_mode)
+{
+    if (wasm_runtime_is_running_mode_supported(running_mode)) {
+        runtime_running_mode = running_mode;
+        return true;
+    }
+    return false;
+}
+
 PackageType
 get_package_type(const uint8 *buf, uint32 size)
 {
@@ -1171,6 +1242,41 @@ wasm_runtime_deinstantiate_internal(WASMModuleInstanceCommon *module_inst,
 #endif
 }
 
+bool
+wasm_runtime_set_running_mode(wasm_module_inst_t module_inst,
+                              RunningMode running_mode)
+{
+#if WASM_ENABLE_AOT != 0
+    if (module_inst->module_type == Wasm_Module_AoT)
+        return true;
+#endif
+
+#if WASM_ENABLE_INTERP != 0
+    if (module_inst->module_type == Wasm_Module_Bytecode) {
+        WASMModuleInstance *module_inst_interp =
+            (WASMModuleInstance *)module_inst;
+
+        return wasm_set_running_mode(module_inst_interp, running_mode);
+    }
+#endif
+
+    return false;
+}
+
+RunningMode
+wasm_runtime_get_running_mode(wasm_module_inst_t module_inst)
+{
+#if WASM_ENABLE_INTERP != 0
+    if (module_inst->module_type == Wasm_Module_Bytecode) {
+        WASMModuleInstance *module_inst_interp =
+            (WASMModuleInstance *)module_inst;
+        return module_inst_interp->e->running_mode;
+    }
+#endif
+
+    return Mode_Default;
+}
+
 void
 wasm_runtime_deinstantiate(WASMModuleInstanceCommon *module_inst)
 {

core/iwasm/common/wasm_runtime_common.h

@@ -25,6 +25,9 @@
 extern "C" {
 #endif
 
+/* Internal use for setting default running mode */
+#define Mode_Default 0
+
 #if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0
 
 #define PUT_I64_TO_ADDR(addr, value)       \
@@ -413,6 +416,13 @@ typedef struct wasm_frame_t {
     const char *func_name_wp;
 } WASMCApiFrame;
 
+#ifdef WASM_ENABLE_JIT
+typedef struct LLVMJITOptions {
+    uint32 opt_level;
+    uint32 size_level;
+} LLVMJITOptions;
+#endif
+
 #ifdef OS_ENABLE_HW_BOUND_CHECK
 /* Signal info passing to interp/aot signal handler */
 typedef struct WASMSignalInfo {
@@ -437,10 +447,28 @@ wasm_runtime_get_exec_env_tls(void);
 WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_init(void);
 
+/* Internal API */
+RunningMode
+wasm_runtime_get_default_running_mode(void);
+
+#if WASM_ENABLE_JIT != 0
+/* Internal API */
+LLVMJITOptions
+wasm_runtime_get_llvm_jit_options(void);
+#endif
+
 /* See wasm_export.h for description */
 WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args);
 
+/* See wasm_export.h for description */
+WASM_RUNTIME_API_EXTERN bool
+wasm_runtime_is_running_mode_supported(RunningMode running_mode);
+
+/* See wasm_export.h for description */
+WASM_RUNTIME_API_EXTERN bool
+wasm_runtime_set_default_running_mode(RunningMode running_mode);
+
 /* See wasm_export.h for description */
 WASM_RUNTIME_API_EXTERN void
 wasm_runtime_destroy(void);
@@ -484,6 +512,15 @@ wasm_runtime_instantiate(WASMModuleCommon *module, uint32 stack_size,
                          uint32 heap_size, char *error_buf,
                          uint32 error_buf_size);
 
+/* See wasm_export.h for description */
+WASM_RUNTIME_API_EXTERN bool
+wasm_runtime_set_running_mode(wasm_module_inst_t module_inst,
+                              RunningMode running_mode);
+
+/* See wasm_export.h for description */
+WASM_RUNTIME_API_EXTERN RunningMode
+wasm_runtime_get_running_mode(wasm_module_inst_t module_inst);
+
 /* See wasm_export.h for description */
 WASM_RUNTIME_API_EXTERN void
 wasm_runtime_deinstantiate(WASMModuleInstanceCommon *module_inst);

core/iwasm/fast-jit/jit_codecache.c

@@ -56,9 +56,31 @@ jit_code_cache_free(void *ptr)
 bool
 jit_pass_register_jitted_code(JitCompContext *cc)
 {
-    uint32 jit_func_idx =
-        cc->cur_wasm_func_idx - cc->cur_wasm_module->import_function_count;
-    cc->cur_wasm_module->fast_jit_func_ptrs[jit_func_idx] =
-        cc->cur_wasm_func->fast_jit_jitted_code = cc->jitted_addr_begin;
+    WASMModuleInstance *instance;
+    WASMModule *module = cc->cur_wasm_module;
+    WASMFunction *func = cc->cur_wasm_func;
+    uint32 jit_func_idx = cc->cur_wasm_func_idx - module->import_function_count;
+
+#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
+    && WASM_ENABLE_LAZY_JIT != 0
+    os_mutex_lock(&module->instance_list_lock);
+#endif
+
+    module->fast_jit_func_ptrs[jit_func_idx] = func->fast_jit_jitted_code =
+        cc->jitted_addr_begin;
+
+#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
+    && WASM_ENABLE_LAZY_JIT != 0
+    instance = module->instance_list;
+    while (instance) {
+        if (instance->e->running_mode == Mode_Fast_JIT)
+            instance->fast_jit_func_ptrs[jit_func_idx] = cc->jitted_addr_begin;
+        instance = instance->e->next;
+    }
+
+    os_mutex_unlock(&module->instance_list_lock);
+#else
+    (void)instance;
+#endif
     return true;
 }

core/iwasm/fast-jit/jit_compiler.c

@@ -254,6 +254,8 @@ jit_compiler_set_call_to_fast_jit(WASMModule *module, uint32 func_idx)
 
     func_ptr = jit_codegen_compile_call_to_fast_jit(module, func_idx);
     if (func_ptr) {
+        uint32 i = func_idx - module->import_function_count;
+        module->functions[i]->call_to_fast_jit_from_llvm_jit = func_ptr;
         jit_compiler_set_llvm_jit_func_ptr(module, func_idx, func_ptr);
     }
 
@@ -267,12 +269,14 @@ jit_compiler_set_llvm_jit_func_ptr(WASMModule *module, uint32 func_idx,
     WASMModuleInstance *instance;
     uint32 i = func_idx - module->import_function_count;
 
-    module->functions[i]->llvm_jit_func_ptr = module->func_ptrs[i] = func_ptr;
-
     os_mutex_lock(&module->instance_list_lock);
+
+    module->func_ptrs[i] = func_ptr;
+
     instance = module->instance_list;
     while (instance) {
-        instance->func_ptrs[func_idx] = func_ptr;
+        if (instance->e->running_mode == Mode_Multi_Tier_JIT)
+            instance->func_ptrs[func_idx] = func_ptr;
         instance = instance->e->next;
     }
     os_mutex_unlock(&module->instance_list_lock);

core/iwasm/include/wasm_export.h

@@ -131,6 +131,14 @@ typedef struct mem_alloc_info_t {
     uint32_t highmark_size;
 } mem_alloc_info_t;
 
+/* Running mode of runtime and module instance*/
+typedef enum RunningMode {
+    Mode_Interp = 1,
+    Mode_Fast_JIT,
+    Mode_LLVM_JIT,
+    Mode_Multi_Tier_JIT,
+} RunningMode;
+
 /* WASM runtime initialize arguments */
 typedef struct RuntimeInitArgs {
     mem_alloc_type_t mem_alloc_type;
@@ -152,6 +160,13 @@ typedef struct RuntimeInitArgs {
 
     /* Fast JIT code cache size */
     uint32_t fast_jit_code_cache_size;
+
+    /* Default running mode of the runtime */
+    RunningMode running_mode;
+
+    /* LLVM JIT opt and size level */
+    uint32_t llvm_jit_opt_level;
+    uint32_t llvm_jit_size_level;
 } RuntimeInitArgs;
 
 #ifndef WASM_VALKIND_T_DEFINED
@@ -195,9 +210,9 @@ WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_init(void);
 
 /**
- * Initialize the WASM runtime environment, and also initialize
- * the memory allocator and register native symbols, which are specified
- * with init arguments
+ * Initialize the WASM runtime environment, WASM running mode,
+ * and also initialize the memory allocator and register native symbols,
+ * which are specified with init arguments
  *
  * @param init_args specifies the init arguments
  *
@@ -206,6 +221,28 @@ wasm_runtime_init(void);
 WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args);
 
+/**
+ * Query whether a certain running mode is supported for the runtime
+ *
+ * @param running_mode the running mode to query
+ *
+ * @return true if this running mode is supported, false otherwise
+ */
+WASM_RUNTIME_API_EXTERN bool
+wasm_runtime_is_running_mode_supported(RunningMode running_mode);
+
+/**
+ * Set the default running mode for the runtime. It is inherited
+ * to set the running mode of a module instance when it is instantiated,
+ * and can be changed by calling wasm_runtime_set_running_mode
+ *
+ * @param running_mode the running mode to set
+ *
+ * @return true if success, false otherwise
+ */
+WASM_RUNTIME_API_EXTERN bool
+wasm_runtime_set_default_running_mode(RunningMode running_mode);
+
 /**
  * Destroy the WASM runtime environment.
  */
@@ -450,6 +487,34 @@ wasm_runtime_instantiate(const wasm_module_t module,
                          uint32_t stack_size, uint32_t heap_size,
                          char *error_buf, uint32_t error_buf_size);
 
+/**
+ * Set the running mode of a WASM module instance, override the
+ * default running mode of the runtime. Note that it only makes sense when
+ * the input is a wasm bytecode file: for the AOT file, runtime always runs
+ * it with AOT engine, and this function always returns true.
+ *
+ * @param module_inst the WASM module instance to set running mode
+ * @param running_mode the running mode to set
+ *
+ * @return true if success, false otherwise
+ */
+WASM_RUNTIME_API_EXTERN bool
+wasm_runtime_set_running_mode(wasm_module_inst_t module_inst,
+                              RunningMode running_mode);
+
+/**
+ * Get the running mode of a WASM module instance, if no running mode
+ * is explicitly set the default running mode of runtime will
+ * be used and returned. Note that it only makes sense when the input is a
+ * wasm bytecode file: for the AOT file, this function always returns 0.
+ *
+ * @param module_inst the WASM module instance to query for running mode
+ *
+ * @return the running mode this module instance currently use
+ */
+WASM_RUNTIME_API_EXTERN RunningMode
+wasm_runtime_get_running_mode(wasm_module_inst_t module_inst);
+
 /**
  * Deinstantiate a WASM module instance, destroy the resources.
  *

core/iwasm/interpreter/wasm.h

@@ -278,9 +278,14 @@ struct WASMFunction {
 #endif
 
 #if WASM_ENABLE_FAST_JIT != 0
+    /* The compiled fast jit jitted code block of this function */
     void *fast_jit_jitted_code;
 #if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
+    /* The compiled llvm jit func ptr of this function */
     void *llvm_jit_func_ptr;
+    /* Code block to call fast jit jitted code of this function
+       from the llvm jit jitted code */
+    void *call_to_fast_jit_from_llvm_jit;
 #endif
 #endif
 };
@@ -512,8 +517,7 @@ struct WASMModule {
      * List of instances referred to this module. When source debugging
      * feature is enabled, the debugger may modify the code section of
      * the module, so we need to report a warning if user create several
-     * instances based on the same module. Sub instances created by
-     * lib-pthread or spawn API won't be added into the list.
+     * instances based on the same module.
      *
      * Also add the instance to the list for Fast JIT to LLVM JIT
      * tier-up, since we need to lazily update the LLVM func pointers
@@ -533,7 +537,22 @@ struct WASMModule {
 #endif
 
 #if WASM_ENABLE_FAST_JIT != 0
-    /* func pointers of Fast JITed (un-imported) functions */
+    /**
+     * func pointers of Fast JITed (un-imported) functions
+     * for non Multi-Tier JIT mode:
+     *   (1) when lazy jit is disabled, each pointer is set to the compiled
+     *       fast jit jitted code
+     *   (2) when lazy jit is enabled, each pointer is firstly inited as
+     *       jit_global->compile_fast_jit_and_then_call, and then set to the
+     *       compiled fast jit jitted code when it is called (the stub will
+     *       compile the jit function and then update itself)
+     * for Multi-Tier JIT mode:
+     *   each pointer is firstly inited as compile_fast_jit_and_then_call,
+     *   and then set to the compiled fast jit jitted code when it is called,
+     *   and when the llvm jit func ptr of the same function is compiled, it
+     *   will be set to call_to_llvm_jit_from_fast_jit of this function type
+     *   (tier-up from fast-jit to llvm-jit)
+     */
     void **fast_jit_func_ptrs;
     /* locks for Fast JIT lazy compilation */
     korp_mutex fast_jit_thread_locks[WASM_ORC_JIT_BACKEND_THREAD_NUM];
@@ -543,7 +562,16 @@ struct WASMModule {
 #if WASM_ENABLE_JIT != 0
     struct AOTCompData *comp_data;
     struct AOTCompContext *comp_ctx;
-    /* func pointers of LLVM JITed (un-imported) functions */
+    /**
+     * func pointers of LLVM JITed (un-imported) functions
+     * for non Multi-Tier JIT mode:
+     *   each pointer is set to the lookuped llvm jit func ptr, note that it
+     *   is a stub and will trigger the actual compilation when it is called
+     * for Multi-Tier JIT mode:
+     *   each pointer is inited as call_to_fast_jit code block, when the llvm
+     *   jit func ptr is actually compiled, it is set to the compiled llvm jit
+     *   func ptr
+     */
     void **func_ptrs;
     /* whether the func pointers are compiled */
     bool *func_ptrs_compiled;
@@ -568,6 +596,12 @@ struct WASMModule {
     korp_tid llvm_jit_init_thread;
     /* whether the llvm jit is initialized */
     bool llvm_jit_inited;
+    /* Whether to enable llvm jit compilation:
+       it is set to true only when there is a module instance starts to
+       run with running mode Mode_LLVM_JIT or Mode_Multi_Tier_JIT,
+       since no need to enable llvm jit compilation for Mode_Interp and
+       Mode_Fast_JIT, so as to improve performance for them */
+    bool enable_llvm_jit_compilation;
 #endif
 };
 

core/iwasm/interpreter/wasm_interp_classic.c

@@ -4195,58 +4195,51 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
         }
     }
     else {
-#if WASM_ENABLE_LAZY_JIT != 0
+        RunningMode running_mode =
+            wasm_runtime_get_running_mode((wasm_module_inst_t)module_inst);
 
-        /* Fast JIT to LLVM JIT tier-up is enabled */
-#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0
-        /* Fast JIT and LLVM JIT are both enabled, call llvm jit function
-           if it is compiled, else call fast jit function */
-        uint32 func_idx = (uint32)(function - module_inst->e->functions);
-        if (module_inst->module->func_ptrs_compiled
-                [func_idx - module_inst->module->import_function_count]) {
+        if (running_mode == Mode_Interp) {
+            wasm_interp_call_func_bytecode(module_inst, exec_env, function,
+                                           frame);
+        }
+#if WASM_ENABLE_FAST_JIT != 0
+        else if (running_mode == Mode_Fast_JIT) {
+            fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
+        }
+#endif
+#if WASM_ENABLE_JIT != 0
+        else if (running_mode == Mode_LLVM_JIT) {
             llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
                                         argv);
             /* For llvm jit, the results have been stored in argv,
                no need to copy them from stack frame again */
             copy_argv_from_frame = false;
         }
-        else {
-            fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
-        }
-#elif WASM_ENABLE_JIT != 0
-        /* Only LLVM JIT is enabled */
-        llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
-                                    argv);
-        /* For llvm jit, the results have been stored in argv,
-           no need to copy them from stack frame again */
-        copy_argv_from_frame = false;
-#elif WASM_ENABLE_FAST_JIT != 0
-        /* Only Fast JIT is enabled */
-        fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
-#else
-        /* Both Fast JIT and LLVM JIT are disabled */
-        wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
 #endif
-
-#else /* else of WASM_ENABLE_LAZY_JIT != 0 */
-
-        /* Fast JIT to LLVM JIT tier-up is enabled */
-#if WASM_ENABLE_JIT != 0
-        /* LLVM JIT is enabled */
-        llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
-                                    argv);
-        /* For llvm jit, the results have been stored in argv,
-           no need to copy them from stack frame again */
-        copy_argv_from_frame = false;
-#elif WASM_ENABLE_FAST_JIT != 0
-        /* Fast JIT is enabled */
-        fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
-#else
-        /* Both Fast JIT and LLVM JIT are disabled */
-        wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame);
+#if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 \
+    && WASM_ENABLE_JIT != 0
+        else if (running_mode == Mode_Multi_Tier_JIT) {
+            /* Tier-up from Fast JIT to LLVM JIT, call llvm jit function
+               if it is compiled, else call fast jit function */
+            uint32 func_idx = (uint32)(function - module_inst->e->functions);
+            if (module_inst->module->func_ptrs_compiled
+                    [func_idx - module_inst->module->import_function_count]) {
+                llvm_jit_call_func_bytecode(module_inst, exec_env, function,
+                                            argc, argv);
+                /* For llvm jit, the results have been stored in argv,
+                   no need to copy them from stack frame again */
+                copy_argv_from_frame = false;
+            }
+            else {
+                fast_jit_call_func_bytecode(module_inst, exec_env, function,
+                                            frame);
+            }
+        }
 #endif
-
-#endif /* end of WASM_ENABLE_LAZY_JIT != 0 */
+        else {
+            /* There should always be a supported running mode selected */
+            bh_assert(0);
+        }
 
         (void)wasm_interp_call_func_bytecode;
 #if WASM_ENABLE_FAST_JIT != 0

core/iwasm/interpreter/wasm_loader.c

@@ -2989,6 +2989,7 @@ static bool
 init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
                                uint32 error_buf_size)
 {
+    LLVMJITOptions llvm_jit_options = wasm_runtime_get_llvm_jit_options();
     AOTCompOption option = { 0 };
     char *aot_last_error;
     uint64 size;
@@ -3027,8 +3028,11 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
     }
 
     option.is_jit_mode = true;
-    option.opt_level = 3;
-    option.size_level = 3;
+
+    llvm_jit_options = wasm_runtime_get_llvm_jit_options();
+    option.opt_level = llvm_jit_options.opt_level;
+    option.size_level = llvm_jit_options.size_level;
+
 #if WASM_ENABLE_BULK_MEMORY != 0
     option.enable_bulk_memory = true;
 #endif
@@ -3112,6 +3116,8 @@ init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf,
         module->func_ptrs[i] = (void *)func_addr;
 
 #if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
+        module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
+
         if (module->orcjit_stop_compiling)
             return false;
 #endif
@@ -3202,9 +3208,9 @@ orcjit_thread_callback(void *arg)
 
     /* Wait until init_llvm_jit_functions_stage2 finishes */
     os_mutex_lock(&module->tierup_wait_lock);
-    while (!module->llvm_jit_inited) {
+    while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation)) {
         os_cond_reltimedwait(&module->tierup_wait_cond,
-                             &module->tierup_wait_lock, 10);
+                             &module->tierup_wait_lock, 10000);
         if (module->orcjit_stop_compiling) {
             /* init_llvm_jit_functions_stage2 failed */
             os_mutex_unlock(&module->tierup_wait_lock);
@@ -4300,9 +4306,9 @@ wasm_loader_unload(WASMModule *module)
                         module->functions[i]->fast_jit_jitted_code);
                 }
 #if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
-                if (module->functions[i]->llvm_jit_func_ptr) {
+                if (module->functions[i]->call_to_fast_jit_from_llvm_jit) {
                     jit_code_cache_free(
-                        module->functions[i]->llvm_jit_func_ptr);
+                        module->functions[i]->call_to_fast_jit_from_llvm_jit);
                 }
 #endif
 #endif

core/iwasm/interpreter/wasm_mini_loader.c

@@ -1835,6 +1835,7 @@ static bool
 init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
                                uint32 error_buf_size)
 {
+    LLVMJITOptions llvm_jit_options = wasm_runtime_get_llvm_jit_options();
     AOTCompOption option = { 0 };
     char *aot_last_error;
     uint64 size;
@@ -1873,8 +1874,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
     }
 
     option.is_jit_mode = true;
-    option.opt_level = 3;
-    option.size_level = 3;
+    option.opt_level = llvm_jit_options.opt_level;
+    option.size_level = llvm_jit_options.size_level;
+
 #if WASM_ENABLE_BULK_MEMORY != 0
     option.enable_bulk_memory = true;
 #endif
@@ -1960,6 +1962,8 @@ init_llvm_jit_functions_stage2(WASMModule *module, char *error_buf,
         module->func_ptrs[i] = (void *)func_addr;
 
 #if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
+        module->functions[i]->llvm_jit_func_ptr = (void *)func_addr;
+
         if (module->orcjit_stop_compiling)
             return false;
 #endif
@@ -2050,9 +2054,9 @@ orcjit_thread_callback(void *arg)
 
     /* Wait until init_llvm_jit_functions_stage2 finishes */
     os_mutex_lock(&module->tierup_wait_lock);
-    while (!module->llvm_jit_inited) {
+    while (!(module->llvm_jit_inited && module->enable_llvm_jit_compilation)) {
         os_cond_reltimedwait(&module->tierup_wait_cond,
-                             &module->tierup_wait_lock, 10);
+                             &module->tierup_wait_lock, 10000);
         if (module->orcjit_stop_compiling) {
             /* init_llvm_jit_functions_stage2 failed */
             os_mutex_unlock(&module->tierup_wait_lock);
@@ -2998,9 +3002,9 @@ wasm_loader_unload(WASMModule *module)
                         module->functions[i]->fast_jit_jitted_code);
                 }
 #if WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
-                if (module->functions[i]->llvm_jit_func_ptr) {
+                if (module->functions[i]->call_to_fast_jit_from_llvm_jit) {
                     jit_code_cache_free(
-                        module->functions[i]->llvm_jit_func_ptr);
+                        module->functions[i]->call_to_fast_jit_from_llvm_jit);
                 }
 #endif
 #endif

core/iwasm/interpreter/wasm_runtime.c

@@ -737,13 +737,12 @@ functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
 
         function++;
     }
+    bh_assert((uint32)(function - functions) == function_count);
 
 #if WASM_ENABLE_FAST_JIT != 0
     module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
 #endif
 
-    bh_assert((uint32)(function - functions) == function_count);
-    (void)module_inst;
     return functions;
 }
 
@@ -1288,9 +1287,8 @@ init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module,
         *func_ptrs = import_func->func_ptr_linked;
     }
 
-    /* Set defined function pointers */
-    bh_memcpy_s(func_ptrs, sizeof(void *) * module->function_count,
-                module->func_ptrs, sizeof(void *) * module->function_count);
+    /* The defined function pointers will be set in
+       wasm_runtime_set_running_mode, no need to set them here */
     return true;
 }
 #endif /* end of WASM_ENABLE_JIT != 0 */
@@ -1336,6 +1334,173 @@ init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf,
 }
 #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */
 
+static bool
+set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode,
+                 bool first_time_set)
+{
+    WASMModule *module = module_inst->module;
+
+    if (running_mode == Mode_Default) {
+#if WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT == 0
+        running_mode = Mode_Interp;
+#elif WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT == 0
+        running_mode = Mode_Fast_JIT;
+#elif WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT != 0
+        running_mode = Mode_LLVM_JIT;
+#else /* WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 */
+#if WASM_ENABLE_LAZY_JIT == 0
+        running_mode = Mode_LLVM_JIT;
+#else
+        running_mode = Mode_Multi_Tier_JIT;
+#endif
+#endif
+    }
+
+    if (!wasm_runtime_is_running_mode_supported(running_mode))
+        return false;
+
+#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
+      && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */
+    module_inst->e->running_mode = running_mode;
+
+    if (running_mode == Mode_Interp) {
+        /* Do nothing for Mode_Interp */
+    }
+    else if (running_mode == Mode_Fast_JIT) {
+        /* Do nothing for Mode_Fast_JIT since
+           module_inst->fast_jit_func_ptrs is same as
+           module->fast_jit_func_ptrs */
+    }
+#if WASM_ENABLE_JIT != 0
+    else if (running_mode == Mode_LLVM_JIT) {
+        /* Set defined function pointers */
+        bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
+                    sizeof(void *) * module->function_count, module->func_ptrs,
+                    sizeof(void *) * module->function_count);
+    }
+#endif
+    else {
+        bh_assert(0);
+    }
+#else /* Possible multi-tier JIT */
+    os_mutex_lock(&module->instance_list_lock);
+
+    module_inst->e->running_mode = running_mode;
+
+    if (running_mode == Mode_Interp) {
+        /* Do nothing for Mode_Interp */
+    }
+#if WASM_ENABLE_FAST_JIT != 0
+    else if (running_mode == Mode_Fast_JIT) {
+        JitGlobals *jit_globals = jit_compiler_get_jit_globals();
+        uint32 i;
+
+        /* Allocate memory for fast_jit_func_ptrs if needed */
+        if (!module_inst->fast_jit_func_ptrs
+            || module_inst->fast_jit_func_ptrs == module->fast_jit_func_ptrs) {
+            uint64 total_size = (uint64)sizeof(void *) * module->function_count;
+            if (!(module_inst->fast_jit_func_ptrs =
+                      runtime_malloc(total_size, NULL, 0))) {
+                os_mutex_unlock(&module->instance_list_lock);
+                return false;
+            }
+        }
+
+        for (i = 0; i < module->function_count; i++) {
+            if (module->functions[i]->fast_jit_jitted_code) {
+                /* current fast jit function has been compiled */
+                module_inst->fast_jit_func_ptrs[i] =
+                    module->functions[i]->fast_jit_jitted_code;
+            }
+            else {
+                module_inst->fast_jit_func_ptrs[i] =
+                    jit_globals->compile_fast_jit_and_then_call;
+            }
+        }
+    }
+#endif
+#if WASM_ENABLE_JIT != 0
+    else if (running_mode == Mode_LLVM_JIT) {
+        void **llvm_jit_func_ptrs;
+        uint32 i;
+
+        /* Notify backend threads to start llvm jit compilation */
+        module->enable_llvm_jit_compilation = true;
+
+        /* Wait until llvm jit finishes initialization */
+        os_mutex_lock(&module->tierup_wait_lock);
+        while (!module->llvm_jit_inited) {
+            os_cond_reltimedwait(&module->tierup_wait_cond,
+                                 &module->tierup_wait_lock, 10);
+            if (module->orcjit_stop_compiling) {
+                /* init_llvm_jit_functions_stage2 failed */
+                os_mutex_unlock(&module->tierup_wait_lock);
+                os_mutex_unlock(&module->instance_list_lock);
+                return false;
+            }
+        }
+        os_mutex_unlock(&module->tierup_wait_lock);
+
+        llvm_jit_func_ptrs =
+            module_inst->func_ptrs + module->import_function_count;
+        for (i = 0; i < module->function_count; i++) {
+            llvm_jit_func_ptrs[i] = module->functions[i]->llvm_jit_func_ptr;
+        }
+    }
+#endif
+    else if (running_mode == Mode_Multi_Tier_JIT) {
+        /* Notify backend threads to start llvm jit compilation */
+        module->enable_llvm_jit_compilation = true;
+
+        /* Free fast_jit_func_ptrs if it is allocated before */
+        if (module_inst->fast_jit_func_ptrs
+            && module_inst->fast_jit_func_ptrs != module->fast_jit_func_ptrs) {
+            wasm_runtime_free(module_inst->fast_jit_func_ptrs);
+        }
+        module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
+
+        /* Copy all llvm jit func ptrs from the module */
+        bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
+                    sizeof(void *) * module->function_count, module->func_ptrs,
+                    sizeof(void *) * module->function_count);
+    }
+    else {
+        bh_assert(0);
+    }
+
+    /* Add module instance into module's instance list if not added */
+    if (first_time_set) {
+        bool found = false;
+        WASMModuleInstance *node = module->instance_list;
+
+        while (node) {
+            if (node == module_inst) {
+                found = true;
+                break;
+            }
+            node = node->e->next;
+        }
+
+        if (!found) {
+            module_inst->e->next = module->instance_list;
+            module->instance_list = module_inst;
+        }
+    }
+
+    os_mutex_unlock(&module->instance_list_lock);
+#endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
+                   && WASM_ENABLE_LAZY_JIT != 0) */
+
+    (void)module;
+    return true;
+}
+
+bool
+wasm_set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode)
+{
+    return set_running_mode(module_inst, running_mode, false);
+}
+
 /**
  * Instantiate module
  */
@@ -1813,33 +1978,29 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
     }
 #endif
 
-#if WASM_ENABLE_DEBUG_INTERP != 0                         \
-    || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
-        && WASM_ENABLE_LAZY_JIT != 0)
+#if WASM_ENABLE_DEBUG_INTERP != 0
     if (!is_sub_inst) {
         /* Add module instance into module's instance list */
         os_mutex_lock(&module->instance_list_lock);
-#if WASM_ENABLE_DEBUG_INTERP != 0
         if (module->instance_list) {
             LOG_WARNING(
                 "warning: multiple instances referencing to the same module "
                 "may cause unexpected behaviour during debugging");
         }
-#endif
-#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
-    && WASM_ENABLE_LAZY_JIT != 0
-        /* Copy llvm func ptrs again in case that they were updated
-           after the module instance was created */
-        bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
-                    sizeof(void *) * module->function_count, module->func_ptrs,
-                    sizeof(void *) * module->function_count);
-#endif
         module_inst->e->next = module->instance_list;
         module->instance_list = module_inst;
         os_mutex_unlock(&module->instance_list_lock);
     }
 #endif
 
+    /* Set running mode before executing wasm functions */
+    if (!set_running_mode(module_inst, wasm_runtime_get_default_running_mode(),
+                          true)) {
+        set_error_buf(error_buf, error_buf_size,
+                      "set instance running mode failed");
+        goto fail;
+    }
+
     if (module->start_function != (uint32)-1) {
         /* TODO: fix start function can be import function issue */
         if (module->start_function >= module->import_function_count)
@@ -1910,6 +2071,14 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
         wasm_runtime_free(module_inst->func_ptrs);
 #endif
 
+#if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
+    && WASM_ENABLE_LAZY_JIT != 0
+    if (module_inst->fast_jit_func_ptrs
+        && module_inst->fast_jit_func_ptrs
+               != module_inst->module->fast_jit_func_ptrs)
+        wasm_runtime_free(module_inst->fast_jit_func_ptrs);
+#endif
+
 #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
     if (module_inst->func_type_indexes)
         wasm_runtime_free(module_inst->func_type_indexes);

core/iwasm/interpreter/wasm_runtime.h

@@ -221,6 +221,7 @@ typedef struct WASMModuleInstanceExtra {
     WASMFunctionInstance *retain_function;
 
     CApiFuncImport *c_api_func_imports;
+    RunningMode running_mode;
 
 #if WASM_ENABLE_SHARED_MEMORY != 0
     /* lock for shared memory atomic operations */
@@ -304,7 +305,11 @@ struct WASMModuleInstance {
        not available in AOTModuleInstance */
     DefPointer(void **, import_func_ptrs);
     /* Array of function pointers to fast jit functions,
-       not available in AOTModuleInstance */
+       not available in AOTModuleInstance:
+       Only when the multi-tier JIT macros are all enabled and the running
+       mode of current module instance is set to Mode_Fast_JIT, runtime
+       will allocate new memory for it, otherwise it always points to the
+       module->fast_jit_func_ptrs */
     DefPointer(void **, fast_jit_func_ptrs);
     /* The custom data that can be set/get by wasm_{get|set}_custom_data */
     DefPointer(void *, custom_data);
@@ -408,6 +413,10 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst);
 void
 wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst);
 
+bool
+wasm_set_running_mode(WASMModuleInstance *module_inst,
+                      RunningMode running_mode);
+
 WASMFunctionInstance *
 wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name,
                      const char *signature);

product-mini/platforms/posix/main.c

@@ -32,12 +32,28 @@ print_help()
 #if WASM_ENABLE_LOG != 0
     printf("  -v=n                     Set log verbose level (0 to 5, default is 2) larger\n"
            "                           level with more log\n");
+#endif
+#if WASM_ENABLE_INTERP != 0
+    printf("  --interp                 Run the wasm app with interpreter mode\n");
+#endif
+#if WASM_ENABLE_FAST_JIT != 0
+    printf("  --fast-jit               Run the wasm app with fast jit mode\n");
+#endif
+#if WASM_ENABLE_JIT != 0
+    printf("  --llvm-jit               Run the wasm app with llvm jit mode\n");
+#endif
+#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
+    printf("  --multi-tier-jit         Run the wasm app with multi-tier jit mode\n");
 #endif
     printf("  --stack-size=n           Set maximum stack size in bytes, default is 64 KB\n");
     printf("  --heap-size=n            Set maximum heap size in bytes, default is 16 KB\n");
 #if WASM_ENABLE_FAST_JIT != 0
     printf("  --jit-codecache-size=n   Set fast jit maximum code cache size in bytes,\n");
     printf("                           default is %u KB\n", FAST_JIT_DEFAULT_CODE_CACHE_SIZE / 1024);
+#endif
+#if WASM_ENABLE_JIT != 0
+    printf("  --llvm-jit-size-level=n  Set LLVM JIT size level, default is 3\n");
+    printf("  --llvm-jit-opt-level=n   Set LLVM JIT optimization level, default is 3\n");
 #endif
     printf("  --repl                   Start a very simple REPL (read-eval-print-loop) mode\n"
            "                           that runs commands in the form of \"FUNC ARG...\"\n");
@@ -347,9 +363,14 @@ main(int argc, char *argv[])
     uint32 stack_size = 64 * 1024, heap_size = 16 * 1024;
 #if WASM_ENABLE_FAST_JIT != 0
     uint32 jit_code_cache_size = FAST_JIT_DEFAULT_CODE_CACHE_SIZE;
+#endif
+#if WASM_ENABLE_JIT != 0
+    uint32 llvm_jit_size_level = 3;
+    uint32 llvm_jit_opt_level = 3;
 #endif
     wasm_module_t wasm_module = NULL;
     wasm_module_inst_t wasm_module_inst = NULL;
+    RunningMode running_mode = 0;
     RuntimeInitArgs init_args;
     char error_buf[128] = { 0 };
 #if WASM_ENABLE_LOG != 0
@@ -387,6 +408,27 @@ main(int argc, char *argv[])
             }
             func_name = argv[0];
         }
+#if WASM_ENABLE_INTERP != 0
+        else if (!strcmp(argv[0], "--interp")) {
+            running_mode = Mode_Interp;
+        }
+#endif
+#if WASM_ENABLE_FAST_JIT != 0
+        else if (!strcmp(argv[0], "--fast-jit")) {
+            running_mode = Mode_Fast_JIT;
+        }
+#endif
+#if WASM_ENABLE_JIT != 0
+        else if (!strcmp(argv[0], "--llvm-jit")) {
+            running_mode = Mode_LLVM_JIT;
+        }
+#endif
+#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 \
+    && WASM_ENABLE_LAZY_JIT != 0
+        else if (!strcmp(argv[0], "--multi-tier-jit")) {
+            running_mode = Mode_Multi_Tier_JIT;
+        }
+#endif
 #if WASM_ENABLE_LOG != 0
         else if (!strncmp(argv[0], "-v=", 3)) {
             log_verbose_level = atoi(argv[0] + 3);
@@ -414,6 +456,38 @@ main(int argc, char *argv[])
             jit_code_cache_size = atoi(argv[0] + 21);
         }
 #endif
+#if WASM_ENABLE_JIT != 0
+        else if (!strncmp(argv[0], "--llvm-jit-size-level=", 22)) {
+            if (argv[0][22] == '\0')
+                return print_help();
+            llvm_jit_size_level = atoi(argv[0] + 22);
+            if (llvm_jit_size_level < 1) {
+                printf("LLVM JIT size level shouldn't be smaller than 1, "
+                       "setting it to 1\n");
+                llvm_jit_size_level = 1;
+            }
+            else if (llvm_jit_size_level > 3) {
+                printf("LLVM JIT size level shouldn't be greater than 3, "
+                       "setting it to 3\n");
+                llvm_jit_size_level = 3;
+            }
+        }
+        else if (!strncmp(argv[0], "--llvm-jit-opt-level=", 21)) {
+            if (argv[0][21] == '\0')
+                return print_help();
+            llvm_jit_opt_level = atoi(argv[0] + 21);
+            if (llvm_jit_opt_level < 1) {
+                printf("LLVM JIT opt level shouldn't be smaller than 1, "
+                       "setting it to 1\n");
+                llvm_jit_opt_level = 1;
+            }
+            else if (llvm_jit_opt_level > 3) {
+                printf("LLVM JIT opt level shouldn't be greater than 3, "
+                       "setting it to 3\n");
+                llvm_jit_opt_level = 3;
+            }
+        }
+#endif
 #if WASM_ENABLE_LIBC_WASI != 0
         else if (!strncmp(argv[0], "--dir=", 6)) {
             if (argv[0][6] == '\0')
@@ -539,6 +613,7 @@ main(int argc, char *argv[])
 
     memset(&init_args, 0, sizeof(RuntimeInitArgs));
 
+    init_args.running_mode = running_mode;
 #if WASM_ENABLE_GLOBAL_HEAP_POOL != 0
     init_args.mem_alloc_type = Alloc_With_Pool;
     init_args.mem_alloc_option.pool.heap_buf = global_heap_buf;
@@ -554,6 +629,11 @@ main(int argc, char *argv[])
     init_args.fast_jit_code_cache_size = jit_code_cache_size;
 #endif
 
+#if WASM_ENABLE_JIT != 0
+    init_args.llvm_jit_size_level = llvm_jit_size_level;
+    init_args.llvm_jit_opt_level = llvm_jit_opt_level;
+#endif
+
 #if WASM_ENABLE_DEBUG_INTERP != 0
     init_args.instance_port = instance_port;
     if (ip_addr)

product-mini/platforms/windows/main.c

@@ -26,9 +26,25 @@ print_help()
 #if WASM_ENABLE_LOG != 0
     printf("  -v=n                   Set log verbose level (0 to 5, default is 2) larger\n"
            "                         level with more log\n");
+#endif
+#if WASM_ENABLE_INTERP != 0
+    printf("  --interp               Run the wasm app with interpreter mode\n");
+#endif
+#if WASM_ENABLE_FAST_JIT != 0
+    printf("  --fast-jit             Run the wasm app with fast jit mode\n");
+#endif
+#if WASM_ENABLE_JIT != 0
+    printf("  --llvm-jit             Run the wasm app with llvm jit mode\n");
+#endif
+#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_LAZY_JIT != 0
+    printf("  --multi-tier-jit       Run the wasm app with multi-tier jit mode\n");
 #endif
     printf("  --stack-size=n         Set maximum stack size in bytes, default is 64 KB\n");
     printf("  --heap-size=n          Set maximum heap size in bytes, default is 16 KB\n");
+#if WASM_ENABLE_JIT != 0
+    printf("  --llvm-jit-size-level=n  Set LLVM JIT size level, default is 3\n");
+    printf("  --llvm-jit-opt-level=n   Set LLVM JIT optimization level, default is 3\n");
+#endif
     printf("  --repl                 Start a very simple REPL (read-eval-print-loop) mode\n"
            "                         that runs commands in the form of `FUNC ARG...`\n");
 #if WASM_ENABLE_LIBC_WASI != 0
@@ -228,8 +244,13 @@ main(int argc, char *argv[])
     uint8 *wasm_file_buf = NULL;
     uint32 wasm_file_size;
     uint32 stack_size = 64 * 1024, heap_size = 16 * 1024;
+#if WASM_ENABLE_JIT != 0
+    uint32 llvm_jit_size_level = 3;
+    uint32 llvm_jit_opt_level = 3;
+#endif
     wasm_module_t wasm_module = NULL;
     wasm_module_inst_t wasm_module_inst = NULL;
+    RunningMode running_mode = 0;
     RuntimeInitArgs init_args;
     char error_buf[128] = { 0 };
 #if WASM_ENABLE_LOG != 0
@@ -257,6 +278,26 @@ main(int argc, char *argv[])
             }
             func_name = argv[0];
         }
+#if WASM_ENABLE_INTERP != 0
+        else if (!strcmp(argv[0], "--interp")) {
+            running_mode = Mode_Interp;
+        }
+#endif
+#if WASM_ENABLE_FAST_JIT != 0
+        else if (!strcmp(argv[0], "--fast-jit")) {
+            running_mode = Mode_Fast_JIT;
+        }
+#endif
+#if WASM_ENABLE_JIT != 0
+        else if (!strcmp(argv[0], "--llvm-jit")) {
+            running_mode = Mode_LLVM_JIT;
+        }
+#endif
+#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_FAST_JIT != 0
+        else if (!strcmp(argv[0], "--multi-tier-jit")) {
+            running_mode = Mode_Multi_Tier_JIT;
+        }
+#endif
 #if WASM_ENABLE_LOG != 0
         else if (!strncmp(argv[0], "-v=", 3)) {
             log_verbose_level = atoi(argv[0] + 3);
@@ -277,6 +318,38 @@ main(int argc, char *argv[])
                 return print_help();
             heap_size = atoi(argv[0] + 12);
         }
+#if WASM_ENABLE_JIT != 0
+        else if (!strncmp(argv[0], "--llvm-jit-size-level=", 22)) {
+            if (argv[0][22] == '\0')
+                return print_help();
+            llvm_jit_size_level = atoi(argv[0] + 22);
+            if (llvm_jit_size_level < 1) {
+                printf("LLVM JIT size level shouldn't be smaller than 1, "
+                       "setting it to 1\n");
+                llvm_jit_size_level = 1;
+            }
+            else if (llvm_jit_size_level > 3) {
+                printf("LLVM JIT size level shouldn't be greater than 3, "
+                       "setting it to 3\n");
+                llvm_jit_size_level = 3;
+            }
+        }
+        else if (!strncmp(argv[0], "--llvm-jit-opt-level=", 21)) {
+            if (argv[0][21] == '\0')
+                return print_help();
+            llvm_jit_opt_level = atoi(argv[0] + 21);
+            if (llvm_jit_opt_level < 1) {
+                printf("LLVM JIT opt level shouldn't be smaller than 1, "
+                       "setting it to 1\n");
+                llvm_jit_opt_level = 1;
+            }
+            else if (llvm_jit_opt_level > 3) {
+                printf("LLVM JIT opt level shouldn't be greater than 3, "
+                       "setting it to 3\n");
+                llvm_jit_opt_level = 3;
+            }
+        }
+#endif
 #if WASM_ENABLE_LIBC_WASI != 0
         else if (!strncmp(argv[0], "--dir=", 6)) {
             if (argv[0][6] == '\0')
@@ -357,6 +430,7 @@ main(int argc, char *argv[])
 
     memset(&init_args, 0, sizeof(RuntimeInitArgs));
 
+    init_args.running_mode = running_mode;
 #if WASM_ENABLE_GLOBAL_HEAP_POOL != 0
     init_args.mem_alloc_type = Alloc_With_Pool;
     init_args.mem_alloc_option.pool.heap_buf = global_heap_buf;
@@ -368,6 +442,11 @@ main(int argc, char *argv[])
     init_args.mem_alloc_option.allocator.free_func = free;
 #endif
 
+#if WASM_ENABLE_JIT != 0
+    init_args.llvm_jit_size_level = llvm_jit_size_level;
+    init_args.llvm_jit_opt_level = llvm_jit_opt_level;
+#endif
+
 #if WASM_ENABLE_DEBUG_INTERP != 0
     init_args.instance_port = instance_port;
     if (ip_addr)