Implement GC for fast interpreter (#1999)

Implement GC for fast interpreter except struct/array related opcodes

core/iwasm/common/gc/gc_object.c

@@ -43,6 +43,7 @@ gc_obj_malloc(void *heap_handle, uint64 size)
 
     if (size >= UINT32_MAX
         || !(mem = mem_allocator_malloc_with_gc(heap_handle, (uint32)size))) {
+        LOG_WARNING("warning: failed to allocate memory for gc object");
         return NULL;
     }
 

core/iwasm/common/wasm_runtime_common.c

@@ -135,6 +135,10 @@ static JitCompOptions jit_options = { 0 };
 static LLVMJITOptions llvm_jit_options = { 3, 3 };
 #endif
 
+#if WASM_ENABLE_GC != 0
+static uint32 gc_heap_size_default = GC_HEAP_SIZE_DEFAULT;
+#endif
+
 static RunningMode runtime_running_mode = Mode_Default;
 
 #ifdef OS_ENABLE_HW_BOUND_CHECK
@@ -537,6 +541,14 @@ wasm_runtime_get_llvm_jit_options(void)
 }
 #endif
 
+#if WASM_ENABLE_GC != 0
+uint32
+wasm_runtime_get_gc_heap_size_default(void)
+{
+    return gc_heap_size_default;
+}
+#endif
+
 bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args)
 {
@@ -553,6 +565,10 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args)
     jit_options.code_cache_size = init_args->fast_jit_code_cache_size;
 #endif
 
+#if WASM_ENABLE_GC != 0
+    gc_heap_size_default = init_args->gc_heap_size;
+#endif
+
 #if WASM_ENABLE_JIT != 0
     llvm_jit_options.size_level = init_args->llvm_jit_size_level;
     llvm_jit_options.opt_level = init_args->llvm_jit_opt_level;

core/iwasm/common/wasm_runtime_common.h

@@ -527,6 +527,12 @@ LLVMJITOptions
 wasm_runtime_get_llvm_jit_options(void);
 #endif
 
+#if WASM_ENABLE_GC != 0
+/* Internal API */
+uint32
+wasm_runtime_get_gc_heap_size_default(void);
+#endif
+
 /* See wasm_export.h for description */
 WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args);

core/iwasm/include/wasm_export.h

@@ -161,6 +161,9 @@ typedef struct RuntimeInitArgs {
     /* Fast JIT code cache size */
     uint32_t fast_jit_code_cache_size;
 
+    /* Default GC heap size */
+    uint32_t gc_heap_size;
+
     /* Default running mode of the runtime */
     RunningMode running_mode;
 

core/iwasm/interpreter/wasm_interp.h

@@ -39,8 +39,11 @@ typedef struct WASMInterpFrame {
        the callee will put return values here continuously */
     uint32 ret_offset;
     uint32 *lp;
+#if WASM_ENABLE_GC != 0
+    uint8 *frame_ref;
+#endif
     uint32 operand[1];
-#else
+#else  /* else of WASM_ENABLE_FAST_INTERP != 0 */
     /* Operand stack top pointer of the current frame. The bottom of
        the stack is the next cell after the last local variable. */
     uint32 *sp_bottom;
@@ -59,7 +62,7 @@ typedef struct WASMInterpFrame {
      *  jit spill cache: only available for fast jit
      */
     uint32 lp[1];
-#endif
+#endif /* end of WASM_ENABLE_FAST_INTERP != 0 */
 } WASMInterpFrame;
 
 /**

core/iwasm/interpreter/wasm_interp_classic.c

@@ -1651,13 +1651,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
 
                 /* clang-format off */
 #if WASM_ENABLE_GC == 0
-                fidx = ((uint32 *)tbl_inst->elems)[val];
+                fidx = tbl_inst->elems[val];
                 if (fidx == (uint32)-1) {
                     wasm_set_exception(module, "uninitialized element");
                     goto got_exception;
                 }
 #else
-                func_obj = ((WASMFuncObjectRef *)tbl_inst->elems)[val];
+                func_obj = (WASMFuncObjectRef)tbl_inst->elems[val];
                 if (!func_obj) {
                     wasm_set_exception(module, "uninitialized element");
                     goto got_exception;
@@ -1684,19 +1684,21 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
                 else
                     cur_func_type = cur_func->u.func->func_type;
 
+                    /* clang-format off */
 #if WASM_ENABLE_GC == 0
                 if (cur_type != cur_func_type) {
                     wasm_set_exception(module, "indirect call type mismatch");
                     goto got_exception;
                 }
 #else
-        if (!wasm_func_type_equal(cur_type, cur_func_type,
-                                  module->module->types,
-                                  module->module->type_count)) {
-            wasm_set_exception(module, "indirect call type mismatch");
-            goto got_exception;
-        }
+                if (!wasm_func_type_equal(cur_type, cur_func_type,
+                                          module->module->types,
+                                          module->module->type_count)) {
+                    wasm_set_exception(module, "indirect call type mismatch");
+                    goto got_exception;
+                }
 #endif
+                /* clang-format on */
 
 #if WASM_ENABLE_TAIL_CALL != 0
                 if (opcode == WASM_OP_RETURN_CALL_INDIRECT)
@@ -1786,9 +1788,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
                 }
 
 #if WASM_ENABLE_GC == 0
-                PUSH_I32(((uint32 *)tbl_inst->elems)[elem_idx]);
+                PUSH_I32(tbl_inst->elems[elem_idx]);
 #else
-                PUSH_REF(((table_elem_type_t *)tbl_inst->elems)[elem_idx]);
+                PUSH_REF(tbl_inst->elems[elem_idx]);
 #endif
                 HANDLE_OP_END();
             }
@@ -1815,7 +1817,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
                     goto got_exception;
                 }
 
-                ((table_elem_type_t *)(tbl_inst->elems))[elem_idx] = elem_val;
+                tbl_inst->elems[elem_idx] = elem_val;
                 HANDLE_OP_END();
             }
 
@@ -2461,7 +2463,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
 #if WASM_ENABLE_THREAD_MGR != 0
                         CHECK_SUSPEND_FLAGS();
 #endif
-                        read_leb_int32(frame_ip, frame_ip_end, depth);
+                        read_leb_uint32(frame_ip, frame_ip_end, depth);
                         read_leb_int32(frame_ip, frame_ip_end, heap_type);
 
                         gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM);
@@ -4205,7 +4207,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
                             (uint32)(n * sizeof(uint32)));
 #else
                         SYNC_ALL_TO_FRAME();
-                        table_elems = (table_elem_type_t *)tbl_inst->elems + d;
+                        table_elems = tbl_inst->elems + d;
                         func_indexes = module->module->table_segments[elem_idx]
                                            .func_indexes
                                        + s;
@@ -4347,8 +4349,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
                         }
 
                         for (; n != 0; i++, n--) {
-                            ((table_elem_type_t *)tbl_inst->elems)[i] =
-                                fill_val;
+                            tbl_inst->elems[i] = fill_val;
                         }
                         break;
                     }

core/iwasm/interpreter/wasm_loader.c

@@ -10410,6 +10410,10 @@ re_scan:
                 if (available_stack_cell
                     > 0) { /* stack isn't in polymorphic state */
                     POP_REF(type);
+#if WASM_ENABLE_FAST_INTERP != 0
+                    /* Erase the opnd offset emitted by POP_REF() */
+                    wasm_loader_emit_backspace(loader_ctx, sizeof(uint16));
+#endif
                 }
                 break;
             }
@@ -11209,6 +11213,9 @@ re_scan:
                 uint32 opcode1;
 
                 read_leb_uint32(p, p_end, opcode1);
+#if WASM_ENABLE_FAST_INTERP != 0
+                emit_byte(loader_ctx, ((uint8)opcode1));
+#endif
 
                 switch (opcode1) {
                     case WASM_OP_STRUCT_NEW_CANON:
@@ -11598,6 +11605,9 @@ re_scan:
                         uint8 type;
 
                         read_leb_int32(p, p_end, heap_type);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        emit_uint32(loader_ctx, (uint32)heap_type);
+#endif
                         if (heap_type >= 0) {
                             if (!check_type_index(module, heap_type, error_buf,
                                                   error_buf_size)) {
@@ -11639,16 +11649,21 @@ re_scan:
 
                         p_org = p;
 
-                        read_leb_int32(p, p_end, depth);
+                        read_leb_uint32(p, p_end, depth);
+                        read_leb_int32(p, p_end, heap_type);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        /* Emit heap_type firstly */
+                        emit_uint32(loader_ctx, (uint32)heap_type);
+#endif
                         (void)depth;
 
+                        /* Pop and backup the stack top's ref type */
                         if (!wasm_loader_pop_heap_obj(loader_ctx, &type_tmp,
                                                       &ref_type_tmp, error_buf,
                                                       error_buf_size)) {
                             goto fail;
                         }
 
-                        read_leb_int32(p, p_end, heap_type);
                         if (heap_type >= 0) {
                             if (!check_type_index(module, heap_type, error_buf,
                                                   error_buf_size)) {
@@ -11671,15 +11686,22 @@ re_scan:
                         }
 
                         p = p_org;
+                        /* Push ref type casted for branch block check */
                         PUSH_REF(wasm_ref_type.ref_type);
                         if (!(frame_csp_tmp = check_branch_block(
                                   loader_ctx, &p, p_end, false, error_buf,
                                   error_buf_size))) {
                             goto fail;
                         }
-                        POP_REF(wasm_ref_type.ref_type);
+                        /* Ignore heap_type */
                         skip_leb_uint32(p, p_end);
 
+                        /* Restore the original stack top's ref type */
+                        POP_REF(wasm_ref_type.ref_type);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        /* Erase the opnd offset emitted by POP_REF() */
+                        wasm_loader_emit_backspace(loader_ctx, sizeof(uint16));
+#endif
                         if (wasm_is_type_multi_byte_type(type_tmp)) {
                             bh_memcpy_s(
                                 &wasm_ref_type,
@@ -11688,33 +11710,56 @@ re_scan:
                                 wasm_reftype_struct_size(&ref_type_tmp));
                         }
                         PUSH_REF(type_tmp);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        /* Erase the opnd offset emitted by PUSH_REF() */
+                        wasm_loader_emit_backspace(loader_ctx, sizeof(uint16));
+#endif
                         break;
                     }
 
                     case WASM_OP_BR_ON_CAST_FAIL:
                     case WASM_OP_BR_ON_CAST_FAIL_NULLABLE:
                     {
-                        WASMRefType ref_type_tmp;
                         uint8 type_tmp;
+                        uint32 depth;
                         bool nullable;
 
                         nullable = (opcode1 == WASM_OP_BR_ON_CAST_FAIL_NULLABLE)
                                        ? true
                                        : false;
 
-                        if (!(frame_csp_tmp = check_branch_block(
-                                  loader_ctx, &p, p_end, false, error_buf,
-                                  error_buf_size))) {
-                            goto fail;
-                        }
+                        p_org = p;
+                        read_leb_uint32(p, p_end, depth);
+                        read_leb_int32(p, p_end, heap_type);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        /* Emit heap_type firstly */
+                        emit_uint32(loader_ctx, (uint32)heap_type);
+#endif
+                        (void)depth;
 
+                        /* Pop and push the stack top's ref type so fast-interp
+                           can get the opnd offset */
                         if (!wasm_loader_pop_heap_obj(loader_ctx, &type_tmp,
-                                                      &ref_type_tmp, error_buf,
+                                                      &wasm_ref_type, error_buf,
                                                       error_buf_size)) {
                             goto fail;
                         }
+                        p = p_org;
+                        PUSH_REF(type_tmp);
+                        if (!(frame_csp_tmp = check_branch_block(
+                                  loader_ctx, &p, p_end, false, error_buf,
+                                  error_buf_size))) {
+                            goto fail;
+                        }
+                        /* Ignore heap_type */
+                        skip_leb_uint32(p, p_end);
 
-                        read_leb_int32(p, p_end, heap_type);
+                        /* Pop the original ref type, and push the casted
+                           ref type */
+                        POP_REF(type_tmp);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        wasm_loader_emit_backspace(loader_ctx, sizeof(uint16));
+#endif
                         if (heap_type >= 0) {
                             if (!check_type_index(module, heap_type, error_buf,
                                                   error_buf_size)) {
@@ -11735,8 +11780,11 @@ re_scan:
                                 &wasm_ref_type.ref_ht_common, nullable,
                                 heap_type);
                         }
-
                         PUSH_REF(wasm_ref_type.ref_type);
+#if WASM_ENABLE_FAST_INTERP != 0
+                        /* Erase the opnd offset emitted by PUSH_REF() */
+                        wasm_loader_emit_backspace(loader_ctx, sizeof(uint16));
+#endif
                         break;
                     }
 

core/iwasm/interpreter/wasm_runtime.c

@@ -1785,7 +1785,12 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size,
 
 #if WASM_ENABLE_GC != 0
     if (!is_sub_inst) {
-        uint32 gc_heap_size = GC_HEAP_SIZE_DEFAULT;
+        uint32 gc_heap_size = wasm_runtime_get_gc_heap_size_default();
+
+        if (gc_heap_size < GC_HEAP_SIZE_MIN)
+            gc_heap_size = GC_HEAP_SIZE_MIN;
+        if (gc_heap_size > GC_HEAP_SIZE_MAX)
+            gc_heap_size = GC_HEAP_SIZE_MAX;
 
         module_inst->e->gc_heap_pool =
             runtime_malloc(gc_heap_size, error_buf, error_buf_size);

product-mini/platforms/posix/main.c

@@ -51,6 +51,10 @@ print_help()
     printf("  --jit-codecache-size=n   Set fast jit maximum code cache size in bytes,\n");
     printf("                           default is %u KB\n", FAST_JIT_DEFAULT_CODE_CACHE_SIZE / 1024);
 #endif
+#if WASM_ENABLE_GC != 0
+    printf("  --gc-heap-size=n         Set maximum gc heap size in bytes,\n");
+    printf("                           default is %u KB\n", GC_HEAP_SIZE_DEFAULT / 1024);
+#endif
 #if WASM_ENABLE_JIT != 0
     printf("  --llvm-jit-size-level=n  Set LLVM JIT size level, default is 3\n");
     printf("  --llvm-jit-opt-level=n   Set LLVM JIT optimization level, default is 3\n");
@@ -364,6 +368,9 @@ main(int argc, char *argv[])
 #if WASM_ENABLE_FAST_JIT != 0
     uint32 jit_code_cache_size = FAST_JIT_DEFAULT_CODE_CACHE_SIZE;
 #endif
+#if WASM_ENABLE_GC != 0
+    uint32 gc_heap_size = GC_HEAP_SIZE_DEFAULT;
+#endif
 #if WASM_ENABLE_JIT != 0
     uint32 llvm_jit_size_level = 3;
     uint32 llvm_jit_opt_level = 3;
@@ -456,6 +463,13 @@ main(int argc, char *argv[])
             jit_code_cache_size = atoi(argv[0] + 21);
         }
 #endif
+#if WASM_ENABLE_GC != 0
+        else if (!strncmp(argv[0], "--gc-heap-size=", 15)) {
+            if (argv[0][21] == '\0')
+                return print_help();
+            gc_heap_size = atoi(argv[0] + 15);
+        }
+#endif
 #if WASM_ENABLE_JIT != 0
         else if (!strncmp(argv[0], "--llvm-jit-size-level=", 22)) {
             if (argv[0][22] == '\0')
@@ -629,6 +643,10 @@ main(int argc, char *argv[])
     init_args.fast_jit_code_cache_size = jit_code_cache_size;
 #endif
 
+#if WASM_ENABLE_GC != 0
+    init_args.gc_heap_size = gc_heap_size;
+#endif
+
 #if WASM_ENABLE_JIT != 0
     init_args.llvm_jit_size_level = llvm_jit_size_level;
     init_args.llvm_jit_opt_level = llvm_jit_opt_level;