Merge pull request #1966 from bytecodealliance/dev/wasi_threads

Merge wasi_threads to main

.github/workflows/compilation_on_android_ubuntu.yml

@@ -358,6 +358,22 @@ jobs:
           sudo tar -xzf wabt-1.0.31-*.tar.gz
           sudo mv wabt-1.0.31 wabt
 
+      - name: build wasi-libc (needed for wasi-threads)
+        run: |
+          mkdir wasi-libc
+          cd wasi-libc
+          git init
+          # "Rename thread_spawn import" commit on main branch
+          git fetch https://github.com/WebAssembly/wasi-libc \
+            8f5275796a82f8ecfd0833a4f3f444fa37ed4546
+          git checkout FETCH_HEAD
+          make \
+            AR=/opt/wasi-sdk/bin/llvm-ar \
+            NM=/opt/wasi-sdk/bin/llvm-nm \
+            CC=/opt/wasi-sdk/bin/clang \
+            THREAD_MODEL=posix
+        working-directory: core/deps
+
       - name: Build Sample [basic]
         run: |
           cd samples/basic
@@ -411,6 +427,14 @@ jobs:
           exit $?
         working-directory: ./samples/simple
 
+      - name: Build Sample [wasi-threads]
+        run: |
+          cd samples/wasi-threads
+          mkdir build && cd build
+          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake --build . --config Release --parallel 4
+          ./iwasm wasm-apps/no_pthread.wasm
+
   test:
     needs: [build_iwasm, build_llvm_libraries_on_ubuntu_2004, build_wamrc]
     runs-on: ubuntu-20.04

.github/workflows/compilation_on_macos.yml

@@ -273,6 +273,22 @@ jobs:
           sudo tar -xzf wabt-1.0.31-*.tar.gz
           sudo mv wabt-1.0.31 wabt
 
+      - name: build wasi-libc (needed for wasi-threads)
+        run: |
+          mkdir wasi-libc
+          cd wasi-libc
+          git init
+          # "Rename thread_spawn import" commit on main branch
+          git fetch https://github.com/WebAssembly/wasi-libc \
+            8f5275796a82f8ecfd0833a4f3f444fa37ed4546
+          git checkout FETCH_HEAD
+          make \
+            AR=/opt/wasi-sdk/bin/llvm-ar \
+            NM=/opt/wasi-sdk/bin/llvm-nm \
+            CC=/opt/wasi-sdk/bin/clang \
+            THREAD_MODEL=posix
+        working-directory: core/deps
+
       - name: Build Sample [basic]
         run: |
           cd samples/basic
@@ -318,3 +334,11 @@ jobs:
           cmake ..
           cmake --build . --config Release --parallel 4
           ./hello
+
+      - name: Build Sample [wasi-threads]
+        run: |
+          cd samples/wasi-threads
+          mkdir build && cd build
+          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake --build . --config Release --parallel 4
+          ./iwasm wasm-apps/no_pthread.wasm

.github/workflows/compilation_on_sgx.yml

@@ -260,6 +260,22 @@ jobs:
           sudo tar -xzf wabt-1.0.31-*.tar.gz
           sudo mv wabt-1.0.31 wabt
 
+      - name: build wasi-libc (needed for wasi-threads)
+        run: |
+          mkdir wasi-libc
+          cd wasi-libc
+          git init
+          # "Rename thread_spawn import" commit on main branch
+          git fetch https://github.com/WebAssembly/wasi-libc \
+            8f5275796a82f8ecfd0833a4f3f444fa37ed4546
+          git checkout FETCH_HEAD
+          make \
+            AR=/opt/wasi-sdk/bin/llvm-ar \
+            NM=/opt/wasi-sdk/bin/llvm-nm \
+            CC=/opt/wasi-sdk/bin/clang \
+            THREAD_MODEL=posix
+        working-directory: core/deps
+
       - name: install SGX SDK and necessary libraries
         run: |
           mkdir -p /opt/intel
@@ -319,6 +335,14 @@ jobs:
           cmake --build . --config Release --parallel 4
           ./hello
 
+      - name: Build Sample [wasi-threads]
+        run: |
+          cd samples/wasi-threads
+          mkdir build && cd build
+          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake --build . --config Release --parallel 4
+          ./iwasm wasm-apps/no_pthread.wasm
+
   spec_test_default:
     needs: [build_iwasm, build_llvm_libraries, build_wamrc]
     runs-on: ubuntu-20.04

CMakeLists.txt

@@ -83,6 +83,11 @@ if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD)
   set (WAMR_BUILD_LIB_PTHREAD 0)
 endif ()
 
+if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS)
+  # Disable wasi threads library by default
+  set (WAMR_BUILD_LIB_WASI_THREADS 0)
+endif ()
+
 if (NOT DEFINED WAMR_BUILD_MINI_LOADER)
   # Disable wasm mini loader by default
   set (WAMR_BUILD_MINI_LOADER 0)

build-scripts/runtime_lib.cmake

@@ -125,6 +125,14 @@ if (WAMR_BUILD_LIB_PTHREAD EQUAL 1)
     set (WAMR_BUILD_SHARED_MEMORY 1)
 endif ()
 
+if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1)
+    include (${IWASM_DIR}/libraries/lib-wasi-threads/lib_wasi_threads.cmake)
+    # Enable the dependent feature if lib wasi threads is enabled
+    set (WAMR_BUILD_THREAD_MGR 1)
+    set (WAMR_BUILD_BULK_MEMORY 1)
+    set (WAMR_BUILD_SHARED_MEMORY 1)
+endif ()
+
 if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
     set (WAMR_BUILD_THREAD_MGR 1)
     include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake)
@@ -191,6 +199,7 @@ set (source_all
     ${WASM_APP_LIB_SOURCE_ALL}
     ${NATIVE_INTERFACE_SOURCE}
     ${APP_MGR_SOURCE}
+    ${LIB_WASI_THREADS_SOURCE}
     ${LIB_PTHREAD_SOURCE}
     ${THREAD_MGR_SOURCE}
     ${LIBC_EMCC_SOURCE}

core/config.h

@@ -161,6 +161,17 @@
 #define WASM_ENABLE_LIB_PTHREAD_SEMAPHORE 0
 #endif
 
+#ifndef WASM_ENABLE_LIB_WASI_THREADS
+#define WASM_ENABLE_LIB_WASI_THREADS 0
+#endif
+
+#ifndef WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION
+#define WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION WASM_ENABLE_LIB_WASI_THREADS
+#elif WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0 \
+    && WASM_ENABLE_LIB_WASI_THREADS == 1
+#error "Heap aux stack allocation must be enabled for WASI threads"
+#endif
+
 #ifndef WASM_ENABLE_BASE_LIB
 #define WASM_ENABLE_BASE_LIB 0
 #endif

core/iwasm/common/wasm_exec_env.h

@@ -196,6 +196,13 @@ wasm_exec_env_create(struct WASMModuleInstanceCommon *module_inst,
 void
 wasm_exec_env_destroy(WASMExecEnv *exec_env);
 
+static inline bool
+wasm_exec_env_is_aux_stack_managed_by_runtime(WASMExecEnv *exec_env)
+{
+    return exec_env->aux_stack_boundary.boundary != 0
+           || exec_env->aux_stack_bottom.bottom != 0;
+}
+
 /**
  * Allocate a WASM frame from the WASM stack.
  *

core/iwasm/common/wasm_native.c

@@ -53,6 +53,17 @@ uint32
 get_lib_pthread_export_apis(NativeSymbol **p_lib_pthread_apis);
 #endif
 
+#if WASM_ENABLE_LIB_WASI_THREADS != 0
+bool
+lib_wasi_threads_init(void);
+
+void
+lib_wasi_threads_destroy(void);
+
+uint32
+get_lib_wasi_threads_export_apis(NativeSymbol **p_lib_wasi_threads_apis);
+#endif
+
 uint32
 get_libc_emcc_export_apis(NativeSymbol **p_libc_emcc_apis);
 
@@ -390,7 +401,7 @@ wasm_native_init()
     || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0      \
     || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0        \
     || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \
-    || WASM_ENABLE_LIB_PTHREAD != 0
+    || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0
     NativeSymbol *native_symbols;
     uint32 n_native_symbols;
 #endif
@@ -445,6 +456,17 @@ wasm_native_init()
         goto fail;
 #endif
 
+#if WASM_ENABLE_LIB_WASI_THREADS != 0
+    if (!lib_wasi_threads_init())
+        goto fail;
+
+    n_native_symbols = get_lib_wasi_threads_export_apis(&native_symbols);
+    if (n_native_symbols > 0
+        && !wasm_native_register_natives("wasi", native_symbols,
+                                         n_native_symbols))
+        goto fail;
+#endif
+
 #if WASM_ENABLE_LIBC_EMCC != 0
     n_native_symbols = get_libc_emcc_export_apis(&native_symbols);
     if (n_native_symbols > 0
@@ -465,7 +487,7 @@ wasm_native_init()
     n_native_symbols = get_wasi_nn_export_apis(&native_symbols);
     if (!wasm_native_register_natives("wasi_nn", native_symbols,
                                       n_native_symbols))
-        return false;
+        goto fail;
 #endif
 
     return true;
@@ -473,7 +495,7 @@ wasm_native_init()
     || WASM_ENABLE_BASE_LIB != 0 || WASM_ENABLE_LIBC_EMCC != 0      \
     || WASM_ENABLE_LIB_RATS != 0 || WASM_ENABLE_WASI_NN != 0        \
     || WASM_ENABLE_APP_FRAMEWORK != 0 || WASM_ENABLE_LIBC_WASI != 0 \
-    || WASM_ENABLE_LIB_PTHREAD != 0
+    || WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0
 fail:
     wasm_native_destroy();
     return false;
@@ -489,6 +511,10 @@ wasm_native_destroy()
     lib_pthread_destroy();
 #endif
 
+#if WASM_ENABLE_LIB_WASI_THREADS != 0
+    lib_wasi_threads_destroy();
+#endif
+
     node = g_native_symbols_list;
     while (node) {
         node_next = node->next;

core/iwasm/interpreter/wasm_loader.c

@@ -4195,7 +4195,20 @@ check_wasi_abi_compatibility(const WASMModule *module,
 
     memory = wasm_loader_find_export(module, "", "memory", EXPORT_KIND_MEMORY,
                                      error_buf, error_buf_size);
-    if (!memory) {
+    if (!memory
+#if WASM_ENABLE_LIB_WASI_THREADS != 0
+        /*
+         * with wasi-threads, it's still an open question if a memory
+         * should be exported.
+         *
+         * https://github.com/WebAssembly/wasi-threads/issues/22
+         * https://github.com/WebAssembly/WASI/issues/502
+         *
+         * Note: this code assumes the number of memories is at most 1.
+         */
+        && module->import_memory_count == 0
+#endif
+    ) {
         set_error_buf(error_buf, error_buf_size,
                       "a module with WASI apis must export memory by default");
         return false;

core/iwasm/interpreter/wasm_runtime.c

@@ -2640,14 +2640,16 @@ wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size)
     WASMModuleInstance *module_inst =
         (WASMModuleInstance *)exec_env->module_inst;
     uint32 stack_top_idx = module_inst->module->aux_stack_top_global_index;
+
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0
+    /* Check the aux stack space */
     uint32 data_end = module_inst->module->aux_data_end;
     uint32 stack_bottom = module_inst->module->aux_stack_bottom;
     bool is_stack_before_data = stack_bottom < data_end ? true : false;
-
-    /* Check the aux stack space, currently we don't allocate space in heap */
     if ((is_stack_before_data && (size > start_offset))
         || ((!is_stack_before_data) && (start_offset - data_end < size)))
         return false;
+#endif
 
     if (stack_top_idx != (uint32)-1) {
         /* The aux stack top is a wasm global,

core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c

@@ -494,7 +494,6 @@ pthread_start_routine(void *arg)
 {
     wasm_exec_env_t exec_env = (wasm_exec_env_t)arg;
     wasm_exec_env_t parent_exec_env;
-    wasm_module_inst_t module_inst = get_module_inst(exec_env);
     ThreadRoutineArgs *routine_args = exec_env->thread_arg;
     ThreadInfoNode *info_node = routine_args->info_node;
     uint32 argv[1];
@@ -504,7 +503,6 @@ pthread_start_routine(void *arg)
     info_node->exec_env = exec_env;
     info_node->u.thread = exec_env->handle;
     if (!append_thread_info_node(info_node)) {
-        wasm_runtime_deinstantiate_internal(module_inst, true);
         delete_thread_info_node(info_node);
         os_cond_signal(&parent_exec_env->wait_cond);
         os_mutex_unlock(&parent_exec_env->wait_lock);
@@ -526,9 +524,6 @@ pthread_start_routine(void *arg)
     /* destroy pthread key values */
     call_key_destructor(exec_env);
 
-    /* routine exit, destroy instance */
-    wasm_runtime_deinstantiate_internal(module_inst, true);
-
     wasm_runtime_free(routine_args);
 
     /* if the thread is joinable, store the result in its info node,
@@ -663,8 +658,9 @@ pthread_create_wrapper(wasm_exec_env_t exec_env,
     routine_args->module_inst = new_module_inst;
 
     os_mutex_lock(&exec_env->wait_lock);
-    ret = wasm_cluster_create_thread(
-        exec_env, new_module_inst, pthread_start_routine, (void *)routine_args);
+    ret =
+        wasm_cluster_create_thread(exec_env, new_module_inst, true,
+                                   pthread_start_routine, (void *)routine_args);
     if (ret != 0) {
         os_mutex_unlock(&exec_env->wait_lock);
         goto fail;

core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads.cmake

@@ -0,0 +1,12 @@
+# Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set (LIB_WASI_THREADS_DIR ${CMAKE_CURRENT_LIST_DIR})
+
+add_definitions (-DWASM_ENABLE_LIB_WASI_THREADS=1 -DWASM_ENABLE_HEAP_AUX_STACK_ALLOCATION=1)
+
+include_directories(${LIB_WASI_THREADS_DIR})
+
+set (LIB_WASI_THREADS_SOURCE
+    ${LIB_WASI_THREADS_DIR}/lib_wasi_threads_wrapper.c
+    ${LIB_WASI_THREADS_DIR}/tid_allocator.c)

core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c

@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "bh_log.h"
+#include "thread_manager.h"
+#include "tid_allocator.h"
+
+#if WASM_ENABLE_INTERP != 0
+#include "wasm_runtime.h"
+#endif
+
+#if WASM_ENABLE_AOT != 0
+#include "aot_runtime.h"
+#endif
+
+static const char *THREAD_START_FUNCTION = "wasi_thread_start";
+static korp_mutex thread_id_lock;
+static TidAllocator tid_allocator;
+
+typedef struct {
+    /* app's entry function */
+    wasm_function_inst_t start_func;
+    /* arg of the app's entry function */
+    uint32 arg;
+    /* thread id passed to the app */
+    int32 thread_id;
+} ThreadStartArg;
+
+static int32
+allocate_thread_id()
+{
+    os_mutex_lock(&thread_id_lock);
+    int32 id = tid_allocator_get_tid(&tid_allocator);
+    os_mutex_unlock(&thread_id_lock);
+
+    return id;
+}
+
+void
+deallocate_thread_id(int32 thread_id)
+{
+    os_mutex_lock(&thread_id_lock);
+    tid_allocator_release_tid(&tid_allocator, thread_id);
+    os_mutex_unlock(&thread_id_lock);
+}
+
+static void *
+thread_start(void *arg)
+{
+    wasm_exec_env_t exec_env = (wasm_exec_env_t)arg;
+    ThreadStartArg *thread_arg = exec_env->thread_arg;
+    uint32 argv[2];
+
+    wasm_exec_env_set_thread_info(exec_env);
+    argv[0] = thread_arg->thread_id;
+    argv[1] = thread_arg->arg;
+
+    if (!wasm_runtime_call_wasm(exec_env, thread_arg->start_func, 2, argv)) {
+        /* Exception has already been spread during throwing */
+    }
+
+    // Routine exit
+    deallocate_thread_id(thread_arg->thread_id);
+    wasm_runtime_free(thread_arg);
+    exec_env->thread_arg = NULL;
+
+    return NULL;
+}
+
+static int32
+thread_spawn_wrapper(wasm_exec_env_t exec_env, uint32 start_arg)
+{
+    wasm_module_t module = wasm_exec_env_get_module(exec_env);
+    wasm_module_inst_t module_inst = get_module_inst(exec_env);
+    wasm_module_inst_t new_module_inst = NULL;
+    ThreadStartArg *thread_start_arg = NULL;
+    wasm_function_inst_t start_func;
+    int32 thread_id;
+    uint32 stack_size = 8192;
+    int32 ret = -1;
+#if WASM_ENABLE_LIBC_WASI != 0
+    WASIContext *wasi_ctx;
+#endif
+
+    bh_assert(module);
+    bh_assert(module_inst);
+
+    stack_size = ((WASMModuleInstance *)module_inst)->default_wasm_stack_size;
+
+    if (!(new_module_inst = wasm_runtime_instantiate_internal(
+              module, true, stack_size, 0, NULL, 0)))
+        return -1;
+
+    wasm_runtime_set_custom_data_internal(
+        new_module_inst, wasm_runtime_get_custom_data(module_inst));
+
+#if WASM_ENABLE_LIBC_WASI != 0
+    wasi_ctx = wasm_runtime_get_wasi_ctx(module_inst);
+    if (wasi_ctx)
+        wasm_runtime_set_wasi_ctx(new_module_inst, wasi_ctx);
+#endif
+
+    start_func = wasm_runtime_lookup_function(new_module_inst,
+                                              THREAD_START_FUNCTION, NULL);
+    if (!start_func) {
+        LOG_ERROR("Failed to find thread start function %s",
+                  THREAD_START_FUNCTION);
+        goto thread_preparation_fail;
+    }
+
+    if (!(thread_start_arg = wasm_runtime_malloc(sizeof(ThreadStartArg)))) {
+        LOG_ERROR("Runtime args allocation failed");
+        goto thread_preparation_fail;
+    }
+
+    thread_start_arg->thread_id = thread_id = allocate_thread_id();
+    if (thread_id < 0) {
+        LOG_ERROR("Failed to get thread identifier");
+        goto thread_preparation_fail;
+    }
+    thread_start_arg->arg = start_arg;
+    thread_start_arg->start_func = start_func;
+
+    os_mutex_lock(&exec_env->wait_lock);
+    ret = wasm_cluster_create_thread(exec_env, new_module_inst, false,
+                                     thread_start, thread_start_arg);
+    if (ret != 0) {
+        LOG_ERROR("Failed to spawn a new thread");
+        goto thread_spawn_fail;
+    }
+    os_mutex_unlock(&exec_env->wait_lock);
+
+    return thread_id;
+
+thread_spawn_fail:
+    os_mutex_unlock(&exec_env->wait_lock);
+    deallocate_thread_id(thread_id);
+
+thread_preparation_fail:
+    if (new_module_inst)
+        wasm_runtime_deinstantiate_internal(new_module_inst, true);
+    if (thread_start_arg)
+        wasm_runtime_free(thread_start_arg);
+
+    return -1;
+}
+
+/* clang-format off */
+#define REG_NATIVE_FUNC(name, func_name, signature) \
+    { name, func_name##_wrapper, signature, NULL }
+/* clang-format on */
+
+static NativeSymbol native_symbols_lib_wasi_threads[] = { REG_NATIVE_FUNC(
+    "thread-spawn", thread_spawn, "(i)i") };
+
+uint32
+get_lib_wasi_threads_export_apis(NativeSymbol **p_lib_wasi_threads_apis)
+{
+    *p_lib_wasi_threads_apis = native_symbols_lib_wasi_threads;
+    return sizeof(native_symbols_lib_wasi_threads) / sizeof(NativeSymbol);
+}
+
+bool
+lib_wasi_threads_init(void)
+{
+    if (0 != os_mutex_init(&thread_id_lock))
+        return false;
+
+    if (!tid_allocator_init(&tid_allocator)) {
+        os_mutex_destroy(&thread_id_lock);
+        return false;
+    }
+
+    return true;
+}
+
+void
+lib_wasi_threads_destroy(void)
+{
+    tid_allocator_deinit(&tid_allocator);
+    os_mutex_destroy(&thread_id_lock);
+}

core/iwasm/libraries/lib-wasi-threads/tid_allocator.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "tid_allocator.h"
+#include "wasm_export.h"
+#include "bh_log.h"
+
+bh_static_assert(TID_MIN <= TID_MAX);
+#define MIN(a, b) (((a) < (b)) ? (a) : (b))
+
+bool
+tid_allocator_init(TidAllocator *tid_allocator)
+{
+    tid_allocator->size = MIN(TID_ALLOCATOR_INIT_SIZE, TID_MAX - TID_MIN + 1);
+    tid_allocator->pos = tid_allocator->size;
+    tid_allocator->ids =
+        wasm_runtime_malloc(tid_allocator->size * sizeof(int32));
+    if (tid_allocator->ids == NULL)
+        return false;
+
+    for (int64 i = tid_allocator->pos - 1; i >= 0; i--)
+        tid_allocator->ids[i] = TID_MIN + (tid_allocator->pos - 1 - i);
+
+    return true;
+}
+
+void
+tid_allocator_deinit(TidAllocator *tid_allocator)
+{
+    wasm_runtime_free(tid_allocator->ids);
+}
+
+int32
+tid_allocator_get_tid(TidAllocator *tid_allocator)
+{
+    if (tid_allocator->pos == 0) { // Resize stack and push new thread ids
+        if (tid_allocator->size == TID_MAX - TID_MIN + 1) {
+            LOG_ERROR("Maximum thread identifier reached");
+            return -1;
+        }
+
+        uint32 old_size = tid_allocator->size;
+        uint32 new_size = MIN(tid_allocator->size * 2, TID_MAX - TID_MIN + 1);
+        if (new_size != TID_MAX - TID_MIN + 1
+            && new_size / 2 != tid_allocator->size) {
+            LOG_ERROR("Overflow detected during new size calculation");
+            return -1;
+        }
+
+        size_t realloc_size = new_size * sizeof(int32);
+        if (realloc_size / sizeof(int32) != new_size) {
+            LOG_ERROR("Overflow detected during realloc");
+            return -1;
+        }
+        int32 *tmp = wasm_runtime_realloc(tid_allocator->ids, realloc_size);
+        if (tmp == NULL) {
+            LOG_ERROR("Thread ID allocator realloc failed");
+            return -1;
+        }
+
+        tid_allocator->size = new_size;
+        tid_allocator->pos = new_size - old_size;
+        tid_allocator->ids = tmp;
+        for (int64 i = tid_allocator->pos - 1; i >= 0; i--)
+            tid_allocator->ids[i] = TID_MIN + (tid_allocator->size - 1 - i);
+    }
+
+    // Pop available thread identifier from the stack
+    return tid_allocator->ids[--tid_allocator->pos];
+}
+
+void
+tid_allocator_release_tid(TidAllocator *tid_allocator, int32 thread_id)
+{
+    // Release thread identifier by pushing it into the stack
+    bh_assert(tid_allocator->pos < tid_allocator->size);
+    tid_allocator->ids[tid_allocator->pos++] = thread_id;
+}

core/iwasm/libraries/lib-wasi-threads/tid_allocator.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef _TID_ALLOCATOR_H
+#define _TID_ALLOCATOR_H
+
+#include "platform_common.h"
+
+#define TID_ALLOCATOR_INIT_SIZE CLUSTER_MAX_THREAD_NUM
+enum {
+    TID_MIN = 1,
+    TID_MAX = 0x1FFFFFFF
+}; // Reserved TIDs (WASI specification)
+
+/* Stack data structure to track available thread identifiers */
+typedef struct {
+    int32 *ids;  // Array used to store the stack
+    uint32 size; // Stack capacity
+    uint32 pos;  // Index of the element after the stack top
+} TidAllocator;
+
+bool
+tid_allocator_init(TidAllocator *tid_allocator);
+
+void
+tid_allocator_deinit(TidAllocator *tid_allocator);
+
+int32
+tid_allocator_get_tid(TidAllocator *tid_allocator);
+
+void
+tid_allocator_release_tid(TidAllocator *tid_allocator, int32 thread_id);
+
+#endif /* _TID_ALLOCATOR_H */

core/iwasm/libraries/thread-mgr/thread_manager.c

@@ -130,8 +130,21 @@ final:
 
 /* The caller must lock cluster->lock */
 static bool
-allocate_aux_stack(WASMCluster *cluster, uint32 *start, uint32 *size)
+allocate_aux_stack(WASMExecEnv *exec_env, uint32 *start, uint32 *size)
 {
+    WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env);
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION != 0
+    WASMModuleInstanceCommon *module_inst =
+        wasm_exec_env_get_module_inst(exec_env);
+    uint32 stack_end;
+
+    stack_end =
+        wasm_runtime_module_malloc(module_inst, cluster->stack_size, NULL);
+    *start = stack_end + cluster->stack_size;
+    *size = cluster->stack_size;
+
+    return stack_end != 0;
+#else
     uint32 i;
 
     /* If the module doesn't have aux stack info,
@@ -151,12 +164,29 @@ allocate_aux_stack(WASMCluster *cluster, uint32 *start, uint32 *size)
     }
 
     return false;
+#endif
 }
 
 /* The caller must lock cluster->lock */
 static bool
-free_aux_stack(WASMCluster *cluster, uint32 start)
+free_aux_stack(WASMExecEnv *exec_env, uint32 start)
 {
+    WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env);
+
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION != 0
+    WASMModuleInstanceCommon *module_inst =
+        wasm_exec_env_get_module_inst(exec_env);
+
+    if (!wasm_exec_env_is_aux_stack_managed_by_runtime(exec_env)) {
+        return true;
+    }
+
+    bh_assert(start >= cluster->stack_size);
+
+    wasm_runtime_module_free(module_inst, start - cluster->stack_size);
+
+    return true;
+#else
     uint32 i;
 
     for (i = 0; i < cluster_max_thread_num; i++) {
@@ -166,14 +196,14 @@ free_aux_stack(WASMCluster *cluster, uint32 start)
         }
     }
     return false;
+#endif
 }
 
 WASMCluster *
 wasm_cluster_create(WASMExecEnv *exec_env)
 {
     WASMCluster *cluster;
-    uint64 total_size;
-    uint32 aux_stack_start, aux_stack_size, i;
+    uint32 aux_stack_start, aux_stack_size;
 
     bh_assert(exec_env->cluster == NULL);
     if (!(cluster = wasm_runtime_malloc(sizeof(WASMCluster)))) {
@@ -209,12 +239,16 @@ wasm_cluster_create(WASMExecEnv *exec_env)
         return cluster;
     }
 
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION != 0
+    cluster->stack_size = aux_stack_size;
+#else
     cluster->stack_size = aux_stack_size / (cluster_max_thread_num + 1);
     if (cluster->stack_size < WASM_THREAD_AUX_STACK_SIZE_MIN) {
         goto fail;
     }
     /* Make stack size 16-byte aligned */
     cluster->stack_size = cluster->stack_size & (~15);
+#endif
 
     /* Set initial aux stack top to the instance and
         aux stack boundary to the main exec_env */
@@ -222,8 +256,10 @@ wasm_cluster_create(WASMExecEnv *exec_env)
                                      cluster->stack_size))
         goto fail;
 
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0
     if (cluster_max_thread_num != 0) {
-        total_size = cluster_max_thread_num * sizeof(uint32);
+        uint64 total_size = cluster_max_thread_num * sizeof(uint32);
+        uint32 i;
         if (total_size >= UINT32_MAX
             || !(cluster->stack_tops =
                      wasm_runtime_malloc((uint32)total_size))) {
@@ -245,6 +281,7 @@ wasm_cluster_create(WASMExecEnv *exec_env)
             cluster->stack_tops[i] = aux_stack_start - cluster->stack_size * i;
         }
     }
+#endif
 
     os_mutex_lock(&cluster_list_lock);
     if (bh_list_insert(cluster_list, cluster) != 0) {
@@ -284,10 +321,12 @@ wasm_cluster_destroy(WASMCluster *cluster)
 
     os_mutex_destroy(&cluster->lock);
 
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0
     if (cluster->stack_tops)
         wasm_runtime_free(cluster->stack_tops);
     if (cluster->stack_segment_occupied)
         wasm_runtime_free(cluster->stack_segment_occupied);
+#endif
 
 #if WASM_ENABLE_DEBUG_INTERP != 0
     wasm_debug_instance_destroy(cluster);
@@ -323,7 +362,13 @@ wasm_cluster_add_exec_env(WASMCluster *cluster, WASMExecEnv *exec_env)
 
     exec_env->cluster = cluster;
 
-    if (bh_list_insert(&cluster->exec_env_list, exec_env) != 0)
+    if (cluster->exec_env_list.len == cluster_max_thread_num + 1) {
+        LOG_ERROR("thread manager error: "
+                  "maximum number of threads exceeded");
+        ret = false;
+    }
+
+    if (ret && bh_list_insert(&cluster->exec_env_list, exec_env) != 0)
         ret = false;
 
     return ret;
@@ -461,7 +506,7 @@ wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env)
     if (!new_exec_env)
         goto fail2;
 
-    if (!allocate_aux_stack(cluster, &aux_stack_start, &aux_stack_size)) {
+    if (!allocate_aux_stack(exec_env, &aux_stack_start, &aux_stack_size)) {
         LOG_ERROR("thread manager error: "
                   "failed to allocate aux stack space for new thread");
         goto fail3;
@@ -482,7 +527,7 @@ wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env)
 
 fail4:
     /* free the allocated aux stack space */
-    free_aux_stack(cluster, aux_stack_start);
+    free_aux_stack(exec_env, aux_stack_start);
 fail3:
     wasm_exec_env_destroy_internal(new_exec_env);
 fail2:
@@ -502,7 +547,7 @@ wasm_cluster_destroy_spawned_exec_env(WASMExecEnv *exec_env)
 
     /* Free aux stack space */
     os_mutex_lock(&cluster->lock);
-    free_aux_stack(cluster, exec_env->aux_stack_bottom.bottom);
+    free_aux_stack(exec_env, exec_env->aux_stack_bottom.bottom);
     wasm_cluster_del_exec_env(cluster, exec_env);
     os_mutex_unlock(&cluster->lock);
     wasm_exec_env_destroy_internal(exec_env);
@@ -517,7 +562,11 @@ thread_manager_start_routine(void *arg)
     void *ret;
     WASMExecEnv *exec_env = (WASMExecEnv *)arg;
     WASMCluster *cluster = wasm_exec_env_get_cluster(exec_env);
+    WASMModuleInstanceCommon *module_inst =
+        wasm_exec_env_get_module_inst(exec_env);
+
     bh_assert(cluster != NULL);
+    bh_assert(module_inst != NULL);
 
     exec_env->handle = os_self_thread();
     ret = exec_env->thread_start_routine(exec_env);
@@ -528,17 +577,22 @@ thread_manager_start_routine(void *arg)
 #endif
 
     /* Routine exit */
+
     /* Detach the native thread here to ensure the resources are freed */
     wasm_cluster_detach_thread(exec_env);
 #if WASM_ENABLE_DEBUG_INTERP != 0
     wasm_cluster_thread_exited(exec_env);
 #endif
+
     os_mutex_lock(&cluster->lock);
     /* Free aux stack space */
-    free_aux_stack(cluster, exec_env->aux_stack_bottom.bottom);
+    free_aux_stack(exec_env, exec_env->aux_stack_bottom.bottom);
+    /* routine exit, destroy instance */
+    wasm_runtime_deinstantiate_internal(module_inst, true);
     /* Remove and exec_env */
     wasm_cluster_del_exec_env(cluster, exec_env);
     os_mutex_unlock(&cluster->lock);
+
     /* destroy exec_env */
     wasm_exec_env_destroy_internal(exec_env);
 
@@ -548,12 +602,12 @@ thread_manager_start_routine(void *arg)
 
 int32
 wasm_cluster_create_thread(WASMExecEnv *exec_env,
-                           wasm_module_inst_t module_inst,
+                           wasm_module_inst_t module_inst, bool alloc_aux_stack,
                            void *(*thread_routine)(void *), void *arg)
 {
     WASMCluster *cluster;
     WASMExecEnv *new_exec_env;
-    uint32 aux_stack_start, aux_stack_size;
+    uint32 aux_stack_start = 0, aux_stack_size;
     korp_tid tid;
 
     cluster = wasm_exec_env_get_cluster(exec_env);
@@ -570,16 +624,23 @@ wasm_cluster_create_thread(WASMExecEnv *exec_env,
     if (!new_exec_env)
         goto fail1;
 
-    if (!allocate_aux_stack(cluster, &aux_stack_start, &aux_stack_size)) {
-        LOG_ERROR("thread manager error: "
-                  "failed to allocate aux stack space for new thread");
-        goto fail2;
-    }
+    if (alloc_aux_stack) {
+        if (!allocate_aux_stack(exec_env, &aux_stack_start, &aux_stack_size)) {
+            LOG_ERROR("thread manager error: "
+                      "failed to allocate aux stack space for new thread");
+            goto fail2;
+        }
 
-    /* Set aux stack for current thread */
-    if (!wasm_exec_env_set_aux_stack(new_exec_env, aux_stack_start,
-                                     aux_stack_size)) {
-        goto fail3;
+        /* Set aux stack for current thread */
+        if (!wasm_exec_env_set_aux_stack(new_exec_env, aux_stack_start,
+                                         aux_stack_size)) {
+            goto fail3;
+        }
+    }
+    else {
+        /* Disable aux stack */
+        new_exec_env->aux_stack_boundary.boundary = 0;
+        new_exec_env->aux_stack_bottom.bottom = UINT32_MAX;
     }
 
     if (!wasm_cluster_add_exec_env(cluster, new_exec_env))
@@ -603,7 +664,8 @@ fail4:
     wasm_cluster_del_exec_env(cluster, new_exec_env);
 fail3:
     /* free the allocated aux stack space */
-    free_aux_stack(cluster, aux_stack_start);
+    if (alloc_aux_stack)
+        free_aux_stack(exec_env, aux_stack_start);
 fail2:
     wasm_exec_env_destroy_internal(new_exec_env);
 fail1:
@@ -849,7 +911,7 @@ wasm_cluster_exit_thread(WASMExecEnv *exec_env, void *retval)
     wasm_cluster_detach_thread(exec_env);
     os_mutex_lock(&cluster->lock);
     /* Free aux stack space */
-    free_aux_stack(cluster, exec_env->aux_stack_bottom.bottom);
+    free_aux_stack(exec_env, exec_env->aux_stack_bottom.bottom);
     /* Remove and destroy exec_env */
     wasm_cluster_del_exec_env(cluster, exec_env);
     os_mutex_unlock(&cluster->lock);

core/iwasm/libraries/thread-mgr/thread_manager.h

@@ -26,14 +26,16 @@ struct WASMCluster {
     korp_mutex lock;
     bh_list exec_env_list;
 
+#if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0
     /* The aux stack of a module with shared memory will be
         divided into several segments. This array store the
         stack top of different segments */
     uint32 *stack_tops;
-    /* Size of every stack segment */
-    uint32 stack_size;
     /* Record which segments are occupied */
     bool *stack_segment_occupied;
+#endif
+    /* Size of every stack segment */
+    uint32 stack_size;
     /* When has_exception == true, this cluster should refuse any spawn thread
      * requests, this flag can be cleared by calling
      * wasm_runtime_clear_exception on instances of any threads of this cluster
@@ -74,7 +76,7 @@ wasm_exec_env_get_cluster(WASMExecEnv *exec_env);
 
 int32
 wasm_cluster_create_thread(WASMExecEnv *exec_env,
-                           wasm_module_inst_t module_inst,
+                           wasm_module_inst_t module_inst, bool alloc_aux_stack,
                            void *(*thread_routine)(void *), void *arg);
 
 int32

product-mini/platforms/linux/CMakeLists.txt

@@ -85,6 +85,12 @@ if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD)
   set (WAMR_BUILD_LIB_PTHREAD 0)
 endif ()
 
+if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS)
+  # Disable wasi threads library by default
+  set (WAMR_BUILD_LIB_WASI_THREADS 0)
+endif()
+
+
 if (NOT DEFINED WAMR_BUILD_MINI_LOADER)
   # Disable wasm mini loader by default
   set (WAMR_BUILD_MINI_LOADER 0)

product-mini/platforms/posix/main.c

@@ -83,7 +83,7 @@ print_help()
     printf("  --module-path=<path>     Indicate a module search path. default is current\n"
            "                           directory('./')\n");
 #endif
-#if WASM_ENABLE_LIB_PTHREAD != 0
+#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0
     printf("  --max-threads=n          Set maximum thread number per cluster, default is 4\n");
 #endif
 #if WASM_ENABLE_DEBUG_INTERP != 0
@@ -573,7 +573,7 @@ main(int argc, char *argv[])
             }
         }
 #endif
-#if WASM_ENABLE_LIB_PTHREAD != 0
+#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0
         else if (!strncmp(argv[0], "--max-threads=", 14)) {
             if (argv[0][14] == '\0')
                 return print_help();

product-mini/platforms/windows/main.c

@@ -59,7 +59,7 @@ print_help()
     printf("  --module-path=<path>   Indicate a module search path. default is current\n"
            "                         directory('./')\n");
 #endif
-#if WASM_ENABLE_LIB_PTHREAD != 0
+#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0
     printf("  --max-threads=n        Set maximum thread number per cluster, default is 4\n");
 #endif
 #if WASM_ENABLE_DEBUG_INTERP != 0
@@ -390,7 +390,7 @@ main(int argc, char *argv[])
             }
         }
 #endif
-#if WASM_ENABLE_LIB_PTHREAD != 0
+#if WASM_ENABLE_LIB_PTHREAD != 0 || WASM_ENABLE_LIB_WASI_THREADS != 0
         else if (!strncmp(argv[0], "--max-threads=", 14)) {
             if (argv[0][14] == '\0')
                 return print_help();

samples/wasi-threads/CMakeLists.txt

@@ -0,0 +1,87 @@
+# Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+cmake_minimum_required(VERSION 3.14)
+
+include(CheckPIESupported)
+
+project(wasi_threads_sample)
+
+if (NOT DEFINED WASI_SYSROOT)
+  message (WARNING "Custom sysroot with threads enabled is required to build wasi threads samples.
+Please note that current wasi-sdk doesn't ship with threads enabled.
+Run cmake command with -DWASI_SYSROOT=/path/to/sysroot/with/threads to compile samples.")
+  return ()
+endif ()
+
+################  runtime settings  ################
+string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM)
+if (APPLE)
+  add_definitions(-DBH_PLATFORM_DARWIN)
+endif ()
+
+# Resetdefault linker flags
+set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
+set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
+
+# WAMR features switch
+
+# Set WAMR_BUILD_TARGET, currently values supported:
+# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]",
+# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]"
+if (NOT DEFINED WAMR_BUILD_TARGET)
+  if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)")
+    set (WAMR_BUILD_TARGET "AARCH64")
+  elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64")
+    set (WAMR_BUILD_TARGET "RISCV64")
+  elseif (CMAKE_SIZEOF_VOID_P EQUAL 8)
+    # Build as X86_64 by default in 64-bit platform
+    set (WAMR_BUILD_TARGET "X86_64")
+  elseif (CMAKE_SIZEOF_VOID_P EQUAL 4)
+    # Build as X86_32 by default in 32-bit platform
+    set (WAMR_BUILD_TARGET "X86_32")
+  else ()
+    message(SEND_ERROR "Unsupported build target platform!")
+  endif ()
+endif ()
+
+if (NOT CMAKE_BUILD_TYPE)
+  set (CMAKE_BUILD_TYPE Release)
+endif ()
+
+set(WAMR_BUILD_INTERP 1)
+set(WAMR_BUILD_AOT 1)
+set(WAMR_BUILD_JIT 0)
+set(WAMR_BUILD_LIBC_BUILTIN 1)
+set(WAMR_BUILD_FAST_INTERP 1)
+set(WAMR_BUILD_LIBC_WASI 1)
+set(WAMR_BUILD_LIB_WASI_THREADS 1)
+
+# compiling and linking flags
+if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang"))
+  set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
+endif ()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security")
+
+# build out vmlib
+set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..)
+include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)
+
+add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE})
+################################################
+
+
+################ wasm application ################
+add_subdirectory(wasm-apps)
+
+################ wamr runtime ################
+include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake)
+
+set (RUNTIME_SOURCE_ALL
+    ${CMAKE_CURRENT_LIST_DIR}/../../product-mini/platforms/linux/main.c
+    ${UNCOMMON_SHARED_SOURCE}
+)
+add_executable (iwasm ${RUNTIME_SOURCE_ALL})
+check_pie_supported()
+set_target_properties (iwasm PROPERTIES POSITION_INDEPENDENT_CODE ON)
+target_link_libraries(iwasm vmlib -lpthread -lm -ldl)

samples/wasi-threads/README.md

@@ -0,0 +1,31 @@
+# "WASI threads" sample introduction
+
+Currently, since the `wasi-sdk` does not have thread support in the latest release, make sure to have [wasi-libc](https://github.com/WebAssembly/wasi-libc) installed. Build it with threads enabled, e.g.
+
+```shell
+make \
+    AR=/opt/wasi-sdk/bin/llvm-ar \
+    NM=/opt/wasi-sdk/bin/llvm-nm \
+    CC=/opt/wasi-sdk/bin/clang \
+    THREAD_MODEL=posix
+```
+
+## Build and run the samples
+
+```shell
+$ mkdir build
+$ cd build
+$ cmake -DWASI_SYSROOT=/path/to/wasi-libc/sysroot ..
+$ make
+...
+$ ./iwasm wasm-apps/no_pthread.wasm
+...
+$ ./iwasm wasm-apps/exception_propagation.wasm
+```
+
+## Run samples in AOT mode
+```shell
+$ ../../../wamr-compiler/build/wamrc \
+    -o wasm-apps/no_pthread.aot wasm-apps/no_pthread.wasm
+$ ./iwasm wasm-apps/no_pthread.aot
+```

samples/wasi-threads/wasm-apps/CMakeLists.txt

@@ -0,0 +1,39 @@
+# Copyright (C) 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+if (APPLE)
+    set (HAVE_FLAG_SEARCH_PATHS_FIRST 0)
+    set (CMAKE_C_LINK_FLAGS "")
+    set (CMAKE_CXX_LINK_FLAGS "")
+endif ()
+
+if (NOT DEFINED WASI_SDK_DIR)
+  set (WASI_SDK_DIR               "/opt/wasi-sdk")
+endif ()
+
+set (CMAKE_SYSROOT                  "${WASI_SYSROOT}")
+set (CMAKE_C_COMPILER               "${WASI_SDK_DIR}/bin/clang")
+set (CMAKE_ASM_COMPILER               "${WASI_SDK_DIR}/bin/clang")
+set (CMAKE_EXE_LINKER_FLAGS         "-target wasm32-wasi-threads")
+
+function (compile_sample SOURCE_FILE)
+  get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE)
+  set (WASM_MODULE ${FILE_NAME}.wasm)
+  add_executable (${WASM_MODULE} ${SOURCE_FILE} ${ARGN})
+
+  target_compile_options (${WASM_MODULE} PRIVATE
+    -pthread -ftls-model=local-exec)
+
+  target_link_options (${WASM_MODULE} PRIVATE
+    -z stack-size=32768
+    LINKER:--export=__heap_base
+    LINKER:--export=__data_end
+    LINKER:--shared-memory,--max-memory=1966080
+    LINKER:--export=wasi_thread_start
+    LINKER:--export=malloc
+    LINKER:--export=free
+  )
+endfunction ()
+
+compile_sample(no_pthread.c wasi_thread_start.S)
+compile_sample(thread_termination.c wasi_thread_start.S)

samples/wasi-threads/wasm-apps/no_pthread.c

@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#ifndef __wasi__
+#error This example only compiles to WASM/WASI target
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+#include "wasi_thread_start.h"
+
+static const int64_t SECOND = 1000 * 1000 * 1000;
+
+typedef struct {
+    start_args_t base;
+    int th_ready;
+    int value;
+    int thread_id;
+} shared_t;
+
+void
+__wasi_thread_start_C(int thread_id, int *start_arg)
+{
+    shared_t *data = (shared_t *)start_arg;
+
+    printf("New thread ID: %d, starting parameter: %d\n", thread_id,
+           data->value);
+
+    data->thread_id = thread_id;
+    data->value += 8;
+    printf("Updated value: %d\n", data->value);
+
+    data->th_ready = 1;
+    __builtin_wasm_memory_atomic_notify(&data->th_ready, 1);
+}
+
+int
+main(int argc, char **argv)
+{
+    shared_t data = { { NULL }, 0, 52, -1 };
+    int thread_id;
+    int ret = EXIT_SUCCESS;
+
+    if (!start_args_init(&data.base)) {
+        printf("Stack allocation for thread failed\n");
+        return EXIT_FAILURE;
+    }
+
+    thread_id = __wasi_thread_spawn(&data);
+    if (thread_id < 0) {
+        printf("Failed to create thread: %d\n", thread_id);
+        ret = EXIT_FAILURE;
+        goto final;
+    }
+
+    if (__builtin_wasm_memory_atomic_wait32(&data.th_ready, 0, SECOND) == 2) {
+        printf("Timeout\n");
+        ret = EXIT_FAILURE;
+        goto final;
+    }
+
+    printf("Thread completed, new value: %d, thread id: %d\n", data.value,
+           data.thread_id);
+
+    assert(thread_id == data.thread_id);
+
+final:
+    start_args_deinit(&data.base);
+
+    return ret;
+}

samples/wasi-threads/wasm-apps/thread_termination.c

@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#ifndef __wasi__
+#error This example only compiles to WASM/WASI target
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <semaphore.h>
+#include <stdbool.h>
+#include <unistd.h>
+
+#include "wasi_thread_start.h"
+
+#define TEST_TERMINATION_BY_TRAP 0 // Otherwise test `proc_exit` termination
+#define TEST_TERMINATION_IN_MAIN_THREAD 1
+
+#define TIMEOUT_SECONDS 10
+#define NUM_THREADS 3
+static sem_t sem;
+
+typedef struct {
+    start_args_t base;
+    bool throw_exception;
+} shared_t;
+
+void
+run_long_task()
+{
+    // Busy waiting to be interruptible by trap or `proc_exit`
+    for (int i = 0; i < TIMEOUT_SECONDS; i++)
+        sleep(1);
+}
+
+void
+start_job()
+{
+    sem_post(&sem);
+    run_long_task(); // Wait to be interrupted
+    assert(false && "Unreachable");
+}
+
+void
+terminate_process()
+{
+    // Wait for all other threads (including main thread) to be ready
+    printf("Waiting before terminating\n");
+    for (int i = 0; i < NUM_THREADS; i++)
+        sem_wait(&sem);
+
+    printf("Force termination\n");
+#if TEST_TERMINATION_BY_TRAP == 1
+    __builtin_trap();
+#else
+    __wasi_proc_exit(1);
+#endif
+}
+
+void
+__wasi_thread_start_C(int thread_id, int *start_arg)
+{
+    shared_t *data = (shared_t *)start_arg;
+
+    if (data->throw_exception) {
+        terminate_process();
+    }
+    else {
+        printf("Thread running\n");
+
+        start_job();
+    }
+}
+
+int
+main(int argc, char **argv)
+{
+    int thread_id = -1, i;
+    shared_t data[NUM_THREADS] = { 0 };
+
+    if (sem_init(&sem, 0, 0) != 0) {
+        printf("Failed to init semaphore\n");
+        return EXIT_FAILURE;
+    }
+
+    for (i = 0; i < NUM_THREADS; i++) {
+        // No graceful memory free to simplify the example
+        if (!start_args_init(&data[i].base)) {
+            printf("Failed to allocate thread's stack\n");
+            return EXIT_FAILURE;
+        }
+    }
+
+// Create a thread that forces termination through trap or `proc_exit`
+#if TEST_TERMINATION_IN_MAIN_THREAD == 1
+    data[0].throw_exception = false;
+#else
+    data[0].throw_exception = true;
+#endif
+    thread_id = __wasi_thread_spawn(&data[0]);
+    if (thread_id < 0) {
+        printf("Failed to create thread: %d\n", thread_id);
+        return EXIT_FAILURE;
+    }
+
+    // Create two additional threads to test exception propagation
+    data[1].throw_exception = false;
+    thread_id = __wasi_thread_spawn(&data[1]);
+    if (thread_id < 0) {
+        printf("Failed to create thread: %d\n", thread_id);
+        return EXIT_FAILURE;
+    }
+    data[2].throw_exception = false;
+    thread_id = __wasi_thread_spawn(&data[2]);
+    if (thread_id < 0) {
+        printf("Failed to create thread: %d\n", thread_id);
+        return EXIT_FAILURE;
+    }
+
+#if TEST_TERMINATION_IN_MAIN_THREAD == 1
+    printf("Force termination (main thread)\n");
+    terminate_process();
+#else  /* TEST_TERMINATION_IN_MAIN_THREAD */
+    printf("Main thread running\n");
+
+    start_job();
+#endif /* TEST_TERMINATION_IN_MAIN_THREAD */
+    return EXIT_SUCCESS;
+}

samples/wasi-threads/wasm-apps/wasi_thread_start.S

@@ -0,0 +1,22 @@
+# A slightly modified copy of the wasi-libc implementation
+# https://github.com/WebAssembly/wasi-libc/pull/376/
+	.globaltype	__stack_pointer, i32
+	.functype	__wasi_thread_start_C (i32, i32) -> ()
+
+	.globl	wasi_thread_start
+
+wasi_thread_start:
+	.functype	wasi_thread_start (i32, i32) -> ()
+
+	# Set up the minimum C environment.
+	# Note: offsetof(start_arg, stack) == 0
+	local.get   1  # start_arg
+	i32.load    0  # stack
+	global.set  __stack_pointer
+
+	# Make the C function do the rest of work.
+	local.get   0  # tid
+	local.get   1  # start_arg
+	call __wasi_thread_start_C
+
+	end_function

samples/wasi-threads/wasm-apps/wasi_thread_start.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+#ifndef WASI_THREAD_START_H
+#define WASI_THREAD_START_H
+
+#define STACK_SIZE 32 * 1024 // same as the main stack
+
+typedef struct {
+    void *stack;
+} start_args_t;
+
+static inline int
+start_args_init(start_args_t *start_args)
+{
+    start_args->stack = malloc(STACK_SIZE);
+    if (!start_args->stack) {
+        return 0;
+    }
+
+    start_args->stack += STACK_SIZE;
+    return 1;
+}
+
+static inline void
+start_args_deinit(start_args_t *start_args)
+{
+    free(start_args->stack - STACK_SIZE);
+}
+
+#endif

wamr-compiler/CMakeLists.txt

@@ -44,6 +44,7 @@ add_definitions(-DWASM_ENABLE_CUSTOM_NAME_SECTION=1)
 add_definitions(-DWASM_ENABLE_DUMP_CALL_STACK=1)
 add_definitions(-DWASM_ENABLE_PERF_PROFILING=1)
 add_definitions(-DWASM_ENABLE_LOAD_CUSTOM_SECTION=1)
+add_definitions(-DWASM_ENABLE_LIB_WASI_THREADS=1)
 
 if (WAMR_BUILD_LLVM_LEGACY_PM EQUAL 1)
   add_definitions(-DWASM_ENABLE_LLVM_LEGACY_PM=1)
@@ -204,6 +205,7 @@ if (NOT MINGW)
   endif()
 endif()
 include (${IWASM_DIR}/libraries/lib-pthread/lib_pthread.cmake)
+include (${IWASM_DIR}/libraries/lib-wasi-threads/lib_wasi_threads.cmake)
 include (${IWASM_DIR}/common/iwasm_common.cmake)
 include (${IWASM_DIR}/interpreter/iwasm_interp.cmake)
 include (${IWASM_DIR}/aot/iwasm_aot.cmake)
@@ -258,6 +260,7 @@ add_library (vmlib
              ${LIBC_BUILTIN_SOURCE}
              ${LIBC_WASI_SOURCE}
              ${LIB_PTHREAD_SOURCE}
+             ${LIB_WASI_THREADS_SOURCE}
              ${IWASM_COMMON_SOURCE}
              ${IWASM_INTERP_SOURCE}
              ${IWASM_AOT_SOURCE})