Add another wamr test (#2411)

Follows up #2364 where we discussed that we might want to have a test
which has really short thread function and creates many threads.

.github/workflows/compilation_on_android_ubuntu.yml

@@ -537,7 +537,7 @@ jobs:
         working-directory: ./core/iwasm/libraries/lib-socket/test/
 
       - name: run tests
-        timeout-minutes: 20
+        timeout-minutes: 30
         run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites
 
@@ -553,7 +553,7 @@ jobs:
           sudo apt install -y g++-multilib lib32gcc-9-dev
 
       - name: run tests x86_32
-        timeout-minutes: 20
+        timeout-minutes: 30
         if: env.TEST_ON_X86_32 == 'true'
         run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites

.github/workflows/nightly_run.yml

@@ -605,7 +605,7 @@ jobs:
         working-directory: ./core/iwasm/libraries/lib-socket/test/
 
       - name: run tests
-        timeout-minutes: 20
+        timeout-minutes: 40
         run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites
 
@@ -621,7 +621,7 @@ jobs:
           sudo apt install -y g++-multilib lib32gcc-9-dev
 
       - name: run tests x86_32
-        timeout-minutes: 20
+        timeout-minutes: 40
         if: env.TEST_ON_X86_32 == 'true'
         run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites

core/iwasm/libraries/lib-wasi-threads/test/build.sh

@@ -34,7 +34,7 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Stress tests names
-thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm")
+thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm" "stress_test_threads_creation.wasm")
 
 for test_c in *.c; do
     test_wasm="$(basename $test_c .c).wasm"
@@ -56,6 +56,7 @@ for test_c in *.c; do
     echo "Compiling $test_c to $test_wasm"
     $CC \
         -target wasm32-wasi-threads \
+        -O2 \
         -pthread -ftls-model=local-exec \
         -z stack-size=32768 \
         -Wl,--export=__heap_base \

core/iwasm/libraries/lib-wasi-threads/test/skip.json

@@ -1,5 +1,6 @@
 {
     "lib-wasi-threads tests": {
-        "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently"
+        "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently",
+        "stress_test_threads_creation": "Stress tests are incompatible with the other part and executed differently"
     }
 }

core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c

@@ -18,8 +18,9 @@
 
 enum CONSTANTS {
     NUM_ITER = 100000,
-    NUM_RETRY = 5,
+    NUM_RETRY = 8,
     MAX_NUM_THREADS = 8,
+    RETRY_SLEEP_TIME_US = 2000,
 };
 
 unsigned prime_numbers_count = 0;
@@ -62,11 +63,13 @@ void
 spawn_thread(pthread_t *thread, unsigned int *arg)
 {
     int status_code = -1;
+    int timeout_us = RETRY_SLEEP_TIME_US;
     for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) {
         status_code = pthread_create(thread, NULL, &check_if_prime, arg);
         assert(status_code == 0 || status_code == EAGAIN);
         if (status_code == EAGAIN) {
-            usleep(2000);
+            usleep(timeout_us);
+            timeout_us *= 2;
         }
     }
 
@@ -95,7 +98,7 @@ main(int argc, char **argv)
 
         args[thread_num] = factorised_number;
 
-        usleep(2000);
+        usleep(RETRY_SLEEP_TIME_US);
         spawn_thread(&threads[thread_num], &args[thread_num]);
         assert(threads[thread_num] != 0);
     }

core/iwasm/libraries/lib-wasi-threads/test/stress_test_threads_creation.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <unistd.h>
+
+enum CONSTANTS {
+    NUM_ITER = 200000,
+    NUM_RETRY = 8,
+    MAX_NUM_THREADS = 8,
+    RETRY_SLEEP_TIME_US = 4000,
+    SECOND = 1000 * 1000 * 1000
+};
+
+int threads_executed = 0;
+unsigned int threads_creation_tried = 0;
+unsigned int threads_in_use = 0;
+
+void *
+thread_func(void *arg)
+{
+    (void)(arg);
+    __atomic_fetch_add(&threads_executed, 1, __ATOMIC_RELAXED);
+    __atomic_fetch_sub(&threads_in_use, 1, __ATOMIC_SEQ_CST);
+    return NULL;
+}
+
+void
+spawn_thread(pthread_t *thread)
+{
+    int status_code = -1;
+    int timeout_us = RETRY_SLEEP_TIME_US;
+    for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) {
+        status_code = pthread_create(thread, NULL, &thread_func, NULL);
+        __atomic_fetch_add(&threads_creation_tried, 1, __ATOMIC_RELAXED);
+
+        assert(status_code == 0 || status_code == EAGAIN);
+        if (status_code == EAGAIN) {
+            usleep(timeout_us);
+            timeout_us *= 2;
+        }
+    }
+
+    assert(status_code == 0 && "Thread creation should succeed");
+}
+
+int
+main(int argc, char **argv)
+{
+    double percentage = 0.1;
+
+    for (int iter = 0; iter < NUM_ITER; ++iter) {
+        if (iter > NUM_ITER * percentage) {
+            fprintf(stderr, "Spawning stress test is %d%% finished\n",
+                    (unsigned int)(percentage * 100));
+            percentage += 0.1;
+        }
+        while (__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST)
+               == MAX_NUM_THREADS) {
+            usleep(100);
+        }
+
+        __atomic_fetch_add(&threads_in_use, 1, __ATOMIC_SEQ_CST);
+        pthread_t tmp;
+        spawn_thread(&tmp);
+        pthread_detach(tmp);
+    }
+
+    while ((__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) != 0)) {
+        __builtin_wasm_memory_atomic_wait32(&threads_in_use, 0, SECOND);
+    }
+
+    assert(__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) == 0);
+
+    // Validation
+    assert(threads_creation_tried >= threads_executed
+           && "Test executed more threads than were created");
+    assert((1. * threads_creation_tried) / threads_executed < 2.5
+           && "Ensuring that we're retrying thread creation less than 2.5 "
+              "times on average ");
+
+    fprintf(stderr,
+            "Spawning stress test finished successfully executed %d threads "
+            "with retry ratio %f\n",
+            threads_creation_tried,
+            (1. * threads_creation_tried) / threads_executed);
+    return 0;
+}

tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh

@@ -14,6 +14,7 @@ readonly WAMR_DIR="${WORK_DIR}/../../../.."
 readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm \
     --allow-resolve=google-public-dns-a.google.com \
     --addr-pool=::1/128,127.0.0.1/32"
+
 readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=8"
 readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc"
 readonly C_TESTS="tests/c/testsuite/"
@@ -21,18 +22,22 @@ readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/"
 readonly THREAD_PROPOSAL_TESTS="tests/proposals/wasi-threads/"
 readonly THREAD_INTERNAL_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/test/"
 readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/"
+readonly STRESS_TESTS=("spawn_stress_test.wasm" "stress_test_threads_creation.wasm")
 
 run_aot_tests () {
     local tests=("$@")
+    local iwasm="${IWASM_CMD}"
     for test_wasm in ${tests[@]}; do
         local extra_stress_flags=""
-        if [[ "$test_wasm" =~ "stress" ]]; then
-            extra_stress_flags="--max-threads=8"
-        fi
+        for stress_test in "${STRESS_TESTS[@]}"; do
+            if [ "$test_wasm" == "$stress_test" ]; then
+                iwasm="${IWASM_CMD_STRESS}"
+            fi  
+        done
 
         test_aot="${test_wasm%.wasm}.aot"
         test_json="${test_wasm%.wasm}.json"
-
+ 
         if [ -f ${test_wasm} ]; then
             expected=$(jq .exit_code ${test_json})
         fi
@@ -48,7 +53,6 @@ run_aot_tests () {
         fi
 
         ${IWASM_CMD} $extra_stress_flags $test_aot
-
         ret=${PIPESTATUS[0]}
 
         echo "expected=$expected, actual=$ret"
@@ -62,15 +66,18 @@ if [[ $MODE != "aot" ]];then
     python3 -m venv wasi-env && source wasi-env/bin/activate
     python3 -m pip install -r test-runner/requirements.txt
 
-    # Stress test requires max-threads=8 so it's run separately
-    if [[ -e "${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm" ]]; then 
-        ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm
-        ret=${PIPESTATUS[0]}
-        if [ "${ret}" -ne 0 ]; then
-            echo "Stress test spawn_stress_test FAILED with code " ${ret}
-            exit_code=${ret}
+    # Stress tests require max-threads=8 so they're executed separately
+    for stress_test in "${STRESS_TESTS[@]}"; do
+        if [[ -e "${THREAD_INTERNAL_TESTS}${stress_test}" ]]; then
+            echo "${stress_test}" is a stress test
+            ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}${stress_test}
+            ret=${PIPESTATUS[0]}
+            if [ "${ret}" -ne 0 ]; then
+                echo "Stress test ${stress_test} FAILED with code " ${ret}
+                exit_code=${ret}
+            fi
         fi
-    fi
+    done
 
     TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \
             -r adapters/wasm-micro-runtime.py \