Add initial stress test (#2364)

We need to make a test that runs longer than the tests we had before to check
some problems that might happen after running for some time (e.g. memory
corruption or something else).

.github/workflows/compilation_on_android_ubuntu.yml

@@ -527,7 +527,7 @@ jobs:
         working-directory: ./core/iwasm/libraries/lib-socket/test/
 
       - name: run tests
-        timeout-minutes: 10
+        timeout-minutes: 20
         run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites
 
@@ -543,7 +543,7 @@ jobs:
           sudo apt install -y g++-multilib lib32gcc-9-dev
 
       - name: run tests x86_32
-        timeout-minutes: 10
+        timeout-minutes: 20
         if: env.TEST_ON_X86_32 == 'true'
         run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites

.github/workflows/nightly_run.yml

@@ -595,7 +595,7 @@ jobs:
         working-directory: ./core/iwasm/libraries/lib-socket/test/
 
       - name: run tests
-        timeout-minutes: 10
+        timeout-minutes: 20
         run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites
 
@@ -611,7 +611,7 @@ jobs:
           sudo apt install -y g++-multilib lib32gcc-9-dev
 
       - name: run tests x86_32
-        timeout-minutes: 10
+        timeout-minutes: 20
         if: env.TEST_ON_X86_32 == 'true'
         run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
         working-directory: ./tests/wamr-test-suites

core/iwasm/libraries/lib-wasi-threads/test/build.sh

@@ -9,10 +9,13 @@ set -eo pipefail
 CC=${CC:=/opt/wasi-sdk/bin/clang}
 WAMR_DIR=../../../../..
 
+# Stress tests names
+thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm")
+
 for test_c in *.c; do
     test_wasm="$(basename $test_c .c).wasm"
 
-    if [ $test_wasm = "linear_memory_size_update.wasm" ]; then
+    if [[ " ${thread_start_file_exclusions[@]} " =~ " ${test_wasm} " ]] ; then
         thread_start_file=""
     else
         thread_start_file=$WAMR_DIR/samples/wasi-threads/wasm-apps/wasi_thread_start.S

core/iwasm/libraries/lib-wasi-threads/test/manifest.json

@@ -0,0 +1,3 @@
+{
+    "name": "lib-wasi-threads tests"
+}

core/iwasm/libraries/lib-wasi-threads/test/skip.json

@@ -0,0 +1,5 @@
+{
+    "lib-wasi-threads tests": {
+        "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently"
+    }
+}

core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef __wasi__
+#error This example only compiles to WASM/WASI target
+#endif
+
+#include <assert.h>
+#include <errno.h>
+#include <math.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+enum CONSTANTS {
+    NUM_ITER = 100000,
+    NUM_RETRY = 5,
+    MAX_NUM_THREADS = 8,
+};
+
+unsigned prime_numbers_count = 0;
+
+bool
+is_prime(unsigned int num)
+{
+    for (unsigned int i = 2; i <= (unsigned int)(sqrt(num)); ++i) {
+        if (num % i == 0) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void *
+check_if_prime(void *value)
+{
+    unsigned int *num = (unsigned int *)(value);
+    usleep(10000);
+    if (is_prime(*num)) {
+        __atomic_fetch_add(&prime_numbers_count, 1, __ATOMIC_SEQ_CST);
+    }
+    return NULL;
+}
+
+unsigned int
+validate()
+{
+    unsigned int counter = 0;
+    for (unsigned int i = 2; i <= NUM_ITER; ++i) {
+        counter += is_prime(i);
+    }
+
+    return counter;
+}
+
+void
+spawn_thread(pthread_t *thread, unsigned int *arg)
+{
+    int status_code = -1;
+    for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) {
+        status_code = pthread_create(thread, NULL, &check_if_prime, arg);
+        assert(status_code == 0 || status_code == EAGAIN);
+        if (status_code == EAGAIN) {
+            usleep(2000);
+        }
+    }
+
+    assert(status_code == 0 && "Thread creation should succeed");
+}
+
+int
+main(int argc, char **argv)
+{
+    pthread_t threads[MAX_NUM_THREADS];
+    unsigned int args[MAX_NUM_THREADS];
+    double percentage = 0.1;
+
+    for (unsigned int factorised_number = 2; factorised_number < NUM_ITER;
+         ++factorised_number) {
+        if (factorised_number > NUM_ITER * percentage) {
+            fprintf(stderr, "Stress test is %d%% finished\n",
+                    (unsigned int)(percentage * 100));
+            percentage += 0.1;
+        }
+
+        unsigned int thread_num = factorised_number % MAX_NUM_THREADS;
+        if (threads[thread_num] != 0) {
+            assert(pthread_join(threads[thread_num], NULL) == 0);
+        }
+
+        args[thread_num] = factorised_number;
+
+        usleep(2000);
+        spawn_thread(&threads[thread_num], &args[thread_num]);
+        assert(threads[thread_num] != 0);
+    }
+
+    for (int i = 0; i < MAX_NUM_THREADS; ++i) {
+        assert(threads[i] == 0 || pthread_join(threads[i], NULL) == 0);
+    }
+
+    // Check the test results
+    assert(
+        prime_numbers_count == validate()
+        && "Answer mismatch between tested code and reference implementation");
+
+    fprintf(stderr, "Stress test finished successfully\n");
+    return 0;
+}

tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh

@@ -14,6 +14,7 @@ readonly WAMR_DIR="${WORK_DIR}/../../../.."
 readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm \
     --allow-resolve=google-public-dns-a.google.com \
     --addr-pool=::1/128,127.0.0.1/32"
+readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=8"
 readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc"
 readonly C_TESTS="tests/c/testsuite/"
 readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/"
@@ -24,6 +25,11 @@ readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/"
 run_aot_tests () {
     local tests=("$@")
     for test_wasm in ${tests[@]}; do
+        local extra_stress_flags=""
+        if [[ "$test_wasm" =~ "stress" ]]; then
+            extra_stress_flags="--max-threads=8"
+        fi
+
         test_aot="${test_wasm%.wasm}.aot"
         test_json="${test_wasm%.wasm}.json"
 
@@ -41,7 +47,7 @@ run_aot_tests () {
             expected=$(jq .exit_code ${test_json})
         fi
 
-        ${IWASM_CMD} $test_aot
+        ${IWASM_CMD} $extra_stress_flags $test_aot
 
         ret=${PIPESTATUS[0]}
 
@@ -55,15 +61,31 @@ run_aot_tests () {
 if [[ $MODE != "aot" ]];then
     python3 -m venv wasi-env && source wasi-env/bin/activate
     python3 -m pip install -r test-runner/requirements.txt
+
+    # Stress test requires max-threads=8 so it's run separately
+    if [[ -e "${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm" ]]; then 
+        ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm
+        ret=${PIPESTATUS[0]}
+        if [ "${ret}" -ne 0 ]; then
+            echo "Stress test spawn_stress_test FAILED with code " ${ret}
+            exit_code=${ret}
+        fi
+    fi
+
     TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \
-                -r adapters/wasm-micro-runtime.py \
-                -t \
-                    ${C_TESTS} \
-                    ${ASSEMBLYSCRIPT_TESTS} \
-                    ${THREAD_PROPOSAL_TESTS} \
-                    ${THREAD_INTERNAL_TESTS} \
-                    ${LIB_SOCKET_TESTS}
-    exit_code=${PIPESTATUS[0]}
+            -r adapters/wasm-micro-runtime.py \
+            -t \
+                ${C_TESTS} \
+                ${ASSEMBLYSCRIPT_TESTS} \
+                ${THREAD_PROPOSAL_TESTS} \
+                ${THREAD_INTERNAL_TESTS} \
+                ${LIB_SOCKET_TESTS} \
+            --exclude-filter "${THREAD_INTERNAL_TESTS}skip.json"
+
+    ret=${PIPESTATUS[0]}
+    if [ "${ret}" -ne 0 ]; then
+        exit_code=${ret}
+    fi
     deactivate
 else
     target_option=""