Add regression tests of BA issue cases (#3462)

tests/regression/ba-issues/.gitignore

@@ -0,0 +1,4 @@
+*.zip
+build
+*.log
+**/out.aot

tests/regression/ba-issues/README.md

@@ -0,0 +1,249 @@
+# BA Issues
+
+For how to add a new test case, you should refer to following steps:
+
+1. [Creating directories for new issue test cases](#helper-shell-script)
+2. If the new issue test cases require new CMake build config of `warmc`/`iwasm` rather than existing ones, modify [build script](#warmc-and-iwasm-build-script) for new build config
+3. Add [running configuration](#add-a-new-configuration-for-how-to-run-your-issue-test-case) for the new issue test cases
+4. [Running tests and check test results](#running-test-cases-and-getting-results)
+
+## helper shell script
+
+`helper.sh` is to help you quickly create corresponding directory for test cases and unzip them if they are in zip file. It's helpful if you want to add many cases all together.
+
+```shell
+# helper scripts will create issues/issue-xxx where xxx is from 2944 to 2966
+./helper.sh 2944 2966
+# helper scripts will create issues/issue-2999
+./helper.sh 2999
+# helper scripts will extract any zip files under issues/issue-xxx where xxx is from 2944 to 2966
+./helper.sh -x 2944 2966
+```
+
+## `warmc` and `iwasm` build script
+
+To build wamrc and iwasm(this could take a while for we are building multiple version of iwasm with different CMake cache variable configurations)
+
+```shell
+./build_wamr.sh
+```
+
+Inside the file, if you want to add a new for you case, you can append a new build command to this file.
+
+```shell
+# format: build_iwasm "CMake cache variable configurations" "runtime name"
+build_iwasm "-DWAMR_BUILD_LIBC_WASI=0 -DWAMR_BUILD_LIBC_BUILTIN=1 -DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_BULK_MEMORY=1 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_FAST_JIT=1" "multi-tier-wasi-disabled"
+```
+
+Above line will compile a `iwasm-multi-tier-wasi-disabled` runtime under directory build, so you can indicate use it in your running config entry in Json.
+
+> PS: if you add some
+
+## Add a new configuration for how to run your issue test case
+
+In `running_config.json`, add new entry for your issue test case
+
+### Here is a simply running configuration that only uses `iwasm`
+
+```Json
+{
+    "deprecated": false,
+    "ids": [
+        2955
+    ],
+    "runtime": "iwasm-default-wasi-disabled",
+    "file": "iwasm_fast_interp_unexpected_value.wasm",
+    "mode": "fast-interp",
+    "options": " --heap-size=0 -f to_test",
+    "argument": "",
+    "expected return": {
+        "ret code": 0,
+        "stdout content": "0x44e5d17eb93a0ce:i64",
+        "description": "expected output 0x44e5d17eb93a0ce:i64"
+    }
+}
+```
+
+Maybe some test cases can shared a running config(for example, they all come from a fuzz reporting issues). When it comes to that, you can simply add their ids all together. And use the wildcard for
+matching file names.
+
+```JSon
+{
+    "deprecated": false,
+    "ids": [
+        2966,
+        2964,
+        2963,
+        2962
+    ],
+    "runtime": "iwasm-multi-tier-wasi-disabled",
+    "file": "*.wasm",
+    "mode": "fast-jit",
+    "options": " --heap-size=0 -f to_test",
+    "argument": "",
+    "expected return": {
+        "ret code": 0,
+        "stdout content": "0x0:i32",
+        "description": "expected output 0x0:i32"
+    }
+}
+```
+
+### Here is a simply running configuration that uses only `wamrc`
+
+TODO: this is only a dummy config example, changes to actual test case configuration
+
+```JSon
+{
+    "deprecated": false,
+    "ids": [
+        2956
+    ],
+    "compile_options": {
+        "compiler": "wamrc", 
+        "only compile": true, 
+        "in file": "*.wasm", 
+        "out file": "out.aot", 
+        "options": "--target=x86_64",
+        "expected return": {
+            "ret code": 0,
+            "stdout content": "",
+            "description": ""
+        }
+    }
+}
+```
+
+### Here is a simply running configuration that uses both `wamrc` and `iwasm`
+
+TODO: this is only a dummy config example, changes to actual test case configuration
+
+```JSon
+{
+    "deprecated": false,
+    "ids": [
+        2956
+    ],
+    "compile_options": {
+        "compiler": "wamrc", 
+        "only compile": false, 
+        "in file": "*.wasm", 
+        "out file": "out.aot", 
+        "options": "--target=x86_64",
+        "expected return": {
+            "ret code": 0,
+            "stdout content": "",
+            "description": ""
+        }
+    },
+    "runtime": "iwasm-multi-tier-wasi-disabled",
+    "file": "out.aot",
+    "mode": "aot",
+    "options": " --heap-size=0 -f to_test",
+    "argument": "",
+    "expected return": {
+        "ret code": 0,
+        "stdout content": "0x0:i32",
+        "description": "expected output 0x0:i32"
+    }
+}
+```
+
+### For deprecated issue test cases
+
+Due to the spec changes, some cases may be deprecated in the future. When the running result is not as expected and after making sure it's not the bug in WAMR but the test cases should be deprecated(maybe using wasm-validate or some other tool). They should be moved into directory `issues-deprecated.` And simply set the `"deprecated": true,` in their corresponding running configuration.
+
+For example:
+
+```JSon
+{
+    "deprecated": true,
+    "ids": [
+        47,
+        48,
+        49,
+        50,
+        51,
+        52,
+        53,
+        54,
+        55,
+        56,
+        57,
+        58,
+        59,
+        60,
+        61,
+        62,
+        63,
+        64,
+        65,
+        66,
+        67,
+        68,
+        69,
+        70,
+        71,
+        72,
+        73,
+        74,
+        75,
+        76,
+        77,
+        78,
+        79,
+        80,
+        81,
+        82,
+        83,
+        84
+    ],
+    "runtime": "iwasm-default",
+    "mode": "classic-interp",
+    "file": "PoC.wasm",
+    "argument": "",
+    "expected return": {
+        "ret code": 0,
+        "stdout content": "",
+        "description": "no segfault"
+    }
+}
+```
+
+## Running test cases and getting results
+
+simply run `run.py`
+
+```shell
+./run.py
+```
+
+If everything went well, you should see similarly output in your command line output
+
+```shell
+Finish testing, 22/22 of test cases passed, no more issues should further test
+```
+
+If you add the test case under directory `issues` but forget to add the running config in json file, the output can be something like
+
+```shell
+Finish testing, 21/21 of test cases passed, {2945} issue(s) should further test
+```
+
+If some test case are failing, then it will be something like
+
+```shell
+Finish testing, 21/22 of test cases passed, no more issue(s) should further test
+```
+
+And a log file named `issues_tests.log` will be generated and inside it will display the details of the failing cases, for example:
+
+```dummy
+=======================================================
+Failing issue id: 2945.
+run with command_lists: ['./build/iwasm-default-wasi-disabled', '--heap-size=0', '-f', 'to_test', '/home/tl/TL/clion_projects/wamr-internal-test/tests/regression/ba-issues/issues/issue-2945/iwasm_fast_interp_moob_unhandled.wasm']
+exit code (actual, expected) : (1, 0)
+stdout (actual, expected) : ('Exception: out of bounds memory access', 'Exception: out of bounds memory access')
+=======================================================
+```
+

tests/regression/ba-issues/build_wamr.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+#
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+PLATFORM=$(uname -s | tr A-Z a-z)
+
+readonly WORK_DIR=$PWD
+readonly WAMR_DIR=${WORK_DIR}/../../..
+
+function build_wamrc() {
+    echo "Build wamrc for spec test under aot compile type"
+    cd ${WAMR_DIR}/wamr-compiler &&
+    ./build_llvm.sh &&
+    cd ${WORK_DIR}/build &&
+    if [ -d build-wamrc ]; then rm -rf build-wamrc; else mkdir build-wamrc; fi &&
+    cd build-wamrc && cmake ${WAMR_DIR}/wamr-compiler && make -j 4
+}
+
+function build_iwasm() {
+    echo "Build iwasm with compile flags " $1 " "
+    cd ${WAMR_DIR}/product-mini/platforms/${PLATFORM} &&
+    cd ${WORK_DIR}/build &&
+    if [ -d build-iwasm-$2 ]; then rm -rf build-iwasm-$2; else mkdir build-iwasm-$2; fi &&
+    cd build-iwasm-$2 &&
+    cmake ${WAMR_DIR}/product-mini/platforms/${PLATFORM} $1 \
+          -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_SANITIZER=asan &&
+    make -j 4
+    if [ "$?" != 0 ]; then
+        echo -e "build iwasm failed"
+        exit 1
+    fi
+}
+
+rm -fr build && mkdir build
+
+# build wamrc
+build_wamrc
+
+# build default iwasm for testing fast-interp and AOT
+build_iwasm "-DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=1" "default"
+
+# build default iwasm for testing fast-interp and AOT with GC enabled
+build_iwasm "-DWAMR_BUILD_GC=1 -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_SPEC_TEST=1" "default-gc-enabled"
+
+# build llvm-jit iwasm for testing llvm-jit
+build_iwasm "-DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_JIT=1" "llvm-jit"
+
+# build multi-tier-jit iwasm for testing classic-interp, fast-jit, llvm-jit and multi-tier-jit
+build_iwasm "-DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1" "multi-tier-jit"
+
+# build default iwasm for testing fast-interp and AOT with libc-wasi disabled
+build_iwasm "-DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_LIBC_WASI=0" "default-wasi-disabled"
+
+# build llvm-jit iwasm for testing llvm-jit with libc-wasi disabled
+build_iwasm "-DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LIBC_WASI=0" "llvm-jit-wasi-disabled"
+
+# build multi-tier-jit iwasm for testing classic-interp, fast-jit, llvm-jit and multi-tier-jit with libc-wasi disabled
+build_iwasm "-DWAMR_BUILD_REF_TYPES=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LIBC_WASI=0" "multi-tier-jit-wasi-disabled"
+
+# TODO: add more version of iwasm, for example, sgx version

tests/regression/ba-issues/helper.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+# Function to create a directory
+create_directory() {
+    dir_name="issue-$1"
+    mkdir -p "$dir_name"
+    echo "Created directory: $dir_name"
+
+    # Unzip files if unzip option is enabled
+    if [ "$unzip" = true ]; then
+        if [ -d "$dir_name" ]; then
+            # /opt/wabt/bin/wasm2wat --enable-all $dir_name/PoC.wasm -o $dir_name/PoC.wast
+            for zipfile in "$dir_name"/*.zip; do
+                if [ -f "$zipfile" ]; then
+                    echo "Unzipping $zipfile in $dir_name"
+                    unzip -o "$zipfile" -d "$dir_name"
+                    rm $zipfile
+                    # /opt/wabt/bin/wasm2wat --enable-all PoC.wasm -o PoC.wast
+                fi
+            done
+        fi
+    fi
+}
+
+# Initialize unzip option to false
+unzip=false
+
+# Parse options
+while getopts ":x" opt; do
+    case $opt in
+    x)
+        unzip=true
+        ;;
+    \?)
+        echo "Invalid option: -$OPTARG" >&2
+        exit 1
+        ;;
+    esac
+done
+
+# Remove the parsed options from the arguments
+shift $((OPTIND - 1))
+
+# Check if at least one argument is provided
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 [-x] <num1> [num2]"
+    exit 1
+fi
+
+num1=$1
+
+# Changes work directories to issues
+cd issues
+
+# If only one argument is provided
+if [ $# -eq 1 ]; then
+    create_directory "$num1"
+else
+    # Extract the second argument
+    num2=$2
+
+    # Check if the second argument is greater than the first
+    if [ "$num2" -lt "$num1" ]; then
+        echo "Second number must be greater than or equal to the first number."
+        exit 1
+    fi
+
+    # Generate directories from num1 to num2
+    for ((i = num1; i <= num2; i++)); do
+        create_directory "$i"
+    done
+fi

tests/regression/ba-issues/issues/issue-2759/case.wat

@@ -0,0 +1,83 @@
+(module
+  (type (;0;) (func (param f64) (result f64)))
+  (type (;1;) (func (param i32 i32 i32) (result i32)))
+  (type (;2;) (func (param i32) (result i32)))
+  (type (;3;) (func (param i32)))
+  (type (;4;) (func (param i32 i32 i32)))
+  (type (;5;) (func (param i32 i32) (result i32)))
+  (type (;6;) (func (param i32 i64 i32) (result i64)))
+  (type (;7;) (func (param i32 i32 i32 i32) (result i32)))
+  (type (;8;) (func))
+  (import "wasi_snapshot_preview1" "args_sizes_get" (func (;0;) (type 5)))
+  (import "wasi_snapshot_preview1" "args_get" (func (;1;) (type 5)))
+  (import "wasi_snapshot_preview1" "proc_exit" (func (;2;) (type 3)))
+  (import "wasi_snapshot_preview1" "fd_write" (func (;3;) (type 7)))
+  (func (;4;) (type 8))
+  (func (;5;) (type 4) (param i32 i32 i32))
+  (func (;6;) (type 8)
+    (local i32 i32 i32 i32 i32)
+    block (result i32)  ;; label = @1
+      loop  ;; label = @5
+        local.get 4
+        local.get 1
+        i32.const 15720
+        i32.add
+        i32.store offset=15576
+        
+        i32.const 0
+        i32.const 0
+        i32.load offset=4
+        i32.const 128
+        i32.sub
+        local.tee 0
+        i32.store offset=4
+
+        ;; i32.const 40
+        ;; i32.load offset=28
+        ;; call 2 ;; same here
+        
+        local.get 2
+        i32.const 5
+        local.get 3
+        call 5
+
+        i32.const 40
+        i32.load offset=28
+        call 2
+        
+        local.get 0
+        i32.const 108
+        i32.eq
+        local.set 0
+        local.get 1
+        i32.const 34
+        i32.eq
+        local.set 0
+        
+        local.get 2
+        i32.const 7
+        local.get 3
+        call 5
+        
+        local.get 4
+        i32.const 1
+        i32.add
+        local.tee 4
+        i32.const 36525
+        i32.ne
+        br_if 0 (;@5;)
+      end
+      i32.const 0
+    end
+    local.set 1
+  )
+  (table (;0;) 6 6 funcref)
+  (memory (;0;) 8192 8192)
+  (global (;0;) (mut i32) (i32.const 76368))
+  (global (;1;) (mut i32) (i32.const 331012996))
+  (global (;2;) (mut i64) (i64.const 575546917))
+  (global (;3;) (mut f32) (f32.const 0x1.b8f31cp-34 (;=1.0026e-10;)))
+  (global (;4;) (mut f64) (f64.const -0x1.8ae9ad59b18a9p-429 (;=-1.11274e-129;)))
+  (export "memory" (memory 0))
+  (export "_start" (func 6))
+)

tests/regression/ba-issues/issues/issue-2787/sock_shutdown.wast

@@ -0,0 +1,17 @@
+(module
+  (type (func))
+  (type (func (param i32 i32) (result i32)))
+  (type (func (param i32)))
+  (import "wasi_snapshot_preview1" "sock_shutdown" (func $sock_shutdown (type 1)))
+  (import "wasi_snapshot_preview1" "proc_exit" (func $proc_exit (type 2)))
+  (func $_start
+    (call $sock_shutdown
+      (i32.const 3)
+      (i32.const 3)
+    )
+    call $proc_exit
+  )
+  (memory 2)
+  (export "memory" (memory 0))
+  (export "_start" (func $_start))
+)

tests/regression/ba-issues/issues/issue-2897/test.wast

@@ -0,0 +1,11 @@
+(module
+  (type (func))
+  (func $_start
+    (memory.grow (i32.const 1))
+    drop
+    (memory.fill (i32.const 65536) (i32.const 0) (i32.const 10))
+  )
+  (memory 1)
+  (export "memory" (memory 0))
+  (export "_start" (func $_start))
+)