Add malformed test cases (#3459)

tests/malformed/README.md

@@ -0,0 +1,18 @@
+# Malformed Test Cases
+This folder contains malformed test cases collected from github and peach fuzzer. More cases should be added to here when found.
+
+# Note
+Malformed tests are currently for testing running mode of interpreter including classic interpreter and fast interpreter.
+
+# Test
+Use the provided python script to test a wasm runtime:
+``` shell
+python malformed_test.py -r "path/to/runtime"
+```
+
+For example you can test wasmtime with:
+``` shell
+python malformed_test.py -r "/usr/bin/wasmtime"
+```
+If no args is passed to the script, the default runtime `/usr/bin/iwasm` will be used.
+

tests/malformed/malformed_test.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+import os
+import sys
+import time
+import subprocess
+from optparse import OptionParser
+
+
+# You can run different runtime by changing the arg '-r':
+# python malformed_test.py -r /path/to/iwasm
+# python malformed_test.py -r /path/to/wasmtime
+# python malformed_test.py -r "/path/to/wasmer run"
+
+
+optParser = OptionParser()
+optParser.add_option("-r", "--run", dest="run",
+                     default="iwasm",
+                     help="specify a runtime [path/to]([iwasm] / wasmtime / wasmer run)")
+
+(options, args) = optParser.parse_args()
+#optParser.usage = "%prog [options]"
+
+succ_cnt = 0
+fail_cnt = 0
+
+test_start = time.time()
+for root, dirs, files in os.walk("."):
+    for file in files:
+        if len(file.split('.')) < 2 or file.split('.')[1] != 'wasm':
+            continue
+        filepath=os.path.join(root, file)
+        cmd = options.run + " " + filepath
+        test_out = subprocess.getoutput(cmd)
+
+        if not test_out.startswith("Segmentation fault"):
+            print("test {:40} ........ [PASSED]".format(filepath))
+            succ_cnt += 1
+        else:
+            print("test {:40} ........ [FAILED]".format(filepath))
+            print(test_out)
+            print('\n')
+            fail_cnt += 1
+test_end = time.time()
+
+print("\n#####################  MALFORMED TEST  ########################")
+print("run {} test cases in {}ms, {} passed, {} failed"
+      .format(succ_cnt + fail_cnt, test_end - test_start, succ_cnt, fail_cnt))

tests/wamr-test-suites/test_wamr.sh

@@ -14,7 +14,7 @@ function help()
 {
     echo "test_wamr.sh [options]"
     echo "-c clean previous test results, not start test"
-    echo "-s {suite_name} test only one suite (spec|wasi_certification|wamr_compiler)"
+    echo "-s {suite_name} test only one suite (spec|malformed|wasi_certification|wamr_compiler)"
     echo "-m set compile target of iwasm(x86_64|x86_32|armv7|armv7_vfp|thumbv7|thumbv7_vfp|"
     echo "                               riscv32|riscv32_ilp32f|riscv32_ilp32d|riscv64|"
     echo "                               riscv64_lp64f|riscv64_lp64d|aarch64|aarch64_vfp)"
@@ -1104,7 +1104,7 @@ if [[ $TEST_CASE_ARR ]];then
     trigger || (echo "TEST FAILED"; exit 1)
 else
     # test all suite, ignore polybench and libsodium because of long time cost
-    TEST_CASE_ARR=("spec")
+    TEST_CASE_ARR=("spec" "malformed")
     : '
     if [[ $COLLECT_CODE_COVERAGE == 1 ]];then
         # add polybench if collecting code coverage data