Update cmake files and wamr-test-suites to support collect code coverage (#1992)

Support collecting code coverage with wamr-test-suites script by using
lcov and genhtml tools, eg.:
  cd tests/wamr-test-suites
  ./test_wamr.sh -s spec -b -P -C

The default code coverage and html files are generated at:
  tests/wamr-test-suites/workspace/wamr.lcov
  tests/wamr-test-suites/workspace/wamr-lcov.zip

And update wamr-test-suites scripts to support testing GC spec cases to
avoid frequent synchronization conflicts between branch main and dev/gc.

CMakeLists.txt

@@ -105,11 +105,6 @@ if (NOT DEFINED WAMR_BUILD_REF_TYPES)
   set (WAMR_BUILD_REF_TYPES 0)
 endif ()
 
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

build-scripts/config_common.cmake

@@ -341,3 +341,12 @@ if (WAMR_BUILD_WASM_CACHE EQUAL 1)
   add_definitions (-DWASM_ENABLE_WASM_CACHE=1)
   message ("     Wasm files cache enabled")
 endif ()
+if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1)
+  add_definitions (-DWASM_ENABLE_GC_VERIFY=1)
+  message ("     GC heap verification enabled")
+endif ()
+if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
+  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+  message ("     Collect code coverage enabled")
+endif ()

core/iwasm/libraries/wasi-nn/test/CMakeLists.txt

@@ -110,11 +110,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
   set (WAMR_BUILD_SIMD 0)
 endif ()
 
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..)
 
 include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

product-mini/platforms/linux-sgx/CMakeLists.txt

@@ -89,10 +89,6 @@ if (NOT DEFINED WAMR_BUILD_SGX_IPFS)
   set (WAMR_BUILD_SGX_IPFS 0)
 endif ()
 
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \
                                      -Wall -Wno-unused-parameter -Wno-pedantic \

product-mini/platforms/linux-sgx/CMakeLists_minimal.txt

@@ -69,10 +69,6 @@ if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD)
   set (WAMR_BUILD_LIB_PTHREAD 0)
 endif ()
 
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \
                                      -Wall -Wno-unused-parameter -Wno-pedantic \

product-mini/platforms/linux/CMakeLists.txt

@@ -117,11 +117,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
   set (WAMR_BUILD_SIMD 0)
 endif ()
 
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 
 include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

product-mini/platforms/windows/CMakeLists.txt

@@ -96,10 +96,6 @@ if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
   set (WAMR_BUILD_SIMD 0)
 endif ()
 
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 
 include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

samples/sgx-ra/CMakeLists.txt

@@ -39,10 +39,6 @@ set (WAMR_BUILD_FAST_INTERP 1)
 set (WAMR_BUILD_LIB_RATS 1)
 
 # compiling and linking flags
-if (COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-endif ()
-
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11 -ffunction-sections -fdata-sections \
                                      -Wall -Wno-unused-parameter -Wno-pedantic \

samples/wasi-threads/wasm-apps/CMakeLists.txt

@@ -16,6 +16,11 @@ set (CMAKE_C_COMPILER               "${WASI_SDK_DIR}/bin/clang")
 set (CMAKE_ASM_COMPILER               "${WASI_SDK_DIR}/bin/clang")
 set (CMAKE_EXE_LINKER_FLAGS         "-target wasm32-wasi-threads")
 
+if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
+  set (CMAKE_C_FLAGS "")
+  set (CMAKE_CXX_FLAGS "")
+endif ()
+
 function (compile_sample SOURCE_FILE)
   get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE)
   set (WASM_MODULE ${FILE_NAME}.wasm)

tests/wamr-test-suites/spec-test-script/all.py

@@ -14,6 +14,18 @@ import time
 
 """
 The script itself has to be put under the same directory with the "spec".
+To run a single non-GC case with interpreter mode:
+  cd workspace
+  python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
+    spec/test/core/xxx.wast
+To run a single non-GC case with aot mode:
+  cd workspace
+  python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \
+    --aot-compiler wamrc spec/test/core/xxx.wast
+To run a single GC case:
+  cd workspace
+  python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \
+    --aot-compiler wamrc --gc spec/test/core/xxx.wast
 """
 
 PLATFORM_NAME = os.uname().sysname.lower()
@@ -22,9 +34,9 @@ IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
 IWASM_QEMU_CMD = "iwasm"
 SPEC_TEST_DIR = "spec/test/core"
 WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
+SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
 WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
 
-
 class TargetAction(argparse.Action):
     TARGET_MAP = {
         "ARMV7_VFP": "armv7",
@@ -51,6 +63,7 @@ def ignore_the_case(
     multi_module_flag=False,
     multi_thread_flag=False,
     simd_flag=False,
+    gc_flag=False,
     xip_flag=False,
     qemu_flag=False
 ):
@@ -63,6 +76,10 @@ def ignore_the_case(
     if "i386" == target and case_name in ["float_exprs"]:
         return True
 
+    if gc_flag:
+        if case_name in ["type-canon", "type-equivalence", "type-rec"]:
+            return True;
+
     if sgx_flag:
         if case_name in ["conversions", "f32_bitwise", "f64_bitwise"]:
             return True
@@ -76,7 +93,9 @@ def ignore_the_case(
             return True
 
     if qemu_flag:
-        if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", "conversions", "f32", "f32_cmp", "float_exprs", "float_misc", "select", "memory_grow"]:
+        if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp",
+                         "conversions", "f32", "f32_cmp", "float_exprs",
+                         "float_misc", "select", "memory_grow"]:
             return True
 
     return False
@@ -109,6 +128,7 @@ def test_case(
     xip_flag=False,
     clean_up_flag=True,
     verbose_flag=True,
+    gc_flag=False,
     qemu_flag=False,
     qemu_firmware='',
     log='',
@@ -124,6 +144,7 @@ def test_case(
         multi_module_flag,
         multi_thread_flag,
         simd_flag,
+        gc_flag,
         xip_flag,
         qemu_flag
     ):
@@ -131,7 +152,7 @@ def test_case(
 
     CMD = ["python3", "runtest.py"]
     CMD.append("--wast2wasm")
-    CMD.append(WAST2WASM_CMD)
+    CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD)
     CMD.append("--interpreter")
     if sgx_flag:
         CMD.append(IWASM_SGX_CMD)
@@ -171,6 +192,9 @@ def test_case(
     if not clean_up_flag:
         CMD.append("--no_cleanup")
 
+    if gc_flag:
+        CMD.append("--gc")
+
     if log != '':
         CMD.append("--log-dir")
         CMD.append(log)
@@ -231,6 +255,7 @@ def test_suite(
     xip_flag=False,
     clean_up_flag=True,
     verbose_flag=True,
+    gc_flag=False,
     parl_flag=False,
     qemu_flag=False,
     qemu_firmware='',
@@ -246,6 +271,10 @@ def test_suite(
         simd_case_list = sorted(suite_path.glob("simd/*.wast"))
         case_list.extend(simd_case_list)
 
+    if gc_flag:
+        gc_case_list = sorted(suite_path.glob("gc/*.wast"))
+        case_list.extend(gc_case_list)
+
     case_count = len(case_list)
     failed_case = 0
     successful_case = 0
@@ -268,6 +297,7 @@ def test_suite(
                         xip_flag,
                         clean_up_flag,
                         verbose_flag,
+                        gc_flag,
                         qemu_flag,
                         qemu_firmware,
                         log,
@@ -304,6 +334,7 @@ def test_suite(
                     xip_flag,
                     clean_up_flag,
                     verbose_flag,
+                    gc_flag,
                     qemu_flag,
                     qemu_firmware,
                     log,
@@ -414,6 +445,13 @@ def main():
         dest="verbose_flag",
         help="Close real time output while running cases, only show last words of failed ones",
     )
+    parser.add_argument(
+        "--gc",
+        action="store_true",
+        default=False,
+        dest="gc_flag",
+        help="Running with GC feature",
+    )
     parser.add_argument(
         "cases",
         metavar="path_to__case",
@@ -446,6 +484,7 @@ def main():
             options.xip_flag,
             options.clean_up_flag,
             options.verbose_flag,
+            options.gc_flag,
             options.parl_flag,
             options.qemu_flag,
             options.qemu_firmware,
@@ -469,6 +508,7 @@ def main():
                     options.xip_flag,
                     options.clean_up_flag,
                     options.verbose_flag,
+                    options.gc_flag,
                     options.qemu_flag,
                     options.qemu_firmware,
                     options.log

tests/wamr-test-suites/spec-test-script/collect_coverage.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+#
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+readonly WORK_DIR=$PWD
+readonly WAMR_DIR=${WORK_DIR}/../../..
+readonly DST_COV_FILE=$1
+readonly SRC_COV_DIR=$2
+readonly SRC_TEMP_COV_FILE=wamr_temp.lcov
+readonly SRC_COV_FILE=wamr.lcov
+
+# get dest folder
+dir=$(dirname ${DST_COV_FILE})
+pushd ${dir} > /dev/null 2>&1
+readonly DST_COV_DIR=${PWD}
+popd > /dev/null 2>&1
+
+if [[ ! -d ${SRC_COV_DIR} ]]; then
+    echo "${SRC_COV_DIR} doesn't exist, ignore code coverage collection"
+    exit
+fi
+
+echo "Start to collect code coverage of ${SRC_COV_DIR} .."
+
+pushd ${SRC_COV_DIR} > /dev/null 2>&1
+
+# collect all code coverage data
+lcov -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1
+# extract code coverage data of WAMR source files
+lcov -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \
+     -rc lcov_branch_coverage=1 \
+     "*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \
+     "*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \
+     "*/app-framework/*" "*/test-tools/*"
+
+if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
+    if [[ -s ${DST_COV_FILE} ]]; then
+        # merge code coverage data
+        lcov --rc lcov_branch_coverage=1 \
+            --add-tracefile ${SRC_TEMP_COV_FILE} \
+            -a ${DST_COV_FILE} -o ${SRC_COV_FILE}
+        # backup the original lcov file
+        cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig"
+        # replace the lcov file
+        cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
+    else
+        cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE}
+        cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
+    fi
+
+    # get ignored prefix path
+    dir=$(dirname ${WAMR_DIR}/../..)
+    pushd ${dir} > /dev/null 2>&1
+    prefix_full_path=${PWD}
+    popd > /dev/null 2>&1
+
+    # generate html output for merged code coverage data
+    rm -fr ${DST_COV_DIR}/wamr-lcov
+    genhtml -t "WAMR Code Coverage" \
+        --rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \
+        -o ${DST_COV_DIR}/wamr-lcov \
+        ${DST_COV_FILE}
+
+    cd ${DST_COV_DIR}
+    rm -f wamr-lcov.zip
+    zip -r -q -o wamr-lcov.zip wamr-lcov
+    rm -fr wamr-lcov
+
+    echo "Code coverage file ${DST_COV_FILE} was generated or appended"
+    echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated"
+else
+    echo "generate code coverage html failed"
+fi
+
+echo ""
+
+popd > /dev/null 2>&1

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -38,7 +38,7 @@ log_file = None
 temp_file_repo = []
 
 # to save the mapping of module files in /tmp by name
-temp_module_table = {} 
+temp_module_table = {}
 
 def debug(data):
     if debug_file:
@@ -230,6 +230,9 @@ parser.add_argument('--multi-module', default=False, action='store_true',
 parser.add_argument('--multi-thread', default=False, action='store_true',
         help="Enable Multi-thread")
 
+parser.add_argument('--gc', default=False, action='store_true',
+        help='Test with GC')
+
 parser.add_argument('--qemu', default=False, action='store_true',
         help="Enable QEMU")
 
@@ -420,11 +423,20 @@ def parse_simple_const_w_type(number, type):
             number = float.fromhex(number) if '0x' in number else float(number)
             return number, "{:.7g}:{}".format(number, type)
     elif type == "ref.null":
-        # hard coding
-        return "extern", "extern:ref.null"
+        if number == "func":
+            return "func", "func:ref.null"
+        elif number == "extern":
+            return "extern", "extern:ref.null"
+        elif number == "any":
+            return "any", "any:ref.null"
+        else:
+            raise Exception("invalid value {} and type {}".format(number, type))
     elif type == "ref.extern":
         number = int(number, 16) if '0x' in number else int(number)
         return number, "0x{:x}:ref.extern".format(number)
+    elif type == "ref.host":
+        number = int(number, 16) if '0x' in number else int(number)
+        return number, "0x{:x}:ref.host".format(number)
     else:
         raise Exception("invalid value {} and type {}".format(number, type))
 
@@ -440,6 +452,10 @@ def parse_assertion_value(val):
     type.const val
     ref.extern val
     ref.null ref_type
+    ref.array
+    ref.struct
+    ref.func
+    ref.i31
     """
     if not val:
         return None, ""
@@ -453,6 +469,8 @@ def parse_assertion_value(val):
     if type in ["i32", "i64", "f32", "f64"]:
         return parse_simple_const_w_type(numbers[0], type)
     elif type == "ref":
+        if splitted[0] in ["ref.array", "ref.struct", "ref.func", "ref.i31"]:
+            return splitted[0]
         # need to distinguish between "ref.null" and "ref.extern"
         return parse_simple_const_w_type(numbers[0], splitted[0])
     else:
@@ -615,6 +633,9 @@ def simple_value_comparison(out, expected):
     elif "ref.extern" == expected_type:
         out_val_binary = out_val
         expected_val_binary = expected_val
+    elif "ref.host" == expected_type:
+        out_val_binary = out_val
+        expected_val_binary = expected_val
     else:
         assert(0), "unknown 'expected_type' {}".format(expected_type)
 
@@ -637,8 +658,10 @@ def value_comparison(out, expected):
     if not expected:
         return False
 
-    assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out)
-    assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected)
+    if not out in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]:
+        assert(':' in out), "out should be in a form likes numbers:type, but {}".format(out)
+    if not expected in ["ref.array", "ref.struct", "ref.func", "ref.any", "ref.i31"]:
+        assert(':' in expected), "expected should be in a form likes numbers:type, but {}".format(expected)
 
     if 'v128' in out:
         return vector_value_comparison(out, expected)
@@ -761,6 +784,9 @@ def test_assert_return(r, opts, form):
                 elif "ref.extern" == splitted[0]:
                     number, _ = parse_simple_const_w_type(splitted[1], splitted[0])
                     args.append(str(number))
+                elif "ref.host" == splitted[0]:
+                    number, _ = parse_simple_const_w_type(splitted[1], splitted[0])
+                    args.append(str(number))
                 else:
                     assert(0), "an unkonwn parameter type"
 
@@ -769,7 +795,15 @@ def test_assert_return(r, opts, form):
         else:
             returns = re.split("\)\s*\(", m.group(3)[1:-1])
         # processed numbers in strings
-        expected = [parse_assertion_value(v)[1] for v in returns]
+        if len(returns) == 1 and returns[0] in ["ref.array", "ref.struct", "ref.i31",
+                                                "ref.eq", "ref.any", "ref.extern",
+                                                "ref.func", "ref.null"]:
+            expected = [returns[0]]
+        elif len(returns) == 1 and returns[0] in ["func:ref.null", "any:ref.null",
+                                                  "extern:ref.null"]:
+            expected = [returns[0]]
+        else:
+            expected = [parse_assertion_value(v)[1] for v in returns]
         expected = ",".join(expected)
 
         test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected)
@@ -800,10 +834,10 @@ def test_assert_return(r, opts, form):
         if n.group(3) == '':
             args=[]
         else:
-            args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n.group(3)[1:-1])]
-
-        # a workaround for "ref.null extern" and "ref.null func"
-        args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args]
+            # convert (ref.null extern/func) into (ref.null null)
+            n1 = n.group(3).replace("(ref.null extern)", "(ref.null null)")
+            n1 = n1.replace("ref.null func)", "(ref.null null)")
+            args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", n1[1:-1])]
 
         _, expected = parse_assertion_value(n.group(4)[1:-1])
         test_assert(r, opts, "return", "%s %s" % (func, " ".join(args)), expected)
@@ -828,10 +862,10 @@ def test_assert_trap(r, opts, form):
         if m.group(2) == '':
             args = []
         else:
-            args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m.group(2)[1:-1])]
-
-        # workaround for "ref.null extern"
-        args = [ arg.replace('extern', 'null').replace('func', 'null') for arg in args]
+            # convert (ref.null extern/func) into (ref.null null)
+            m1 = m.group(2).replace("(ref.null extern)", "(ref.null null)")
+            m1 = m1.replace("ref.null func)", "(ref.null null)")
+            args = [re.split(' +', v)[1] for v in re.split("\)\s*\(", m1[1:-1])]
 
         expected = "Exception: %s" % m.group(3)
         test_assert(r, opts, "trap", "%s %s" % (func, " ".join(args)), expected)
@@ -918,10 +952,11 @@ def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts):
     log("Compiling WASM to '%s'" % wasm_tempfile)
 
     # default arguments
-    cmd = [opts.wast2wasm,
-            "--enable-thread",
-            "--no-check",
-            wast_tempfile, "-o", wasm_tempfile ]
+    if opts.gc:
+        cmd = [opts.wast2wasm, "-u", "-d", wast_tempfile, "-o", wasm_tempfile]
+    else:
+        cmd = [opts.wast2wasm, "--enable-thread", "--no-check",
+               wast_tempfile, "-o", wasm_tempfile ]
 
     # remove reference-type and bulk-memory enabling options since a WABT
     # commit 30c1e983d30b33a8004b39fd60cbd64477a7956c
@@ -1023,18 +1058,18 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
     if (r != None):
         r.cleanup()
     r = Runner(cmd, no_pty=opts.no_pty)
-    
+
     if opts.qemu:
         r.read_to_prompt(['nsh> '], 10)
         r.writeline("mount -t hostfs -o fs={} /tmp".format(tempfile.gettempdir()))
         r.read_to_prompt(['nsh> '], 10)
         r.writeline(" ".join(cmd_iwasm))
-    
+
     return r
 
 def create_tmpfiles(wast_name):
     tempfiles = []
-    
+
     (t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
     (t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
     tempfiles.append(wast_tempfile)

tests/wamr-test-suites/test_wamr.sh

@@ -14,19 +14,23 @@ function help()
 {
     echo "test_wamr.sh [options]"
     echo "-c clean previous test results, not start test"
-    echo "-s {suite_name} test only one suite (spec)"
-    echo "-m set compile target of iwasm(x86_64\x86_32\armv7_vfp\thumbv7_vfp\riscv64_lp64d\riscv64_lp64)"
-    echo "-t set compile type of iwasm(classic-interp\fast-interp\jit\aot\fast-jit\multi-tier-jit)"
+    echo "-s {suite_name} test only one suite (spec|wasi_certification)"
+    echo "-m set compile target of iwasm(x86_64|x86_32|armv7_vfp|thumbv7_vfp|riscv64_lp64d|riscv64_lp64)"
+    echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)"
     echo "-M enable multi module feature"
     echo "-p enable multi thread feature"
     echo "-S enable SIMD feature"
+    echo "-G enable GC feature"
     echo "-X enable XIP feature"
     echo "-x test SGX"
+    echo "-w enable WASI threads"
     echo "-b use the wabt binary release package instead of compiling from the source code"
+    echo "-g build iwasm with debug version"
+    echo "-v enable GC heap verification"
     echo "-P run the spec test parallelly"
     echo "-Q enable qemu"
     echo "-F set the firmware path used by qemu"
-    echo "-w enable WASI threads"
+    echo "-C enable code coverage collect"
 }
 
 OPT_PARSED=""
@@ -40,7 +44,10 @@ ENABLE_MULTI_MODULE=0
 ENABLE_MULTI_THREAD=0
 COLLECT_CODE_COVERAGE=0
 ENABLE_SIMD=0
+ENABLE_GC=0
 ENABLE_XIP=0
+ENABLE_DEBUG_VERSION=0
+ENABLE_GC_HEAP_VERIFY=0
 #unit test case arrary
 TEST_CASE_ARR=()
 SGX_OPT=""
@@ -50,7 +57,7 @@ ENABLE_QEMU=0
 QEMU_FIRMWARE=""
 WASI_TESTSUITE_COMMIT="b18247e2161bea263fe924b8189c67b1d2d10a10"
 
-while getopts ":s:cabt:m:wMCpSXxPQF:" opt
+while getopts ":s:cabgvt:m:MCpSXxwPGQF:" opt
 do
     OPT_PARSED="TRUE"
     case $opt in
@@ -72,8 +79,9 @@ do
         c)
         read -t 5 -p "Are you sure to delete all reports. y/n    " cmd
         if [[ $cmd == "y" && $(ls -A workspace/report) ]];then
-            rm -r workspace/report/*
-            echo "cleaned all reports"
+            rm -fr workspace/report/*
+            rm -fr /tmp/*.wasm /tmp/*.wast /tmp/*.aot
+            echo "cleaned all reports and temp files"
         fi
         exit 0;;
         a)
@@ -128,6 +136,18 @@ do
         echo "test SGX"
         SGX_OPT="--sgx"
         ;;
+        g)
+        echo "enable build iwasm with debug version"
+        ENABLE_DEBUG_VERSION=1
+        ;;
+        v)
+        echo "enable GC heap verification"
+        ENABLE_GC_HEAP_VERIFY=1
+        ;;
+        G)
+        echo "enable GC feature"
+        ENABLE_GC=1
+        ;;
         P)
         PARALLELISM=1
         ;;
@@ -164,7 +184,6 @@ readonly DATE=$(date +%Y-%m-%d_%H:%M:%S)
 readonly REPORT_DIR=${WORK_DIR}/report/${DATE}
 mkdir -p ${REPORT_DIR}
 
-# TODO: a strong assumation about a link to the WAMR project
 readonly WAMR_DIR=${WORK_DIR}/../../..
 
 if [[ ${SGX_OPT} == "--sgx" ]];then
@@ -198,14 +217,16 @@ readonly ORC_EAGER_JIT_COMPILE_FLAGS="\
     -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \
     -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \
     -DWAMR_BUILD_LAZY_JIT=0 \
-    -DWAMR_BUILD_SPEC_TEST=1"
+    -DWAMR_BUILD_SPEC_TEST=1 \
+    -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
 
 readonly ORC_LAZY_JIT_COMPILE_FLAGS="\
     -DWAMR_BUILD_TARGET=${TARGET} \
     -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_FAST_INTERP=0 \
     -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_AOT=1 \
     -DWAMR_BUILD_LAZY_JIT=1 \
-    -DWAMR_BUILD_SPEC_TEST=1"
+    -DWAMR_BUILD_SPEC_TEST=1 \
+    -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
 
 readonly AOT_COMPILE_FLAGS="\
     -DWAMR_BUILD_TARGET=${TARGET} \
@@ -219,13 +240,15 @@ readonly FAST_JIT_COMPILE_FLAGS="\
     -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \
     -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_AOT=0 \
     -DWAMR_BUILD_FAST_JIT=1 \
-    -DWAMR_BUILD_SPEC_TEST=1"
+    -DWAMR_BUILD_SPEC_TEST=1 \
+    -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
 
 readonly MULTI_TIER_JIT_COMPILE_FLAGS="\
     -DWAMR_BUILD_TARGET=${TARGET} \
     -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 \
     -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \
-    -DWAMR_BUILD_SPEC_TEST=1"
+    -DWAMR_BUILD_SPEC_TEST=1 \
+    -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}"
 
 readonly COMPILE_FLAGS=(
         "${CLASSIC_INTERP_COMPILE_FLAGS}"
@@ -237,39 +260,19 @@ readonly COMPILE_FLAGS=(
         "${MULTI_TIER_JIT_COMPILE_FLAGS}"
     )
 
-# TODO: with libiwasm.so only
 function unit_test()
 {
     echo "Now start unit tests"
 
     cd ${WORK_DIR}
-    readonly UNIT_CASES="wasm-vm host-tool utils"
+    rm -fr unittest-build && mkdir unittest-build
+    cd unittest-build
 
     echo "Build unit test"
     touch ${REPORT_DIR}/unit_test_report.txt
-
-    for compile_flag in "${COMPILE_FLAGS[@]}"; do
-        echo "Build unit test with compile flags with " ${compile_flag}
-
-        # keep going and do not care if it is success or not
-        make -ki clean | true
-        cmake ${compile_flag} ${WORK_DIR}/../../unit && make -j 4
-        if [ "$?" != 0 ];then
-            echo -e "build unit test failed, you may need to change wamr into dev/aot branch and ensure llvm is built"
-            exit 1
-        fi
-
-        echo ${compile_flag} >> ${REPORT_DIR}/unit_test_report.txt
-
-        for case in ${UNIT_CASES}
-        do
-            echo "run ${case} ..."
-            cd ./${case}/
-            ./${case/-/_}"_test" | tee -a ${REPORT_DIR}/unit_test_report.txt
-            cd -
-            echo "finish ${case}"
-        done
-    done
+    cmake ${WORK_DIR}/../../unit -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE}
+    make -j
+    make test | tee -a ${REPORT_DIR}/unit_test_report.txt
 
     echo "Finish unit tests"
 }
@@ -347,6 +350,27 @@ function spec_test()
         git apply ../../spec-test-script/thread_proposal_fix_atomic_case.patch
     fi
 
+    # update GC cases
+    if [[ ${ENABLE_GC} == 1 ]]; then
+        echo "checkout spec for GC proposal"
+
+        popd
+        rm -fr spec
+        # check spec test cases for GC
+        git clone -b main --single-branch https://github.com/WebAssembly/gc.git spec
+        pushd spec
+
+        git restore . && git clean -ffd .
+        # Sync constant expression descriptions
+        git reset --hard 62beb94ddd41987517781732f17f213d8b866dcc
+        git apply ../../spec-test-script/gc_ignore_cases.patch
+
+        echo "compile the reference intepreter"
+        pushd interpreter
+        make opt
+        popd
+    fi
+
     popd
     echo $(pwd)
 
@@ -446,6 +470,10 @@ function spec_test()
         ARGS_FOR_SPEC_TEST+="--parl "
     fi
 
+    if [[ ${ENABLE_GC} == 1 ]]; then
+        ARGS_FOR_SPEC_TEST+="--gc "
+    fi
+
     if [[ ${ENABLE_QEMU} == 1 ]]; then
         ARGS_FOR_SPEC_TEST+="--qemu "
         ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} "
@@ -484,7 +512,7 @@ function wasi_test()
 
 function wasi_certification_test()
 {
-    echo  "Now start wasi tests"
+    echo  "Now start wasi certification tests"
 
     cd ${WORK_DIR}
     if [ ! -d "wasi-testsuite" ]; then
@@ -514,7 +542,6 @@ function polybench_test()
     if [[ $1 == "aot" || $1 == "jit" ]];then
         ./build.sh AOT ${SGX_OPT}
         ./test_aot.sh $1 ${SGX_OPT}
-
     else
         ./build.sh
         ./test_interp.sh ${SGX_OPT}
@@ -524,6 +551,22 @@ function polybench_test()
     echo "Finish polybench tests"
 }
 
+function libsodium_test()
+{
+    echo "Now start libsodium tests"
+
+    cd ${WORK_DIR}/../libsodium
+    if [[ $1 == "aot" || $1 == "jit" ]];then
+        ./build.sh ${SGX_OPT}
+        ./test_aot.sh $1 ${SGX_OPT}
+    else
+        ./test_interp.sh ${SGX_OPT}
+    fi
+    cp report.txt ${REPORT_DIR}/libsodium_$1_test_report.txt
+
+    echo "Finish libsodium tests"
+}
+
 function malformed_test()
 {
     # build iwasm firstly
@@ -535,14 +578,13 @@ function standalone_test()
 {
     cd ${WORK_DIR}/../../standalone
 
-    args=""
+    args="--$1"
 
-    [[ $1 == "aot" ]] && args="$args --aot" || args="$args --no-aot"
     [[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx"
 
-    if [[ ${ENABLE_MULTI_THREAD} == 1 ]];then
-        args="$args --thread"
-    fi
+    [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" && args="$args --no-thread"
+
+    [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" && args="$args --no-simd"
 
     ./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt
 }
@@ -589,7 +631,7 @@ function build_wamrc()
         && ./build_llvm.sh \
         && if [ -d build ]; then rm -r build/*; else mkdir build; fi \
         && cd build \
-        && cmake .. \
+        && cmake .. -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \
         && make -j 4
 }
 
@@ -602,15 +644,33 @@ function build_wamrc()
 
 function collect_coverage()
 {
-    if [[ ${COLLECT_CODE_COVERAGE} == 1 ]];then
-        cd ${IWASM_LINUX_ROOT_DIR}/build
-        lcov -t "iwasm code coverage" -o iwasm.info -c -d .
-        genhtml -o iwasm-gcov iwasm.info
-        [[ -d iwasm-gcov ]] && \
-                cp -r iwasm-gcov ${REPORT_DIR}/$1_iwasm_gcov || \
-                echo "generate code coverage html failed"
+    if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then
+        ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR}
+
+        CODE_COV_FILE=""
+        if [[ -z "${COV_FILE}" ]]; then
+            CODE_COV_FILE="${WORK_DIR}/wamr.lcov"
+        else
+            CODE_COV_FILE="${COV_FILE}"
+        fi
+
+        pushd ${WORK_DIR} > /dev/null 2>&1
+        echo "Collect code coverage of iwasm"
+        ./collect_coverage.sh ${CODE_COV_FILE} ${IWASM_LINUX_ROOT_DIR}/build
+        if [[ $1 == "llvm-aot" ]]; then
+            echo "Collect code coverage of wamrc"
+            ./collect_coverage.sh ${CODE_COV_FILE} ${WAMR_DIR}/wamr-compiler/build
+        fi
+        for suite in "${TEST_CASE_ARR[@]}"; do
+            if [[ ${suite} = "unit" ]]; then
+                echo "Collect code coverage of unit test"
+                ./collect_coverage.sh ${CODE_COV_FILE} ${WORK_DIR}/unittest-build
+                break
+            fi
+        done
+        popd > /dev/null 2>&1
     else
-        echo "will not collect code coverage"
+        echo "code coverage isn't collected"
     fi
 }
 
@@ -639,10 +699,22 @@ function trigger()
         EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_SIMD=0"
     fi
 
+    if [[ ${ENABLE_GC} == 1 ]]; then
+        EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC=1"
+        EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_REF_TYPES=1"
+        EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_BULK_MEMORY=1"
+    fi
+
+    if [[ ${ENABLE_DEBUG_VERSION} == 1 ]]; then
+        EXTRA_COMPILE_FLAGS+=" -DCMAKE_BUILD_TYPE=Debug"
+    fi
+
+    if [[ ${ENABLE_GC_HEAP_VERIFY} == 1 ]]; then
+        EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_GC_HEAP_VERIFY=1"
+    fi
+
     if [[ ${ENABLE_WASI_THREADS} == 1 ]]; then
         EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=1"
-    else
-        EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_WASI_THREADS=0"
     fi
 
     for t in "${TYPE[@]}"; do
@@ -692,18 +764,18 @@ function trigger()
                 echo "work in orc jit eager compilation mode"
                 BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
                 build_iwasm_with_cfg $BUILD_FLAGS
-                build_wamrc
                 for suite in "${TEST_CASE_ARR[@]}"; do
                     $suite"_test" jit
                 done
+                collect_coverage llvm-jit
 
                 echo "work in orc jit lazy compilation mode"
                 BUILD_FLAGS="$ORC_EAGER_JIT_COMPILE_FLAGS $EXTRA_COMPILE_FLAGS"
                 build_iwasm_with_cfg $BUILD_FLAGS
-                build_wamrc
                 for suite in "${TEST_CASE_ARR[@]}"; do
                     $suite"_test" jit
                 done
+                collect_coverage llvm-jit
             ;;
 
             "aot")
@@ -717,7 +789,7 @@ function trigger()
                 for suite in "${TEST_CASE_ARR[@]}"; do
                     $suite"_test" aot
                 done
-                collect_coverage aot
+                collect_coverage llvm-aot
             ;;
 
             "fast-jit")
@@ -728,6 +800,7 @@ function trigger()
                 for suite in "${TEST_CASE_ARR[@]}"; do
                     $suite"_test" fast-jit
                 done
+                collect_coverage fast-jit
             ;;
 
             "multi-tier-jit")
@@ -738,6 +811,7 @@ function trigger()
                 for suite in "${TEST_CASE_ARR[@]}"; do
                     $suite"_test" multi-tier-jit
                 done
+                collect_coverage multi-tier-jit
             ;;
 
             *)
@@ -748,11 +822,19 @@ function trigger()
 }
 
 # if collect code coverage, ignore -s, test all test cases.
-if [[ $TEST_CASE_ARR && $COLLECT_CODE_COVERAGE != 1 ]];then
+if [[ $TEST_CASE_ARR ]];then
     trigger || (echo "TEST FAILED"; exit 1)
 else
-    # test all suite, ignore polybench because of long time cost
-    TEST_CASE_ARR=("sightglass" "spec" "wasi" "malformed" "standalone")
+    # test all suite, ignore polybench and libsodium because of long time cost
+    TEST_CASE_ARR=("spec")
+    : '
+    if [[ $COLLECT_CODE_COVERAGE == 1 ]];then
+        # add polybench if collecting code coverage data
+        TEST_CASE_ARR+=("polybench")
+        # add libsodium if needed, which takes long time to run
+        TEST_CASE_ARR+=("libsodium")
+    fi
+    '
     trigger || (echo "TEST FAILED"; exit 1)
     # Add more suites here
 fi

wamr-compiler/CMakeLists.txt

@@ -166,6 +166,12 @@ if (WAMR_BUILD_DEBUG_AOT EQUAL 1)
   message(STATUS "find lldb ${LLDB_ALL_PLUGINS} in: ${LLVM_LIBRARY_DIRS}")
 endif()
 
+if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
+  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+  message ("-- Collect code coverage enabled")
+endif ()
+
 if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang"))
   if(NOT MSVC)
     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")