Enable to collect code coverage of samples/simple (#2003)

build-scripts/config_common.cmake

@@ -348,5 +348,6 @@ endif ()
 if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
   set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
   set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+  add_definitions (-DCOLLECT_CODE_COVERAGE)
   message ("     Collect code coverage enabled")
 endif ()

core/app-mgr/app-manager/module_wasm_app.c

@@ -993,6 +993,14 @@ wasm_app_module_uninstall(request_t *msg)
 
     app_manager_printf("Uninstall WASM app successful!\n");
 
+#ifdef COLLECT_CODE_COVERAGE
+    /* Exit app manager so as to collect code coverage data */
+    if (!strcmp(m_name, "__exit_app_manager__")) {
+        app_manager_printf("Exit app manager\n");
+        bh_queue_exit_loop_run(get_app_manager_queue());
+    }
+#endif
+
 #if VALGRIND_CHECK != 0
     bh_queue_exit_loop_run(get_app_manager_queue());
 #endif

samples/simple/CMakeLists.txt

@@ -12,6 +12,11 @@ message(STATUS "WAMR_BUILD_SDK_PROFILE=${WAMR_BUILD_SDK_PROFILE}")
 set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
 set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
 
+if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
+endif ()
+
 set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..)
 
 ## use library and headers in the SDK

samples/simple/build.sh

@@ -56,7 +56,7 @@ done
 
 
 if [ "$CLEAN" = "TRUE" ]; then
-    rm -rf $CURR_DIR/cmake_build
+    rm -rf $CURR_DIR/cmake-build
 fi
 
 
@@ -107,8 +107,8 @@ cd ${WAMR_DIR}/wamr-sdk
 
 echo "#####################build simple project"
 cd ${CURR_DIR}
-mkdir -p cmake_build/$PROFILE
-cd cmake_build/$PROFILE
+mkdir -p cmake-build/$PROFILE
+cd cmake-build/$PROFILE
 cmake ../.. -DWAMR_BUILD_SDK_PROFILE=$PROFILE $CM_TOOLCHAIN $CM_BUILD_TYPE
 make
 if [ $? != 0 ];then

samples/simple/sample_test_run.py

@@ -11,6 +11,7 @@ import sys
 import time
 import traceback
 
+WAMRC_CMD = "../../wamr-compiler/build/wamrc"
 
 def start_server(cwd):
     """
@@ -80,8 +81,31 @@ def main():
     """
     parser = argparse.ArgumentParser(description="run the sample and examine outputs")
     parser.add_argument("working_directory", type=str)
+    parser.add_argument("--aot", action='store_true', help="Test with AOT")
     args = parser.parse_args()
 
+    test_aot = False
+    suffix = ".wasm"
+    if not args.aot:
+        print("Test with interpreter mode")
+    else:
+        print("Test with AOT mode")
+        test_aot = True
+        suffix = ".aot"
+        wasm_files = [ "timer", "sensor", "connection",
+                       "event_publisher", "event_subscriber",
+                       "request_handler", "request_sender" ]
+        work_dir = args.working_directory
+        wasm_apps_dir = work_dir + "/wasm-apps"
+        print("Compile wasm app into aot files")
+        for wasm_file in wasm_files:
+            CMD = []
+            CMD.append(WAMRC_CMD)
+            CMD.append("-o")
+            CMD.append(wasm_apps_dir + "/" + wasm_file + ".aot")
+            CMD.append(wasm_apps_dir + "/" + wasm_file + ".wasm")
+            subprocess.check_call(CMD)
+
     ret = 1
     app_server = None
     try:
@@ -90,35 +114,47 @@ def main():
         # wait for a second
         time.sleep(1)
 
-        print("--> Install timer.wasm...")
+        print("--> Install timer" + suffix + "...")
         install_wasm_application(
-            "timer", "./wasm-apps/timer.wasm", args.working_directory
+            "timer", "./wasm-apps/timer" + suffix, args.working_directory
         )
 
-        print("--> Install event_publisher.wasm...")
+        # wait for a second
+        time.sleep(3)
+
+        print("--> Query all installed applications...")
+        query_installed_application(args.working_directory)
+
+        print("--> Install event_publisher" + suffix + "...")
         install_wasm_application(
             "event_publisher",
-            "./wasm-apps/event_publisher.wasm",
+            "./wasm-apps/event_publisher" + suffix,
             args.working_directory,
         )
 
-        print("--> Install event_subscriber.wasm...")
+        print("--> Install event_subscriber" + suffix + "...")
         install_wasm_application(
             "event_subscriber",
-            "./wasm-apps/event_subscriber.wasm",
+            "./wasm-apps/event_subscriber" + suffix,
             args.working_directory,
         )
 
-        print("--> Uninstall timer.wasm...")
+        print("--> Query all installed applications...")
+        query_installed_application(args.working_directory)
+
+        print("--> Uninstall timer" + suffix + "...")
         uninstall_wasm_application("timer", args.working_directory)
 
-        print("--> Uninstall event_publisher.wasm...")
+        print("--> Query all installed applications...")
+        query_installed_application(args.working_directory)
+
+        print("--> Uninstall event_publisher" + suffix + "...")
         uninstall_wasm_application(
             "event_publisher",
             args.working_directory,
         )
 
-        print("--> Uninstall event_subscriber.wasm...")
+        print("--> Uninstall event_subscriber" + suffix + "...")
         uninstall_wasm_application(
             "event_subscriber",
             args.working_directory,
@@ -127,26 +163,55 @@ def main():
         print("--> Query all installed applications...")
         query_installed_application(args.working_directory)
 
-        print("--> Install request_handler.wasm...")
+        print("--> Install request_handler" + suffix + "...")
         install_wasm_application(
             "request_handler",
-            "./wasm-apps/request_handler.wasm",
+            "./wasm-apps/request_handler" + suffix,
             args.working_directory,
         )
 
         print("--> Query again...")
         query_installed_application(args.working_directory)
 
-        print("--> Install request_sender.wasm...")
+        print("--> Install request_sender" + suffix + "...")
         install_wasm_application(
             "request_sender",
-            "./wasm-apps/request_sender.wasm",
+            "./wasm-apps/request_sender" + suffix,
             args.working_directory,
         )
 
         print("--> Send GET to the Wasm application named request_handler...")
         send_get_to_wasm_application("request_handler", "/url1", args.working_directory)
 
+        print("--> Uninstall request_handler" + suffix + "...")
+        uninstall_wasm_application(
+            "request_handler",
+            args.working_directory,
+        )
+
+        print("--> Uninstall request_sender" + suffix + "...")
+        uninstall_wasm_application(
+            "request_sender",
+            args.working_directory,
+        )
+
+        # Install a wasm app named "__exit_app_manager__" just to make app manager exit
+        # while the wasm app is uninstalled, so as to collect the code coverage data.
+        # Only available when collecting code coverage is enabled.
+        print("--> Install timer" + suffix + "...")
+        install_wasm_application(
+            "__exit_app_manager__", "./wasm-apps/timer" + suffix, args.working_directory
+        )
+
+        print("--> Uninstall timer" + suffix + "...")
+        uninstall_wasm_application(
+            "__exit_app_manager__",
+            args.working_directory,
+        )
+
+        # wait for a second
+        time.sleep(1)
+
         print("--> All pass")
         ret = 0
     except AssertionError:

tests/wamr-test-suites/spec-test-script/collect_coverage.sh

@@ -28,13 +28,14 @@ echo "Start to collect code coverage of ${SRC_COV_DIR} .."
 pushd ${SRC_COV_DIR} > /dev/null 2>&1
 
 # collect all code coverage data
-lcov -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1
+lcov -q -o ${SRC_TEMP_COV_FILE} -c -d . --rc lcov_branch_coverage=1
 # extract code coverage data of WAMR source files
-lcov -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \
+lcov -q -r ${SRC_TEMP_COV_FILE} -o ${SRC_TEMP_COV_FILE} \
      -rc lcov_branch_coverage=1 \
      "*/usr/*" "*/_deps/*" "*/deps/*" "*/tests/unit/*" \
      "*/llvm/include/*" "*/include/llvm/*" "*/samples/*" \
-     "*/app-framework/*" "*/test-tools/*"
+     "*/app-framework/*" "*/app-mgr/*" "*/test-tools/*" \
+     "*/tests/standalone/*" "*/tests/*"
 
 if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
     if [[ -s ${DST_COV_FILE} ]]; then
@@ -46,9 +47,11 @@ if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
         cp -a ${DST_COV_FILE} "${DST_COV_FILE}.orig"
         # replace the lcov file
         cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
+        echo "Code coverage file ${DST_COV_FILE} was appended"
     else
         cp -a ${SRC_TEMP_COV_FILE} ${SRC_COV_FILE}
         cp -a ${SRC_COV_FILE} ${DST_COV_FILE}
+        echo "Code coverage file ${DST_COV_FILE} was generated"
     fi
 
     # get ignored prefix path
@@ -59,7 +62,7 @@ if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
 
     # generate html output for merged code coverage data
     rm -fr ${DST_COV_DIR}/wamr-lcov
-    genhtml -t "WAMR Code Coverage" \
+    genhtml -q -t "WAMR Code Coverage" \
         --rc lcov_branch_coverage=1 --prefix=${prefix_full_path} \
         -o ${DST_COV_DIR}/wamr-lcov \
         ${DST_COV_FILE}
@@ -69,7 +72,6 @@ if [[ -s ${SRC_TEMP_COV_FILE} ]]; then
     zip -r -q -o wamr-lcov.zip wamr-lcov
     rm -fr wamr-lcov
 
-    echo "Code coverage file ${DST_COV_FILE} was generated or appended"
     echo "Code coverage html ${DST_COV_DIR}/wamr-lcov.zip was generated"
 else
     echo "generate code coverage html failed"

tests/wamr-test-suites/test_wamr.sh

@@ -574,19 +574,64 @@ function malformed_test()
     ./malformed_test.py --run ${IWASM_CMD} | tee ${REPORT_DIR}/malfomed_$1_test_report.txt
 }
 
+function collect_standalone()
+{
+    if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then
+        pushd ${WORK_DIR} > /dev/null 2>&1
+
+        CODE_COV_FILE=""
+        if [[ -z "${CODE_COV_FILE}" ]]; then
+            CODE_COV_FILE="${WORK_DIR}/wamr.lcov"
+        else
+            CODE_COV_FILE="${CODE_COV_FILE}"
+        fi
+
+        STANDALONE_DIR=${WORK_DIR}/../../standalone
+
+        echo "Collect code coverage of standalone dump-call-stack"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-call-stack/build"
+        echo "Collect code coverage of standalone dump-mem-profiling"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-mem-profiling/build"
+        echo "Collect code coverage of standalone dump-perf-profiling"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/dump-perf-profiling/build"
+        if [[ $1 == "aot" ]]; then
+            echo "Collect code coverage of standalone pad-test"
+            ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/pad-test/build"
+        fi
+        echo "Collect code coverage of standalone test-invoke-native"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-invoke-native/build"
+        echo "Collect code coverage of standalone test-running-modes"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-running-modes/build"
+        echo "Collect code coverage of standalone test-running-modes/c-embed"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-running-modes/c-embed/build"
+        echo "Collect code coverage of standalone test-ts2"
+        ./collect_coverage.sh "${CODE_COV_FILE}" "${STANDALONE_DIR}/test-ts2/build"
+
+        popd > /dev/null 2>&1
+    fi
+}
+
 function standalone_test()
 {
+    if [[ ${COLLECT_CODE_COVERAGE} == 1 ]]; then
+        export COLLECT_CODE_COVERAGE=1
+    fi
+
     cd ${WORK_DIR}/../../standalone
 
     args="--$1"
 
     [[ ${SGX_OPT} == "--sgx" ]] && args="$args --sgx" || args="$args --no-sgx"
 
-    [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" && args="$args --no-thread"
+    [[ ${ENABLE_MULTI_THREAD} == 1 ]] && args="$args --thread" || args="$args --no-thread"
 
-    [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" && args="$args --no-simd"
+    [[ ${ENABLE_SIMD} == 1 ]] && args="$args --simd" || args="$args --no-simd"
+
+    args="$args ${TARGET}"
 
     ./standalone.sh $args | tee ${REPORT_DIR}/standalone_$1_test_report.txt
+
+    collect_standalone "$1"
 }
 
 function build_iwasm_with_cfg()
@@ -648,10 +693,10 @@ function collect_coverage()
         ln -sf ${WORK_DIR}/../spec-test-script/collect_coverage.sh ${WORK_DIR}
 
         CODE_COV_FILE=""
-        if [[ -z "${COV_FILE}" ]]; then
+        if [[ -z "${CODE_COV_FILE}" ]]; then
             CODE_COV_FILE="${WORK_DIR}/wamr.lcov"
         else
-            CODE_COV_FILE="${COV_FILE}"
+            CODE_COV_FILE="${CODE_COV_FILE}"
         fi
 
         pushd ${WORK_DIR} > /dev/null 2>&1

wamr-sdk/build_sdk.sh

@@ -224,8 +224,8 @@ echo -e "\n\n"
 
 echo "##############  Start to build runtime sdk  ###############"
 cd ${sdk_root}/runtime
-rm -fr build_runtime_sdk && mkdir build_runtime_sdk
-cd build_runtime_sdk
+rm -fr build-runtime-sdk && mkdir build-runtime-sdk
+cd build-runtime-sdk
 cmake .. $CM_DEXTRA_SDK_INCLUDE_PATH \
        -DWAMR_BUILD_SDK_PROFILE=${PROFILE} \
        -DCONFIG_PATH=${wamr_config_cmake_file} \
@@ -249,6 +249,6 @@ fi
 
 
 cd ..
-rm -fr build_runtime_sdk
+rm -fr build-runtime-sdk
 
 exit 0