Enable spec test on nuttx and daily run it (#1763)

- Spec test needs about 15 min
- Skip some test cases that cannot pass now, most of which are FP relative
  (due to potential bugs in NuttX's libc)

.github/workflows/spec_test_on_nuttx.yml

@@ -0,0 +1,108 @@
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: spec test on nuttx
+
+on:
+  schedule:
+    - cron:  '0 0 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  spec_test_on_qemu:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        nuttx_board_config: [
+          # cortex-a9
+          "boards/arm/imx6/sabre-6quad/configs/nsh",
+          # riscv32imac
+          "boards/risc-v/qemu-rv/rv-virt/configs/nsh",
+          # riscv64imac
+          # "boards/risc-v/qemu-rv/rv-virt/configs/nsh64",
+        ]
+        wamr_test_option: [
+          # "-t fast-interp",
+          "-t aot",
+          # "-t aot -X"
+        ]
+    steps:
+      - name: Install Utilities
+        run: |
+          sudo apt install -y kconfig-frontends-nox genromfs
+
+      - name: Install ARM Compilers
+        if: contains(matrix.nuttx_board_config, 'arm')
+        run: sudo apt install -y gcc-arm-none-eabi
+
+      - name: Install RISC-V Compilers
+        if: contains(matrix.nuttx_board_config, 'risc-v')
+        run: |
+          curl -L https://static.dev.sifive.com/dev-tools/freedom-tools/v2020.12/riscv64-unknown-elf-toolchain-10.2.0-2020.12.8-x86_64-linux-ubuntu14.tar.gz > riscv.tar.gz
+          tar xvf riscv.tar.gz
+          echo "$PWD/riscv64-unknown-elf-toolchain-10.2.0-2020.12.8-x86_64-linux-ubuntu14/bin" >> $GITHUB_PATH
+
+      - name: Checkout NuttX
+        uses: actions/checkout@v3
+        with:
+          repository: apache/incubator-nuttx
+          path: nuttx
+
+      - name: Checkout NuttX Apps
+        uses: actions/checkout@v3
+        with:
+          repository: apache/incubator-nuttx-apps
+          path: apps
+
+      - name: Checkout WAMR
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ github.repository }}
+          path: apps/interpreters/wamr/wamr
+
+      - name: Enable WAMR for NuttX
+        run: |
+          find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_INTERPRETERS_WAMR=y\nCONFIG_INTERPRETERS_WAMR_AOT=y\nCONFIG_INTERPRETERS_WAMR_FAST=y\nCONFIG_INTERPRETERS_WAMR_LOG=y\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\n'
+          find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\nCONFIG_FS_HOSTFS=y\nCONFIG_LIBC_FLOATINGPOINT=y\n'
+
+      - name: Build wamrc
+        working-directory: apps/interpreters/wamr/wamr/wamr-compiler
+        run: |
+          sudo apt install llvm-13-dev
+          cmake -Bbuild -DWAMR_BUILD_WITH_CUSTOM_LLVM=1 .
+          cmake --build build
+
+      - name: Build
+        run: |
+          cd nuttx
+          tools/configure.sh ${{ matrix.nuttx_board_config }}
+          make -j$(nproc)
+          echo "firmware=$PWD/nuttx" >> $GITHUB_ENV
+
+      - name: Test on ARM
+        if: endsWith(matrix.nuttx_board_config, 'sabre-6quad/configs/nsh')
+        run: |
+          curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.1.0-1/xpack-qemu-arm-7.1.0-1-linux-x64.tar.gz > xpack-qemu-arm.tar.gz
+          tar xvf xpack-qemu-arm.tar.gz
+          export PATH=$PATH:$PWD/xpack-qemu-arm-7.1.0-1/bin
+          cd apps/interpreters/wamr/wamr/tests/wamr-test-suites
+          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m thumbv7_vfp -b -Q -P -F ${{ env.firmware }}
+
+      - name: Test on RISCV32
+        if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh')
+        run: |
+          curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz
+          tar xvf xpack-qemu-riscv.tar.gz
+          export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin
+          cd apps/interpreters/wamr/wamr/tests/wamr-test-suites
+          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m RISCV32 -b -Q -P -F ${{ env.firmware }}
+
+      - name: Test on RISCV64
+        if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh64')
+        run: |
+          curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz
+          tar xvf xpack-qemu-riscv.tar.gz
+          export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin
+          cd apps/interpreters/wamr/wamr/tests/wamr-test-suites
+          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m riscv64 -b -Q -P -F ${{ env.firmware }}

tests/wamr-test-suites/spec-test-script/all.py

@@ -52,6 +52,7 @@ def ignore_the_case(
     multi_thread_flag=False,
     simd_flag=False,
     xip_flag=False,
+    qemu_flag=False
 ):
     if case_name in ["comments", "inline-module", "names"]:
         return True
@@ -74,6 +75,10 @@ def ignore_the_case(
         ]:
             return True
 
+    if qemu_flag:
+        if case_name in ["f32_bitwise", "f64_bitwise", "loop", "f64", "f64_cmp", "conversions", "f32", "f32_cmp", "float_exprs", "float_misc", "select", "memory_grow"]:
+            return True
+
     return False
 
 
@@ -119,6 +124,7 @@ def test_case(
         multi_thread_flag,
         simd_flag,
         xip_flag,
+        qemu_flag
     ):
         return True