Fix several issues in CI binary releasing (#2064)

.github/workflows/build_iwasm_release.yml

@@ -14,6 +14,10 @@ on:
         description: workfing directory
         type: string
         required: true
+      llvm_cache_key:
+        description: the cache key of llvm libraries
+        type: string
+        required: true
       runner:
         description: OS of compilation
         type: string
@@ -33,6 +37,22 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: get cached LLVM libraries
+        id: retrieve_llvm_libs
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
+          key: ${{ inputs.llvm_cache_key }}
+
+      - name: Quit if cache miss
+        if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
+        run: echo "::error::can not get prebuilt llvm libraries" && exit 1
+
       - name: generate iwasm binary release
         run: |
           cmake -S . -B build \

.github/workflows/build_wamr_vscode_ext.yml

@@ -24,22 +24,25 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 14.x
-      
-      - name: set vscode extension to correct version 
+
+      - name: set vscode extension to correct version
         run: |
           npm install -g json
           json -I -f package.json -e "this.version=\"${{ inputs.ver_num }}\""
         working-directory: test-tools/wamr-ide/VSCode-Extension
 
+      # [!workflow]
+      # bypass the step of publishing the extension to the Market.
+      # recover it after creating the secret in the Environment
       - name: generate wamr ide vscode extension
-        env: 
+        env:
           credentials: ${{ secrets.TOKEN }}
         run: |
           npm install -g vsce
           rm -rf node_modules
           npm install
           vsce package
-          vsce publish -p ${{ secrets.TOKEN }}
+          # vsce publish -p ${{ secrets.TOKEN }}
         working-directory: test-tools/wamr-ide/VSCode-Extension
 
       - name: compress the vscode extension

.github/workflows/build_wamrc.yml

@@ -38,7 +38,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: get cached LLVM libraries
-        id: cache_llvm
+        id: retrieve_llvm_libs
         uses: actions/cache@v3
         with:
           path: |
@@ -49,10 +49,9 @@ jobs:
             ./core/deps/llvm/build/share
           key: ${{ inputs.llvm_cache_key }}
 
-      - name: Build llvm and clang from source
-        if: steps.cache_llvm.outputs.cache-hit != 'true'
-        run: /usr/bin/env python3 ./build_llvm.py --arch X86
-        working-directory: build-scripts
+      - name: Quit if cache miss
+        if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
+        run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
       - name: generate wamrc binary release
         run: |

.github/workflows/release_process.yml

@@ -49,36 +49,56 @@ jobs:
           draft: false
           body: ${{ env.RELEASE_NOTE }}
 
+  #
+  # LLVM_LIBRARIES
+  build_llvm_libraries_on_ubuntu_2004:
+    needs: [create_tag, create_release]
+    uses: ./.github/workflows/build_llvm_libraries.yml
+    with:
+      os: "ubuntu-20.04"
+      arch: "X86"
+
+  build_llvm_libraries_on_ubuntu_2204:
+    needs: [create_tag, create_release]
+    uses: ./.github/workflows/build_llvm_libraries.yml
+    with:
+      os: "ubuntu-22.04"
+      arch: "X86"
+
+  build_llvm_libraries_on_macos:
+    needs: [create_tag, create_release]
+    uses: ./.github/workflows/build_llvm_libraries.yml
+    with:
+      os: "macos-latest"
+      arch: "X86"
+
   #
   # WAMRC
   release_wamrc_on_ubuntu_2004:
-    needs: [create_tag, create_release]
+    needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2004]
     uses: ./.github/workflows/build_wamrc.yml
     with:
-      # can't take an env variable here
-      llvm_cache_key: ubuntu-20.04-build-llvm_libraries_ex
+      llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
       release: true
       runner: ubuntu-20.04
       upload_url: ${{ needs.create_release.outputs.upload_url }}
       ver_num: ${{ needs.create_tag.outputs.new_ver}}
 
   release_wamrc_on_ubuntu_2204:
-    needs: [create_tag, create_release]
+    needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2204 ]
     uses: ./.github/workflows/build_wamrc.yml
     with:
-      # can't take an env variable here
-      llvm_cache_key: ubuntu-22.04-build-llvm_libraries_ex
+      llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
       release: true
       runner: ubuntu-22.04
       upload_url: ${{ needs.create_release.outputs.upload_url }}
       ver_num: ${{ needs.create_tag.outputs.new_ver }}
 
   release_wamrc_on_ubuntu_macos:
-    needs: [create_tag, create_release]
+    needs: [create_tag, create_release, build_llvm_libraries_on_macos]
     uses: ./.github/workflows/build_wamrc.yml
     with:
-      # can't take an env variable here
-      llvm_cache_key: macos-latest-build-llvm_libraries_ex
+      llvm_cache_key: ${{ needs.build_llvm_libraries_on_macos.outputs.cache_key }}
       release: true
       runner: macos-latest
       upload_url: ${{ needs.create_release.outputs.upload_url }}
@@ -87,28 +107,31 @@ jobs:
   #
   # IWASM
   release_iwasm_on_ubuntu_2004:
-    needs: [create_tag, create_release]
+    needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2004]
     uses: ./.github/workflows/build_iwasm_release.yml
     with:
       cwd: product-mini/platforms/linux
+      llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
       runner: ubuntu-20.04
       upload_url: ${{ needs.create_release.outputs.upload_url }}
       ver_num: ${{ needs.create_tag.outputs.new_ver}}
 
   release_iwasm_on_ubuntu_2204:
-    needs: [create_tag, create_release]
+    needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2204]
     uses: ./.github/workflows/build_iwasm_release.yml
     with:
       cwd: product-mini/platforms/linux
+      llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
       runner: ubuntu-22.04
       upload_url: ${{ needs.create_release.outputs.upload_url }}
       ver_num: ${{ needs.create_tag.outputs.new_ver}}
 
   release_iwasm_on_macos:
-    needs: [create_tag, create_release]
+    needs: [create_tag, create_release, build_llvm_libraries_on_macos]
     uses: ./.github/workflows/build_iwasm_release.yml
     with:
       cwd: product-mini/platforms/darwin
+      llvm_cache_key: ${{ needs.build_llvm_libraries_on_macos.outputs.cache_key }}
       runner: macos-latest
       upload_url: ${{ needs.create_release.outputs.upload_url }}
       ver_num: ${{ needs.create_tag.outputs.new_ver}}

product-mini/platforms/android/build_llvm.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt
 /usr/bin/env python3 ../../../build-scripts/build_llvm.py --platform android "$@"

product-mini/platforms/darwin/build_llvm.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt
 /usr/bin/env python3 ../../../build-scripts/build_llvm.py --platform darwin "$@"

product-mini/platforms/freebsd/build_llvm.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt
 /usr/bin/env python3 ../../../build-scripts/build_llvm.py "$@"

product-mini/platforms/linux/build_llvm.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt
 /usr/bin/env python3 ../../../build-scripts/build_llvm.py "$@"

test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile

@@ -12,7 +12,7 @@ COPY resource /root/
 ##  - download cmake with wget and set up
 # hadolint ignore=DL3008
 RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.tar.gz \
-    && tar -zxvf cmake-3.21.1-linux-x86_64.tar.gz \
+    && tar -zxf cmake-3.21.1-linux-x86_64.tar.gz \
     && rm -f cmake-3.21.1-linux-x86_64.tar.gz \
     && mv cmake-3.21.1-linux-x86_64 /opt/cmake \
     && ln -s /opt/cmake/bin/cmake /bin/cmake \
@@ -26,13 +26,17 @@ RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/
 
 ##  - download wasi-sdk with wget and set up to /opt/wasi-sdk
 RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz \
-    && tar -zxvf wasi-sdk-*-linux.tar.gz \
+    && tar -zxf wasi-sdk-*-linux.tar.gz \
     && mv wasi-sdk-19.0 /opt/wasi-sdk/ \
     && rm -f wasi-sdk-*-linux.tar.gz
 
 ##  - clone wamr repo
 RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git
 
+WORKDIR /root/wasm-micro-runtime/build-scripts
+RUN apt-get update && apt-get install --no-install-recommends -y ccache ninja-build python3-pip
+RUN pip3 install --user -r requirements.txt
+
 WORKDIR /root/wasm-micro-runtime/wamr-compiler
 RUN ./build_llvm.sh \
     && mkdir build

wamr-compiler/build_llvm.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt
 /usr/bin/env python3 ../build-scripts/build_llvm.py "$@"

wamr-compiler/build_llvm_arc.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt
 /usr/bin/env python3 ../build-scripts/build_llvm.py --platform arc "$@"

wamr-compiler/build_llvm_xtensa.sh

@@ -3,4 +3,5 @@
 # Copyright (C) 2020 Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt
 /usr/bin/env python3 ../build-scripts/build_llvm.py --platform xtensa "$@"