Implement Inclavare Containers PAL interface in WAMR Linux-SGX (#429)

* Implement the PAL interface for rune

Work in progress

Signed-off-by: Le Yao <le.yao@intel.com>

* Support PAL for one runtime with multi-instances

Load runtime into enclave and run multi-instances

Signed-off-by: Le Yao <le.yao@intel.com>

product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp

@@ -16,6 +16,7 @@
 
 #include "Enclave_u.h"
 #include "sgx_urts.h"
+#include "pal_api.h"
 
 #ifndef TRUE
 #define TRUE 1
@@ -33,6 +34,12 @@
 
 static sgx_enclave_id_t g_eid = 0;
 
+sgx_enclave_id_t
+pal_get_enclave_id(void)
+{
+    return g_eid;
+}
+
 void
 ocall_print(const char* str)
 {
@@ -734,3 +741,163 @@ fail1:
     return 0;
 }
 
+int
+wamr_pal_get_version(void)
+{
+    return WAMR_PAL_VERSION;
+}
+
+int
+wamr_pal_init(const struct wamr_pal_attr *args)
+{
+    sgx_enclave_id_t *p_eid = &g_eid;
+
+    if (enclave_init(&g_eid) < 0) {
+        std::cout << "Fail to initialize enclave." << std::endl;
+        return 1;
+    }
+}
+
+int
+wamr_pal_create_process(struct wamr_pal_create_process_args *args)
+{
+    uint32_t stack_size = 16 * 1024, heap_size = 16 * 1024;
+    int log_verbose_level = 2;
+    bool is_repl_mode = false, alloc_with_pool = false;
+    const char *dir_list[8] = { NULL };
+    uint32_t dir_list_size = 0;
+    const char *env_list[8] = { NULL };
+    uint32_t env_list_size = 0;
+    uint32_t max_thread_num = 4;
+    char *wasm_files[16];
+    void *wasm_module_inst[16];
+
+    int argc = 2;
+    char *argv[argc] = { (char*)"./iwasm", (char *)args->argv[0] };
+
+    uint8_t *wasm_files_buf = NULL;
+    void *wasm_modules = NULL;
+    int len = 0, i;
+
+    char *temp = (char *)args->argv[0];
+    while(temp) {
+        len++;
+        temp=(char *)args->argv[len];
+    }
+
+    if (len > sizeof(wasm_files)/sizeof(char *)) {
+        printf("Number of input files is out of range\n");
+        return -1;
+    }
+
+    for (i = 0; i < len; ++i) {
+        wasm_files[i] = (char *)args->argv[i];
+    }
+
+    /* Init runtime */
+    if (!init_runtime(alloc_with_pool, max_thread_num)) {
+        printf("Failed to init runtime\n");
+        return -1;
+    }
+
+    /* Set log verbose level */
+    if (!set_log_verbose_level(log_verbose_level)) {
+        printf("Failed to set log level\n");
+        destroy_runtime();
+        return -1;
+    }
+
+    for (i = 0; i < len; ++i) {
+        uint8_t *wasm_file_buf = NULL;
+        uint32_t wasm_file_size;
+        void *wasm_module = NULL;
+        char error_buf[128] = { 0 };
+
+        /* Load WASM byte buffer from WASM bin file */
+        if (!(wasm_file_buf = (uint8_t *)read_file_to_buffer
+                              (wasm_files[i], &wasm_file_size))) {
+            printf("Failed to read file to buffer\n");
+            destroy_runtime();
+            return -1;
+        }
+
+        /* Load module */
+        if (!(wasm_module = load_module(wasm_file_buf, wasm_file_size,
+                                        error_buf, sizeof(error_buf)))) {
+            printf("%s\n", error_buf);
+            free(wasm_file_buf);
+            destroy_runtime();
+            return -1;
+        }
+
+        /* Set wasi arguments */
+        if (!set_wasi_args(wasm_module, dir_list, dir_list_size,
+                           env_list, env_list_size, argv, argc)) {
+            printf("%s\n", "set wasi arguments failed.\n");
+            unload_module(wasm_module);
+            free(wasm_file_buf);
+            destroy_runtime();
+            return -1;
+        }
+
+        /* Instantiate module */
+        if (!(wasm_module_inst[i] = instantiate_module(wasm_module,
+                                                stack_size, heap_size,
+                                                error_buf,
+                                                sizeof(error_buf)))) {
+            printf("%s\n", error_buf);
+            unload_module(wasm_module);
+            free(wasm_file_buf);
+            destroy_runtime();
+            return -1;
+        }
+
+        app_instance_main(wasm_module_inst[i], argc, argv);
+
+        /* Deinstantiate module */
+        deinstantiate_module(wasm_module_inst[i]);
+        unload_module(wasm_module);
+        free(wasm_file_buf);
+    }
+
+    destroy_runtime();
+    return 0;
+}
+
+int
+wamr_pal_destroy(void)
+{
+    //sgx_destroy_enclave(g_eid);
+    return 0;
+}
+
+int
+wamr_pal_exec(struct wamr_pal_exec_args *args)
+{
+    //app_instance_main(wasm_module_inst[i], argc, argv);
+    return 0;
+}
+
+int
+wamr_pal_kill(int pid, int sig)
+{
+    //deinstantiate_module(wasm_module_inst[i]);
+    //unload_module(wasm_module);
+    //free(wasm_file_buf);
+    return 0;
+}
+
+int pal_get_version(void) __attribute__((weak, alias ("wamr_pal_get_version")));
+
+int pal_init(const struct wamr_pal_attr *attr)\
+__attribute__ ((weak, alias ("wamr_pal_init")));
+
+int pal_create_process(struct wamr_pal_create_process_args *args)\
+__attribute__ ((weak, alias ("wamr_pal_create_process")));
+
+int pal_exec(struct wamr_pal_exec_args *args)\
+__attribute__ ((weak, alias ("wamr_pal_exec")));
+
+int pal_kill(int pid, int sig) __attribute__ ((weak, alias ("wamr_pal_kill")));
+
+int pal_destroy(void) __attribute__ ((weak, alias ("wamr_pal_destroy")));

product-mini/platforms/linux-sgx/enclave-sample/App/README.md

@@ -0,0 +1,206 @@
+# WAMR as an Enclave Runtime for Rune
+
+## Build WAMR vmcore (iwasm) for Linux-SGX
+
+### SIM Mode
+
+The default SGX mode in WAMR is the SIM mode. Build the source code and enclave example, please refer to [this guild](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/linux_sgx.md#build-wamr-vmcore-iwasm-for-linux-sgx).
+
+### HW Mode
+
+Please do the following changes before execute [this guild](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/linux_sgx.md#build-wamr-vmcore-iwasm-for-linux-sgx).
+
+```shell
+diff --git a/product-mini/platforms/linux-sgx/enclave-sample/Makefile b/product-mini/platforms/linux-sgx/enclave-sample/Makefile
+index f06b5b8..f247f3e 100644
+--- a/product-mini/platforms/linux-sgx/enclave-sample/Makefile
++++ b/product-mini/platforms/linux-sgx/enclave-sample/Makefile
+@@ -4,7 +4,7 @@
+ ######## SGX SDK Settings ########
+
+ SGX_SDK ?= /opt/intel/sgxsdk
+-SGX_MODE ?= SIM
++SGX_MODE ?= HW
+ SGX_ARCH ?= x64
+ SGX_DEBUG ?= 0
+ SPEC_TEST ?= 0
+```
+
+```shell
+diff --git a/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal b/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal
+index a64d577..747d995 100644
+--- a/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal
++++ b/product-mini/platforms/linux-sgx/enclave-sample/Makefile_minimal
+@@ -4,7 +4,7 @@
+ ######## SGX SDK Settings ########
+
+ SGX_SDK ?= /opt/intel/sgxsdk
+-SGX_MODE ?= SIM
++SGX_MODE ?= HW
+ SGX_ARCH ?= x64
+ SGX_DEBUG ?= 0
+ SPEC_TEST ?= 0
+
+```
+
+```shell
+diff --git a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp
+index c321575..3b41c30 100644
+--- a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp
++++ b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp
+@@ -31,6 +31,7 @@
+ #define MAX_PATH 1024
+
+ #define TEST_OCALL_API 0
++#define SGX_DEBUG_FLAG 1
+
+```
+
+After building, please sign enclave.so to generate enclave.signed.so which is needed in PAL
+
+```shell
+/opt/intel/sgxsdk/bin/x64/sgx_sign sign -key Enclave/Enclave_private.pem -enclave enclave.so -out enclave.signed.so -config Enclave/Enclave.config.xml
+```
+
+---
+
+## Build PAL dynamically linked shared object
+
+To build WAMR as an Enclave Runtime for [Inclavare Containers](https://github.com/alibaba/inclavare-containers), we should implement the [PAL interface](https://github.com/alibaba/inclavare-containers/blob/master/rune/libenclave/internal/runtime/pal/spec_v2.md) in WAMR for rune to call the PAL to create the enclave with WAMR and run applications.
+
+```shell
+g++ -shared -fPIC -o libwamr-pal.so App/*.o libvmlib_untrusted.a -L/opt/intel/sgxsdk/lib64 -lsgx_urts -lpthread -lssl -lcrypto
+cp ./libwamr-pal.so /usr/lib/libwamr-pal.so
+```
+
+Note: `/opt/intel/sgxsdk/` is where you installed the SGX SDK
+
+---
+
+## Build WAMR application
+
+To Build a WAMR application, please refer to [this guide](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/build_wasm_app.md#build-wasm-applications)
+
+To run a WAMR application with Intel SGX enclave by `rune`, please compile the `.wasm` file to `.aot` file, refer to [this guide](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/build_wasm_app.md#compile-wasm-to-aot-module) 
+
+---
+
+## Build WAMR container image
+
+Under the `enclave-sample` directory, to create the WAMR docker images to load the `enclave.signed.so` and target application wasm files, please type the following commands to create a `Dockerfile`:
+
+For centos:
+
+```shell
+cat >Dockerfile <<EOF
+FROM centos:8.1.1911
+
+RUN mkdir -p /run/rune
+WORKDIR /run/rune
+
+COPY enclave.signed.so .
+COPY ${wasm_app1.aot} .
+#COPY ${wasm_app2.aot} .
+#...
+EOF
+```
+
+ For ubuntu:
+
+```shell
+cat > Dockerfile <<EOF
+FROM ubuntu:18.04
+
+RUN mkdir -p /run/rune
+WORKDIR /run/rune
+
+COPY enclave.signed.so .
+COPY ${wasm_app1.aot} .
+#COPY ${wasm_app2.aot} .
+#...
+EOF
+```
+
+`${wasm_app.aot}` files are the applications you want to run in WAMR. 
+
+Then build the WAMR container image with the command:
+
+```shell
+docker build . -t wamr-app
+```
+
+---
+
+## Run WAMR SGX with Docker and OCI Runtime rune
+
+The following guide provides the steps to run WAMR with Docker and OCI Runtime `rune`.
+
+[rune](https://github.com/alibaba/inclavare-containers/tree/master/rune) is a novel OCI Runtime used to run trusted applications in containers with the hardware-assisted enclave technology.
+
+### Requirements
+
+- Ensure that you have one of the following required operating systems to build a WAMR container image:
+
+  - CentOS 8.1
+  - Ubuntu 18.04-server
+
+- Please follow [Intel SGX Installation Guide](https://download.01.org/intel-sgx/sgx-linux/2.11/docs/Intel_SGX_Installation_Guide_Linux_2.11_Open_Source.pdf) to install Intel SGX driver, Intel SGX SDK & PSW for Linux.
+
+  - For CentOS 8.1, UAE service libraries are needed but may not installed if SGX PSW installer is used. Please manually install it:
+
+    ```shell
+    rpm -i libsgx-uae-service-2.11.100.2-1.el8.x86_64.rpm
+    ```
+
+### Configuring OCI Runtime rune for Docker
+
+Add the assocated configuration for `rune` in dockerd config file, e.g, `/etc/docker/daemon.json`, on your system.
+
+```
+{
+	"runtimes": {
+		"rune": {
+			"path": "/usr/bin/rune",
+			"runtimeArgs": []
+		}
+	}
+}
+```
+
+then restart dockerd on your system.
+
+You can check whether `rune` is correctly enabled or not with:
+
+```
+docker info | grep rune
+```
+
+The expected result would be:
+
+```
+Runtimes: rune runc
+```
+
+### Run WAMR container image
+
+You need to specify a set of parameters to `docker run` to run:
+
+```shell
+docker run -it --rm --runtime=rune \
+  -e ENCLAVE_TYPE=intelSgx \
+  -e ENCLAVE_RUNTIME_PATH=/usr/lib/libwamr-pal.so \
+  -e ENCLAVE_RUNTIME_ARGS=debug \
+  wamr-app
+```
+
+where:
+
+- @ENCLAVE_TYPE: specify the type of enclave hardware to use, such as `intelSgx`.
+- @ENCLAVE_RUNTIME_PATH: specify the path to enclave runtime to launch. For an WAMR application, you need to specify the path to `libwamr-pal.so`.
+- @ENCLAVE_RUNTIME_ARGS: specify the specific arguments to enclave runtime, separated by the comma.
+
+---
+
+## (Optional) Run WAMR bundle for Rune
+
+Please refer to [this guide](https://github.com/leyao-daily/wasm-micro-runtime/blob/main/product-mini/platforms/linux-sgx/enclave-sample/App/wamr-bundle.md)

product-mini/platforms/linux-sgx/enclave-sample/App/pal_api.h

@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2019 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef __WAMR_PAL_API_H__
+#define __WAMR_PAL_API_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * WAMR PAL API version number
+ */
+#define WAMR_PAL_VERSION 2
+
+/*
+ * @brief Get version of WAMR PAL API
+ *
+ * @retval If > 0, then success; otherwise, it is an invalid version.
+ */
+int wamr_pal_get_version(void);
+
+/*
+ * WAMR PAL attributes
+ */
+typedef struct wamr_pal_attr {
+    // WAMR instance directory.
+    //
+    // The default value is "."; that is, the current working directory
+    const char     *instance_dir;
+    // Log level.
+    //
+    // Specifies the log verbose level (0 to 5, default is 2)
+    // large level with more log
+    //
+    const char     *log_level;
+} wamr_pal_attr_t;
+
+#define WAMR_PAL_ATTR_INITVAL { \
+    .instance_dir = ".",        \
+    .log_level = 2              \
+}
+
+/*
+ * The struct which consists of file descriptors of standard I/O
+ */
+typedef struct wamr_stdio_fds {
+    int stdin_fd;
+    int stdout_fd;
+    int stderr_fd;
+} wamr_stdio_fds_t;
+
+/*
+ * The struct which consists of arguments needed by wamr_pal_create_process
+ */
+struct wamr_pal_create_process_args {
+
+    // Path to new process.
+    //
+    // The path of the command which will be created as a new process.
+    //
+    // Mandatory field. Must not be NULL.
+    const char *path;
+
+    // Argments array pass to new process.
+    //
+    // The arguments to the command. By convention, the argv[0] should be the program name.
+    // And the last element of the array must be NULL to indicate the length of array.
+    //
+    // Mandatory field. Must not be NULL.
+    const char **argv;
+
+    // Untrusted environment variable array pass to new process.
+    //
+    // The untrusted env vars to the command. And the last element of the array must be
+    // NULL to indicate the length of array.
+    //
+    // Optional field.
+    const char **env;
+
+    // File descriptors of the redirected standard I/O (i.e., stdin, stdout, stderr)
+    //
+    // If set to NULL, will use the original standard I/O file descriptors.
+    //
+    // Optional field.
+    const struct wamr_stdio_fds *stdio;
+
+    // Output. Pid of new process in libos.
+    //
+    // If wamr_pal_create_process returns success, pid of the new process will
+    // be updated.
+    //
+    // Mandatory field. Must not be NULL.
+    int *pid;
+};
+
+/*
+ * The struct which consists of arguments needed by wamr_pal_exec
+ */
+struct wamr_pal_exec_args {
+    // Pid of new process created with wamr_pal_create_process.
+    //
+    // Mandatory field.
+    int pid;
+
+    // Output. The exit status of the command. The semantic of
+    // this value follows the one described in wait(2) man
+    // page. For example, if the program terminated normally,
+    // then WEXITSTATUS(exit_status) gives the value returned
+    // from a main function.
+    //
+    // Mandatory field. Must not be NULL.
+    int *exit_value;
+};
+
+int wamr_pal_init(const struct wamr_pal_attr *args);
+
+int wamr_pal_create_process(struct wamr_pal_create_process_args *args);
+
+int wamr_pal_destroy(void);
+
+int wamr_pal_exec(struct wamr_pal_exec_args *args);
+
+int wamr_pal_kill(int pid, int sig);
+
+int pal_get_version(void);
+
+int pal_init(const struct wamr_pal_attr *attr);
+
+int pal_create_process(struct wamr_pal_create_process_args *args);
+
+int pal_exec(struct wamr_pal_exec_args *args);
+
+int pal_kill(int pid, int sig);
+
+int pal_destroy(void);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __WAMR_PAL_API_H__ */
+

product-mini/platforms/linux-sgx/enclave-sample/App/wamr-bundle.md

@@ -0,0 +1,65 @@
+# Run WAMR bundle for Rune
+
+## Create WAMR Application bundle
+
+In order to use `rune` you must have your container image in the format of an OCI bundle. If you have Docker installed you can use its `export` method to acquire a root filesystem from an existing WAMR application container image.
+
+```shell
+# create the top most bundle directory
+mkdir -p "$HOME/rune_workdir"
+cd "$HOME/rune_workdir"
+mkdir rune-container
+cd rune-container
+
+# create the rootfs directory
+mkdir rootfs
+
+# export wamr application image via Docker into the rootfs directory
+docker export $(docker create ${wamr_application_image}) | sudo tar -C rootfs -xvf -
+```
+
+After a root filesystem is populated you just generate a spec in the format of a config.json file inside your bundle. `rune` provides a spec command which is similar to `runc` to generate a template file that you are then able to edit.
+
+```shell
+rune spec
+```
+
+To find features and documentation for fields in the spec please refer to the [specs](https://github.com/opencontainers/runtime-spec) repository.
+
+In order to run the target applications in WAMR with `rune`, you need to change the entrypoint from `sh` to `/run/rune/${wasm_app1.wasm}`, and in order to run multi-applications in one runtime with enclave, change it to `/run/rune/${wasm_app1.aot}`, `/run/rune/${wasm_app2.aot}` ... 
+
+```yaml
+  "process": {
+      "args": [
+          "/run/rune/demo.aot"
+      ],
+  }
+```
+
+and then configure enclave runtime as following:
+
+```yaml
+  "annotations": {
+      "enclave.type": "intelSgx",
+      "enclave.runtime.path": "/usr/lib/libwamr-pal.so",
+      "enclave.runtime.args": "./"
+  }
+```
+
+where:
+
+- @enclave.type: specify the type of enclave hardware to use, such as `intelSgx`.
+- @enclave.runtime.path: specify the path to enclave runtime to launch. For an WAMR application, you need to specify the path to `libwamr-pal.so`.
+- @enclave.runtime.args: specify the specific arguments to enclave runtime, separated by the comma.
+
+---
+
+## Run WAMR Application
+
+Assuming you have an OCI bundle from the previous step you can execute the container in this way.
+
+```shell
+cd "$HOME/rune_workdir/rune-container"
+sudo rune run ${wamr_application_container_name}
+```
+