Implement ecall to handle commands from host to call enclave runtime APIs (#320)

product-mini/platforms/linux-sgx/CMakeLists.txt

@@ -32,9 +32,9 @@ if (NOT DEFINED WAMR_BUILD_INTERP)
 endif ()
 
 if (NOT DEFINED WAMR_BUILD_AOT)
-  # Disable AOT by default.
-  # If enabling AOT, please install Intel SGX SDKv2.8 or later.
-  set (WAMR_BUILD_AOT 0)
+  # Enable AOT by default
+  # Please install Intel SGX SDKv2.8 or later.
+  set (WAMR_BUILD_AOT 1)
 endif ()
 
 if (NOT DEFINED WAMR_BUILD_JIT)
@@ -52,6 +52,10 @@ if (NOT DEFINED WAMR_BUILD_LIBC_WASI)
   set (WAMR_BUILD_LIBC_WASI 0)
 endif ()
 
+if (COLLECT_CODE_COVERAGE EQUAL 1)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
+endif ()
+
 set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -ffunction-sections -fdata-sections \
                                      -Wall -Wno-unused-parameter -Wno-pedantic \

product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp

@@ -4,11 +4,15 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <pwd.h>
+#include <assert.h>
 #include <iostream>
 #include <cstdio>
 #include <cstring>
-#include <unistd.h>
-#include <pwd.h>
 
 #include "Enclave_u.h"
 #include "sgx_urts.h"
@@ -25,7 +29,9 @@
 #define ENCLAVE_FILENAME "enclave.signed.so"
 #define MAX_PATH FILENAME_MAX
 
-sgx_enclave_id_t g_eid = 0;
+#define TEST_OCALL_API 0
+
+static sgx_enclave_id_t g_eid = 0;
 
 void
 ocall_print(const char* str)
@@ -53,7 +59,7 @@ enclave_init(sgx_enclave_id_t *p_eid)
     sgx_status_t ret = SGX_ERROR_UNEXPECTED;
     int updated = 0;
 
-    /* Step 1: try to retrieve the launch token saved by last transaction 
+    /* Step 1: try to retrieve the launch token saved by last transaction
      *         if there is no token, then create a new one.
      */
     /* try to get the token saved in $HOME */
@@ -116,15 +122,568 @@ enclave_init(sgx_enclave_id_t *p_eid)
     return 0;
 }
 
+static unsigned char *
+read_file_to_buffer(const char *filename, uint32_t *ret_size)
+{
+    unsigned char *buffer;
+    FILE *file;
+    int file_size, read_size;
+
+    if (!(file = fopen(filename, "r")))
+        return NULL;
+
+    fseek(file, 0, SEEK_END);
+    file_size = ftell(file);
+    fseek(file, 0, SEEK_SET);
+
+    if (!(buffer = (unsigned char*)malloc(file_size))) {
+        fclose(file);
+        return NULL;
+    }
+
+    read_size = fread(buffer, 1, file_size, file);
+    fclose(file);
+
+    if (read_size < file_size) {
+        free(buffer);
+        return NULL;
+    }
+
+    *ret_size = file_size;
+
+    return buffer;
+}
+
+static int
+print_help()
+{
+    printf("Usage: iwasm [-options] wasm_file [args...]\n");
+    printf("options:\n");
+    printf("  -f|--function name     Specify a function name of the module to run rather\n"
+           "                         than main\n");
+    printf("  -v=n                   Set log verbose level (0 to 5, default is 2) larger\n"
+           "                         level with more log\n");
+    printf("  --stack-size=n         Set maximum stack size in bytes, default is 16 KB\n");
+    printf("  --heap-size=n          Set maximum heap size in bytes, default is 16 KB\n");
+    printf("  --repl                 Start a very simple REPL (read-eval-print-loop) mode\n"
+           "                         that runs commands in the form of `FUNC ARG...`\n");
+    printf("  --env=<env>            Pass wasi environment variables with \"key=value\"\n");
+    printf("                         to the program, for example:\n");
+    printf("                           --env=\"key1=value1\" --env=\"key2=value2\"\n");
+    printf("  --dir=<dir>            Grant wasi access to the given host directories\n");
+    printf("                         to the program, for example:\n");
+    printf("                           --dir=<dir1> --dir=<dir2>\n");
+    printf("  --max-threads=n        Set maximum thread number per cluster, default is 4\n");
+    return 1;
+}
+
+/**
+ * Split a space separated strings into an array of strings
+ * Returns NULL on failure
+ * Memory must be freed by caller
+ * Based on: http://stackoverflow.com/a/11198630/471795
+ */
+static char **
+split_string(char *str, int *count)
+{
+    char **res = NULL;
+    char *p;
+    int idx = 0;
+
+    /* split string and append tokens to 'res' */
+    do {
+        p = strtok(str, " ");
+        str = NULL;
+        res = (char **)realloc(res, sizeof(char *) * (unsigned)(idx + 1));
+        if (res == NULL) {
+            return NULL;
+        }
+        res[idx++] = p;
+    } while (p);
+
+    /**
+     * since the function name,
+     * res[0] might be contains a '\' to indicate a space
+     * func\name -> func name
+     */
+    p = strchr(res[0], '\\');
+    while (p) {
+        *p = ' ';
+        p = strchr(p, '\\');
+    }
+
+    if (count) {
+        *count = idx - 1;
+    }
+    return res;
+}
+
+typedef enum EcallCmd {
+    CMD_INIT_RUNTIME = 0,     /* wasm_runtime_init/full_init() */
+    CMD_LOAD_MODULE,          /* wasm_runtime_load() */
+    CMD_INSTANTIATE_MODULE,   /* wasm_runtime_instantiate() */
+    CMD_LOOKUP_FUNCTION,      /* wasm_runtime_lookup_function() */
+    CMD_CREATE_EXEC_ENV,      /* wasm_runtime_create_exec_env() */
+    CMD_CALL_WASM,            /* wasm_runtime_call_wasm */
+    CMD_EXEC_APP_FUNC,        /* wasm_application_execute_func() */
+    CMD_EXEC_APP_MAIN,        /* wasm_application_execute_main() */
+    CMD_GET_EXCEPTION,        /* wasm_runtime_get_exception() */
+    CMD_DEINSTANTIATE_MODULE, /* wasm_runtime_deinstantiate() */
+    CMD_UNLOAD_MODULE,        /* wasm_runtime_unload() */
+    CMD_DESTROY_RUNTIME,      /* wasm_runtime_destroy() */
+    CMD_SET_WASI_ARGS,        /* wasm_runtime_set_wasi_args() */
+    CMD_SET_LOG_LEVEL,        /* bh_log_set_verbose_level() */
+    CMD_SET_MAX_THREAD_NUM,   /* wasm_runtime_set_max_thread_num() */
+} EcallCmd;
+
+static void
+app_instance_func(void *wasm_module_inst, const char *func_name,
+                  int app_argc, char **app_argv);
+
+static void *
+app_instance_repl(void *module_inst, int app_argc, char **app_argv)
+{
+    char *cmd = NULL;
+    size_t len = 0;
+    ssize_t n;
+
+    while ((printf("webassembly> "), n = getline(&cmd, &len, stdin)) != -1) {
+        assert(n > 0);
+        if (cmd[n - 1] == '\n') {
+            if (n == 1)
+                continue;
+            else
+                cmd[n - 1] = '\0';
+        }
+        if (!strcmp(cmd, "__exit__")) {
+            printf("exit repl mode\n");
+            break;
+        }
+        app_argv = split_string(cmd, &app_argc);
+        if (app_argv == NULL) {
+            printf("Wasm prepare param failed: split string failed.\n");
+            break;
+        }
+        if (app_argc != 0) {
+            app_instance_func(module_inst, app_argv[0],
+                              app_argc - 1, app_argv + 1);
+        }
+        free(app_argv);
+    }
+    free(cmd);
+    return NULL;
+}
+
+static bool
+validate_env_str(char *env)
+{
+    char *p = env;
+    int key_len = 0;
+
+    while (*p != '\0' && *p != '=') {
+        key_len++;
+        p++;
+    }
+
+    if (*p != '=' || key_len == 0)
+        return false;
+
+    return true;
+}
+
+static bool
+set_log_verbose_level(int log_verbose_level)
+{
+    uint64_t ecall_args[1];
+
+    /* Set log verbose level */
+    if (log_verbose_level != 2) {
+        ecall_args[0] = log_verbose_level;
+        if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_SET_LOG_LEVEL,
+                                                (uint8_t *)ecall_args,
+                                                sizeof(uint64_t))) {
+            printf("Call ecall_handle_command() failed.\n");
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool
+init_runtime(bool alloc_with_pool)
+{
+    uint64_t ecall_args[1];
+
+    ecall_args[0] = alloc_with_pool;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_INIT_RUNTIME,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t))) {
+        printf("Call ecall_handle_command() failed.\n");
+        return false;
+    }
+    if (!(bool)ecall_args[0]) {
+        printf("Init runtime environment failed.\n");
+        return false;
+    }
+    return true;
+}
+
+static void
+destroy_runtime()
+{
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_DESTROY_RUNTIME,
+                                            NULL, 0)) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+}
+
+static void *
+load_module(uint8_t *wasm_file_buf, uint32_t wasm_file_size,
+            char *error_buf, uint32_t error_buf_size)
+{
+    uint64_t ecall_args[4];
+
+    ecall_args[0] = (uint64_t)(uintptr_t)wasm_file_buf;
+    ecall_args[1] = wasm_file_size;
+    ecall_args[2] = (uint64_t)(uintptr_t)error_buf;
+    ecall_args[3] = error_buf_size;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_LOAD_MODULE,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t) * 4)) {
+        printf("Call ecall_handle_command() failed.\n");
+        return NULL;
+    }
+
+    return (void *)(uintptr_t)ecall_args[0];
+}
+
+static void
+unload_module(void *wasm_module)
+{
+    uint64_t ecall_args[1];
+
+    ecall_args[0] = (uint64_t)(uintptr_t)wasm_module;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_UNLOAD_MODULE,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t))) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+}
+
+static void *
+instantiate_module(void *wasm_module,
+                   uint32_t stack_size, uint32_t heap_size,
+                   char *error_buf, uint32_t error_buf_size)
+{
+    uint64_t ecall_args[5];
+
+    ecall_args[0] = (uint64_t)(uintptr_t)wasm_module;
+    ecall_args[1] = stack_size;
+    ecall_args[2] = heap_size;
+    ecall_args[3] = (uint64_t)(uintptr_t)error_buf;
+    ecall_args[4] = error_buf_size;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_INSTANTIATE_MODULE,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t) * 5)) {
+        printf("Call ecall_handle_command() failed.\n");
+        return NULL;
+    }
+
+    return (void *)(uintptr_t)ecall_args[0];
+}
+
+static void
+deinstantiate_module(void *wasm_module_inst)
+{
+    uint64_t ecall_args[1];
+
+    ecall_args[0] = (uint64_t)(uintptr_t)wasm_module_inst;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_DEINSTANTIATE_MODULE,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t))) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+}
+
+static bool
+get_exception(void *wasm_module_inst,
+              char *exception, uint32_t exception_size)
+{
+    uint64_t ecall_args[3];
+
+    ecall_args[0] = (uint64_t)(uintptr_t)wasm_module_inst;
+    ecall_args[1] = (uint64_t)(uintptr_t)exception;
+    ecall_args[2] = exception_size;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_GET_EXCEPTION,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t) * 3)) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+
+    return (bool)ecall_args[0];
+}
+
+static void
+app_instance_main(void *wasm_module_inst,
+                  int app_argc, char **app_argv)
+{
+    char exception[128];
+    uint64_t ecall_args_buf[16], *ecall_args = ecall_args_buf;
+    int i, size;
+
+    if (app_argc + 2 > sizeof(ecall_args_buf) / sizeof(uint64_t)) {
+        if (!(ecall_args = (uint64_t *)
+                           malloc(sizeof(uint64_t) * (app_argc + 2)))) {
+            printf("Allocate memory failed.\n");
+            return;
+        }
+    }
+
+    ecall_args[0] = (uintptr_t)wasm_module_inst;
+    ecall_args[1] = app_argc;
+    for (i = 0; i < app_argc; i++) {
+        ecall_args[i + 2] = (uintptr_t)app_argv[i];
+    }
+
+    size = (uint32_t)sizeof(uint64_t) * (app_argc + 2);
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_EXEC_APP_MAIN,
+                                            (uint8_t *)ecall_args,
+                                            size)) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+
+    if (get_exception(wasm_module_inst, exception, sizeof(exception))) {
+        printf("%s\n", exception);
+    }
+
+    if (ecall_args != ecall_args_buf) {
+        free(ecall_args);
+    }
+}
+
+static void
+app_instance_func(void *wasm_module_inst, const char *func_name,
+                  int app_argc, char **app_argv)
+{
+    uint64_t ecall_args_buf[16], *ecall_args = ecall_args_buf;
+    int i, size;
+
+    if (app_argc + 3 > sizeof(ecall_args_buf) / sizeof(uint64_t)) {
+        if (!(ecall_args = (uint64_t *)
+                           malloc(sizeof(uint64_t) * (app_argc + 3)))) {
+            printf("Allocate memory failed.\n");
+            return;
+        }
+    }
+
+    ecall_args[0] = (uintptr_t)wasm_module_inst;
+    ecall_args[1] = (uintptr_t)func_name;
+    ecall_args[2] = (uintptr_t)app_argc;
+    for (i = 0; i < app_argc; i++) {
+        ecall_args[i + 3] = (uintptr_t)app_argv[i];
+    }
+
+    size = (uint32_t)sizeof(uint64_t) * (app_argc + 3);
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_EXEC_APP_FUNC,
+                                            (uint8_t *)ecall_args,
+                                            size)) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+
+    if (ecall_args != ecall_args_buf) {
+        free(ecall_args);
+    }
+}
+
+static bool
+set_wasi_args(void *wasm_module,
+              const char **dir_list, uint32_t dir_list_size,
+              const char **env_list, uint32_t env_list_size,
+              char **argv, uint32_t argc)
+{
+    uint64_t ecall_args[7];
+
+    ecall_args[0] = (uint64_t)(uintptr_t)wasm_module;
+    ecall_args[1] = (uint64_t)(uintptr_t)dir_list;
+    ecall_args[2] = dir_list_size;
+    ecall_args[3] = (uint64_t)(uintptr_t)env_list;
+    ecall_args[4] = env_list_size;
+    ecall_args[5] = (uint64_t)(uintptr_t)argv;
+    ecall_args[6] = argc;
+    if (SGX_SUCCESS != ecall_handle_command(g_eid, CMD_SET_WASI_ARGS,
+                                            (uint8_t *)ecall_args,
+                                            sizeof(uint64_t) * 7)) {
+        printf("Call ecall_handle_command() failed.\n");
+    }
+
+    return (bool)ecall_args[0];
+}
+
 int
-main(int argc, char const *argv[])
+main(int argc, char *argv[])
 {
+    char *wasm_file = NULL;
+    const char *func_name = NULL;
+    uint8_t *wasm_file_buf = NULL;
+    uint32_t wasm_file_size;
+    uint32_t stack_size = 16 * 1024, heap_size = 16 * 1024;
+    void *wasm_module = NULL;
+    void *wasm_module_inst = NULL;
+    char error_buf[128] = { 0 };
+    int log_verbose_level = 2;
+    bool is_repl_mode = false, alloc_with_pool = false;
+    const char *dir_list[8] = { NULL };
+    uint32_t dir_list_size = 0;
+    const char *env_list[8] = { NULL };
+    uint32_t env_list_size = 0;
+
     if (enclave_init(&g_eid) < 0) {
         std::cout << "Fail to initialize enclave." << std::endl;
         return 1;
     }
 
-    ecall_iwasm_main(g_eid);
+#if TEST_OCALL_API != 0
+    {
+        ecall_iwasm_test(g_eid);
+        return 0;
+    }
+#endif
+
+    /* Process options. */
+    for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) {
+        if (!strcmp(argv[0], "-f") || !strcmp(argv[0], "--function")) {
+            argc--, argv++;
+            if (argc < 2) {
+                print_help();
+                return 0;
+            }
+            func_name = argv[0];
+        }
+        else if (!strncmp(argv[0], "-v=", 3)) {
+            log_verbose_level = atoi(argv[0] + 3);
+            if (log_verbose_level < 0 || log_verbose_level > 5)
+                return print_help();
+        }
+        else if (!strcmp(argv[0], "--repl")) {
+            is_repl_mode = true;
+        }
+        else if (!strncmp(argv[0], "--stack-size=", 13)) {
+            if (argv[0][13] == '\0')
+                return print_help();
+            stack_size = atoi(argv[0] + 13);
+        }
+        else if (!strncmp(argv[0], "--heap-size=", 12)) {
+            if (argv[0][12] == '\0')
+                return print_help();
+            heap_size = atoi(argv[0] + 12);
+        }
+        else if (!strncmp(argv[0], "--dir=", 6)) {
+            if (argv[0][6] == '\0')
+                return print_help();
+            if (dir_list_size >= sizeof(dir_list) / sizeof(char *)) {
+                printf("Only allow max dir number %d\n",
+                       (int)(sizeof(dir_list) / sizeof(char *)));
+                return -1;
+            }
+            dir_list[dir_list_size++] = argv[0] + 6;
+        }
+        else if (!strncmp(argv[0], "--env=", 6)) {
+            char *tmp_env;
+
+            if (argv[0][6] == '\0')
+                return print_help();
+            if (env_list_size >= sizeof(env_list) / sizeof(char *)) {
+                printf("Only allow max env number %d\n",
+                       (int)(sizeof(env_list) / sizeof(char *)));
+                return -1;
+            }
+            tmp_env = argv[0] + 6;
+            if (validate_env_str(tmp_env))
+                env_list[env_list_size++] = tmp_env;
+            else {
+                printf("Wasm parse env string failed: expect \"key=value\", "
+                       "got \"%s\"\n",
+                       tmp_env);
+                return print_help();
+            }
+        }
+        else if (!strncmp(argv[0], "--max-threads=", 14)) {
+            if (argv[0][14] == '\0')
+                return print_help();
+            /*wasm_runtime_set_max_thread_num(atoi(argv[0] + 14));*/
+            /* TODO */
+        }
+        else
+            return print_help();
+    }
+
+    if (argc == 0)
+        return print_help();
+
+    wasm_file = argv[0];
+
+    /* Init runtime */
+    if (!init_runtime(alloc_with_pool)) {
+        return -1;
+    }
+
+    /* Set log verbose level */
+    if (!set_log_verbose_level(log_verbose_level)) {
+        goto fail1;
+    }
+
+    /* Load WASM byte buffer from WASM bin file */
+    if (!(wasm_file_buf =
+            (uint8_t *)read_file_to_buffer(wasm_file, &wasm_file_size))) {
+        goto fail1;
+    }
+
+    /* Load module */
+    if (!(wasm_module = load_module(wasm_file_buf, wasm_file_size,
+                                    error_buf, sizeof(error_buf)))) {
+        printf("%s\n", error_buf);
+        goto fail2;
+    }
+
+    /* Set wasi arguments */
+    if (!set_wasi_args(wasm_module, dir_list, dir_list_size,
+                       env_list, env_list_size, argv, argc)) {
+        printf("%s\n", "set wasi arguments failed.\n");
+        goto fail3;
+    }
+
+    /* Instantiate module */
+    if (!(wasm_module_inst = instantiate_module(wasm_module,
+                                                stack_size, heap_size,
+                                                error_buf,
+                                                sizeof(error_buf)))) {
+        printf("%s\n", error_buf);
+        goto fail3;
+    }
+
+    if (is_repl_mode)
+        app_instance_repl(wasm_module_inst, argc, argv);
+    else if (func_name)
+        app_instance_func(wasm_module_inst, func_name,
+                          argc - 1, argv + 1);
+    else
+        app_instance_main(wasm_module_inst, argc, argv);
+
+    /* Deinstantiate module */
+    deinstantiate_module(wasm_module_inst);
+
+fail3:
+    /* Unload module */
+    unload_module(wasm_module);
+
+fail2:
+    /* Free the file buffer */
+    free(wasm_file_buf);
+
+fail1:
+    /* Destroy runtime environment */
+    destroy_runtime();
+
     return 0;
 }
 

product-mini/platforms/linux-sgx/enclave-sample/Enclave/Enclave.config.xml

@@ -3,7 +3,7 @@
   <ProdID>0</ProdID>
   <ISVSVN>0</ISVSVN>
   <StackMaxSize>0x40000</StackMaxSize>
-  <HeapMaxSize>0x200000</HeapMaxSize>
+  <HeapMaxSize>0x400000</HeapMaxSize>
   <ReservedMemMaxSize>0x100000</ReservedMemMaxSize>
   <ReservedMemExecutable>1</ReservedMemExecutable>
   <TCSNum>10</TCSNum>

product-mini/platforms/linux-sgx/enclave-sample/Enclave/Enclave.cpp

@@ -5,50 +5,461 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <inttypes.h>
+#include <stdbool.h>
+
 #include "Enclave_t.h"
-#include "test_wasm.h"
 #include "wasm_export.h"
+#include "bh_platform.h"
+
+extern "C" {
+    typedef void (*os_print_function_t)(const char* message);
+    extern void os_set_print_function(os_print_function_t pf);
 
-static char global_heap_buf[2* 1024 * 1024] = { 0 };
+    void
+    enclave_print(const char *message)
+    {
+        ocall_print(message);
+    }
+}
 
-static int app_argc;
-static char **app_argv;
+typedef enum EcallCmd {
+    CMD_INIT_RUNTIME = 0,     /* wasm_runtime_init/full_init() */
+    CMD_LOAD_MODULE,          /* wasm_runtime_load() */
+    CMD_INSTANTIATE_MODULE,   /* wasm_runtime_instantiate() */
+    CMD_LOOKUP_FUNCTION,      /* wasm_runtime_lookup_function() */
+    CMD_CREATE_EXEC_ENV,      /* wasm_runtime_create_exec_env() */
+    CMD_CALL_WASM,            /* wasm_runtime_call_wasm */
+    CMD_EXEC_APP_FUNC,        /* wasm_application_execute_func() */
+    CMD_EXEC_APP_MAIN,        /* wasm_application_execute_main() */
+    CMD_GET_EXCEPTION,        /* wasm_runtime_get_exception() */
+    CMD_DEINSTANTIATE_MODULE, /* wasm_runtime_deinstantiate() */
+    CMD_UNLOAD_MODULE,        /* wasm_runtime_unload() */
+    CMD_DESTROY_RUNTIME,      /* wasm_runtime_destroy() */
+    CMD_SET_WASI_ARGS,        /* wasm_runtime_set_wasi_args() */
+    CMD_SET_LOG_LEVEL,        /* bh_log_set_verbose_level() */
+    CMD_SET_MAX_THREAD_NUM,   /* wasm_runtime_set_max_thread_num() */
+} EcallCmd;
 
-static void*
-app_instance_main(wasm_module_inst_t module_inst)
+typedef struct EnclaveModule {
+    wasm_module_t module;
+    uint8 *wasm_file;
+    uint32 wasm_file_size;
+    char *wasi_arg_buf;
+    char **wasi_dir_list;
+    uint32 wasi_dir_list_size;
+    char **wasi_env_list;
+    uint32 wasi_env_list_size;
+    char **wasi_argv;
+    uint32 wasi_argc;
+} EnclaveModule;
+
+static char global_heap_buf[2 * 1024 * 1024] = { 0 };
+
+static void
+set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
 {
-    const char *exception;
+    if (error_buf != NULL)
+        snprintf(error_buf, error_buf_size, "%s", string);
+}
 
-    wasm_application_execute_main(module_inst, app_argc, app_argv);
-    if ((exception = wasm_runtime_get_exception(module_inst))) {
-        ocall_print(exception);
-        ocall_print("\n");
+static void
+handle_cmd_init_runtime(uint64 *args, uint32 argc)
+{
+    bool alloc_with_pool = true;
+    RuntimeInitArgs init_args;
+
+    os_set_print_function(enclave_print);
+
+    if (argc > 0) {
+        alloc_with_pool = (bool)args[0];
+    }
+
+    memset(&init_args, 0, sizeof(RuntimeInitArgs));
+
+    if (alloc_with_pool) {
+        init_args.mem_alloc_type = Alloc_With_Pool;
+        init_args.mem_alloc_option.pool.heap_buf = global_heap_buf;
+        init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf);
+    }
+    else {
+        init_args.mem_alloc_type = Alloc_With_System_Allocator;
+    }
+
+    /* initialize runtime environment */
+    if (!wasm_runtime_full_init(&init_args)) {
+        LOG_ERROR("Init runtime environment failed.\n");
+        args[0] = false;
+        return;
     }
-    return NULL;
+
+    args[0] = true;
+
+    LOG_VERBOSE("Init runtime environment success.\n");
 }
 
-extern "C" {
+static void
+handle_cmd_destroy_runtime()
+{
+    wasm_runtime_destroy();
+
+    LOG_VERBOSE("Destroy runtime success.\n");
+}
+
+static void
+handle_cmd_load_module(uint64 *args, uint32 argc)
+{
+    uint64 *args_org = args;
+    char *wasm_file = *(char **)args++;
+    uint32 wasm_file_size = *(uint32 *)args++;
+    char *error_buf = *(char **)args++;
+    uint32 error_buf_size = *(uint32 *)args++;
+    uint64 total_size = sizeof(EnclaveModule) + (uint64)wasm_file_size;
+    EnclaveModule *enclave_module;
+
+    bh_assert(argc == 4);
+
+    if (total_size >= UINT32_MAX
+        || !(enclave_module = (EnclaveModule *)
+                    wasm_runtime_malloc((uint32)total_size))) {
+        set_error_buf(error_buf, error_buf_size,
+                      "WASM module load failed: "
+                      "allocate memory failed.");
+        *(void **)args_org = NULL;
+        return;
+    }
+
+    memset(enclave_module, 0, (uint32)total_size);
+    enclave_module->wasm_file = (uint8 *)enclave_module
+                                + sizeof(EnclaveModule);
+    bh_memcpy_s(enclave_module->wasm_file, wasm_file_size,
+                wasm_file, wasm_file_size);
+
+    if (!(enclave_module->module =
+                wasm_runtime_load(enclave_module->wasm_file, wasm_file_size,
+                                  error_buf, error_buf_size))) {
+        *(void **)args_org = NULL;
+        return;
+    }
+
+    *(EnclaveModule **)args_org = enclave_module;
+
+    LOG_VERBOSE("Load module success.\n");
+}
+
+static void
+handle_cmd_unload_module(uint64 *args, uint32 argc)
+{
+    EnclaveModule *enclave_module = *(EnclaveModule **)args++;
+    uint32 i;
+
+    bh_assert(argc == 1);
+
+    if (enclave_module->wasi_arg_buf)
+        wasm_runtime_free(enclave_module->wasi_arg_buf);
+
+    wasm_runtime_unload(enclave_module->module);
+    wasm_runtime_free(enclave_module);
 
-typedef void (*os_print_function_t)(const char* message);
-extern void os_set_print_function(os_print_function_t pf);
+    LOG_VERBOSE("Unload module success.\n");
+}
 
-void enclave_print(const char *message)
+static void
+handle_cmd_instantiate_module(uint64 *args, uint32 argc)
 {
-    ocall_print(message);
+    uint64 *args_org = args;
+    EnclaveModule *enclave_module = *(EnclaveModule **)args++;
+    uint32 stack_size = *(uint32 *)args++;
+    uint32 heap_size = *(uint32 *)args++;
+    char *error_buf = *(char **)args++;
+    uint32 error_buf_size = *(uint32 *)args++;
+    wasm_module_inst_t module_inst;
+
+    bh_assert(argc == 5);
+
+    if (!(module_inst =
+                wasm_runtime_instantiate(enclave_module->module,
+                                         stack_size, heap_size,
+                                         error_buf, error_buf_size))) {
+        *(void **)args_org = NULL;
+        return;
+    }
+
+    *(wasm_module_inst_t *)args_org = module_inst;
+
+    LOG_VERBOSE("Instantiate module success.\n");
 }
 
+static void
+handle_cmd_deinstantiate_module(uint64 *args, uint32 argc)
+{
+    wasm_module_inst_t module_inst = *(wasm_module_inst_t *)args++;
+
+    bh_assert(argc == 1);
+
+    wasm_runtime_deinstantiate(module_inst);
+
+    LOG_VERBOSE("Deinstantiate module success.\n");
 }
 
-void ecall_iwasm_main()
+static void
+handle_cmd_get_exception(uint64 *args, uint32 argc)
 {
-    os_set_print_function(enclave_print);
+    uint64 *args_org = args;
+    wasm_module_inst_t module_inst = *(wasm_module_inst_t *)args++;
+    char *exception = *(char **)args++;
+    uint32 exception_size = *(uint32 *)args++;
+    const char *exception1;
+
+    bh_assert(argc == 3);
+
+    if ((exception1 = wasm_runtime_get_exception(module_inst))) {
+        snprintf(exception, exception_size,
+                 "%s", exception1);
+        args_org[0] = true;
+    }
+    else {
+        args_org[0] = false;
+    }
+}
 
-    uint8_t *wasm_file_buf = NULL;
-    int wasm_file_size;
+static void
+handle_cmd_exec_app_main(uint64 *args, int32 argc)
+{
+    wasm_module_inst_t module_inst = *(wasm_module_inst_t *)args++;
+    uint32 app_argc = *(uint32 *)args++;
+    char **app_argv = NULL;
+    uint64 total_size;
+    int32 i;
+
+    bh_assert(argc >= 3);
+    bh_assert(app_argc >= 1);
+
+    total_size = sizeof(char *) * (app_argc > 2 ? (uint64)app_argc : 2);
+
+    if (total_size >= UINT32_MAX
+        || !(app_argv = (char **)wasm_runtime_malloc(total_size))) {
+        wasm_runtime_set_exception(module_inst, "allocate memory failed.");
+        return;
+    }
+
+    for (i = 0; i < app_argc; i++) {
+        app_argv[i] = (char *)(uintptr_t)args[i];
+    }
+
+    wasm_application_execute_main(module_inst, app_argc - 1, app_argv + 1);
+
+    wasm_runtime_free(app_argv);
+}
+
+static void
+handle_cmd_exec_app_func(uint64 *args, int32 argc)
+{
+    wasm_module_inst_t module_inst = *(wasm_module_inst_t *)args++;
+    char *func_name = *(char **)args++;
+    uint32 app_argc = *(uint32 *)args++;
+    char **app_argv = NULL;
+    uint64 total_size;
+    int32 i, func_name_len = strlen(func_name);
+
+    bh_assert(argc == app_argc + 3);
+
+    total_size = sizeof(char *) * (app_argc > 2 ? (uint64)app_argc : 2);
+
+    if (total_size >= UINT32_MAX
+        || !(app_argv = (char **)wasm_runtime_malloc(total_size))) {
+        wasm_runtime_set_exception(module_inst, "allocate memory failed.");
+        return;
+    }
+
+    for (i = 0; i < app_argc; i++) {
+        app_argv[i] = (char *)(uintptr_t)args[i];
+    }
+
+    wasm_application_execute_func(module_inst, func_name, app_argc, app_argv);
+
+    wasm_runtime_free(app_argv);
+}
+
+static void
+handle_cmd_set_log_level(uint64 *args, uint32 argc)
+{
+#if WASM_ENABLE_LOG != 0
+    LOG_VERBOSE("Set log verbose level to %d.\n", (int)args[0]);
+    bh_log_set_verbose_level((int)args[0]);
+#endif
+}
+
+static void
+handle_cmd_set_max_thread_num(uint64 *args, uint32 argc)
+{
+#if WASM_ENABLE_LIB_PTHREAD != 0
+    LOG_VERBOSE("Set max thread num to %d.\n", (uint32)args[0]);
+    wasm_runtime_set_max_thread_num((uint32)args[0]);
+#endif
+}
+
+static void
+handle_cmd_set_wasi_args(uint64 *args, int32 argc)
+{
+    uint64 *args_org = args;
+    EnclaveModule *enclave_module = *(EnclaveModule **)args++;
+    char **dir_list = *(char ***)args++;
+    uint32 dir_list_size = *(uint32 *)args++;
+    char **env_list = *(char ***)args++;
+    uint32 env_list_size = *(uint32 *)args++;
+    char **wasi_argv = *(char ***)args++;
+    char *p, *p1;
+    uint32 wasi_argc = *(uint32 *)args++;
+    uint64 total_size = 0;
+    int32 i, str_len;
+
+    bh_assert(argc == 7);
+
+    total_size += sizeof(char *) * (uint64)dir_list_size
+                  + sizeof(char *) * (uint64)env_list_size
+                  + sizeof(char *) * (uint64)wasi_argc;
+
+    for (i = 0; i < dir_list_size; i++) {
+        total_size += strlen(dir_list[i]) + 1;
+    }
+
+    for (i = 0; i < env_list_size; i++) {
+        total_size += strlen(env_list[i]) + 1;
+    }
+
+    for (i = 0; i < wasi_argc; i++) {
+        total_size += strlen(wasi_argv[i]) + 1;
+    }
+
+    if (total_size >= UINT32_MAX
+        || !(enclave_module->wasi_arg_buf = p = (char *)
+                    wasm_runtime_malloc((uint32)total_size))) {
+        *args_org = false;
+        return;
+    }
+
+    p1 = p + sizeof(char *) * dir_list_size
+           + sizeof(char *) * env_list_size
+           + sizeof(char *) * wasi_argc;
+
+    if (dir_list_size > 0) {
+        enclave_module->wasi_dir_list = (char **)p;
+        enclave_module->wasi_dir_list_size = dir_list_size;
+        for (i = 0; i < dir_list_size; i++) {
+            enclave_module->wasi_dir_list[i] = p1;
+            str_len = strlen(dir_list[i]);
+            bh_memcpy_s(p1, str_len + 1, dir_list[i], str_len + 1);
+            p1 += str_len + 1;
+        }
+        p += sizeof(char *) * dir_list_size;
+    }
+
+    if (env_list_size > 0) {
+        enclave_module->wasi_env_list = (char **)p;
+        enclave_module->wasi_env_list_size = env_list_size;
+        for (i = 0; i < env_list_size; i++) {
+            enclave_module->wasi_env_list[i] = p1;
+            str_len = strlen(env_list[i]);
+            bh_memcpy_s(p1, str_len + 1, env_list[i], str_len + 1);
+            p1 += str_len + 1;
+        }
+        p += sizeof(char *) * env_list_size;
+    }
+
+    if (wasi_argc > 0) {
+        enclave_module->wasi_argv = (char **)p;
+        enclave_module->wasi_argc = wasi_argc;
+        for (i = 0; i < wasi_argc; i++) {
+            enclave_module->wasi_argv[i] = p1;
+            str_len = strlen(wasi_argv[i]);
+            bh_memcpy_s(p1, str_len + 1, wasi_argv[i], str_len + 1);
+            p1 += str_len + 1;
+        }
+        p += sizeof(char *) * wasi_argc;
+    }
+
+#if WASM_ENABLE_LIBC_WASI != 0
+    wasm_runtime_set_wasi_args(enclave_module->module,
+                               (const char **)enclave_module->wasi_dir_list,
+                               dir_list_size,
+                               NULL, 0,
+                               (const char **)enclave_module->wasi_env_list,
+                               env_list_size,
+                               envclave_module->wasi_argv,
+                               envclave_module->wasi_argc);
+#endif
+
+    *args_org = true;
+}
+
+void
+ecall_handle_command(unsigned cmd,
+                     unsigned char *cmd_buf,
+                     unsigned cmd_buf_size)
+{
+    uint64 *args = (uint64 *)cmd_buf;
+    uint32 argc = cmd_buf_size / sizeof(uint64);
+
+    switch (cmd) {
+        case CMD_INIT_RUNTIME:
+            handle_cmd_init_runtime(args, argc);
+            break;
+        case CMD_LOAD_MODULE:
+            handle_cmd_load_module(args, argc);
+            break;
+        case CMD_SET_WASI_ARGS:
+            handle_cmd_set_wasi_args(args, argc);
+            break;
+        case CMD_INSTANTIATE_MODULE:
+            handle_cmd_instantiate_module(args, argc);
+            break;
+        case CMD_LOOKUP_FUNCTION:
+            break;
+        case CMD_CREATE_EXEC_ENV:
+            break;
+        case CMD_CALL_WASM:
+            break;
+        case CMD_EXEC_APP_FUNC:
+            handle_cmd_exec_app_func(args, argc);
+            break;
+        case CMD_EXEC_APP_MAIN:
+            handle_cmd_exec_app_main(args, argc);
+            break;
+        case CMD_GET_EXCEPTION:
+            handle_cmd_get_exception(args, argc);
+            break;
+        case CMD_DEINSTANTIATE_MODULE:
+            handle_cmd_deinstantiate_module(args, argc);
+            break;
+        case CMD_UNLOAD_MODULE:
+            handle_cmd_unload_module(args, argc);
+            break;
+        case CMD_DESTROY_RUNTIME:
+            handle_cmd_destroy_runtime();
+            break;
+        case CMD_SET_LOG_LEVEL:
+            handle_cmd_set_log_level(args, argc);
+            break;
+        case CMD_SET_MAX_THREAD_NUM:
+            handle_cmd_set_max_thread_num(args, argc);
+            break;
+        default:
+            LOG_ERROR("Unknown command %d\n", cmd);
+            break;
+    }
+}
+
+void
+ecall_iwasm_main(uint8_t *wasm_file_buf, uint32_t wasm_file_size)
+{
     wasm_module_t wasm_module = NULL;
     wasm_module_inst_t wasm_module_inst = NULL;
     RuntimeInitArgs init_args;
     char error_buf[128];
+    const char *exception;
+
+    os_set_print_function(enclave_print);
 
     memset(&init_args, 0, sizeof(RuntimeInitArgs));
 
@@ -63,10 +474,6 @@ void ecall_iwasm_main()
         return;
     }
 
-    /* load WASM byte buffer from byte buffer of include file */
-    wasm_file_buf = (uint8_t*) wasm_test_file;
-    wasm_file_size = sizeof(wasm_test_file);
-
     /* load WASM module */
     if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size,
                                           error_buf, sizeof(error_buf)))) {
@@ -87,7 +494,11 @@ void ecall_iwasm_main()
     }
 
     /* execute the main function of wasm app */
-    app_instance_main(wasm_module_inst);
+    wasm_application_execute_main(wasm_module_inst, 0, NULL);
+    if ((exception = wasm_runtime_get_exception(wasm_module_inst))) {
+        ocall_print(exception);
+        ocall_print("\n");
+    }
 
     /* destroy the module instance */
     wasm_runtime_deinstantiate(wasm_module_inst);
@@ -100,4 +511,3 @@ fail1:
     /* destroy runtime environment */
     wasm_runtime_destroy();
 }
-

product-mini/platforms/linux-sgx/enclave-sample/Enclave/Enclave.edl

@@ -8,7 +8,11 @@ enclave {
 
     trusted {
         /* define ECALLs here. */
-        public void ecall_iwasm_main(void);
+        public void ecall_handle_command(unsigned cmd,
+                                         [in, out, size=cmd_buf_size]uint8_t *cmd_buf,
+                                         unsigned cmd_buf_size);
+        public void ecall_iwasm_main([user_check]uint8_t *wasm_file_buf,
+                                     uint32_t wasm_file_size);
     };
 
     untrusted {

product-mini/platforms/linux-sgx/enclave-sample/Makefile

@@ -6,6 +6,7 @@
 SGX_SDK ?= /opt/intel/sgxsdk
 SGX_MODE ?= SIM
 SGX_ARCH ?= x64
+SGX_DEBUG ?= 0
 
 ifeq ($(shell getconf LONG_BIT), 32)
 	SGX_ARCH := x86
@@ -73,7 +74,7 @@ endif
 
 App_Cpp_Objects := $(App_Cpp_Files:.cpp=.o)
 
-App_Name := app
+App_Name := iwasm
 
 ######## Enclave Settings ########
 
@@ -89,15 +90,20 @@ Crypto_Library_Name := sgx_tcrypto
 WAMR_ROOT := $(CURDIR)/../../../../
 
 Enclave_Cpp_Files := Enclave/Enclave.cpp
+
 Enclave_Include_Paths := -IEnclave -I$(WAMR_ROOT)/core/iwasm/include \
-                         -I$(SGX_SDK)/include -I$(SGX_SDK)/include/tlibc -I$(SGX_SDK)/include/stlport
+                         -I$(WAMR_ROOT)/core/shared/utils \
+                         -I$(WAMR_ROOT)/core/shared/platform/linux-sgx \
+                         -I$(SGX_SDK)/include \
+                         -I$(SGX_SDK)/include/tlibc \
+                         -I$(SGX_SDK)/include/stlport
 
 Enclave_C_Flags := $(SGX_COMMON_CFLAGS) -nostdinc -fvisibility=hidden -fpie -fstack-protector $(Enclave_Include_Paths)
 Enclave_Cpp_Flags := $(Enclave_C_Flags) -std=c++03 -nostdinc++
 Enclave_Link_Flags := $(SGX_COMMON_CFLAGS) -Wl,--no-undefined -nostdlib -nodefaultlibs -nostartfiles -L$(SGX_LIBRARY_PATH) \
 	-Wl,--whole-archive -l$(Trts_Library_Name) -Wl,--no-whole-archive \
     libvmlib.a \
-	-Wl,--start-group -lsgx_tstdc -lsgx_tcxx -l$(Crypto_Library_Name) -l$(Service_Library_Name) -Wl,--end-group \
+	-Wl,--start-group -lsgx_tstdc -lsgx_tcxx -lsgx_pthread -l$(Crypto_Library_Name) -l$(Service_Library_Name) -Wl,--end-group \
 	-Wl,-Bstatic -Wl,-Bsymbolic -Wl,--no-undefined \
 	-Wl,-pie,-eenclave_entry -Wl,--export-dynamic  \
 	-Wl,--defsym,__ImageBase=0
@@ -140,7 +146,7 @@ endif
 ######## App Objects ########
 
 App/Enclave_u.c: $(SGX_EDGER8R) Enclave/Enclave.edl
-	@cd App && $(SGX_EDGER8R) --untrusted ../Enclave/Enclave.edl --search-path ../Enclave --search-path $(SGX_SDK)/include
+	@cd App && $(SGX_EDGER8R) --untrusted ../Enclave/Enclave.edl --search-path ../Enclave --search-path $(SGX_SDK)/include --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx
 	@echo "GEN  =>  $@"
 
 App/Enclave_u.o: App/Enclave_u.c
@@ -159,7 +165,7 @@ $(App_Name): App/Enclave_u.o $(App_Cpp_Objects)
 ######## Enclave Objects ########
 
 Enclave/Enclave_t.c: $(SGX_EDGER8R) Enclave/Enclave.edl
-	@cd Enclave && $(SGX_EDGER8R) --trusted ../Enclave/Enclave.edl --search-path ../Enclave --search-path $(SGX_SDK)/include
+	@cd Enclave && $(SGX_EDGER8R) --trusted ../Enclave/Enclave.edl --search-path ../Enclave --search-path $(SGX_SDK)/include --search-path $(WAMR_ROOT)/core/shared/platform/linux-sgx
 	@echo "GEN  =>  $@"
 
 Enclave/Enclave_t.o: Enclave/Enclave_t.c
@@ -185,4 +191,4 @@ $(Signed_Enclave_Name): $(Enclave_Name)
 .PHONY: clean
 
 clean:
-	@rm -f $(App_Name) $(Enclave_Name) $(Signed_Enclave_Name) $(App_Cpp_Objects) App/Enclave_u.* $(Enclave_Cpp_Objects) Enclave/Enclave_t.*
+	@rm -f $(App_Name) $(Enclave_Name) $(Signed_Enclave_Name) $(App_Cpp_Objects) App/Enclave_u.* $(Enclave_Cpp_Objects) Enclave/Enclave_t.* libvmlib.a