sdk/gramine: add (experimental!) running instructions.

Gramine support is still experimental and (mostly) untested.. Currently helloworld example runs, but it seems to crash once the custom AEP stub calls into libc.
jovanbulck · Oct 12, 2022 · 4a425f0 · 4a425f0
1 parent 5bad772
commit 4a425f0
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 41 deletions.
diff --git a/sdk/gramine/0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch b/sdk/gramine/0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch
@@ -1,67 +1,67 @@
-From f3d05a71a513445e9c3eec9e2ab6d4ab30f52d46 Mon Sep 17 00:00:00 2001
+From 01802a7641e0090e3a4926239ca004d7de2e6a5c Mon Sep 17 00:00:00 2001
 From: Jo Van Bulck <jo.vanbulck@cs.kuleuven.be>
-Date: Wed, 12 Oct 2022 16:58:07 +0200
+Date: Wed, 12 Oct 2022 19:14:24 +0200
 Subject: [PATCH 1/2] SGX-Step Gramine patches to reconfigure AEP/TCS.
 
 ---
- pal/src/host/linux-sgx/host_entry.S | 50 +++++++++++++++++++++++++++--
- pal/src/host/linux-sgx/meson.build  | 11 +++++++
- 2 files changed, 58 insertions(+), 3 deletions(-)
+ pal/src/host/linux-sgx/host_entry.S     | 52 +++++++++++++++++++++++--
+ pal/src/host/linux-sgx/host_exception.c | 13 +++++++
+ pal/src/host/linux-sgx/meson.build      | 11 ++++++
+ 3 files changed, 73 insertions(+), 3 deletions(-)
 
 diff --git a/pal/src/host/linux-sgx/host_entry.S b/pal/src/host/linux-sgx/host_entry.S
-index bb38ced4..b84803d2 100644
+index bb38ced4..013323d7 100644
 --- a/pal/src/host/linux-sgx/host_entry.S
 +++ b/pal/src/host/linux-sgx/host_entry.S
-@@ -2,6 +2,21 @@
+@@ -2,6 +2,18 @@
 
  #include "asm-offsets.h"
 
++/************************************************************************/
++/* XXX AEP hook for SGX-Step support */
 +    .data
++    .global g_aep_pointer
 +g_aep_pointer:
 +    .word 0x0
 +    .word 0x0
 +    .word 0x0
 +    .word 0x0
-+
-+    .data
-+g_tcs:
-+    .word 0x0
-+    .word 0x0
-+    .word 0x0
-+    .word 0x0
++/************************************************************************/
 +
 +    .text
      .extern tcs_base
      .extern g_in_aex_profiling
 
-@@ -36,11 +51,18 @@ sgx_ecall:
-
-     # RBX has to be the TCS of the thread
+@@ -38,9 +50,17 @@ sgx_ecall:
      movq %gs:PAL_HOST_TCB_TCS, %rbx
-+    leaq        g_tcs(%rip), %rax
-+    movq        %rbx, (%rax)
 
      # RCX has to be the AEP (Asynchronous Exit Pointer)
 -    leaq async_exit_pointer(%rip), %rcx
 -
 -    movq $EENTER, %rax
++    /************************************************************************/
++    /* XXX support hookable AEP for SGX-Step */
 +    leaq        g_aep_pointer(%rip), %rax
 +    movq        (%rax), %rcx                    /* aep addr */
 +    cmp         $0x0, %rcx
 +    jnz 1f
 +    leaq        async_exit_pointer(%rip), %rcx
 +    movq        %rcx, (%rax)
++    /************************************************************************/
 +
 +1:  movq        $EENTER, %rax                /* EENTER leaf */
      ENCLU
 
      # currently only ECALL_THREAD_RESET returns
-@@ -178,3 +200,25 @@ sgx_raise:
+@@ -178,3 +198,29 @@ sgx_raise:
      # RSI - external event
      jmp .Ldo_ecall
      .cfi_endproc
 +
 +
++/************************************************************************/
++/* XXX getter functions to be used in libsgxstep */
++
 +.macro DECLARE_GLOBAL_FUNC name
 +    .globl \name
 +    .type \name, @function
@@ -79,11 +79,39 @@ index bb38ced4..b84803d2 100644
 +    ret
 +
 +DECLARE_GLOBAL_FUNC sgx_get_tcs
-+    leaq        g_tcs(%rip), %rax
-+    movq        (%rax), %rax
++    movq %gs:PAL_HOST_TCB_TCS, %rax
 +    ret
++
++/************************************************************************/
+diff --git a/pal/src/host/linux-sgx/host_exception.c b/pal/src/host/linux-sgx/host_exception.c
+index f0aae5bd..7bdb092d 100644
+--- a/pal/src/host/linux-sgx/host_exception.c
++++ b/pal/src/host/linux-sgx/host_exception.c
+@@ -85,9 +85,22 @@ static enum pal_event signal_to_pal_event(int sig) {
+     }
+ }
+
++/* XXX support hookable AEP for SGX-Step */
++extern void *g_aep_pointer;
++
+ static bool interrupted_in_enclave(struct ucontext* uc) {
+     unsigned long rip = ucontext_get_ip(uc);
+
++    /************************************************************************/
++    if (g_aep_pointer)
++    {
++        /* XXX Note: we do _not_ report to the enclave if an exception happens
++         * _inside_ our custom SGX-Step AEP (i.e., contrary to the original
++         * Gramine code below). */
++        return (rip == (unsigned long) g_aep_pointer);
++    }
++    /************************************************************************/
++
+     /* in case of AEX, RIP can point to any instruction in the AEP/ERESUME trampoline code, i.e.,
+      * RIP can point to anywhere in [async_exit_pointer, async_exit_pointer_end) interval */
+     return rip >= (unsigned long)async_exit_pointer && rip < (unsigned long)async_exit_pointer_end;
 diff --git a/pal/src/host/linux-sgx/meson.build b/pal/src/host/linux-sgx/meson.build
-index 3e77bdcc..c40794fa 100644
+index 3e77bdcc..a1776fb8 100644
 --- a/pal/src/host/linux-sgx/meson.build
 +++ b/pal/src/host/linux-sgx/meson.build
 @@ -152,10 +152,21 @@ libpal_sgx_host = executable('loader',
@@ -92,7 +120,7 @@ index 3e77bdcc..c40794fa 100644
          '-fstack-protector-strong',
 +
 +        # XXX add libsgxstep include path
-+        '-I../../..',
++        '-I../../../..',
 +        # XXX FORTIFY_SOURCE conflicts with regular libc header includes in libsgxstep
 +        '-D_FORTIFY_SOURCE=0',
 +        # XXX libsgxstep needs Intel SDK headers (only for error reporting; can be hacked out if needed)
@@ -103,7 +131,7 @@ index 3e77bdcc..c40794fa 100644
          '-Wl,-znow',
 +
 +        # XXX link in libsgxstep library
-+        '-L../../../libsgxstep',
++        '-L../../../../libsgxstep',
 +        '-lsgx-step',
      ],
      pie: true,

diff --git a/sdk/gramine/0002-Example-usage-of-libsgxstep-functionality-on-Gramine.patch b/sdk/gramine/0002-Example-usage-of-libsgxstep-functionality-on-Gramine.patch
@@ -1,18 +1,18 @@
-From 0a276da7b6ba82ced095a795f66ac7f3ad0870a1 Mon Sep 17 00:00:00 2001
+From e973524bb67c1d6ff1668c62a097f4fee8260e8b Mon Sep 17 00:00:00 2001
 From: Jo Van Bulck <jo.vanbulck@cs.kuleuven.be>
-Date: Wed, 12 Oct 2022 16:58:44 +0200
+Date: Wed, 12 Oct 2022 19:17:45 +0200
 Subject: [PATCH 2/2] Example usage of libsgxstep functionality on Gramine's
  enclave entry.
 
 ---
- pal/src/host/linux-sgx/host_ecalls.c | 35 ++++++++++++++++++++++++++++
- 1 file changed, 35 insertions(+)
+ pal/src/host/linux-sgx/host_ecalls.c | 30 ++++++++++++++++++++++++++++
+ 1 file changed, 30 insertions(+)
 
 diff --git a/pal/src/host/linux-sgx/host_ecalls.c b/pal/src/host/linux-sgx/host_ecalls.c
-index 9387266b..a740df24 100644
+index 9387266b..d05487b9 100644
 --- a/pal/src/host/linux-sgx/host_ecalls.c
 +++ b/pal/src/host/linux-sgx/host_ecalls.c
-@@ -6,6 +6,31 @@
+@@ -6,6 +6,26 @@
  #include "pal_ecall_types.h"
  #include "pal_rpc_queue.h"
 
@@ -29,22 +29,17 @@ index 9387266b..a740df24 100644
 +    
 +    void aep_cb_func(void)
 +    {
-+        gprsgx_region_t gprsgx = {0};
-+        uint64_t erip = edbgrd_erip() - (uint64_t) get_enclave_base();
-+        info("Hello world from AEP callback %d with erip=%#llx! Resuming enclave..", aep_fired, erip); 
-+    
-+        edbgrd(get_enclave_ssa_gprsgx_adrs(), &gprsgx, sizeof(gprsgx_region_t));
-+        dump_gprsgx_region(&gprsgx);
-+    
 +        aep_fired++;
++        return;
++        /* NOTE: somehow calling libc functions here crashes the untrusted runtime... */
 +    }
 +#endif
 +/*****************************************************************************/
 +
  int ecall_enclave_start(char* libpal_uri, char* args, size_t args_size, char* env,
                          size_t env_size, int parent_stream_fd, sgx_target_info_t* qe_targetinfo,
                          struct pal_topo_info* topo_info, struct pal_dns_host_conf* dns_conf) {
-@@ -32,6 +57,16 @@ int ecall_enclave_start(char* libpal_uri, char* args, size_t args_size, char* en
+@@ -32,6 +52,16 @@ int ecall_enclave_start(char* libpal_uri, char* args, size_t args_size, char* en
      ms.ms_topo_info        = topo_info;
      ms.ms_dns_host_conf    = dns_conf;
      ms.rpc_queue           = g_rpc_queue;

diff --git a/sdk/gramine/README.md b/sdk/gramine/README.md
@@ -9,8 +9,17 @@
 > Intel SDK. As always, issues/PRs are welcome if you want to contribute
 > improvements for a work-in-progress Gramine port.
 
-1. First, apply the patches in the untrusted Gramine runtime `host_entry.S` to
-be able to link to `libsgxstep`:
+### Building the patched Gramine
+
+0. First, make sure to build `libsgxstep.a`:
+
+```bash
+$ cd ../../libsgxstep
+$ make clean all
+```
+
+1. Apply the patches in the untrusted Gramine runtime `host_entry.S` to
+be able to link to `libsgxstep.a`:
 
 ```bash
 $ ./patch_entry.sh
@@ -21,6 +30,7 @@ properly applied in the modified Gramine loader:
 
 ```bash
 $ cd gramine
+$ meson setup build/ --buildtype=release -Dsgx=enabled
 $ meson configure build/ -Dsgx=enabled
 $ ninja -C build
 $ objdump -d build/pal/src/host/linux-sgx/loader | grep sgx_set_aep
@@ -38,3 +48,20 @@ $ ninja -C build
 $ objdump -d build/pal/src/host/linux-sgx/loader | grep sgx_step
    18631:	48 8d 05 2c 1c 00 00 	lea    0x1c2c(%rip),%rax        # 1a264 <sgx_step_aep_trampoline>
 ```
+
+4. Install the patched Gramine.
+
+```bash
+$ sudo ninja -C build/ install
+```
+### Running a sample application with the patched Gramine
+
+Proceed as follows:
+
+```bash
+$ export PYTHONPATH=/usr/local  # Fix https://github.com/gramineproject/gramine/issues/492
+$ gramine-sgx-gen-private-key
+$ cd CI-Examples/helloworld
+$ make SGX=1
+$ gramine-sgx helloworld
+```