From ed4f06d4bbfb002a955360ba54e8b8a04f505b0c Mon Sep 17 00:00:00 2001 From: Anja Rabich Date: Thu, 15 Dec 2022 20:33:43 +0100 Subject: [PATCH] Gramine single-stepping example --- sdk/gramine/0000-Libsgxstep-patches.patch | 43 +++ ...amine-patches-to-reconfigure-AEP-TCS.patch | 87 +++--- ...2-helloworld-with-Nemesis-in-Gramine.patch | 269 ++++++++++++++++++ sdk/gramine/README.md | 98 +++++-- sdk/gramine/patch_ecall.sh | 4 +- sdk/gramine/patch_entry.sh | 2 - sdk/gramine/patch_sgxstep.sh | 4 + 7 files changed, 449 insertions(+), 58 deletions(-) create mode 100644 sdk/gramine/0000-Libsgxstep-patches.patch create mode 100644 sdk/gramine/0002-helloworld-with-Nemesis-in-Gramine.patch mode change 100755 => 100644 sdk/gramine/patch_ecall.sh mode change 100755 => 100644 sdk/gramine/patch_entry.sh create mode 100755 sdk/gramine/patch_sgxstep.sh diff --git a/sdk/gramine/0000-Libsgxstep-patches.patch b/sdk/gramine/0000-Libsgxstep-patches.patch new file mode 100644 index 0000000..d716a2e --- /dev/null +++ b/sdk/gramine/0000-Libsgxstep-patches.patch @@ -0,0 +1,43 @@ +diff --git a/libsgxstep/apic.c b/libsgxstep/apic.c +index 67e4f5e..a4b707e 100644 +--- a/libsgxstep/apic.c ++++ b/libsgxstep/apic.c +@@ -27,7 +27,7 @@ + #include "../kernel/sgxstep_ioctl.h" + + extern void *apic_base; +-void *dummy_pt = NULL; ++//void *dummy_pt = NULL; + uint32_t apic_lvtt = 0x0, apic_tdcr = 0x0; + + /* +diff --git a/libsgxstep/apic.h b/libsgxstep/apic.h +index 94a9a52..20857c6 100644 +--- a/libsgxstep/apic.h ++++ b/libsgxstep/apic.h +@@ -72,7 +72,7 @@ static inline int apic_write(uint32_t reg, uint32_t v) + if (!apic_base) apic_init(); + + addr = (volatile uint32_t *)(apic_base + reg); +- asm volatile ("movl %1, %0\n\t" ++ __asm__ volatile ("movl %1, %0\n\t" + :"=m"(*addr):"r"(v):); + + return 0; +diff --git a/libsgxstep/enclave.h b/libsgxstep/enclave.h +index 16855ec..c78bfbf 100644 +--- a/libsgxstep/enclave.h ++++ b/libsgxstep/enclave.h +@@ -58,10 +58,10 @@ int edbgrdwr(void *adrs, void* res, int len, int write); + + /* HACK: to avoid having to retrieve the SSA framesize from the untrusted + runtime (driver), we assume a standard/hard-coded SSA framesize of 1 page */ +-#define SGX_SSAFRAMESIZE 4096 ++//#define SGX_SSAFRAMESIZE 4096 + //TODO determine this at runtime.. + // SSA framesize for Gramine seems to be as follows +-// #define SGX_SSAFRAMESIZE 16384 ++#define SGX_SSAFRAMESIZE 16384 + + struct gprsgx_region { + uint64_t rax; diff --git a/sdk/gramine/0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch b/sdk/gramine/0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch index 580f6fa..71da8bd 100644 --- a/sdk/gramine/0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch +++ b/sdk/gramine/0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch @@ -1,22 +1,27 @@ -From 01802a7641e0090e3a4926239ca004d7de2e6a5c Mon Sep 17 00:00:00 2001 -From: Jo Van Bulck -Date: Wed, 12 Oct 2022 19:14:24 +0200 -Subject: [PATCH 1/2] SGX-Step Gramine patches to reconfigure AEP/TCS. - ---- - pal/src/host/linux-sgx/host_entry.S | 52 +++++++++++++++++++++++-- - pal/src/host/linux-sgx/host_exception.c | 13 +++++++ - pal/src/host/linux-sgx/meson.build | 11 ++++++ - 3 files changed, 73 insertions(+), 3 deletions(-) - +diff --git a/meson.build b/meson.build +index 85dfcc6b..faae77cc 100644 +--- a/meson.build ++++ b/meson.build +@@ -272,6 +272,10 @@ if sgx + sgx_dcap_quoteverify_dep = cc.find_library('sgx_dcap_quoteverify') + endif + ++ # XXX taken from: https://stackoverflow.com/questions/34663124/link-static-library-to-shared-library-or-to-a-binary ++ libsgxstepdir = '' ++ libsgxstep_dep = cc.find_library('libsgx-step', dirs : libsgxstepdir) ++ + vtune_dep = [] + dl_dep = [] + diff --git a/pal/src/host/linux-sgx/host_entry.S b/pal/src/host/linux-sgx/host_entry.S -index bb38ced4..013323d7 100644 +index bb38ced4..f53ea26c 100644 --- a/pal/src/host/linux-sgx/host_entry.S +++ b/pal/src/host/linux-sgx/host_entry.S -@@ -2,6 +2,18 @@ +@@ -1,7 +1,18 @@ + #include "sgx_arch.h" #include "asm-offsets.h" - +- +/************************************************************************/ +/* XXX AEP hook for SGX-Step support */ + .data @@ -32,7 +37,7 @@ index bb38ced4..013323d7 100644 .extern tcs_base .extern g_in_aex_profiling -@@ -38,9 +50,17 @@ sgx_ecall: +@@ -38,9 +49,17 @@ sgx_ecall: movq %gs:PAL_HOST_TCB_TCS, %rbx # RCX has to be the AEP (Asynchronous Exit Pointer) @@ -53,7 +58,7 @@ index bb38ced4..013323d7 100644 ENCLU # currently only ECALL_THREAD_RESET returns -@@ -178,3 +198,29 @@ sgx_raise: +@@ -178,3 +197,29 @@ sgx_raise: # RSI - external event jmp .Ldo_ecall .cfi_endproc @@ -84,10 +89,10 @@ index bb38ced4..013323d7 100644 + +/************************************************************************/ diff --git a/pal/src/host/linux-sgx/host_exception.c b/pal/src/host/linux-sgx/host_exception.c -index f0aae5bd..7bdb092d 100644 +index f0aae5bd..670ec638 100644 --- a/pal/src/host/linux-sgx/host_exception.c +++ b/pal/src/host/linux-sgx/host_exception.c -@@ -85,9 +85,22 @@ static enum pal_event signal_to_pal_event(int sig) { +@@ -85,8 +85,21 @@ static enum pal_event signal_to_pal_event(int sig) { } } @@ -96,7 +101,7 @@ index f0aae5bd..7bdb092d 100644 + static bool interrupted_in_enclave(struct ucontext* uc) { unsigned long rip = ucontext_get_ip(uc); - ++ + /************************************************************************/ + if (g_aep_pointer) + { @@ -106,34 +111,48 @@ index f0aae5bd..7bdb092d 100644 + return (rip == (unsigned long) g_aep_pointer); + } + /************************************************************************/ -+ + /* in case of AEX, RIP can point to any instruction in the AEP/ERESUME trampoline code, i.e., * RIP can point to anywhere in [async_exit_pointer, async_exit_pointer_end) interval */ - return rip >= (unsigned long)async_exit_pointer && rip < (unsigned long)async_exit_pointer_end; diff --git a/pal/src/host/linux-sgx/meson.build b/pal/src/host/linux-sgx/meson.build -index 3e77bdcc..a1776fb8 100644 +index 3444d9b4..2d188bcd 100644 --- a/pal/src/host/linux-sgx/meson.build +++ b/pal/src/host/linux-sgx/meson.build -@@ -152,10 +152,19 @@ libpal_sgx_host = executable('loader', +@@ -10,12 +10,15 @@ pal_sgx_inc = [ + ), + ] + ++sgxstepdir = '' ++ + sgx_inc = [ + includes_pal_common, + pal_sgx_inc, + include_directories( + '../../../include/arch/@0@/linux'.format(host_machine.cpu_family()), + '../../../include/host/linux-common', ++ join_paths(sgxstepdir, 'sgx-step/') + ), + ] + +@@ -157,6 +160,12 @@ libpal_sgx_host = executable('loader', # host part of PAL uses stack protector with standard parameters (not the ones defined for # PAL/LibOS) '-fstack-protector-strong', -+ -+ # XXX add libsgxstep include path -+ '-I../../../..', ++ + # XXX FORTIFY_SOURCE conflicts with regular libc header includes in libsgxstep + '-D_FORTIFY_SOURCE=0', ++ # XXX libsgxstep needs Intel SDK headers (only for error reporting; can be hacked out if needed) ++ '-I/opt/intel/sgxsdk/include/' ++ ], link_args: [ '-Wl,-zrelro', - '-Wl,-znow', -+ +@@ -169,6 +178,8 @@ libpal_sgx_host = executable('loader', + protobuf_dep, + vtune_dep, + dl_dep, + # XXX link in libsgxstep library -+ '-L../../../../libsgxstep', -+ '-lsgx-step', ++ libsgxstep_dep ], - pie: true, --- -2.34.1 - + install: true, diff --git a/sdk/gramine/0002-helloworld-with-Nemesis-in-Gramine.patch b/sdk/gramine/0002-helloworld-with-Nemesis-in-Gramine.patch new file mode 100644 index 0000000..bc6f0aa --- /dev/null +++ b/sdk/gramine/0002-helloworld-with-Nemesis-in-Gramine.patch @@ -0,0 +1,269 @@ +diff --git a/CI-Examples/helloworld/helloworld.manifest.template b/CI-Examples/helloworld/helloworld.manifest.template +index 64c908e1..205860da 100644 +--- a/CI-Examples/helloworld/helloworld.manifest.template ++++ b/CI-Examples/helloworld/helloworld.manifest.template +@@ -6,6 +6,9 @@ loader.log_level = "{{ log_level }}" + + loader.env.LD_LIBRARY_PATH = "/lib" + ++loader.insecure__use_cmdline_argv = true ++loader.insecure__disable_aslr = true ++ + fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "/helloworld", uri = "file:helloworld" }, +@@ -13,6 +16,8 @@ fs.mounts = [ + + sgx.debug = true + sgx.nonpie_binary = true ++sgx.preheat_enclave = true ++sgx.thread_num = 4 + + sgx.trusted_files = [ + "file:{{ gramine.libos }}", +diff --git a/pal/src/host/linux-sgx/host_ecalls.c b/pal/src/host/linux-sgx/host_ecalls.c +index 201a0a27..f97ea21b 100644 +--- a/pal/src/host/linux-sgx/host_ecalls.c ++++ b/pal/src/host/linux-sgx/host_ecalls.c +@@ -6,6 +6,185 @@ + #include "pal_ecall_types.h" + #include "pal_rpc_queue.h" + ++ ++#define SGX_STEP_ENABLE 1 ++ ++/*****************************************************************************/ ++#if SGX_STEP_ENABLE ++ #include ++ #include ++ #include ++ #include "libsgxstep/apic.h" ++ #include "libsgxstep/pt.h" ++ #include "libsgxstep/debug.h" ++ #include "libsgxstep/enclave.h" ++ #include "libsgxstep/sched.h" ++ #include "libsgxstep/config.h" ++ #include "libsgxstep/idt.h" ++ #include "libsgxstep/cpu.h" ++ #include "libsgxstep/gdt.h" ++ ++ #define THREAD_END 4 ++ ++ #ifndef NUM_RUNS ++ #define NUM_RUNS 500 ++ #endif ++ ++ void fault_handler(int signal); ++ void attacker_config_runtime(void); ++ void attacker_config_page_table(void); ++ void aep_cb_func(void); ++ ++ static spinlock_t g_step_lock = INIT_SPINLOCK_UNLOCKED; ++ ++ static int thread_ctr = 0; ++ ++ uint64_t *pte_encl; ++ uint64_t *pmd_encl; ++ uint64_t *code_adrs; ++ uint64_t erip_prev = 0x0; ++ ++ /* ++ * This are the start and end offsets of the function main() in the helloworld CI-Example. ++ * You can find these in the compiled binary using objdump. ++ */ ++ const uint64_t main_start_offset = 0x1149; ++ const uint64_t main_end_offset = 0x116e; ++ ++ /* ++ * This is the address of the libos in the enclave. You can find this value by executing the ++ * binary with Gramine in DEBUG mode. Note that ASLR must be turned off in the manifest for ++ * the addresses to remain constant. ++ */ ++ const uint64_t libos = 0xfd2b000; ++ /* ++ * This is the address in the enclave where the binary starts. Gramine will also tell you this ++ * offset in DEBUG mode. ++ */ ++ const uint64_t bin_offset = 0xfc84000; ++ ++ int irq_cnt = 0, do_irq = 1, reset_fault_handler = 0; ++ idt_t idt = {0}; ++ ++ int ignored_fault_cnt = 0; ++ int fault_cnt = 0; ++ ++ /* Called upon SIGSEGV caused by untrusted page tables. */ ++ void fault_handler(int signal) ++ { ++ ASSERT(fault_cnt++ < 10); ++ switch ( signal ) ++ { ++ case SIGSEGV: ++ info("Caught fault %d! Restoring enclave page permissions..", signal); ++ //info("Fault count: %d", fault_cnt); ++ *pte_encl = MARK_NOT_EXECUTE_DISABLE(*pte_encl); ++ *pte_encl = MARK_NOT_ACCESSED(*pte_encl); ++ break; ++ default: ++ info("Caught unknown signal '%d'", signal); ++ //abort(); ++ } ++ ++ // NOTE: return eventually continues at aep_cb_func and initiates ++ // single-stepping mode. ++ } ++ ++ void attacker_config_runtime(void) ++ { ++ ASSERT( !claim_cpu(VICTIM_CPU) ); ++ ASSERT( !prepare_system_for_benchmark(PSTATE_PCT) ); ++ ASSERT(signal(SIGSEGV, fault_handler) != SIG_ERR); ++ print_system_settings(); ++ register_enclave_info(); ++ print_enclave_info(); ++ } ++ ++ void aep_cb_func(void) ++ { ++ uint64_t erip = edbgrd_erip() - (uint64_t) get_enclave_base(); ++ ++ if(erip >= libos){ ++ /* NOTE: we only want to step through the loaded binary */ ++ //info("Ignoring RIP"); ++ ignored_fault_cnt++; ++ return; ++ } ++ ++ /** ++ * We only log instructions that are within the offset of the function we are interested in. ++ * For the helloworld example, this is the function main(). ++ */ ++ if(erip_prev >= (bin_offset + main_start_offset) && erip_prev <= (bin_offset + main_end_offset)) { ++ /* ++ * NOTE: the stored SSA.rip is the instruction that will be executed upon ++ * ERESUME (i.e., _not_ the one we just measured) ++ */ ++ info("^^ enclave prev RIP=%#llx; cur RIP=%#llx; ACCESSED=%d; latency=%d", erip_prev, erip, ACCESSED(*pte_encl), ++ nemesis_tsc_aex - nemesis_tsc_eresume); ++ } ++ ++ erip_prev = erip; ++ irq_cnt++; ++ ++ if (do_irq && (irq_cnt > NUM_RUNS * 1000)) { ++ info( ++ "excessive interrupt rate detected (try adjusting timer interval " ++ "to avoid getting stuck in zero-stepping); aborting..."); ++ do_irq = 0; ++ } ++ ++ /* ++ * NOTE: We explicitly clear the "accessed" bit of the _unprotected_ PTE ++ * referencing the enclave code page about to be executed, so as to be able ++ * to filter out "zero-step" results that won't set the accessed bit. ++ */ ++ *pte_encl = MARK_NOT_ACCESSED(*pte_encl); ++ ++ /* ++ * Configure APIC timer interval for next interrupt. ++ * ++ * On our evaluation platforms, we explicitly clear the enclave's ++ * _unprotected_ PMD "accessed" bit below, so as to slightly slow down ++ * ERESUME such that the interrupt reliably arrives in the first subsequent ++ * enclave instruction. ++ * ++ */ ++ if (do_irq) { ++ *pmd_encl = MARK_NOT_ACCESSED(*pmd_encl); ++ apic_timer_irq(SGX_STEP_TIMER_INTERVAL); ++ } ++ ++ return; ++ /* NOTE: somehow calling libc functions here crashes the untrusted runtime... */ ++ } ++ ++ /* Provoke page fault on enclave entry to initiate single-stepping mode. */ ++ void attacker_config_page_table(void) ++ { ++ code_adrs = (uint64_t*) (bin_offset + main_start_offset); ++ print_page_table( code_adrs ); ++ info("enclave trigger code adrs at %p\n", code_adrs); ++ ++ ASSERT( pte_encl = remap_page_table_level( code_adrs, PTE) ); ++ ++ // enable single-stepping ++ #if SINGLE_STEP_ENABLE ++ *pte_encl = MARK_NOT_ACCESSED(*pte_encl); ++ *pte_encl = MARK_EXECUTE_DISABLE(*pte_encl); ++ print_pte(pte_encl); ++ ASSERT( PRESENT(*pte_encl) ); ++ #endif ++ ++ //print_page_table( get_enclave_base() ); ++ ASSERT( pmd_encl = remap_page_table_level( get_enclave_base(), PMD) ); ++ ASSERT( PRESENT(*pmd_encl) ); ++ } ++ ++#endif ++/*****************************************************************************/ ++ ++ + int ecall_enclave_start(char* libpal_uri, char* args, size_t args_size, char* env, + size_t env_size, int parent_stream_fd, sgx_target_info_t* qe_targetinfo, + struct pal_topo_info* topo_info, struct pal_dns_host_conf* dns_conf, +@@ -36,13 +215,55 @@ int ecall_enclave_start(char* libpal_uri, char* args, size_t args_size, char* en + .reserved_mem_ranges_size = reserved_mem_ranges_size, + .rpc_queue = g_rpc_queue, + }; ++ ++/*****************************************************************************/ ++#if SGX_STEP_ENABLE ++ /* NOTE: set sgx.preheat_enclave = true in manifest to prefault pages (occurs in pal_main.c) */ ++ /* Configure and check attacker untrusted runtime environment. */ ++ attacker_config_runtime(); ++#endif ++/*****************************************************************************/ ++ + return sgx_ecall(ECALL_ENCLAVE_START, &start_args); + } + + int ecall_thread_start(void) { ++/*****************************************************************************/ ++#if SGX_STEP_ENABLE ++ thread_ctr++; ++ //info_event("Thread %d started!", thread_ctr); ++ ++ if(thread_ctr == 4) { ++ ++ spinlock_lock(&g_step_lock); ++ /* 1. Setup attack execution environment. */ ++ attacker_config_page_table(); ++ register_aep_cb(aep_cb_func); ++ ++ info_event("Establishing user-space APIC/IDT mappings"); ++ map_idt(&idt); ++ ++ install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR); ++ apic_timer_oneshot(IRQ_VECTOR); ++ spinlock_unlock(&g_step_lock); ++ info_event("calling enclave num_runs=%d; TIMER=%d", NUM_RUNS, SGX_STEP_TIMER_INTERVAL); ++ /* 2. Single-step enclaved execution. */ ++ } ++#endif ++/*****************************************************************************/ + return sgx_ecall(ECALL_THREAD_START, NULL); + } + + int ecall_thread_reset(void) { ++/*****************************************************************************/ ++#if SGX_STEP_ENABLE ++ //info_event("Thread %d stopped!", thread_ctr); ++ if(thread_ctr == THREAD_END){ ++ /* 3. Restore normal execution environment. */ ++ apic_timer_deadline(); ++ info_event("all done; counted %d(+%d ignored)/%d IRQs (AEP/IDT)", irq_cnt, ignored_fault_cnt, __ss_irq_count); ++ } ++#endif ++/*****************************************************************************/ + return sgx_ecall(ECALL_THREAD_RESET, NULL); + } diff --git a/sdk/gramine/README.md b/sdk/gramine/README.md index 46e84cc..22ca8a1 100644 --- a/sdk/gramine/README.md +++ b/sdk/gramine/README.md @@ -1,32 +1,21 @@ -## Proof-of-concept _untested_ Gramine support - -> :warning: **Note.** Integration with Gramine (v1.3.1) is currently _untested_ -> and only provided as an example/starter for people wishing to experiment with -> SGX-Step on Gramine (e.g., see issue #47). Particularly, the patches below -> were validated to successfully compile but were never actually ran(!) -> Furthermore, single-stepping itself is not currently provided for Gramine, -> but should be straightforwardly feasible based on the existing code for the -> Intel SDK. As always, issues/PRs are welcome if you want to contribute -> improvements for a work-in-progress Gramine port. - ### Building the patched Gramine -0. First, make sure to build `libsgxstep.a`: +0. First, make sure to build `libsgxstep.a` with some minor adjustments: ```bash +$ cd sdk/gramine +$ ./patch_sgxstep.sh $ cd ../../libsgxstep $ make clean all ``` -1. Apply the patches in the untrusted Gramine runtime `host_entry.S` to -be able to link to `libsgxstep.a`: +1. Apply the patches in the untrusted Gramine runtime `host_entry.S` to be able to link to `libsgxstep.a`. ```bash $ ./patch_entry.sh ``` -2. Build the patched Gramine runtime and validate that the patches were -properly applied in the modified Gramine loader: +2. Build the patched Gramine runtime and validate that the patches were properly applied in the modified Gramine loader. ```bash $ cd gramine @@ -37,6 +26,36 @@ $ objdump -d build/pal/src/host/linux-sgx/loader | grep sgx_set_aep 000000000000924c : ``` +For Building with the out-of-tree (OOT) driver, in DEBUG mode: +```bash +$ meson setup build-debug/ --werror --buildtype=debug -Dsgx=enabled -Dsgx_driver=oot -Dsgx_driver_include_path=/kernel/linux-sgx-driver +$ meson configure build-debug/ --werror --buildtype=debug -Dsgx=enabled -Dsgx_driver=oot -Dsgx_driver_include_path=/kernel/linux-sgx-driver +``` + +You will also need to set: +```bash +$ sudo sysctl vm.mmap_min_addr=0 +``` +Prior to running anything in Gramine. + + +For Building with upstream drivers (Kernel >= 5.11). Note that these have not been tested with SGX-Step and may not work at the moment because Gramine uses the [VDSO](https://github.com/jovanbulck/sgx-step#0-system-requirements): + +```bash +$ meson setup build-debug/ --werror --buildtype=debug -Dsgx=enabled -Dsgx_driver=upstream +$ meson configure build-debug/ --werror --buildtype=debug -Dsgx=enabled -Dsgx_driver=upstream +``` + +Note that only the OOT was tested on Ubuntu 20.04.4 LTS with Kernel 5.9 loaded. You may also have to experiment with the [kernel parameters](https://github.com/jovanbulck/sgx-step#0-system-requirements) before SGX-Step runs smoothly. The following worked on the tested configuration: + +```bash +$ cat /proc/cmdline +... nox2apic iomem=relaxed no_timer_check nosmep nosmap clearcpuid=514 isolcpus=1 noexec=off nmi_watchdog=0 rcuupdate.rcu_cpu_stall_suppress=1 msr.allow_writes=on intel_idle.max_cstate=1 processor.max_cstate=1 dis_ucode_ldr vt.handoff=7 +``` + +Setting `kpti=0` on the test system didn't work reliably and caused system freezes after a few rounds of single stepping. Instead, the parameter `noexec=off` was set. + + 3. Now, you can implement the required attack code in Gramine's untrusted runtime using `libsgxstep` functionality as usual. For example, the following patch in `host_ecalls.c` demonstrates some basic usages: @@ -51,17 +70,58 @@ $ objdump -d build/pal/src/host/linux-sgx/loader | grep sgx_step 4. Install the patched Gramine. + ```bash $ sudo ninja -C build/ install ``` -### Running a sample application with the patched Gramine +### Running a sample application with the patched Gramine and with the latency collection: -Proceed as follows: +Proceed as follows. First Generate a private key: ```bash $ export PYTHONPATH=/usr/local # Fix https://github.com/gramineproject/gramine/issues/492 $ gramine-sgx-gen-private-key +``` + +The following changes are patched into the target application's manifest template with `0002-helloworld-with-Nemesis-in-Gramine.patch`: + +|`loader.insecure__disable_aslr = true` | For setting function offsets found through objdump in the binary | +|`loader.insecure__use_cmdline_argv = true` | For passing command line arguments to a binary | +|`sgx.debug = true` | For getting debug output such as the offset of the libos and the binary | +|`sgx.preheat_enclave = true` | From my understanding of the [documentation](https://gramine.readthedocs.io/en/latest/manifest-syntax.html#pre-heating-enclave) this is equivalent to performing a dry run with the SDK | +|`sgx.thread_num = 4` | This is probably optional, but Gramine needs 3 threads according to the [docs](https://gramine.readthedocs.io/en/latest/manifest-syntax.html#number-of-threads-deprecated-syntax), so we set it to 4 (assuming the binary is single-threaded) | + + +The example explicitly filters out all steps in the address range of the libOS (the assumption being that we are interested in the binary and not Gramine) and only logs stepping the `main()` function. The offsets of the LibOS and binary included in the Gramine patch will most likely not work. To retrieve them, execute `helloworld` in DEBUG mode. When determining the offset, it is recommended to turn off stepping (set `#define SGX_STEP_ENABLE` to 0 in `host_ecalls.c`). Don't forget to run `ninja` after making changes to Gramine. + +```bash $ cd CI-Examples/helloworld -$ make SGX=1 +$ make SGX=1 DEBUG=1 $ gramine-sgx helloworld ``` +Which will print plenty of debug output. For the libOS, you should see: + +``` +debug: LibOS loaded at
, ready to initialize +``` +And for the `helloworld` binary: + +``` +[P1:T1:helloworld] debug: append_r_debug: adding file:helloworld at
+``` + +The addresses have to be replaced in `host_ecalls.c` (again, don't forget to run `ninja`). + +Now you can run the binary and watch it step through `main()`: + +```bash +$ make clean && make SGX=1 DEBUG=1 +$ sudo gramine-sgx ./helloworld > out +``` + +You may also want to pin it to the isolated core for less noise: +```bash +$ sudo taskset -c 1 gramine-sgx ./helloworld > out +``` + +Happy hacking! diff --git a/sdk/gramine/patch_ecall.sh b/sdk/gramine/patch_ecall.sh old mode 100755 new mode 100644 index a985d93..5c05103 --- a/sdk/gramine/patch_ecall.sh +++ b/sdk/gramine/patch_ecall.sh @@ -1,6 +1,4 @@ #!/bin/bash -cd gramine - echo "=== patching ECALL ===" -patch -p1 < ../0002-Example-usage-of-libsgxstep-functionality-on-Gramine.patch +patch -p1 < ../0002-helloworld-with-Nemesis-in-Gramine.patch diff --git a/sdk/gramine/patch_entry.sh b/sdk/gramine/patch_entry.sh old mode 100755 new mode 100644 index 865e53b..b5c27e2 --- a/sdk/gramine/patch_entry.sh +++ b/sdk/gramine/patch_entry.sh @@ -1,6 +1,4 @@ #!/bin/bash -cd gramine - echo "=== patching AEP/TCS ===" patch -p1 < ../0001-SGX-Step-Gramine-patches-to-reconfigure-AEP-TCS.patch diff --git a/sdk/gramine/patch_sgxstep.sh b/sdk/gramine/patch_sgxstep.sh new file mode 100755 index 0000000..c2dbd6b --- /dev/null +++ b/sdk/gramine/patch_sgxstep.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "=== patching LIBSGXSTEP ===" +patch -p1 < ../0000-Libsgxstep-patches.patch