Support upstream /dev/sgx_enclave driver Linux >5.11

SGX-Step should now support both the legacy /dev/isgx out-of-tree driver, as
well as the upstreamed /dev/sgx_enclave driver (for platforms with recent Linux
kernels >5.11 and hardware support for flexible-launch control).

The libsgxstep/enclave.c code has been refactored to use the standard Linux
interfaces /proc/self/maps and /proc/self/mem to retrieve enclave layout and
access enclave debug memory respectively. This means we don't have to rely
anymore on any SGX driver-specific hacks in /dev/sgx-step for this.

Fixes #39.

README.md

@@ -119,14 +119,13 @@ below.
 | `rcuupdate.rcu_cpu_stall_suppress=1` | Disable the kernel's read-copy update (RCU) CPU stall detector (to avoid warnings when single-stepping for a long time without calling the kernel's timer interrupt handler.) |
 | `msr.allow_writes=on`                | Suppress kernel warning messages for model-specific register (MSR) writes by SGX-Step. |
 | `vdso=0`                             | Only on recent Linux kernels: disable vdso_sgx_enter_enclave library (not compatible with AEP interception patches). |
-| `nosgx`                              | Only on recent Linux kernels: disable in-kernel SGX driver (until #39 is resolved). |
 | `dis_ucode_ldr`                      | Optionally disable CPU microcode updates (recent transient-execution attack mitigations may necessitate re-calibrating the single-stepping interval).                  |
 
 Pass the desired boot parameters to the kernel as follows:
 
 ```bash
 $ sudo vim /etc/default/grub
-  # Add the following line: GRUB_CMDLINE_LINUX_DEFAULT="quiet splash nox2apic iomem=relaxed no_timer_check nosmep nosmap clearcpuid=514 kpti=0 isolcpus=1 nmi_watchdog=0 rcupdate.rcu_cpu_stall_suppress=1 msr.allow_writes=on vdso=0 nosgx"
+  # Add the following line: GRUB_CMDLINE_LINUX_DEFAULT="quiet splash nox2apic iomem=relaxed no_timer_check nosmep nosmap clearcpuid=514 kpti=0 isolcpus=1 nmi_watchdog=0 rcupdate.rcu_cpu_stall_suppress=1 msr.allow_writes=on vdso=0"
 $ sudo update-grub && reboot
   # to inspect the boot parameters of the currently running kernel, execute:
 $ cat /proc/cmdline
@@ -148,14 +147,17 @@ To build and load the `/dev/sgx-step` driver, execute:
 
 ```bash
 $ cd kernel/
-$ ./install_SGX_driver.sh              # tested on Ubuntu 18.04/20.04
+$ ./install_SGX_driver.sh              # tested on Ubuntu 18.04/20.04/22.04
 $ make clean load
 ```
-
-**Note (/dev/isgx).** Our driver uses some internal symbols and data structures
-from the official Intel `/dev/isgx` out-of-tree driver. We therefore include a
-git submodule that points to an unmodified v2.14
-[linux-sgx-driver](https://github.com/intel/linux-sgx-driver).
+**Note (/dev/sgx_enclave).** SGX-Step supports both the legacy Intel
+`/dev/isgx` out-of-tree driver that should work on all platforms, as well as
+well as the upstream `/dev/sgx_enclave` driver for platforms with recent Linux
+kernels >5.11 plus hardware support for flexible-launch control. The
+`install_SGX_driver.sh` script should automatically detect whether an in-tree
+`/dev/sgx_enclave` driver is available, and, if not, build and load the
+out-of-tree `/dev/isgx` driver via the git submodule that points to an
+unmodified v2.14 [linux-sgx-driver](https://github.com/intel/linux-sgx-driver).
 
 **Note (/dev/mem).** We rely on Linux's virtual `/dev/mem` device to construct
 user-level virtual memory mappings for APIC physical memory-mapped I/O

app/aep-redirect/main.c

@@ -26,13 +26,15 @@
 #include "libsgxstep/debug.h"
 #include "libsgxstep/pt.h"
 
+#define DBG_ENCL        1
+
 void *data_pt = NULL, *data_page = NULL, *code_pt = NULL;
 int fault_fired = 0, aep_fired = 0;
 sgx_enclave_id_t eid = 0;
 
 void aep_cb_func(void)
 {
-    gprsgx_region_t gprsgx;
+    gprsgx_region_t gprsgx = {0};
     uint64_t erip = edbgrd_erip() - (uint64_t) get_enclave_base();
     info("Hello world from AEP callback with erip=%#llx! Resuming enclave..", erip); 
 
@@ -116,16 +118,19 @@ void attacker_config_page_table(void)
 
 int main( int argc, char **argv )
 {
-	sgx_launch_token_t token = {0};
-	int retval = 0, updated = 0;
+    sgx_launch_token_t token = {0};
+    int retval = 0, updated = 0;
     char old = 0x00, new = 0xbb;
 
-   	info("Creating enclave...");
-	SGX_ASSERT( sgx_create_enclave( "./Enclave/encl.so", /*debug=*/1,
-                                    &token, &updated, &eid, NULL ) );
+    info("Creating enclave...");
+    SGX_ASSERT( sgx_create_enclave( "./Enclave/encl.so", /*debug=*/DBG_ENCL,
+                                &token, &updated, &eid, NULL ) );
 
-    attacker_config_page_table();
+    info("Dry run to allocate pages");
+    SGX_ASSERT( enclave_dummy_call(eid, &retval) );
+    SGX_ASSERT( page_aligned_func(eid) );
 
+    attacker_config_page_table();
     register_aep_cb(aep_cb_func);
     print_enclave_info();
 

app/bench/main.c

@@ -102,7 +102,7 @@ void aep_cb_func(void)
 /* Called upon SIGSEGV caused by untrusted page tables. */
 void fault_handler(int signal)
 {
-	info("Caught fault %d! Restoring enclave page permissions..", signal);
+    info("Caught fault %d! Restoring enclave page permissions..", signal);
     *pte_encl = MARK_NOT_EXECUTE_DISABLE(*pte_encl);
     ASSERT(fault_cnt++ < 10);
 
@@ -118,7 +118,7 @@ void attacker_config_runtime(void)
     ASSERT( !claim_cpu(VICTIM_CPU) );
     ASSERT( !prepare_system_for_benchmark(PSTATE_PCT) );
     ASSERT(signal(SIGSEGV, fault_handler) != SIG_ERR);
-	print_system_settings();
+    print_system_settings();
 
     if (isatty(fileno(stdout)))
     {
@@ -127,7 +127,6 @@ void attacker_config_runtime(void)
         info("precise single-stepping results...");
     }
 
-    register_aep_cb(aep_cb_func);
     register_enclave_info();
     print_enclave_info();
 }
@@ -141,6 +140,7 @@ void attacker_config_page_table(void)
         SGX_ASSERT( get_str_adrs( eid, &str_adrs) );
         info("enclave string adrs at %p", str_adrs);
         ASSERT( pte_str_encl = remap_page_table_level( str_adrs, PTE) );
+        ASSERT( PRESENT(*pte_str_encl) );
     #endif
 
     #if (ATTACK_SCENARIO == STRLEN)
@@ -156,36 +156,46 @@ void attacker_config_page_table(void)
     ASSERT( pte_encl = remap_page_table_level( code_adrs, PTE) );
     #if SINGLE_STEP_ENABLE
         *pte_encl = MARK_EXECUTE_DISABLE(*pte_encl);
+        print_pte(pte_encl);
+        ASSERT( PRESENT(*pte_encl) );
     #endif
 
     //print_page_table( get_enclave_base() );
     ASSERT( pmd_encl = remap_page_table_level( get_enclave_base(), PMD) );
+    ASSERT( PRESENT(*pmd_encl) );
 }
 
 /* ================== ATTACKER MAIN ================= */
 
 /* Untrusted main function to create/enter the trusted enclave. */
 int main( int argc, char **argv )
 {
-	sgx_launch_token_t token = {0};
+    sgx_launch_token_t token = {0};
     int apic_fd, encl_strlen = 0, updated = 0, vec=0;
     idt_t idt = {0};
 
-   	info_event("Creating enclave...");
-	SGX_ASSERT( sgx_create_enclave( "./Enclave/encl.so", /*debug=*/1,
-                                    &token, &updated, &eid, NULL ) );
+    info_event("Creating enclave...");
+    SGX_ASSERT( sgx_create_enclave( "./Enclave/encl.so", /*debug=*/1,
+                                &token, &updated, &eid, NULL ) );
+
+    /* 0. dry run */
+    info("Dry run to allocate pages");
+    SGX_ASSERT( do_zigzagger(eid, NUM_RUNS) );
+    SGX_ASSERT( do_strlen(eid, &encl_strlen, NUM_RUNS) );
+    SGX_ASSERT( do_nop_slide(eid) );
 
     /* 1. Setup attack execution environment. */
     attacker_config_runtime();
     attacker_config_page_table();
+    register_aep_cb(aep_cb_func);
 
     info_event("Establishing user-space APIC/IDT mappings");
     map_idt(&idt);
     install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
     apic_timer_oneshot(IRQ_VECTOR);
 
     /* 2. Single-step enclaved execution. */
-    info("calling enclave: attack=%d; num_runs=%d; timer=%d",
+    info_event("calling enclave: attack=%d; num_runs=%d; timer=%d",
         ATTACK_SCENARIO, NUM_RUNS, SGX_STEP_TIMER_INTERVAL);
 
     #if (ATTACK_SCENARIO == ZIGZAGGER)

app/memcmp/main.c

@@ -16,6 +16,7 @@
 #define MAX_LEN            15
 #define DO_STEP            1
 #define DEBUG              0
+#define DBG_ENCL           1
 #define ANIMATION_DELAY    50000000
 
 sgx_enclave_id_t eid = 0;
@@ -28,13 +29,13 @@ void *code_adrs, *trigger_adrs;
 /* Called before resuming the enclave after an Asynchronous Enclave eXit. */
 void aep_cb_func(void)
 {
-    uint64_t erip = edbgrd_erip() - (uint64_t) get_enclave_base();
     #if DEBUG
+        uint64_t erip = edbgrd_erip() - (uint64_t) get_enclave_base();
         info("^^ enclave RIP=%#llx; ACCESSED=%d", erip, ACCESSED(*pte_encl));
     #endif
     irq_cnt++;
 
-    if (do_irq && (irq_cnt > NUM_RUNS*500))
+    if (do_irq && (irq_cnt > NUM_RUNS*1000))
     {
         info("excessive interrupt rate detected (try adjusting timer interval " \
              "to avoid getting stuck in zero-stepping); aborting...");
@@ -123,7 +124,6 @@ void attacker_config_runtime(void)
     ASSERT( !prepare_system_for_benchmark(PSTATE_PCT) );
     //print_system_settings();
 
-    register_aep_cb(aep_cb_func);
     register_enclave_info();
     print_enclave_info();
 
@@ -145,17 +145,20 @@ void attacker_config_page_table(void)
     info("enclave trigger at %p; code at %p", trigger_adrs, code_adrs);
 
     ASSERT( pte_encl    = remap_page_table_level( code_adrs, PTE) );
+    ASSERT( PRESENT(*pte_encl) );
     *pte_encl = MARK_NOT_ACCESSED(*pte_encl);
     info("enclave code at %p with PTE", code_adrs);
     print_pte_adrs( code_adrs );
 
     ASSERT( pte_trigger = remap_page_table_level( trigger_adrs, PTE) );
+    ASSERT( PRESENT(*pte_trigger) );
     *pte_trigger = MARK_NOT_ACCESSED(*pte_trigger);
     ASSERT(!mprotect(trigger_adrs, 4096, PROT_NONE ));
     info("enclave trigger at %p with PTE", trigger_adrs);
     print_pte_adrs( trigger_adrs );
 
     ASSERT( pmd_encl = remap_page_table_level( get_enclave_base(), PMD) );
+    ASSERT( PRESENT(*pmd_encl) );
 }
 
 /* ================== ATTACKER MAIN ================= */
@@ -170,7 +173,7 @@ int main( int argc, char **argv )
     int step_cnt_prev = 0;
 
     info_event("Creating enclave...");
-    SGX_ASSERT( sgx_create_enclave( "./Enclave/encl.so", /*debug=*/1,
+    SGX_ASSERT( sgx_create_enclave( "./Enclave/encl.so", /*debug=*/DBG_ENCL,
                                     &token, &updated, &eid, NULL ) );
 
     /* 0. dry run */
@@ -179,6 +182,7 @@ int main( int argc, char **argv )
     /* 1. Setup attack execution environment. */
     attacker_config_runtime();
     attacker_config_page_table();
+    register_aep_cb(aep_cb_func);
 
 #if DO_STEP
     info_event("Establishing user-space APIC/IDT mappings");

kernel/Makefile

@@ -1,14 +1,8 @@
 LKM             = sgx-step
-SGX_DRIVER_DIR  = ../
 PWD            := $(shell pwd)
 
-ifndef NO_SGX
-   export NO_SGX=0
-endif
-
 obj-m          += $(LKM).o
 $(LKM)-objs     = sgxstep.o
-ccflags-y      := -I$(src)/$(SGX_DRIVER_DIR) -DNO_SGX=$(NO_SGX)
 
 all:
 	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules

kernel/install_SGX_driver.sh

@@ -2,6 +2,17 @@
 
 set -e
 
+if [ -e /dev/sgx_enclave ]; then
+    echo "Found in-kernel /dev/sgx_enclave driver; skipping installation of out-of-tree isgx driver!"
+    exit 0
+fi
+
+if [ -e /dev/isgx ]; then
+    echo "Found pre-installed out-of-tree /dev/isgx driver!"
+    exit 0
+fi
+echo "Building and installing out-of-tree isgx driver.."
+
 git submodule init
 git submodule update
 cd linux-sgx-driver

kernel/sgxstep.c

@@ -35,10 +35,6 @@
 #include <linux/clockchips.h>
 #include <linux/version.h>
 
-#if !NO_SGX
-    #include "linux-sgx-driver/sgx.h"
-#endif
-
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Jo Van Bulck <jo.vanbulck@cs.kuleuven.be>, Raoul Strackx <raoul.strackx@cs.kuleuven.be>");
 MODULE_DESCRIPTION("SGX-Step: A Practical Attack Framework for Precise Enclave Execution Control");
@@ -64,49 +60,6 @@ int step_release(struct inode *inode, struct file *file)
     return 0;
 }
 
-long sgx_step_ioctl_info(struct file *filep, unsigned int cmd, unsigned long arg)
-{
-    #if !NO_SGX
-        struct sgx_encl *enclave;
-        struct vm_area_struct *vma = NULL;
-        struct sgx_step_enclave_info *info = (struct sgx_step_enclave_info *) arg;
-
-        vma = find_vma(current->mm, (uint64_t) info->tcs);
-        RET_ASSERT(vma && (enclave = vma->vm_private_data));
-        RET_ASSERT(info->aep && info->tcs);
-
-        info->base = enclave->base;
-        info->size = enclave->size;
-    #endif
-
-    return 0;
-}
-
-long edbgrdwr(unsigned long addr, void *buf, int len, int write)
-{
-    struct vm_area_struct *vma = NULL;
-
-    /* use the vm_operations defined by the isgx driver
-     * (so we don't have to worry about illegal ptrs or #PFs etc) */
-    vma = find_vma(current->mm, addr);
-    RET_ASSERT(vma && vma->vm_ops && vma->vm_ops->access);
-    return vma->vm_ops->access(vma, addr, buf, len, write);
-}
-
-long sgx_step_ioctl_edbgrd(struct file *filep, unsigned int cmd, unsigned long arg)
-{
-    edbgrd_t *data = (edbgrd_t*) arg;
-    uint8_t buf[data->len];
-    if (data->write && copy_from_user(buf, (void __user *) data->val, data->len))
-        return -EFAULT;
-
-    edbgrdwr((unsigned long) data->adrs, &buf, data->len, data->write);
-
-    if (!data->write && copy_to_user((void __user *) data->val, buf, data->len))
-        return -EFAULT;
-    return 0;
-}
-
 /* Convenience function when editing PTEs from user space (but normally not
  * needed, since SGX already flushes the TLB on enclave entry/exit) */
 long sgx_step_ioctl_invpg(struct file *filep, unsigned int cmd, unsigned long arg)
@@ -188,15 +141,9 @@ long step_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 
     switch (cmd)
     {
-        case SGX_STEP_IOCTL_VICTIM_INFO:
-            handler = sgx_step_ioctl_info;
-            break;
         case SGX_STEP_IOCTL_GET_PT_MAPPING:
             handler = sgx_step_get_pt_mapping;
             break;
-        case SGX_STEP_IOCTL_EDBGRD:
-            handler = sgx_step_ioctl_edbgrd;
-            break;
         case SGX_STEP_IOCTL_INVPG:
             handler = sgx_step_ioctl_invpg;
             break;

kernel/sgxstep_internal.h

@@ -37,13 +37,4 @@
         }                                           \
     } while(0)
 
-/* NOTE: incorrect GPRSGX size in Intel manual vol. 3D June 2016 p.38-7 */
-#define SGX_TCS_OSSA_OFFSET         16
-#define SGX_GPRSGX_SIZE             184
-#define SGX_GPRSGX_RIP_OFFSET       136
-
-/* HACK: to avoid having to retrieve the SSA framesize from the untrusted
-   runtime (driver), we assume a standard/hard-coded SSA framesize of 1 page */
-#define SGX_SSAFRAMESIZE            4096
-
 #endif

kernel/sgxstep_ioctl.h

@@ -24,18 +24,8 @@
 #include <linux/ioctl.h>
 
 #define SGX_STEP_IOCTL_MAGIC            'L'
-#define SGX_STEP_IOCTL_VICTIM_INFO      _IOWR(SGX_STEP_IOCTL_MAGIC, 0, struct sgx_step_enclave_info)
-#define SGX_STEP_IOCTL_GET_PT_MAPPING   _IOWR(SGX_STEP_IOCTL_MAGIC, 1, address_mapping_t)
-#define SGX_STEP_IOCTL_EDBGRD           _IOWR(SGX_STEP_IOCTL_MAGIC, 2, edbgrd_t)
-#define SGX_STEP_IOCTL_INVPG            _IOWR(SGX_STEP_IOCTL_MAGIC, 3, invpg_t)
-
-struct sgx_step_enclave_info
-{
-    uint64_t base;
-    uint64_t size;
-    uint64_t aep;
-    uint64_t tcs;
-};
+#define SGX_STEP_IOCTL_GET_PT_MAPPING   _IOWR(SGX_STEP_IOCTL_MAGIC, 0, address_mapping_t)
+#define SGX_STEP_IOCTL_INVPG            _IOWR(SGX_STEP_IOCTL_MAGIC, 1, invpg_t)
 
 typedef struct {
 	uint64_t virt;
@@ -47,13 +37,6 @@ typedef struct {
 	uint64_t pte;
 } address_mapping_t;
 
-typedef struct {
-    uint64_t adrs;
-    uint64_t val;
-    int64_t  len;
-    int      write;
-} edbgrd_t;
-
 typedef struct {
     uint64_t adrs;
 } invpg_t;