diff --git a/app/idt_isr_map/.gitignore b/app/idt_isr_map/.gitignore new file mode 100644 index 0000000..f1aeaf0 --- /dev/null +++ b/app/idt_isr_map/.gitignore @@ -0,0 +1,13 @@ +app +measurements.txt +measurements_raw.txt +outlier_idx.txt +plot.pdf +xlabels.gp + +*.swp + +out.txt +parsed.txt +parsed_zz.txt +parsed_strlen.txt diff --git a/app/idt_isr_map/Makefile b/app/idt_isr_map/Makefile new file mode 100644 index 0000000..45ba47a --- /dev/null +++ b/app/idt_isr_map/Makefile @@ -0,0 +1,64 @@ +LIBSGXSTEP_DIR = ../.. +LIBSGXSTEP = $(LIBSGXSTEP_DIR)/libsgxstep +-include $(LIBSGXSTEP)/Makefile.config + +URTS_LIB_PATH = $(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux + +ifeq ($(SGX_SDK),) + SGX_SDK = /opt/intel/sgxsdk +endif +export SGX_SDK +ifneq ($(SGX_SDK), /opt/intel/sgxsdk) + URTS_LD_LIBRARY_PATH = LD_LIBRARY_PATH=$(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux +endif + +SUBDIRS = $(LIBSGXSTEP) + +CC = gcc +AS = gcc +LD = gcc + +CFLAGS += -fPIC -fno-stack-protector -fno-builtin -fno-jump-tables \ + -fno-common -Wno-attributes -g -D_GNU_SOURCE -O0 +INCLUDE = -I$(SGX_SDK)/include/ -I$(LIBSGXSTEP_DIR) +LDFLAGS += -lsgx-step -lsgx_urts \ + -lsgx_uae_service -pthread $(SUBDIRS:%=-L %) -L$(SGX_SDK)/lib64/ + +SOURCES = $(shell ls *.c) +OBJECTS = $(SOURCES:.c=.o) +OUTPUT = app + +BUILDDIRS = $(SUBDIRS:%=build-%) +CLEANDIRS = $(SUBDIRS:%=clean-%) + + +MAKEFLAGS += --silent + +all: $(OUTPUT) + +run: clean all + sudo $(URTS_LD_LIBRARY_PATH) ./app + +$(OUTPUT): $(BUILDDIRS) $(OBJECTS) + echo "$(INDENT)[LD]" $(OBJECTS) $(LIBS) -o $(OUTPUT) + $(LD) $(OBJECTS) $(LDFLAGS) -o $(OUTPUT) + +%.o : %.c + echo "$(INDENT)[CC] " $< + $(CC) $(CFLAGS) $(INCLUDE) -c $< + +%.o : %.S + echo "$(INDENT)[AS] " $< + $(AS) $(INCLUDE) -c $< -o $@ + +clean: $(CLEANDIRS) + echo "$(INDENT)[RM]" $(OBJECTS) $(OUTPUT) + rm -f $(OBJECTS) $(OUTPUT) + +$(BUILDDIRS): + echo "$(INDENT)[===] $(@:build-%=%) [===]" + $(MAKE) -C $(@:build-%=%) INDENT+="$(INDENT_STEP)" curr-dir=$(curr-dir)/$(@:build-%=%) + +$(CLEANDIRS): + echo "$(INDENT)[===] $(@:clean-%=%) [===]" + $(MAKE) clean -C $(@:clean-%=%) INDENT+="$(INDENT_STEP)" curr-dir=$(curr-dir)/$(@:build-%=%) diff --git a/app/idt_isr_map/README.md b/app/idt_isr_map/README.md new file mode 100644 index 0000000..8a4e791 --- /dev/null +++ b/app/idt_isr_map/README.md @@ -0,0 +1,41 @@ +# Test for the kernel mapped interupt handlers + +This test uses the kernel mapped ISR region to demonstrate sw IRQ handling from other processes and cores. + +## Usage: +Start the listener: +``` +sudo ./app +``` + +Trigger the SW IRQ from another process: +``` +taskset -c 1 ./app trigger +``` + + +# Sample output: + + +``` +[idt.c] locking IRQ handler pages 0x564b69f22000/0x564b69f21000 +[idt.c] setting up isr mapping: from 0x564b69f21000 to 0x564b69f2205b +[pt.c] /dev/sgx-step opened! +[idt.c] we received the base address from kernel 0xffffc900201ea000 +[idt.c] the offset to the kernel mapped ISR region is 0xffff72b4b62c9000 +[pt.c] /dev/mem opened! +[sched.c] continuing on CPU 1 +[idt.c] DTR.base=0xfffffe0000000000/size=4095 (256 entries) +[idt.c] established user space IDT mapping at 0x7f4302014000 + +-------------------------------------------------------------------------------- +[main.c] Installing and testing ring0 IDT handler +-------------------------------------------------------------------------------- + +[idt.c] using kernel mapped ISR handler: 0x564b69f22000 -> 0xffffc900201eb000 +[idt.c] installed asm IRQ handler at 10:0x564b69f22000 +[idt.c] IDT[ 45] @0x7f43020142d0 = 0xffffc900201eb000 (seg sel 0x10); p=1; dpl=3; type=14; ist=0 +[main.c] wating for sw IRQ on vector 45 ... +[main.c] returned from IRQ: my_cpl=3; irq_cpl=0; count=01; flags=0x246; nemesis=835151372 +[main.c] all is well; irq_count=1; exiting.. +``` diff --git a/app/idt_isr_map/main.c b/app/idt_isr_map/main.c new file mode 100644 index 0000000..a08faf3 --- /dev/null +++ b/app/idt_isr_map/main.c @@ -0,0 +1,90 @@ +/* + * This file is part of the SGX-Step enclave execution control framework. + * + * Copyright (C) 2017 Jo Van Bulck , + * Raoul Strackx + * + * SGX-Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SGX-Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SGX-Step. If not, see . + */ + +#include "libsgxstep/idt.h" +#include "libsgxstep/gdt.h" +#include "libsgxstep/apic.h" +#include "libsgxstep/cpu.h" +#include "libsgxstep/sched.h" +#include "libsgxstep/config.h" +#include + +#define DO_APIC_SW_IRQ 1 +#define DO_APIC_TMR_IRQ 0 +#define DO_EXEC_PRIV 0 +#define NUM 1000 +#define NEMESIS_HIGH 1 + +/* ------------------------------------------------------------ */ +/* This code may execute with ring0 privileges */ +int my_cpl = -1; +extern uint64_t nemesis_tsc_aex, nemesis_tsc_eresume; + +void pre_irq(void) +{ + my_cpl = get_cpl(); + __ss_irq_fired = 0; + nemesis_tsc_eresume = rdtsc_begin(); +} + +void do_irq_sw(void) +{ + asm("int %0\n\t" ::"i"(IRQ_VECTOR):); +} + + +/* ------------------------------------------------------------ */ + +void post_irq(void) +{ + ASSERT(__ss_irq_fired); + info("returned from IRQ: my_cpl=%d; irq_cpl=%d; count=%02d; flags=%p; nemesis=%d", + my_cpl, __ss_irq_cpl, __ss_irq_count, read_flags(), nemesis_tsc_aex - nemesis_tsc_eresume); +} + +int main( int argc, char **argv ) +{ + if (argc == 2) { + info("triggering sw IRQ on vector %d!", IRQ_VECTOR); + do_irq_sw(); + return 0; + } + + idt_t idt = {0}; + ASSERT( !claim_cpu(VICTIM_CPU) ); + map_idt(&idt); + + info_event("Installing and testing ring0 IDT handler"); + install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR); + + info("wating for sw IRQ on vector %d ...", IRQ_VECTOR); + + pre_irq(); + + while (__ss_irq_fired == 0) { + sleep(1); + } + + post_irq(); + + info("all is well; irq_count=%d; exiting..", __ss_irq_count); + return 0; + +} diff --git a/kernel/sgxstep.c b/kernel/sgxstep.c index 293a730..78d3f5f 100644 --- a/kernel/sgxstep.c +++ b/kernel/sgxstep.c @@ -31,6 +31,9 @@ #include #include #include +#include +#include +#include #include #include @@ -39,24 +42,78 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Jo Van Bulck , Raoul Strackx "); MODULE_DESCRIPTION("SGX-Step: A Practical Attack Framework for Precise Enclave Execution Control"); -int target_cpu = -1; +static struct page **isr_pages = NULL; +static uint64_t isr_nr_pages = 0; +static void *isr_kernel_vbase = NULL; + +static int in_use = 0; + +typedef struct { + uint16_t size; + uint64_t base; +} __attribute__((packed)) dtr_t; + +static dtr_t original_dtr = {0}; + +static void *original_idt_copy = NULL; int step_open(struct inode *inode, struct file *file) { - if (target_cpu != -1) + if (in_use) { err("Device is already opened"); return -EBUSY; } - target_cpu = smp_processor_id(); + asm volatile ("sidt %0\n\t" + :"=m"(original_dtr) :: ); + + log("read idt: 0x%llx with size %u", original_dtr.base, original_dtr.size+1); + + original_idt_copy = kmalloc(original_dtr.size+1, GFP_KERNEL); + RET_ASSERT(original_idt_copy); + + memcpy(original_idt_copy, (void*)original_dtr.base, original_dtr.size+1); + log("copied original idt"); + + in_use = 1; return 0; } int step_release(struct inode *inode, struct file *file) { - target_cpu = -1; + uint64_t cr0; + asm volatile("mov %%cr0, %0" : "=r"(cr0)); + + // disable write protection + asm volatile("mov %0, %%cr0" : : "r"(cr0 & ~(1llu << 16))); + + // restore original idt to ensure no user pointers are left + memcpy((void*)original_dtr.base, original_idt_copy, original_dtr.size+1); + log("restored original idt at 0x%llx with size %u", original_dtr.base, original_dtr.size+1); + + // restore original cr0 + asm volatile("mov %0, %%cr0" : : "r"(cr0)); + + kfree(original_idt_copy); + original_idt_copy = NULL; + + if (isr_kernel_vbase) { + // freeup the kernel vbase mapping for isrs + vunmap(isr_kernel_vbase); + + // unpin the isr user physical pages + unpin_user_pages(isr_pages, isr_nr_pages); + + // free the isr user physical page structure + kfree(isr_pages); + + isr_kernel_vbase = NULL; + isr_pages = NULL; + isr_nr_pages = 0; + } + in_use = 0; return 0; } @@ -131,6 +188,46 @@ long sgx_step_get_pt_mapping(struct file *filep, unsigned int cmd, unsigned long return 0; } + +long sgx_step_ioctl_setup_isr_map(struct file *filep, unsigned int cmd, unsigned long arg) +{ + uint64_t nr_pinned_pages; + + setup_isr_map_t *data = (setup_isr_map_t*) arg; + + isr_nr_pages = (data->isr_stop - data->isr_start + PAGE_SIZE - 1) / PAGE_SIZE; + + isr_pages = kmalloc(isr_nr_pages * sizeof(struct page *), GFP_KERNEL); + RET_ASSERT_GOTO(isr_pages, "cannot allocate memory", out); + + nr_pinned_pages = pin_user_pages(data->isr_start & ~(PAGE_SIZE - 1), isr_nr_pages, FOLL_LONGTERM | FOLL_WRITE, isr_pages, NULL); + log("nr_pinned_pages = %llu should be %llu", nr_pinned_pages, isr_nr_pages); + + RET_ASSERT_GOTO(nr_pinned_pages == isr_nr_pages, "could not pin all isr pages", cleanup_pages); + + isr_kernel_vbase = vmap(isr_pages, isr_nr_pages, VM_READ | VM_EXEC | VM_SHARED, PAGE_SHARED_EXEC); + RET_ASSERT_GOTO(isr_kernel_vbase, "could not vmap isr", cleanup_pin); + + log("mapped isr to kernel virtual address 0x%llx", (uint64_t)isr_kernel_vbase); + + data->isr_kernel_base = isr_kernel_vbase; + + return 0; + +cleanup_pin: + unpin_user_pages(isr_pages, isr_nr_pages); + +cleanup_pages: + kfree(isr_pages); + +out: + isr_kernel_vbase = NULL; + isr_pages = NULL; + isr_nr_pages = 0; + return -EINVAL; +} + + typedef long (*ioctl_t)(struct file *filep, unsigned int cmd, unsigned long arg); long step_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) @@ -147,6 +244,9 @@ long step_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) case SGX_STEP_IOCTL_INVPG: handler = sgx_step_ioctl_invpg; break; + case SGX_STEP_IOCTL_SETUP_ISR_MAP: + handler = sgx_step_ioctl_setup_isr_map; + break; default: return -EINVAL; } diff --git a/kernel/sgxstep_internal.h b/kernel/sgxstep_internal.h index bcd531d..fedc2b9 100644 --- a/kernel/sgxstep_internal.h +++ b/kernel/sgxstep_internal.h @@ -37,4 +37,13 @@ } \ } while(0) +#define RET_ASSERT_GOTO(cond, message, label) \ + do { \ + if (!(cond)) \ + { \ + err("assertion '" #cond "' failed: %s", message); \ + goto label; \ + } \ + } while(0) + #endif diff --git a/kernel/sgxstep_ioctl.h b/kernel/sgxstep_ioctl.h index 86dcb9c..da1c440 100644 --- a/kernel/sgxstep_ioctl.h +++ b/kernel/sgxstep_ioctl.h @@ -26,6 +26,7 @@ #define SGX_STEP_IOCTL_MAGIC 'L' #define SGX_STEP_IOCTL_GET_PT_MAPPING _IOWR(SGX_STEP_IOCTL_MAGIC, 0, address_mapping_t) #define SGX_STEP_IOCTL_INVPG _IOWR(SGX_STEP_IOCTL_MAGIC, 1, invpg_t) +#define SGX_STEP_IOCTL_SETUP_ISR_MAP _IOWR(SGX_STEP_IOCTL_MAGIC, 2, setup_isr_map_t) typedef struct { uint64_t virt; @@ -41,4 +42,10 @@ typedef struct { uint64_t adrs; } invpg_t; +typedef struct { + uint64_t isr_start; // in + uint64_t isr_stop; // in + uint64_t isr_kernel_base; // out +} setup_isr_map_t; + #endif diff --git a/libsgxstep/idt.c b/libsgxstep/idt.c index 2fba4f9..b836a23 100644 --- a/libsgxstep/idt.c +++ b/libsgxstep/idt.c @@ -3,11 +3,56 @@ #include "apic.h" #include "sched.h" #include +#include +#include /* See irq_entry.S to see how these are used. */ void sgx_step_irq_gate_func(void); exec_priv_cb_t sgx_step_irq_gate_cb = NULL; +uint64_t sgx_step_isr_kernel_map_offset = 0; + +// this externs will be created by the compiler pointing to the ISR section. +extern void *__start_isr_section; +extern void *__stop_isr_section; + +extern int fd_step; + +static int is_in_isr_section(void *handler) { + return (void*)(&__start_isr_section) <= handler && handler < (void*)(&__stop_isr_section); +} + +static void setup_isr_map() +{ + setup_isr_map_t param; + + param.isr_start = (uint64_t)&__start_isr_section; + param.isr_stop = (uint64_t)&__stop_isr_section; + + info("setting up isr mapping: from 0x%lx to 0x%lx", param.isr_start, param.isr_stop); + + step_open(); + ASSERT( ioctl(fd_step, SGX_STEP_IOCTL_SETUP_ISR_MAP, ¶m) >= 0 ); + + info("we received the base address from kernel 0x%lx", param.isr_kernel_base); + + // calculate the virtual address space offset + sgx_step_isr_kernel_map_offset = param.isr_kernel_base - param.isr_start; + + info("the offset to the kernel mapped ISR region is 0x%lx", sgx_step_isr_kernel_map_offset); + + // This is currently a workaround: We are currently use one section for both data and code. + // Therefore, we need executable and writable pages which are per definition a security risk. + // Linux detects this and removes one of the flags, therefore we directly modify the page tables. + for (uint64_t address = param.isr_start; address < param.isr_stop; address += 0x1000) { + void *page = (void*)address + sgx_step_isr_kernel_map_offset; + uint64_t* pte = remap_page_table_level(page, PTE); + *pte = MARK_EXECUTABLE(*pte); + *pte = MARK_WRITABLE(*pte); + flush_tlb(page); + } +} + void dump_gate(gate_desc_t *gate, int idx) { info("IDT[%3d] @%p = %p (seg sel 0x%x); p=%d; dpl=%d; type=%02d; ist=%d", @@ -48,10 +93,23 @@ void install_irq_handler(idt_t *idt, void* asm_handler, int vector, cs_t seg, ga { ASSERT(vector >= 0 && vector < idt->entries); + uint64_t handler = (uint64_t)asm_handler; + + // check if the ISR section is mapped + if (sgx_step_isr_kernel_map_offset == 0) { + setup_isr_map(); + } + + // only if the handler is within the ISR section we use the kernel mapped handler + if (is_in_isr_section(asm_handler)) { + handler += sgx_step_isr_kernel_map_offset; + libsgxstep_info("using kernel mapped ISR handler: %p -> %p", asm_handler, (void*)handler); + } + gate_desc_t *gate = gate_ptr(idt->base, vector); - gate->offset_low = PTR_LOW(asm_handler); - gate->offset_middle = PTR_MIDDLE(asm_handler); - gate->offset_high = PTR_HIGH(asm_handler); + gate->offset_low = PTR_LOW(handler); + gate->offset_middle = PTR_MIDDLE(handler); + gate->offset_high = PTR_HIGH(handler); gate->p = 1; gate->segment = seg; @@ -65,6 +123,7 @@ void install_irq_handler(idt_t *idt, void* asm_handler, int vector, cs_t seg, ga #endif } + void install_user_irq_handler(idt_t *idt, void* asm_handler, int vector) { /* @@ -104,4 +163,5 @@ void __attribute__((constructor)) init_sgx_step( void ) info("locking IRQ handler pages %p/%p", &__ss_irq_handler, &__ss_irq_fired); ASSERT( !mlock(&__ss_irq_handler, 0x1000) ); ASSERT( !mlock((void*) &__ss_irq_fired, 0x1000) ); + // TODO: maybe call setup_isr_map here? this would require to open sgx_step in the constructor. } diff --git a/libsgxstep/irq_entry.S b/libsgxstep/irq_entry.S index f4b2ed8..c83a5a0 100644 --- a/libsgxstep/irq_entry.S +++ b/libsgxstep/irq_entry.S @@ -1,5 +1,5 @@ /* ********************************************************************** */ - .data + .section isr_section,"awx",@progbits .align 0x1000 .global __ss_irq_fired, __ss_irq_count, __ss_irq_cpl, apic_base, nemesis_tsc_aex __ss_irq_fired: @@ -19,7 +19,7 @@ __ss_irq_rdx: .quad 0x0 /* ********************************************************************** */ - .text + .section isr_section,"awx",@progbits .align 0x1000 .global __ss_irq_handler __ss_irq_handler: @@ -46,8 +46,11 @@ __ss_irq_handler: mov __ss_irq_rax(%rip), %rax mov __ss_irq_rdx(%rip), %rdx iretq - /* ********************************************************************** */ +/* don't put the custom user handler in the mapped ISR region (yet)! */ +/* the pointer could point to data outside the ISR region */ + .text + .align 0x1000 .global sgx_step_irq_gate_func sgx_step_irq_gate_func: call *sgx_step_irq_gate_cb(%rip)