Merge pull request #60 from jovanbulck/isr-kernel-map

ISR kernel mapping

.gitignore

@@ -15,3 +15,4 @@
 *.o.*
 *.mk
 *.mod
+*.swn

README.md

@@ -115,7 +115,7 @@ below.
 | `isolcpus=1`                         | Affinitize the victim process to an isolated CPU core.             |
 | `nosmap nosmep`                      | Disable Supervisor Mode Access/Execution Prevention (to allow SGX-Step to execute ring-0 IRQ handlers on user pages). |
 | `clearcpuid=514`                     | Disable User-Mode Instruction Prevention (on newer CPUs).          |
-| `kpti=0`                             | Disable Kernel Page-Table Isolation (to avoid kernel panics with user IRQ handlers). |
+| `pti=off`                            | Disable Kernel Page-Table Isolation (to avoid kernel panics with user IRQ handlers). |
 | `rcuupdate.rcu_cpu_stall_suppress=1` | Disable the kernel's read-copy update (RCU) CPU stall detector (to avoid warnings when single-stepping for a long time without calling the kernel's timer interrupt handler.) |
 | `msr.allow_writes=on`                | Suppress kernel warning messages for model-specific register (MSR) writes by SGX-Step. |
 | `vdso=0`                             | Only on recent Linux kernels: disable vdso_sgx_enter_enclave library (not compatible with AEP interception patches). |
@@ -125,7 +125,7 @@ Pass the desired boot parameters to the kernel as follows:
 
 ```bash
 $ sudo vim /etc/default/grub
-  # Add the following line: GRUB_CMDLINE_LINUX_DEFAULT="quiet splash nox2apic iomem=relaxed no_timer_check nosmep nosmap clearcpuid=514 kpti=0 isolcpus=1 nmi_watchdog=0 rcupdate.rcu_cpu_stall_suppress=1 msr.allow_writes=on vdso=0"
+  # Add the following line: GRUB_CMDLINE_LINUX_DEFAULT="quiet splash nox2apic iomem=relaxed no_timer_check nosmep nosmap clearcpuid=514 pti=off isolcpus=1 nmi_watchdog=0 rcupdate.rcu_cpu_stall_suppress=1 msr.allow_writes=on vdso=0"
 $ sudo update-grub && reboot
   # to inspect the boot parameters of the currently running kernel, execute:
 $ cat /proc/cmdline

app/idt/main.c

@@ -29,16 +29,19 @@
 #define DO_APIC_TMR_IRQ             1
 #define DO_EXEC_PRIV                1
 #define NUM                         100
+#define INFINITE_LOOP               1
 #define NEMESIS_HIGH                1
 
 /* ------------------------------------------------------------ */
 /* This code may execute with ring0 privileges */
 int my_cpl = -1;
+uint64_t my_flags = 0;
 extern uint64_t nemesis_tsc_aex, nemesis_tsc_eresume;
 
 void pre_irq(void)
 {
     my_cpl = get_cpl();
+    my_flags = read_flags();
     __ss_irq_fired = 0;
     nemesis_tsc_eresume = rdtsc_begin();
 }
@@ -49,12 +52,20 @@ void do_irq_sw(void)
     asm("int %0\n\t" ::"i"(IRQ_VECTOR):);
 }
 
-
 void do_irq_tmr(void)
 {
     pre_irq();
     apic_timer_irq(10);
-    while(!__ss_irq_fired)
+
+    /*
+     * Ring-0 `exec_priv` handler executes with interrupts disabled FLAGS.IF=0
+     * (as the kernel may freeze when interrupting a ring-0 trap gate), so the
+     * APIC timer IRQ should be handled after returning.
+     */
+    if (!(my_flags & (0x1 << 9)))
+        return;
+
+    while (!__ss_irq_fired)
     {
         #if NEMESIS_HIGH
             asm("rdrand %rax\n\t");
@@ -66,58 +77,60 @@ void do_irq_tmr(void)
 
 /* ------------------------------------------------------------ */
 
-void post_irq(void)
+void post_irq(char *s)
 {
     ASSERT(__ss_irq_fired);
-    info("returned from IRQ: my_cpl=%d; irq_cpl=%d; count=%02d; flags=%p; nemesis=%d",
-            my_cpl, __ss_irq_cpl, __ss_irq_count, read_flags(), nemesis_tsc_aex - nemesis_tsc_eresume);
+    info("returned from %s IRQ: my_cpl=%d; irq_cpl=%d; my_flags=%p; count=%02d; nemesis=%d", s,
+            my_cpl, __ss_irq_cpl, my_flags, __ss_irq_count, nemesis_tsc_aex - nemesis_tsc_eresume);
 }
 
 void do_irq_test(int do_exec_priv)
 {
     #if DO_APIC_SW_IRQ
         printf("\n");
-        info("Triggering ring3 software interrupts..");
+        info_event("Triggering ring-3 software interrupts..");
         for (int i=0; i < NUM; i++)
         {
             do_irq_sw();
-            post_irq();
+            post_irq("software");
         }
 
         if (do_exec_priv)
         {
             printf("\n");
-            info("Triggering ring0 software interrupts..");
+            info_event("Triggering ring-0 software interrupts..");
             for (int i=0; i < NUM; i++)
             {
+                my_cpl = -1;
                 exec_priv(do_irq_sw);
-                post_irq();
+                post_irq("software");
             }
         }
     #endif
 
     #if DO_APIC_TMR_IRQ
         printf("\n");
-        info("Triggering ring3 APIC timer interrupts..");
+        info_event("Triggering ring-3 APIC timer interrupts..");
         apic_timer_oneshot(IRQ_VECTOR);
 
         for (int i=0; i < NUM; i++)
         {
             do_irq_tmr();
-            post_irq();
+            post_irq("APIC timer");
         }
 
         if (do_exec_priv)
         {
             printf("\n");
-            info("Triggering ring0 APIC timer interrupts..");
+            info_event("Triggering ring-0 APIC timer interrupts..");
             for (int i=0; i < NUM; i++)
             {
+                my_cpl = -1;
                 exec_priv(do_irq_tmr);
-                post_irq();
+                while (!__ss_irq_fired);
+                post_irq("APIC timer");
             }
         }
-
         apic_timer_deadline();
     #endif
 }
@@ -129,19 +142,24 @@ int main( int argc, char **argv )
     map_idt(&idt);
 
     #if 0
-    // ring 0 timer irq handlers may #GP when interrupting the kernel..
+    /* ring 3 timer IRQ handlers may #GP when interrupting the kernel.. */
     info_event("Installing and testing ring3 IDT handler");
     install_user_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
     do_irq_test(/*do_exec_priv=*/ 0);
     #endif
 
-    info_event("Installing and testing ring0 IDT handler");
+    info_event("Installing and testing ring-0 IDT handler");
     install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
+
     #if DO_EXEC_PRIV
         exec_priv(pre_irq);
         info("back from exec_priv(pre_irq) with CPL=%d", my_cpl);
     #endif
-    do_irq_test(/*do_exec_priv=*/ DO_EXEC_PRIV);
+
+    #if INFINITE_LOOP
+    while(1)
+    #endif
+        do_irq_test(/*do_exec_priv=*/ DO_EXEC_PRIV);
 
     info("all is well; irq_count=%d; exiting..", __ss_irq_count);
     return 0;

app/idt_isr_map/.gitignore

@@ -0,0 +1,13 @@
+app
+measurements.txt
+measurements_raw.txt
+outlier_idx.txt
+plot.pdf
+xlabels.gp
+
+*.swp
+
+out.txt
+parsed.txt
+parsed_zz.txt
+parsed_strlen.txt

app/idt_isr_map/Makefile

@@ -0,0 +1,64 @@
+LIBSGXSTEP_DIR       = ../..
+LIBSGXSTEP           = $(LIBSGXSTEP_DIR)/libsgxstep
+-include $(LIBSGXSTEP)/Makefile.config
+
+URTS_LIB_PATH        = $(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux
+
+ifeq ($(SGX_SDK),)
+    SGX_SDK          = /opt/intel/sgxsdk
+endif
+export SGX_SDK
+ifneq ($(SGX_SDK), /opt/intel/sgxsdk)
+    URTS_LD_LIBRARY_PATH = LD_LIBRARY_PATH=$(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux
+endif
+
+SUBDIRS              = $(LIBSGXSTEP)
+
+CC                   = gcc
+AS                   = gcc
+LD                   = gcc
+
+CFLAGS              += -fPIC -fno-stack-protector -fno-builtin -fno-jump-tables \
+                       -fno-common -Wno-attributes -g -D_GNU_SOURCE -O0
+INCLUDE              = -I$(SGX_SDK)/include/ -I$(LIBSGXSTEP_DIR) 
+LDFLAGS             += -lsgx-step -lsgx_urts \
+                       -lsgx_uae_service -pthread $(SUBDIRS:%=-L %) -L$(SGX_SDK)/lib64/
+
+SOURCES              = $(shell ls *.c)
+OBJECTS              = $(SOURCES:.c=.o)
+OUTPUT               = app
+
+BUILDDIRS            = $(SUBDIRS:%=build-%)
+CLEANDIRS            = $(SUBDIRS:%=clean-%)
+
+
+MAKEFLAGS           += --silent
+
+all: $(OUTPUT)
+
+run: clean all
+	sudo $(URTS_LD_LIBRARY_PATH) ./app
+
+$(OUTPUT): $(BUILDDIRS) $(OBJECTS)
+	echo "$(INDENT)[LD]" $(OBJECTS) $(LIBS) -o $(OUTPUT) 
+	$(LD) $(OBJECTS) $(LDFLAGS) -o $(OUTPUT) 
+
+%.o : %.c
+	echo "$(INDENT)[CC] " $<
+	$(CC) $(CFLAGS) $(INCLUDE) -c $<
+
+%.o : %.S
+	echo "$(INDENT)[AS] " $<
+	$(AS) $(INCLUDE) -c $< -o $@
+
+clean: $(CLEANDIRS)
+	echo "$(INDENT)[RM]" $(OBJECTS) $(OUTPUT)
+	rm -f $(OBJECTS) $(OUTPUT)
+
+$(BUILDDIRS):
+	echo "$(INDENT)[===] $(@:build-%=%) [===]"
+	$(MAKE) -C $(@:build-%=%) INDENT+="$(INDENT_STEP)" curr-dir=$(curr-dir)/$(@:build-%=%)
+
+$(CLEANDIRS):
+	echo "$(INDENT)[===] $(@:clean-%=%) [===]"
+	$(MAKE) clean -C $(@:clean-%=%) INDENT+="$(INDENT_STEP)" curr-dir=$(curr-dir)/$(@:build-%=%)

app/idt_isr_map/README.md

@@ -0,0 +1,41 @@
+# Test for the kernel mapped interupt handlers
+
+This test uses the kernel mapped ISR region to demonstrate sw IRQ handling from other processes and cores.
+
+## Usage:
+Start the listener:
+```
+sudo ./app   
+```
+
+Trigger the SW IRQ from another process:
+```
+taskset -c 1 ./app trigger
+```
+
+
+# Sample output:
+
+
+```
+[idt.c] locking IRQ handler pages 0x564b69f22000/0x564b69f21000
+[idt.c] setting up isr mapping: from 0x564b69f21000 to 0x564b69f2205b
+[pt.c] /dev/sgx-step opened!
+[idt.c] we received the base address from kernel 0xffffc900201ea000
+[idt.c] the offset to the kernel mapped ISR region is 0xffff72b4b62c9000
+[pt.c] /dev/mem opened!
+[sched.c] continuing on CPU 1
+[idt.c] DTR.base=0xfffffe0000000000/size=4095 (256 entries)
+[idt.c] established user space IDT mapping at 0x7f4302014000
+
+--------------------------------------------------------------------------------
+[main.c] Installing and testing ring0 IDT handler
+--------------------------------------------------------------------------------
+
+[idt.c] using kernel mapped ISR handler: 0x564b69f22000 -> 0xffffc900201eb000
+[idt.c] installed asm IRQ handler at 10:0x564b69f22000
+[idt.c] IDT[ 45] @0x7f43020142d0 = 0xffffc900201eb000 (seg sel 0x10); p=1; dpl=3; type=14; ist=0
+[main.c] wating for sw IRQ on vector 45 ...
+[main.c] returned from IRQ: my_cpl=3; irq_cpl=0; count=01; flags=0x246; nemesis=835151372
+[main.c] all is well; irq_count=1; exiting..
+```

app/idt_isr_map/main.c

@@ -0,0 +1,90 @@
+/*
+ *  This file is part of the SGX-Step enclave execution control framework.
+ *
+ *  Copyright (C) 2017 Jo Van Bulck <jo.vanbulck@cs.kuleuven.be>,
+ *                     Raoul Strackx <raoul.strackx@cs.kuleuven.be>
+ *
+ *  SGX-Step is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  SGX-Step is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with SGX-Step. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "libsgxstep/idt.h"
+#include "libsgxstep/gdt.h"
+#include "libsgxstep/apic.h"
+#include "libsgxstep/cpu.h"
+#include "libsgxstep/sched.h"
+#include "libsgxstep/config.h"
+#include <unistd.h>
+
+#define DO_APIC_SW_IRQ              1
+#define DO_APIC_TMR_IRQ             0
+#define DO_EXEC_PRIV                0
+#define NUM                         1000
+#define NEMESIS_HIGH                1
+
+/* ------------------------------------------------------------ */
+/* This code may execute with ring0 privileges */
+int my_cpl = -1;
+extern uint64_t nemesis_tsc_aex, nemesis_tsc_eresume;
+
+void pre_irq(void)
+{
+    my_cpl = get_cpl();
+    __ss_irq_fired = 0;
+    nemesis_tsc_eresume = rdtsc_begin();
+}
+
+void do_irq_sw(void)
+{
+    asm("int %0\n\t" ::"i"(IRQ_VECTOR):);
+}
+
+
+/* ------------------------------------------------------------ */
+
+void post_irq(void)
+{
+    ASSERT(__ss_irq_fired);
+    info("returned from IRQ: my_cpl=%d; irq_cpl=%d; count=%02d; flags=%p; nemesis=%d",
+            my_cpl, __ss_irq_cpl, __ss_irq_count, read_flags(), nemesis_tsc_aex - nemesis_tsc_eresume);
+}
+
+int main( int argc, char **argv )
+{
+    if (argc == 2) {
+        info("triggering sw IRQ on vector %d!", IRQ_VECTOR);
+        do_irq_sw();
+        return 0;
+    }
+
+    idt_t idt = {0};
+    ASSERT( !claim_cpu(VICTIM_CPU) );
+    map_idt(&idt);
+
+    info_event("Installing and testing ring0 IDT handler");
+    install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
+
+    info("wating for sw IRQ on vector %d ...", IRQ_VECTOR);
+
+    pre_irq();
+
+    while (__ss_irq_fired == 0) {
+        sleep(1);
+    }
+
+    post_irq();
+
+    info("all is well; irq_count=%d; exiting..", __ss_irq_count);
+    return 0;
+
+}