diff --git a/app/apic/.gitignore b/app/apic/.gitignore new file mode 100644 index 0000000..75d4672 --- /dev/null +++ b/app/apic/.gitignore @@ -0,0 +1,4 @@ +*.txt +*.pdf +log +copy_icx.sh diff --git a/app/apic/Makefile b/app/apic/Makefile new file mode 100644 index 0000000..06394f7 --- /dev/null +++ b/app/apic/Makefile @@ -0,0 +1,98 @@ +LIBSGXSTEP_DIR = ../.. +LIBSGXSTEP = $(LIBSGXSTEP_DIR)/libsgxstep +-include $(LIBSGXSTEP)/Makefile.config + +LIBSGXSTEP_SILENT = 1 + +ifeq ($(SGX_SDK),) + SGX_SDK = /opt/intel/sgxsdk +endif +export SGX_SDK +ifneq ($(SGX_SDK), /opt/intel/sgxsdk) + URTS_LD_LIBRARY_PATH = LD_LIBRARY_PATH=$(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux +endif + +SUBDIRS = $(LIBSGXSTEP) + +CC = gcc +AS = gcc +LD = gcc + +ifeq ($(M32), 1) + ASFLAGS = -m32 -DM32=$(M32) + CFLAGS = -m32 -DM32=$(M32) + LDFLAGS = -m32 +else + LIB_SUFX = 64 +endif + +CFLAGS += -fPIC -fno-stack-protector -fno-builtin -fno-jump-tables \ + -fno-common -Wno-attributes -g -D_GNU_SOURCE -O0 +INCLUDE = -I$(SGX_SDK)/include/ -I$(LIBSGXSTEP_DIR) +LDFLAGS += -lsgx-step -lsgx_urts \ + -lsgx_uae_service -pthread $(SUBDIRS:%=-L %) -L$(SGX_SDK)/lib$(LIB_SUFX)/ \ + -L$(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux + +C_SOURCES = $(shell ls *.c) +ASM_SOURCES = $(shell ls *.S) +C_OBJECTS = $(C_SOURCES:.c=.o) +ASM_OBJECTS = $(ASM_SOURCES:.S=.o) +OBJECTS = $(C_OBJECTS) $(ASM_OBJECTS) +OUTPUT = app + +BUILDDIRS = $(SUBDIRS:%=build-%) +CLEANDIRS = $(SUBDIRS:%=clean-%) + +ATTACK = 1 +PARSE = nop +ifeq ($(STRLEN), 1) + ATTACK = 2 + PARSE = strlen +endif +ifeq ($(ZIGZAG), 1) + ATTACK = 3 + PARSE = zz +endif + +ifeq ($(NUM),) + NUM = 100 +endif +export NUM + +CFLAGS += -DATTACK_SCENARIO=$(ATTACK) -DNUM_RUNS=$(NUM) + +MAKEFLAGS += --silent + +#.SILENT: +all: $(OUTPUT) + +run: clean all + sudo $(URTS_LD_LIBRARY_PATH) ./app > out.txt + cat out.txt + +parse: run + SGX_STEP_PLATFORM=$(SGX_STEP_PLATFORM) ./parse_$(PARSE).py $(NUM) + +$(OUTPUT): $(BUILDDIRS) $(OBJECTS) + echo "$(INDENT)[LD]" $(OBJECTS) $(LIBS) -o $(OUTPUT) + $(LD) $(OBJECTS) $(LDFLAGS) -o $(OUTPUT) + +%.o : %.c + echo "$(INDENT)[CC] " $< + $(CC) $(CFLAGS) $(INCLUDE) -c $< + +%.o : %.S + echo "$(INDENT)[AS] " $< + $(AS) $(ASFLAGS) $(INCLUDE) -c $< -o $@ + +clean: $(CLEANDIRS) + echo "$(INDENT)[RM]" $(OBJECTS) $(OUTPUT) + rm -f $(OBJECTS) $(OUTPUT) + +$(BUILDDIRS): + echo "$(INDENT)[===] $(@:build-%=%) [===]" + $(MAKE) -C $(@:build-%=%) INDENT+="$(INDENT_STEP)" M32=$(M32) curr-dir=$(curr-dir)/$(@:build-%=%) + +$(CLEANDIRS): + echo "$(INDENT)[===] $(@:clean-%=%) [===]" + $(MAKE) clean -C $(@:clean-%=%) INDENT+="$(INDENT_STEP)" curr-dir=$(curr-dir)/$(@:build-%=%) diff --git a/app/apic/README.md b/app/apic/README.md new file mode 100644 index 0000000..7dddf6c --- /dev/null +++ b/app/apic/README.md @@ -0,0 +1,30 @@ +# APIC Precision Microbenchmarks + +This directory contains the experiments to determine the accuracy of the local +APIC timer, as described in Appendix A of the [AEX-Notify +paper](https://jovanbulck.github.io/files/usenix23-aexnotify.pdf). + +The local APIC timer can be configured in one-shot or periodic mode to send an +interrupt when an MMIO counter register reaches zero, or in TSC-deadline mode +when the processor's timestamp counter exceeds a value specified in the +dedicated `IA32_TSC_DEADLINE_MSR` model-specific register. Depending on the +processor model, the APIC timer operates at the frequency of either the +processor’s bus clock or its core crystal clock. Neither time source is +synchronized with the core clock, which typically operates at a much higher +frequency (e.g., more than 100x higher). + +**TSC-Deadline Mode.** As a contribution of independent interest, we devised an +improved setup to considerably reduce the noise from additional kernel context +switches. For this, we registered a custom ring-0 interrupt gate in the +processor's interrupt-descriptor table. This allows to essentially bypass the +OS kernel altogether by directly invoking a minimal assembly handler that +programs a specified value in `IA32_TSC_DEADLINE_MSR` using the privileged +`WRMSR` instruction via a software interrupt. + +**Comparison Results.** We conclude that our novel interrupt-gate TSC-deadline +timer configuration technique yields a considerable improvement in terms of +standard deviation of elapsed cycles for both the existing one-shot technique +that SGX-Step currently uses, as well as naive kernel-level configuration +approaches (cf. as is also visually evident in the figure below) + +![APIC distribution histogram](apic-hist.png) diff --git a/app/apic/apic-hist.png b/app/apic/apic-hist.png new file mode 100644 index 0000000..974e66b Binary files /dev/null and b/app/apic/apic-hist.png differ diff --git a/app/apic/asm.S b/app/apic/asm.S new file mode 100644 index 0000000..5553697 --- /dev/null +++ b/app/apic/asm.S @@ -0,0 +1,15 @@ +.text +.global incs +incs: + xor %rax, %rax + .rept 1000000 + inc %rax + .endr + ret + +.data +.align 0x1000 +.global my_page +my_page: +.word 0xdead +.space 0x1000 diff --git a/app/apic/config.h b/app/apic/config.h new file mode 100644 index 0000000..a82b8d5 --- /dev/null +++ b/app/apic/config.h @@ -0,0 +1,17 @@ +#ifndef CONFIG_H_INC +#define CONFIG_H_INC + +#define TSC_DEADLINE 1 + +#define TEST_ITERATIONS 100001 +#define APIC_ONESHOT_TICKS 30*2 // NOTE: sgx-step uses APIC timer divide 2(!) +#define APIC_TSC_INTERVAL 5750 + +#define USE_IRQ_GATE 1 +#define WRMSR_ON_CPU_LATENCY 0 // is about 8000 + +#ifndef IA32_TSC_DEADLINE_MSR + #define IA32_TSC_DEADLINE_MSR 0x6e0 +#endif + +#endif diff --git a/app/apic/irq_entry.S b/app/apic/irq_entry.S new file mode 100644 index 0000000..8ab3c38 --- /dev/null +++ b/app/apic/irq_entry.S @@ -0,0 +1,58 @@ +#include "config.h" + +/* ********************************************************************** */ + .section isr_section,"awx",@progbits + .global __apic_irq_tsc, __apic_irq_rax, __apic_deadline_tsc +__apic_irq_tsc: + .quad 0x0 +__apic_deadline_tsc: + .quad 0x0 +__apic_irq_rax: + .quad 0x0 +__apic_irq_rcx: + .quad 0x0 +__apic_irq_rdx: + .quad 0x0 + +/* ********************************************************************** */ + .section isr_section,"awx",@progbits + .global __apic_irq_handler_timer +__apic_irq_handler_timer: + mov %rax, __apic_irq_rax(%rip) + mov %rdx, __apic_irq_rdx(%rip) + rdtsc + mov %eax, __apic_irq_tsc(%rip) + mov %edx, __apic_irq_tsc+4(%rip) + + /* IRQ bookkeeping */ + incl __ss_irq_fired(%rip) + + /* apic_write(APIC_EOI, 0x0); */ + lea apic_base(%rip), %rax + mov (%rax),%rax + test %rax, %rax + jz 1f + add $0xb0, %rax + movl $0x0, (%rax) +1: + mov __apic_irq_rax(%rip), %rax + mov __apic_irq_rdx(%rip), %rdx + iretq + + .section isr_section,"awx",@progbits + .global __apic_priv_irq_gate +__apic_priv_irq_gate: + push %rax + push %rcx + push %rdx + + /* wrmsr(IA32_TSC_DEADLINE_MSR, __apic_deadline_tsc) */ + mov __apic_deadline_tsc(%rip), %eax + mov __apic_deadline_tsc+4(%rip), %edx + mov $IA32_TSC_DEADLINE_MSR, %ecx + wrmsr + + pop %rdx + pop %rcx + pop %rax + iretq diff --git a/app/apic/main.c b/app/apic/main.c new file mode 100644 index 0000000..3ddfb2f --- /dev/null +++ b/app/apic/main.c @@ -0,0 +1,118 @@ +/* + * This file is part of the SGX-Step enclave execution control framework. + * + * Copyright (C) 2017 Jo Van Bulck , + * Raoul Strackx + * + * SGX-Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * SGX-Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with SGX-Step. If not, see . + */ + +#include +#include +#include +#include "libsgxstep/apic.h" +#include "libsgxstep/cpu.h" +#include "libsgxstep/pt.h" +#include "libsgxstep/sched.h" +#include "libsgxstep/enclave.h" +#include "libsgxstep/debug.h" +#include "libsgxstep/config.h" +#include "libsgxstep/idt.h" +#include "libsgxstep/config.h" +#include "config.h" + +unsigned long long test_tsc_results[TEST_ITERATIONS]; +unsigned long long test_inc_results[TEST_ITERATIONS]; +unsigned long long test_deadline_results[TEST_ITERATIONS]; + +void incs(void); +void __apic_irq_handler_timer(void); +void __apic_priv_irq_gate(void); +extern unsigned long long __apic_irq_tsc; +extern unsigned long long __apic_deadline_tsc; +extern unsigned long long __apic_irq_rax; + +/* ================== ATTACKER INIT/SETUP ================= */ + +/* Configure and check attacker untrusted runtime environment. */ +void attacker_config_runtime(void) +{ + ASSERT( !claim_cpu(VICTIM_CPU) ); + ASSERT( !prepare_system_for_benchmark(PSTATE_PCT) ); + print_system_settings(); + + if (isatty(fileno(stdout))) + { + info("WARNING: interactive terminal detected; known to cause "); + info("unstable timer intervals! Use stdout file redirection for "); + info("precise single-stepping results..."); + } +} + +/* ================== ATTACKER MAIN ================= */ + +/* Untrusted main function to create/enter the trusted enclave. */ +int main( int argc, char **argv ) +{ + idt_t idt = {0}; + attacker_config_runtime(); + map_idt(&idt); + install_kernel_irq_handler(&idt, __apic_irq_handler_timer, IRQ_VECTOR); + + #if TSC_DEADLINE + const char *filename = "deadline_results.txt"; + apic_timer_deadline(IRQ_VECTOR); + + #if USE_IRQ_GATE + install_kernel_irq_handler(&idt, __apic_priv_irq_gate, IRQ_PRIV_VECTOR); + #endif + #else + const char *filename = "oneshot_results.txt"; + apic_timer_oneshot(IRQ_VECTOR); + #endif + + for (int i = 0; i < TEST_ITERATIONS; ++i) + { + __ss_irq_fired = 0; + unsigned long long begin_time = __rdtsc(); + + #if TSC_DEADLINE && USE_IRQ_GATE + __apic_deadline_tsc = begin_time + APIC_TSC_INTERVAL; + asm("int %0\n\t" ::"i"(IRQ_PRIV_VECTOR):); + #elif TSC_DEADLINE + __apic_deadline_tsc = begin_time + APIC_TSC_INTERVAL + WRMSR_ON_CPU_LATENCY; + wrmsr_on_cpu(IA32_TSC_DEADLINE_MSR, get_cpu(), __apic_deadline_tsc); + //unsigned long long wrmsr_time = __rdtsc(); + #else + apic_timer_irq(APIC_ONESHOT_TICKS); + #endif + + incs(); + ASSERT(__ss_irq_fired); + + test_tsc_results[i] = __apic_irq_tsc - begin_time; + test_inc_results[i] = __apic_irq_rax; + test_deadline_results[i] = __apic_irq_tsc - __apic_deadline_tsc; + } + + // record results + FILE *fp = fopen(filename, "w"); + fprintf(fp, "#tsc_diff,inc_count,tsc_deadline_drift\n"); + for (int i = 1; i < TEST_ITERATIONS; ++i) { + fprintf(fp, "%llu,%llu,%llu\n", test_tsc_results[i], test_inc_results[i], test_deadline_results[i]); + } + fclose(fp); + + return 0; +} diff --git a/app/apic/parse.py b/app/apic/parse.py new file mode 100755 index 0000000..5d40df5 --- /dev/null +++ b/app/apic/parse.py @@ -0,0 +1,139 @@ +#!/usr/bin/python3 + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.font_manager as font_manager +import statistics + +# ensure no type 3 fonts +import matplotlib +matplotlib.rcParams['pdf.fonttype'] = 42 +matplotlib.rcParams['ps.fonttype'] = 42 + +# TSC deadline + IRQ gate; deadline diff +LATENCY_MIN = 1000 +LATENCY_MAX = 1500 + +# TSC deadline + IRQ gate; tsc diff +#LATENCY_MIN = 11000 +#LATENCY_MAX = 11500 + +# TSC deadline w/o IRQ gate; deadline diff +#LATENCY_MIN = 900 +#LATENCY_MAX = 1300 + +# TSC oneshot; tsc diff +LATENCY_MIN = 14600 +LATENCY_MAX = 15200 + +# ICX TSC oneshot; tsc diff +LATENCY_MIN = 10800 +LATENCY_MAX = 11200 + +pd.set_option('display.precision', 0) +np.set_printoptions(precision=0) + +def hist(latencies, labels=[], title='', filename='hist', xlabel='Latency (cycles)', legend_loc='best'): + plt.figure(figsize=(10,5)) + #colors = [] + + for i in range(0,len(labels)): + d = latencies[i] + labels[i] += f' (μ={int(np.mean(d))}, σ={int(np.std(d))})' #, M={int(np.median(d))})' + #colors.append(f'C{i}') + + plt.hist(latencies, + label=labels, + bins=500, + #histtype='step', + histtype='stepfilled', + alpha=0.5, + range=[LATENCY_MIN, LATENCY_MAX]) + + #for i in range(0,len(latencies)): + # x = latencies[i] + # plt.axvline(np.mean(x), color=colors[i], linestyle='dashed', linewidth=1) + + font = font_manager.FontProperties(family='monospace', size=10) #weight='bold', style='normal') + if len(labels) > 0: + plt.legend(prop=font,loc=legend_loc) + + plt.xlabel(xlabel) + plt.ylabel('Frequency') + plt.title(title) + + print(f".. writing histogram '{title}' to '{filename}.pdf'") + plt.savefig(filename + '.pdf', bbox_inches='tight') + plt.show() + +def reject_outliers(data, m=3): + stdev = np.std(data) + mean = np.mean(data) + mask_min = mean - stdev * m + mask_max = mean + stdev * m + + outliers = [d for d in data if d < mask_min or d > mask_max ] + print(f'Warning: removing {len(outliers)} outliers:') + print(outliers) + + return [d for d in data if d >= mask_min and d <= mask_max ] + +def reject_syscall_inc_outliers(data): + for i in range(0, len(data)): + if data[i] > 1000000: + print(f'Warning: removing outlier: {data[i]}') + data[i] = 0 + return data + +def load_data(file, col): + print(f".. loading data from '{file}'") + #data = np.loadtxt(file, dtype=int, skiprows=1) + d = pd.read_csv(file) + data = d[col] + #data = reject_outliers(data) + + #HACK: if the IRQ arrived before the slide the counter in RAX is invalid (zero) + if file == 'logs/icx/deadline_results_syscall.txt' and col == 'inc_count': + data = reject_syscall_inc_outliers(data) + + print('---------------------------------------------------------------------------') + s = pd.Series(data) + print(s.describe()) + print(f'med {int(np.median(data))}') + print('---------------------------------------------------------------------------') + return data + + +#LATENCY_MIN = 10700 +#LATENCY_MAX = 11500 +#s0 = load_data('oneshot_results.txt', col='#tsc_diff') +#hist([s0], ['oneshot']) +#exit() + +##s0 = load_data('deadline_results.txt', col='tsc_deadline_drift') + +#s0 = load_data('logs/icx/oneshot_results.txt', col='inc_count') +#s1 = load_data('logs/icx/deadline_results_syscall.txt', col='inc_count') +#s2 = load_data('logs/icx/deadline_results_irq_gate.txt', col='inc_count') +#LATENCY_MIN = 2500 +#LATENCY_MAX = 10000 +#hist([s2,s1,s0], ['TSC IRQ gate', 'TSC syscall', 'oneshot'], xlabel='ADD instructions executed before IRQ', legend_loc='upper left') + +## Appendix figure in paper +LATENCY_MIN = 10680 +LATENCY_MAX = 11120 +s0 = load_data('logs/icx/oneshot_results.txt', col='#tsc_diff') +s1 = load_data('logs/icx/deadline_results_syscall.txt', col='#tsc_diff') +s2 = load_data('logs/icx/deadline_results_irq_gate.txt', col='#tsc_diff') +hist([s2,s1,s0], ['TSC IRQ gate', 'TSC syscall', 'oneshot']) + +#s0 = load_data('oneshot_results.txt', col='#tsc_diff') +#hist([s0], ['oneshot'], 'APIC timer oneshot TSC distribution') + +#LATENCY_MIN = 650 +#LATENCY_MAX = 680 +##s0 = load_data('pt_results.txt', col='pte-not-a') +#s0 = load_data('pt_results.txt', col='pte-a') +#hist([s0], ['pte-not-a'], 'Assisted page-table walk latency distribution') +