-
Notifications
You must be signed in to change notification settings - Fork 145
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature: LPMP support to break the enclave number limitation
This PR introduces LPMP, which breaks the number limitation via an efficient PMP virtualization mechanism. As a proof-of-concept, our primary prototype enables Keystone to host 32 enclaves simultaneously. Co-authored-by: Mingde Ren <mingde-ren@outlook.com>
- Loading branch information
1 parent
b40dbb6
commit 27b5fd9
Showing
15 changed files
with
685 additions
and
74 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| set(eapp_bin hello-lpmp) | ||
| set(eapp_src eapp/hello_lpmp.c) | ||
| set(host_bin hello-lpmp-runner) | ||
| set(host_src host/host_lpmp.cpp) | ||
| set(package_name "hello-lpmp.ke") | ||
| set(package_script "./hello-lpmp-runner hello-lpmp eyrie-rt loader.bin") | ||
| set(eyrie_plugins "io_syscall linux_syscall env_setup") | ||
|
|
||
| # eapp | ||
|
|
||
| add_executable(${eapp_bin} ${eapp_src}) | ||
| target_link_libraries(${eapp_bin} "-static") | ||
|
|
||
| # host | ||
|
|
||
| add_executable(${host_bin} ${host_src}) | ||
| target_link_libraries(${host_bin} ${KEYSTONE_LIB_HOST} ${KEYSTONE_LIB_EDGE}) | ||
|
|
||
| # add target for Eyrie runtime (see keystone.cmake) | ||
|
|
||
| set(eyrie_files_to_copy .options_log eyrie-rt loader.bin) | ||
| add_eyrie_runtime(${eapp_bin}-eyrie | ||
| ${eyrie_plugins} | ||
| ${eyrie_files_to_copy}) | ||
|
|
||
| # add target for packaging (see keystone.cmake) | ||
|
|
||
| add_keystone_package(${eapp_bin}-package | ||
| ${package_name} | ||
| ${package_script} | ||
| ${eyrie_files_to_copy} ${eapp_bin} ${host_bin}) | ||
|
|
||
| add_dependencies(${eapp_bin}-package ${eapp_bin}-eyrie) | ||
|
|
||
| # add package to the top-level target | ||
| add_dependencies(examples ${eapp_bin}-package) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #include <stdio.h> | ||
| #include <unistd.h> | ||
|
|
||
| int main() | ||
| { | ||
| printf("Hello, Enclave!\n"); | ||
| sleep(30); | ||
| printf("Goodbye, Enclave!\n"); | ||
| return 0; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| //****************************************************************************** | ||
| // Copyright (c) 2018, The Regents of the University of California (Regents). | ||
| // All Rights Reserved. See LICENSE for license details. | ||
| //------------------------------------------------------------------------------ | ||
| #include "edge/edge_call.h" | ||
| #include "host/keystone.h" | ||
|
|
||
| using namespace Keystone; | ||
|
|
||
| int | ||
| main(int argc, char** argv) { | ||
| Enclave enclave; | ||
| Params params; | ||
|
|
||
| params.setFreeMemSize(256 * 1024); | ||
| params.setUntrustedSize(256 * 1024); | ||
|
|
||
| enclave.init(argv[1], argv[2], argv[3], params); | ||
|
|
||
| enclave.registerOcallDispatch(incoming_call_dispatch); | ||
| edge_call_init_internals( | ||
| (uintptr_t)enclave.getSharedBuffer(), enclave.getSharedBufferSize()); | ||
|
|
||
| enclave.run(); | ||
|
|
||
| return 0; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| diff --git a/include/sbi/sbi_lpmp.h b/include/sbi/sbi_lpmp.h | ||
| new file mode 100644 | ||
| index 0000000..d56ad96 | ||
| --- /dev/null | ||
| +++ b/include/sbi/sbi_lpmp.h | ||
| @@ -0,0 +1,21 @@ | ||
| +#ifndef __SBI_LPMP_H__ | ||
| +#define __SBI_LPMP_H__ | ||
| + | ||
| +#define PTE_V (1L << 0) | ||
| +#define PTE_R (1L << 1) | ||
| +#define PTE_W (1L << 2) | ||
| +#define PTE_X (1L << 3) | ||
| + | ||
| +#define PPNSHIFT 9 | ||
| +#define PGSHIFT 12 | ||
| +#define PTE2PA(pte) (((pte) >> 10) << 12) | ||
| +#define PXMASK 0x1FF // 9 bits | ||
| +#define PXSHIFT(level) (PGSHIFT + (9 * (level))) | ||
| +#define PX(level, va) ((((uint64_t)(va)) >> PXSHIFT(level)) & PXMASK) | ||
| + | ||
| +typedef uint64_t *pagetable_t; | ||
| +typedef uint64_t pte_t; | ||
| + | ||
| +int pmp_fault_handler(ulong mtval); | ||
| + | ||
| +#endif | ||
| diff --git a/lib/sbi/objects.mk b/lib/sbi/objects.mk | ||
| index 1ed1983..9bdc555 100644 | ||
| --- a/lib/sbi/objects.mk | ||
| +++ b/lib/sbi/objects.mk | ||
| @@ -44,3 +44,4 @@ libsbi-objs-y += sbi_tlb.o | ||
| libsbi-objs-y += sbi_trap.o | ||
| libsbi-objs-y += sbi_unpriv.o | ||
| libsbi-objs-y += sbi_expected_trap.o | ||
| +libsbi-objs-y += sbi_lpmp.o | ||
| diff --git a/lib/sbi/sbi_lpmp.c b/lib/sbi/sbi_lpmp.c | ||
| new file mode 100644 | ||
| index 0000000..35837e3 | ||
| --- /dev/null | ||
| +++ b/lib/sbi/sbi_lpmp.c | ||
| @@ -0,0 +1,75 @@ | ||
| +#include "lpmp.h" | ||
| +#include <sbi/riscv_asm.h> | ||
| +#include <sbi/riscv_encoding.h> | ||
| +#include <sbi/sbi_console.h> | ||
| +#include <sbi/sbi_error.h> | ||
| +#include <sbi/sbi_types.h> | ||
| +#include <sbi/sbi_lpmp.h> | ||
| + | ||
| +static uint64_t get_pt_root(void) { | ||
| + return ((csr_read(satp) & 0xFFFFFFFFFFF) << 12); | ||
| +} | ||
| + | ||
| +static uint64_t walkaddr(pagetable_t pagetable, uint64_t va) { | ||
| + if (pagetable == 0) | ||
| + return 0; | ||
| + pte_t *pte; | ||
| + uint64_t level; | ||
| + uint64_t pa; | ||
| + | ||
| + // make sure page tables in PMP. | ||
| + host_hit_region((uint64_t)pagetable); | ||
| + for (level = 4; level > 0; level--) { | ||
| + pte = &pagetable[PX(level, va)]; | ||
| + if (*pte & (PTE_X | PTE_W | PTE_R)) { | ||
| + goto found; // A leaf pte has been found. | ||
| + } else if (*pte & PTE_V) { | ||
| + pagetable = (pagetable_t)PTE2PA(*pte); | ||
| + host_hit_region((uint64_t)pagetable); | ||
| + } else { | ||
| + sbi_panic("invalid va=0x%lx\n", va); | ||
| + } | ||
| + } | ||
| + pte = &pagetable[PX(0, va)]; | ||
| + | ||
| +found: | ||
| + if (pte == 0) | ||
| + return 0; | ||
| + if ((*pte & PTE_V) == 0) | ||
| + return 0; | ||
| + uint64_t number_of_ones = PGSHIFT + level * PPNSHIFT; | ||
| + uint64_t offset_mask = (1 << number_of_ones) - 1; | ||
| + uint64_t offset = (va & offset_mask); | ||
| + pa = PTE2PA(*pte) + offset; | ||
| + | ||
| + return pa; | ||
| +} | ||
| + | ||
| +static inline void flush_tlb() | ||
| +{ | ||
| + asm volatile("sfence.vma"); | ||
| +} | ||
| + | ||
| +int pmp_fault_handler(ulong mtval) { | ||
| + if (!mtval) { | ||
| + sbi_printf("mepc = 0x%lx\n", csr_read(CSR_MEPC)); | ||
| + sbi_printf("Null pointer!\n"); | ||
| + return -1; | ||
| + } | ||
| + pagetable_t pt_root = (pagetable_t)get_pt_root(); | ||
| + uintptr_t pa = pt_root ? walkaddr(pt_root, mtval) : mtval; | ||
| + | ||
| + if (pa && host_hit_region(pa)) { | ||
| + activate_host_lpmp(); | ||
| + // Option 1. enable TLB cached PMP. | ||
| + asm volatile("sfence.vma %0, zero \n\t" : : "r"(mtval)); | ||
| + | ||
| + // Option 2. disable TLB cached PMP. | ||
| + // flush_tlb(); | ||
| + | ||
| + return 0; | ||
| + } else { | ||
| + sbi_printf("Error: Host should not access this pa\n"); | ||
| + return -1; | ||
| + } | ||
| +} | ||
| diff --git a/lib/sbi/sbi_trap.c b/lib/sbi/sbi_trap.c | ||
| index ee3e4e9..2a4fd44 100644 | ||
| --- a/lib/sbi/sbi_trap.c | ||
| +++ b/lib/sbi/sbi_trap.c | ||
| @@ -21,6 +21,7 @@ | ||
| #include <sbi/sbi_scratch.h> | ||
| #include <sbi/sbi_timer.h> | ||
| #include <sbi/sbi_trap.h> | ||
| +#include <sbi/sbi_lpmp.h> | ||
|
|
||
| static void __noreturn sbi_trap_error(const char *msg, int rc, | ||
| ulong mcause, ulong mtval, ulong mtval2, | ||
| @@ -304,6 +305,10 @@ struct sbi_trap_regs *sbi_trap_handler(struct sbi_trap_regs *regs) | ||
| sbi_pmu_ctr_incr_fw(mcause == CAUSE_LOAD_ACCESS ? | ||
| SBI_PMU_FW_ACCESS_LOAD : SBI_PMU_FW_ACCESS_STORE); | ||
| /* fallthrough */ | ||
| + case CAUSE_FETCH_ACCESS: | ||
| + rc = pmp_fault_handler(mtval); | ||
| + msg = "access fault (PMP)"; | ||
| + break; | ||
| default: | ||
| /* If the trap came from S or U mode, redirect it there */ | ||
| trap.epc = regs->mepc; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.