From d7e97ecc56a5265c8eb16bd8cd8c47845f13aa13 Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Fri, 12 Nov 2021 15:24:12 +0100 Subject: [PATCH 1/9] elfloader/risc-v: clarify parameters and types - Use word_t to adapt to architecture - sbi_hart_start() takes a custom argument. - hart ID and custom argument is passed when starting a secondary hart. Signed-off-by: Axel Heider --- elfloader-tool/include/arch-riscv/sbi.h | 44 ++++++++++++------------- elfloader-tool/src/arch-riscv/boot.c | 23 ++++++------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/elfloader-tool/include/arch-riscv/sbi.h b/elfloader-tool/include/arch-riscv/sbi.h index 3cf90b03..9c87057d 100644 --- a/elfloader-tool/include/arch-riscv/sbi.h +++ b/elfloader-tool/include/arch-riscv/sbi.h @@ -19,10 +19,10 @@ #define SBI_SHUTDOWN 8 #define SBI_CALL(which, arg0, arg1, arg2) ({ \ - register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); \ - register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); \ - register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); \ - register uintptr_t a7 asm ("a7") = (uintptr_t)(which); \ + register word_t a0 asm ("a0") = (word_t)(arg0); \ + register word_t a1 asm ("a1") = (word_t)(arg1); \ + register word_t a2 asm ("a2") = (word_t)(arg2); \ + register word_t a7 asm ("a7") = (word_t)(which); \ asm volatile ("ecall" \ : "+r" (a0) \ : "r" (a1), "r" (a2), "r" (a7) \ @@ -34,11 +34,11 @@ #define SBI_HSM_HART_START 0 #define SBI_EXT_CALL(extension, which, arg0, arg1, arg2) ({ \ - register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); \ - register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); \ - register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); \ - register uintptr_t a6 asm ("a6") = (uintptr_t)(which); \ - register uintptr_t a7 asm ("a7") = (uintptr_t)(extension); \ + register word_t a0 asm ("a0") = (word_t)(arg0); \ + register word_t a1 asm ("a1") = (word_t)(arg1); \ + register word_t a2 asm ("a2") = (word_t)(arg2); \ + register word_t a6 asm ("a6") = (word_t)(which); \ + register word_t a7 asm ("a7") = (word_t)(extension); \ asm volatile ("ecall" \ : "+r" (a0) \ : "r" (a1), "r" (a2), "r" (a6), "r" (a7) \ @@ -86,34 +86,34 @@ static inline void sbi_clear_ipi(void) SBI_CALL_0(SBI_CLEAR_IPI); } -static inline void sbi_send_ipi(const unsigned long *hart_mask) +static inline void sbi_send_ipi(const word_t *hart_mask) { SBI_CALL_1(SBI_SEND_IPI, hart_mask); } -static inline void sbi_remote_fence_i(const unsigned long *hart_mask) +static inline void sbi_remote_fence_i(const word_t *hart_mask) { SBI_CALL_1(SBI_REMOTE_FENCE_I, hart_mask); } -static inline void sbi_remote_sfence_vma(const unsigned long *hart_mask, - UNUSED unsigned long start, - UNUSED unsigned long size) +static inline void sbi_remote_sfence_vma(const word_t *hart_mask, + UNUSED word_t start, + UNUSED word_t size) { SBI_CALL_1(SBI_REMOTE_SFENCE_VMA, hart_mask); } -static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, - UNUSED unsigned long start, - UNUSED unsigned long size, - UNUSED unsigned long asid) +static inline void sbi_remote_sfence_vma_asid(const word_t *hart_mask, + UNUSED word_t start, + UNUSED word_t size, + UNUSED word_t asid) { SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask); } -static inline void sbi_hart_start(const unsigned long hart_id, - void (*start)(unsigned long), - unsigned long privilege) +static inline void sbi_hart_start(const word_t hart_id, + void (*start)(word_t hart_id, word_t arg), + word_t arg) { - SBI_HSM_CALL(SBI_HSM_HART_START, hart_id, start, privilege); + SBI_HSM_CALL(SBI_HSM_HART_START, hart_id, start, arg); } diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index f485594a..7edac2a3 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -147,17 +147,17 @@ int hsm_exists = 0; #if CONFIG_MAX_NUM_NODES > 1 -extern void secondary_harts(unsigned long); +extern void secondary_harts(word_t hart_id, word_t core_id); int secondary_go = 0; int next_logical_core_id = 1; int mutex = 0; int core_ready[CONFIG_MAX_NUM_NODES] = { 0 }; -static void set_and_wait_for_ready(int hart_id, int core_id) +static void set_and_wait_for_ready(word_t hart_id, word_t core_id) { /* Acquire lock to update core ready array */ while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); - printf("Hart ID %d core ID %d\n", hart_id, core_id); + printf("Hart ID %"PRIu_word" core ID %"PRIu_word"\n", hart_id, core_id); core_ready[core_id] = 1; __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); @@ -190,7 +190,7 @@ static inline void enable_virtual_memory(void) ifence(); } -static int run_elfloader(UNUSED int hart_id, void *bootloader_dtb) +static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) { int ret; @@ -217,14 +217,14 @@ static int run_elfloader(UNUSED int hart_id, void *bootloader_dtb) #if CONFIG_MAX_NUM_NODES > 1 while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); - printf("Main entry hart_id:%d\n", hart_id); + printf("Main entry hart_id:%"PRIu_word"\n", hart_id); __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); /* Unleash secondary cores */ __atomic_store_n(&secondary_go, 1, __ATOMIC_RELEASE); /* Start all cores */ - int i = 0; + word_t i = 0; while (i < CONFIG_MAX_NUM_NODES && hsm_exists) { i++; if (i != hart_id) { @@ -259,12 +259,13 @@ static int run_elfloader(UNUSED int hart_id, void *bootloader_dtb) #if CONFIG_MAX_NUM_NODES > 1 -void secondary_entry(int hart_id, int core_id) +void secondary_entry(word_t hart_id, word_t core_id) { while (__atomic_load_n(&secondary_go, __ATOMIC_ACQUIRE) == 0) ; while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); - printf("Secondary entry hart_id:%d core_id:%d\n", hart_id, core_id); + printf("Secondary entry hart_id:%"PRIu_word" core_id:%"PRIu_word"\n", + hart_id, core_id); __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); set_and_wait_for_ready(hart_id, core_id); @@ -287,11 +288,11 @@ void secondary_entry(int hart_id, int core_id) #endif -void main(int hart_id, void *bootloader_dtb) +void main(word_t hart_id, void *bootloader_dtb) { /* Printing uses SBI, so there is no need to initialize any UART. */ - printf("ELF-loader started on (HART %d) (NODES %d)\n", - hart_id, CONFIG_MAX_NUM_NODES); + printf("ELF-loader started on (HART %"PRIu_word") (NODES %d)\n", + hart_id, (unsigned int)CONFIG_MAX_NUM_NODES); printf(" paddr=[%p..%p]\n", _text, _end - 1); From 852f00ed84eea64fd85c069c9cd23b92a91a99f4 Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Sun, 6 Feb 2022 21:16:38 +0100 Subject: [PATCH 2/9] elfloader/risc-v: no need to use t0 This also makes the comment about a1 correct again. Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/crt0.S | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/crt0.S b/elfloader-tool/src/arch-riscv/crt0.S index 3447cc38..69e8808e 100644 --- a/elfloader-tool/src/arch-riscv/crt0.S +++ b/elfloader-tool/src/arch-riscv/crt0.S @@ -147,13 +147,14 @@ secondary_harts: #if CONFIG_MAX_NUM_NODES > 1 la a1, next_logical_core_id li t2, 1 - amoadd.w t0, t2, (a1) - /* now a1 has the logical core id */ + /* atomically increment next_logical_core_id by one, afterwards a1 holds value + * before the update - which is our logical core ID. + */ + amoadd.w a1, t2, (a1) li t2, CONFIG_MAX_NUM_NODES - bge t0, t2, hsm_suspend_hart + bge a1, t2, hsm_suspend_hart - mv a1, t0 - slli t0, t0, 12 + slli t0, a1, 12 la sp, bootstack_secondary_cores add sp, sp, t0 la s0, secondary_entry From aec5ef1783c6daf7da35f88fa5157be58a8d6e1c Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Sun, 6 Feb 2022 21:17:31 +0100 Subject: [PATCH 3/9] elfloader/risc-v: unify multicore stacks - declare stack fully in C code - use one array that holds all stack - use CONFIG_KERNEL_STACK_BITS for size Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/boot.c | 3 ++- elfloader-tool/src/arch-riscv/crt0.S | 25 ++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index 7edac2a3..48085b8e 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -57,7 +57,8 @@ unsigned long l2pt[PTES_PER_PT] __attribute__((aligned(4096))); unsigned long l2pt_elf[PTES_PER_PT] __attribute__((aligned(4096))); #endif -char elfloader_stack_alloc[BIT(CONFIG_KERNEL_STACK_BITS)]; +/* Stacks for each core are set up in the assembly startup code. */ +char elfloader_stack[CONFIG_MAX_NUM_NODES * BIT(CONFIG_KERNEL_STACK_BITS)] __attribute__((aligned(4096))); /* first HART will initialise these */ void const *dtb = NULL; diff --git a/elfloader-tool/src/arch-riscv/crt0.S b/elfloader-tool/src/arch-riscv/crt0.S index 69e8808e..74e39a6e 100644 --- a/elfloader-tool/src/arch-riscv/crt0.S +++ b/elfloader-tool/src/arch-riscv/crt0.S @@ -9,8 +9,11 @@ .extern main .extern __global_pointer$ -.extern elfloader_stack_alloc +.extern elfloader_stack .extern hsm_exists +#if CONFIG_MAX_NUM_NODES > 1 +.extern next_logical_core_id +#endif #define BIT(n) (1 << (n)) @@ -56,7 +59,7 @@ _start: mv s2, a1 /* preserve a1 (dtb) in s2 */ /* Attach the stack to sp before calling any C functions */ - la sp, (elfloader_stack_alloc + BIT(12)) + la sp, (elfloader_stack + BIT(CONFIG_KERNEL_STACK_BITS)) #ifdef CONFIG_IMAGE_BINARY /* Clear the BSS before we get to do anything more specific */ @@ -117,7 +120,7 @@ _start1: /* a0 must hold current hard ID passed by bootloader */ /* This HART may be a different HART to the one that started at _start * If we've switched HARTs then the other HART will get a different stack * region in secondary_harts. */ - la sp, (elfloader_stack_alloc + BIT(12)) + la sp, (elfloader_stack + BIT(CONFIG_KERNEL_STACK_BITS)) /* The C code expects the registers to be set up as: * a0 = hart id * a1 = dtb @@ -125,15 +128,6 @@ _start1: /* a0 must hold current hard ID passed by bootloader */ la s0, main jr s0 -#if CONFIG_MAX_NUM_NODES > 1 -.extern next_logical_core_id -.data -bootstack_secondary_cores: -.align 12 -.space 4096 * (CONFIG_MAX_NUM_NODES - 1) -#endif - -.text .global secondary_harts secondary_harts: @@ -153,9 +147,10 @@ secondary_harts: amoadd.w a1, t2, (a1) li t2, CONFIG_MAX_NUM_NODES bge a1, t2, hsm_suspend_hart - - slli t0, a1, 12 - la sp, bootstack_secondary_cores + /* setup the hart specific stack pointer */ + la sp, elfloader_stack + addi t0, a1, 1 /* increment by one because we need to set sp to the end */ + slli t0, t0, CONFIG_KERNEL_STACK_BITS /* t0 = t0 * BIT(CONFIG_KERNEL_STACK_BITS) */ add sp, sp, t0 la s0, secondary_entry jr s0 From c2ec6d2518419a05a1200eed30795cb885af36f5 Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Sun, 6 Feb 2022 21:08:57 +0100 Subject: [PATCH 4/9] elfloader/risc-v: improve comments Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/boot.c | 19 +++++++++++-------- elfloader-tool/src/arch-riscv/crt0.S | 26 +++++++++++++------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index 48085b8e..1d5b7f12 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -144,14 +144,14 @@ uint64_t vm_mode = 0x9llu << 60; #error "Wrong PT level" #endif -int hsm_exists = 0; +int hsm_exists = 0; /* assembly startup code will initialise this */ #if CONFIG_MAX_NUM_NODES > 1 extern void secondary_harts(word_t hart_id, word_t core_id); int secondary_go = 0; -int next_logical_core_id = 1; +int next_logical_core_id = 1; /* incremented by assembly code */ int mutex = 0; int core_ready[CONFIG_MAX_NUM_NODES] = { 0 }; static void set_and_wait_for_ready(word_t hart_id, word_t core_id) @@ -162,12 +162,15 @@ static void set_and_wait_for_ready(word_t hart_id, word_t core_id) core_ready[core_id] = 1; __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); - /* Wait untill all cores are go */ + /* Wait until all cores are go */ for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) { - while (__atomic_load_n(&core_ready[i], __ATOMIC_RELAXED) == 0) ; + while (__atomic_load_n(&core_ready[i], __ATOMIC_RELAXED) == 0) { + /* busy waiting loop */ + } } } -#endif + +#endif /* CONFIG_MAX_NUM_NODES > 1 */ static inline void sfence_vma(void) { @@ -234,7 +237,7 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) } set_and_wait_for_ready(hart_id, 0); -#endif +#endif /* CONFIG_MAX_NUM_NODES > 1 */ printf("Enabling MMU and paging\n"); enable_virtual_memory(); @@ -250,7 +253,7 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) , hart_id, 0 -#endif +#endif /* CONFIG_MAX_NUM_NODES > 1 */ ); /* We should never get here. */ @@ -287,7 +290,7 @@ void secondary_entry(word_t hart_id, word_t core_id) ); } -#endif +#endif /* CONFIG_MAX_NUM_NODES > 1 */ void main(word_t hart_id, void *bootloader_dtb) { diff --git a/elfloader-tool/src/arch-riscv/crt0.S b/elfloader-tool/src/arch-riscv/crt0.S index 74e39a6e..2814abb6 100644 --- a/elfloader-tool/src/arch-riscv/crt0.S +++ b/elfloader-tool/src/arch-riscv/crt0.S @@ -107,28 +107,28 @@ hsm_switch_hart: mv a0, s0 /* restore a0 to hold hart ID passed by OpenSBI */ j secondary_harts - -_start1: /* a0 must hold current hard ID passed by bootloader */ - /* a1 must hold dtb address passed by bootloader */ +/*----------------------------------------------------------------------------*/ +_start1: +/* This is basically an asm wrapper to jump to the C code at main(). The + * registers are already set up with the perameters for the C code: + * a0 holds current hard ID passed by bootloader + * a1 holds dtb address passed by bootloader + * All that is left to be done here is setting up the registers gp and sp to + * have a proper C environment. The hart we are running on now could be a + * different HART to the one that we have been on in _start. The original hart + * we came from will get a different stack in secondary_harts. + */ .option push .option norelax 1:auipc gp, %pcrel_hi(__global_pointer$) addi gp, gp, %pcrel_lo(1b) .option pop - - /* Attach the stack to sp before calling any C functions */ - /* This HART may be a different HART to the one that started at _start - * If we've switched HARTs then the other HART will get a different stack - * region in secondary_harts. */ la sp, (elfloader_stack + BIT(CONFIG_KERNEL_STACK_BITS)) - /* The C code expects the registers to be set up as: - * a0 = hart id - * a1 = dtb - */ + /* Jump via a register, as this can cover a bigger range. */ la s0, main jr s0 - +/*----------------------------------------------------------------------------*/ .global secondary_harts secondary_harts: From 88ff2ec8e4b67e33311df8cf9d3be5b4d9813ca3 Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Thu, 11 Nov 2021 07:39:13 +0100 Subject: [PATCH 5/9] elfloader/risc-v: Improve code readability Reorder code blocks and improve comments. Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/boot.c | 66 ++++++++++++++-------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index 1d5b7f12..042f8eec 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -64,6 +64,38 @@ char elfloader_stack[CONFIG_MAX_NUM_NODES * BIT(CONFIG_KERNEL_STACK_BITS)] __att void const *dtb = NULL; size_t dtb_size = 0; +static inline void sfence_vma(void) +{ + asm volatile("sfence.vma" ::: "memory"); +} + +static inline void ifence(void) +{ + asm volatile("fence.i" ::: "memory"); +} + +#if CONFIG_PT_LEVELS == 2 +uint64_t vm_mode = 0x1llu << 31; +#elif CONFIG_PT_LEVELS == 3 +uint64_t vm_mode = 0x8llu << 60; +#elif CONFIG_PT_LEVELS == 4 +uint64_t vm_mode = 0x9llu << 60; +#else +#error "Wrong PT level" +#endif + +static inline void enable_virtual_memory(void) +{ + sfence_vma(); + asm volatile( + "csrw satp, %0\n" + : + : "r"(vm_mode | (uintptr_t)l1pt >> RISCV_PGSHIFT) + : + ); + ifence(); +} + /* * overwrite the default implementation for abort() */ @@ -134,16 +166,6 @@ static int map_kernel_window(struct image_info *kernel_info) return 0; } -#if CONFIG_PT_LEVELS == 2 -uint64_t vm_mode = 0x1llu << 31; -#elif CONFIG_PT_LEVELS == 3 -uint64_t vm_mode = 0x8llu << 60; -#elif CONFIG_PT_LEVELS == 4 -uint64_t vm_mode = 0x9llu << 60; -#else -#error "Wrong PT level" -#endif - int hsm_exists = 0; /* assembly startup code will initialise this */ #if CONFIG_MAX_NUM_NODES > 1 @@ -172,28 +194,6 @@ static void set_and_wait_for_ready(word_t hart_id, word_t core_id) #endif /* CONFIG_MAX_NUM_NODES > 1 */ -static inline void sfence_vma(void) -{ - asm volatile("sfence.vma" ::: "memory"); -} - -static inline void ifence(void) -{ - asm volatile("fence.i" ::: "memory"); -} - -static inline void enable_virtual_memory(void) -{ - sfence_vma(); - asm volatile( - "csrw satp, %0\n" - : - : "r"(vm_mode | (uintptr_t)l1pt >> RISCV_PGSHIFT) - : - ); - ifence(); -} - static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) { int ret; @@ -220,6 +220,7 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) } #if CONFIG_MAX_NUM_NODES > 1 + while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); printf("Main entry hart_id:%"PRIu_word"\n", hart_id); __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); @@ -237,6 +238,7 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) } set_and_wait_for_ready(hart_id, 0); + #endif /* CONFIG_MAX_NUM_NODES > 1 */ printf("Enabling MMU and paging\n"); From e3f90b1e948bbf1c3f8f0a68b5a4e0c9ec5714f4 Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Fri, 12 Nov 2021 18:14:59 +0100 Subject: [PATCH 6/9] elfloader/risc-v: add multi core helper functions Use wrapper function to Improve code readability. Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/boot.c | 65 +++++++++++++++++++--------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index 042f8eec..83605408 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -176,17 +176,52 @@ int secondary_go = 0; int next_logical_core_id = 1; /* incremented by assembly code */ int mutex = 0; int core_ready[CONFIG_MAX_NUM_NODES] = { 0 }; + +static void acquire_multicore_lock(void) +{ + while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0) { + /* busy waiting loop */ + } +} + +static void release_multicore_lock(void) +{ + __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); +} + +static void set_secondary_cores_go(void) +{ + __atomic_store_n(&secondary_go, 1, __ATOMIC_RELEASE); +} + +static void block_until_secondary_cores_go(void) +{ + while (__atomic_load_n(&secondary_go, __ATOMIC_ACQUIRE) == 0) { + /* busy waiting loop */ + } +} + +static void mark_core_ready(int core_id) +{ + core_ready[core_id] = 1; +} + +static int is_core_ready(int core_id) +{ + return (0 != __atomic_load_n(&core_ready[core_id], __ATOMIC_RELAXED)); +} + static void set_and_wait_for_ready(word_t hart_id, word_t core_id) { /* Acquire lock to update core ready array */ - while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); + acquire_multicore_lock(); printf("Hart ID %"PRIu_word" core ID %"PRIu_word"\n", hart_id, core_id); - core_ready[core_id] = 1; - __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); + mark_core_ready(core_id); + release_multicore_lock(); /* Wait until all cores are go */ for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) { - while (__atomic_load_n(&core_ready[i], __ATOMIC_RELAXED) == 0) { + while (!is_core_ready(i)) { /* busy waiting loop */ } } @@ -220,14 +255,11 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) } #if CONFIG_MAX_NUM_NODES > 1 - - while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); + acquire_multicore_lock(); printf("Main entry hart_id:%"PRIu_word"\n", hart_id); - __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); - + release_multicore_lock(); /* Unleash secondary cores */ - __atomic_store_n(&secondary_go, 1, __ATOMIC_RELEASE); - + set_secondary_cores_go(); /* Start all cores */ word_t i = 0; while (i < CONFIG_MAX_NUM_NODES && hsm_exists) { @@ -236,9 +268,7 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) sbi_hart_start(i, secondary_harts, i); } } - set_and_wait_for_ready(hart_id, 0); - #endif /* CONFIG_MAX_NUM_NODES > 1 */ printf("Enabling MMU and paging\n"); @@ -267,18 +297,13 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) void secondary_entry(word_t hart_id, word_t core_id) { - while (__atomic_load_n(&secondary_go, __ATOMIC_ACQUIRE) == 0) ; - - while (__atomic_exchange_n(&mutex, 1, __ATOMIC_ACQUIRE) != 0); + block_until_secondary_cores_go(); + acquire_multicore_lock(); printf("Secondary entry hart_id:%"PRIu_word" core_id:%"PRIu_word"\n", hart_id, core_id); - __atomic_store_n(&mutex, 0, __ATOMIC_RELEASE); - + release_multicore_lock(); set_and_wait_for_ready(hart_id, core_id); - enable_virtual_memory(); - - /* If adding or modifying these parameters you will need to update the registers in head.S */ ((init_riscv_kernel_t)kernel_info.virt_entry)(user_info.phys_region_start, From 4652182751ba93e0ed358e466f20cc454af29a2b Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Fri, 12 Nov 2021 15:27:35 +0100 Subject: [PATCH 7/9] elfloader/risc-v: add helper function boot_hart() Rework boot code flow to simplify it and avoid redundancy. Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/boot.c | 144 ++++++++++++++------------- elfloader-tool/src/arch-riscv/crt0.S | 3 +- 2 files changed, 79 insertions(+), 68 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index 83605408..872dac7a 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -211,25 +211,10 @@ static int is_core_ready(int core_id) return (0 != __atomic_load_n(&core_ready[core_id], __ATOMIC_RELAXED)); } -static void set_and_wait_for_ready(word_t hart_id, word_t core_id) -{ - /* Acquire lock to update core ready array */ - acquire_multicore_lock(); - printf("Hart ID %"PRIu_word" core ID %"PRIu_word"\n", hart_id, core_id); - mark_core_ready(core_id); - release_multicore_lock(); - - /* Wait until all cores are go */ - for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) { - while (!is_core_ready(i)) { - /* busy waiting loop */ - } - } -} #endif /* CONFIG_MAX_NUM_NODES > 1 */ -static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) +static int run_elfloader(void *bootloader_dtb) { int ret; @@ -248,75 +233,86 @@ static int run_elfloader(UNUSED word_t hart_id, void *bootloader_dtb) return -1; } + /* Setup MMU tables. */ ret = map_kernel_window(&kernel_info); if (0 != ret) { printf("ERROR: could not map kernel window, code %d\n", ret); return -1; } + return 0; +} + +static NORETURN void boot_hart(word_t hart_id, word_t core_id) +{ + /* Caller must hold the multicore lock here. */ + + if (0 == core_id) { + printf("Enabling MMU and paging\n"); + } + enable_virtual_memory(); + #if CONFIG_MAX_NUM_NODES > 1 - acquire_multicore_lock(); - printf("Main entry hart_id:%"PRIu_word"\n", hart_id); + /* We are ready to hand over control to the kernel on this hart. Sync with + * all other harts before doing this. + */ + mark_core_ready(core_id); release_multicore_lock(); - /* Unleash secondary cores */ - set_secondary_cores_go(); - /* Start all cores */ - word_t i = 0; - while (i < CONFIG_MAX_NUM_NODES && hsm_exists) { - i++; - if (i != hart_id) { - sbi_hart_start(i, secondary_harts, i); + for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) { + while (!is_core_ready(i)) { + /* busy waiting loop */ } } - set_and_wait_for_ready(hart_id, 0); #endif /* CONFIG_MAX_NUM_NODES > 1 */ - printf("Enabling MMU and paging\n"); - enable_virtual_memory(); + if (0 == core_id) { + printf("Jumping to kernel-image entry point...\n\n"); + } - printf("Jumping to kernel-image entry point...\n\n"); - ((init_riscv_kernel_t)kernel_info.virt_entry)(user_info.phys_region_start, - user_info.phys_region_end, - user_info.phys_virt_offset, - user_info.virt_entry, - (word_t)dtb, - dtb_size + /* The hand over interface is the same on all cores. We avoid making + * assumption how the parameters are used. The current seL4 kernel + * implementation only cares about the DTB on the primary core. + */ + ((init_riscv_kernel_t)kernel_info.virt_entry)( + user_info.phys_region_start, + user_info.phys_region_end, + user_info.phys_virt_offset, + user_info.virt_entry, + (word_t)dtb, + dtb_size #if CONFIG_MAX_NUM_NODES > 1 - , - hart_id, - 0 + , + hart_id, + core_id #endif /* CONFIG_MAX_NUM_NODES > 1 */ - ); + ); /* We should never get here. */ - printf("ERROR: Kernel returned back to the ELF Loader\n"); - return -1; + printf("ERROR: back in ELF-loader hart %"PRIu_word" (core ID %"PRIu_word")\n", + hart_id, core_id); + abort(); + UNREACHABLE(); } #if CONFIG_MAX_NUM_NODES > 1 - -void secondary_entry(word_t hart_id, word_t core_id) +NORETURN void secondary_hart_main(word_t hart_id, word_t core_id) { block_until_secondary_cores_go(); acquire_multicore_lock(); - printf("Secondary entry hart_id:%"PRIu_word" core_id:%"PRIu_word"\n", + printf("started hart %"PRIu_word" (core id %"PRIu_word")\n", hart_id, core_id); - release_multicore_lock(); - set_and_wait_for_ready(hart_id, core_id); - enable_virtual_memory(); - /* If adding or modifying these parameters you will need to update - the registers in head.S */ - ((init_riscv_kernel_t)kernel_info.virt_entry)(user_info.phys_region_start, - user_info.phys_region_end, - user_info.phys_virt_offset, - user_info.virt_entry, - (word_t)dtb, - dtb_size, - hart_id, - core_id - ); -} + if (core_id >= CONFIG_MAX_NUM_NODES) { + printf("ERROR: max number of harts exceeded (%d)\n", + (int)CONFIG_MAX_NUM_NODES); + abort(); + UNREACHABLE(); + } + + boot_hart(hart_id, core_id); + UNREACHABLE(); + +} #endif /* CONFIG_MAX_NUM_NODES > 1 */ void main(word_t hart_id, void *bootloader_dtb) @@ -327,10 +323,7 @@ void main(word_t hart_id, void *bootloader_dtb) printf(" paddr=[%p..%p]\n", _text, _end - 1); - /* Run the actual ELF loader, this is not expected to return unless there - * was an error. - */ - int ret = run_elfloader(hart_id, bootloader_dtb); + int ret = run_elfloader(bootloader_dtb); if (0 != ret) { printf("ERROR: ELF-loader failed, code %d\n", ret); /* There is nothing we can do to recover. */ @@ -338,8 +331,25 @@ void main(word_t hart_id, void *bootloader_dtb) UNREACHABLE(); } - /* We should never get here. */ - printf("ERROR: ELF-loader didn't hand over control\n"); - abort(); +#if CONFIG_MAX_NUM_NODES > 1 + /* Take the multicore lock first, then start all secondary cores. This + * ensures the boot process on the primary core can continue without running + * into concurrency issues, until things can really run in parallel. The + * main use case for this currently is printing nicely serialized boot + * messages, + */ + acquire_multicore_lock(); + set_secondary_cores_go(); + word_t i = 0; + while (i < CONFIG_MAX_NUM_NODES && hsm_exists) { + i++; + if (i != hart_id) { + sbi_hart_start(i, secondary_harts, i); + } + } +} +#endif /* CONFIG_MAX_NUM_NODES > 1 */ + + boot_hart(hart_id, 0); UNREACHABLE(); } diff --git a/elfloader-tool/src/arch-riscv/crt0.S b/elfloader-tool/src/arch-riscv/crt0.S index 2814abb6..105b88be 100644 --- a/elfloader-tool/src/arch-riscv/crt0.S +++ b/elfloader-tool/src/arch-riscv/crt0.S @@ -12,6 +12,7 @@ .extern elfloader_stack .extern hsm_exists #if CONFIG_MAX_NUM_NODES > 1 +.extern secondary_hart_main .extern next_logical_core_id #endif @@ -152,7 +153,7 @@ secondary_harts: addi t0, a1, 1 /* increment by one because we need to set sp to the end */ slli t0, t0, CONFIG_KERNEL_STACK_BITS /* t0 = t0 * BIT(CONFIG_KERNEL_STACK_BITS) */ add sp, sp, t0 - la s0, secondary_entry + la s0, secondary_hart_main jr s0 #endif /* If we get here then the HSM extension exists and the current From a12093a7b5590de006cbaeda09e4faf326398c5c Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Thu, 11 Nov 2021 03:59:23 +0100 Subject: [PATCH 8/9] elfloader/risc-v: fix multicore hart starting - stop wrong boot hart, it will be started alter again. - abort if a secondary hart cannot be started. - check hsm_exists only once. - log message if HSM extension is missing. - improve comments. Signed-off-by: Axel Heider --- elfloader-tool/include/arch-riscv/sbi.h | 60 ++++++++++++++------- elfloader-tool/src/arch-riscv/boot.c | 70 +++++++++++++++++++------ elfloader-tool/src/arch-riscv/crt0.S | 14 ++--- 3 files changed, 100 insertions(+), 44 deletions(-) diff --git a/elfloader-tool/include/arch-riscv/sbi.h b/elfloader-tool/include/arch-riscv/sbi.h index 9c87057d..3af5f6b5 100644 --- a/elfloader-tool/include/arch-riscv/sbi.h +++ b/elfloader-tool/include/arch-riscv/sbi.h @@ -30,24 +30,43 @@ a0; \ }) +typedef enum { + SBI_SUCCESS = 0, + SBI_ERR_FAILED = -1, + SBI_ERR_NOT_SUPPORTED = -2, + SBI_ERR_INVALID_PARAM = -3, + SBI_ERR_DENIED = -4, + SBI_ERR_INVALID_ADDRESS = -5, + SBI_ERR_ALREADY_AVAILABLE = -6, + SBI_ERR_ALREADY_STARTED = -7, + SBI_ERR_ALREADY_STOPPED = -8 +} sbi_call_ret_t; + #define SBI_HSM 0x48534DULL #define SBI_HSM_HART_START 0 -#define SBI_EXT_CALL(extension, which, arg0, arg1, arg2) ({ \ - register word_t a0 asm ("a0") = (word_t)(arg0); \ - register word_t a1 asm ("a1") = (word_t)(arg1); \ - register word_t a2 asm ("a2") = (word_t)(arg2); \ - register word_t a6 asm ("a6") = (word_t)(which); \ - register word_t a7 asm ("a7") = (word_t)(extension); \ - asm volatile ("ecall" \ - : "+r" (a0) \ - : "r" (a1), "r" (a2), "r" (a6), "r" (a7) \ - : "memory"); \ - a0; \ -}) - -#define SBI_HSM_CALL(which, arg0, arg1, arg2) \ - SBI_EXT_CALL(SBI_HSM, (which), (arg0), (arg1), (arg2)) +typedef struct { + sbi_call_ret_t code; + word_t data; +} sbi_hsm_ret_t; + +#define SBI_EXT_CALL(extension, which, arg0, arg1, arg2, var_sbi_hsm_ret) \ + do { \ + register word_t a0 asm ("a0") = (word_t)(arg0); \ + register word_t a1 asm ("a1") = (word_t)(arg1); \ + register word_t a2 asm ("a2") = (word_t)(arg2); \ + register word_t a6 asm ("a6") = (word_t)(which); \ + register word_t a7 asm ("a7") = (word_t)(extension); \ + asm volatile ("ecall" \ + : "+r" (a0), "+r" (a1) \ + : "r" (a2), "r" (a6), "r" (a7) \ + : "memory"); \ + (var_sbi_hsm_ret).code = a0; \ + (var_sbi_hsm_ret).data = a1; \ + } while(0) + +#define SBI_HSM_CALL(which, arg0, arg1, arg2, var_sbi_hsm_ret) \ + SBI_EXT_CALL(SBI_HSM, (which), (arg0), (arg1), (arg2), var_sbi_hsm_ret) /* Lazy implementations until SBI is finalized */ #define SBI_CALL_0(which) SBI_CALL(which, 0, 0, 0) @@ -111,9 +130,12 @@ static inline void sbi_remote_sfence_vma_asid(const word_t *hart_mask, SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask); } -static inline void sbi_hart_start(const word_t hart_id, - void (*start)(word_t hart_id, word_t arg), - word_t arg) +static inline sbi_hsm_ret_t sbi_hart_start(const word_t hart_id, + void (*start)(word_t hart_id, + word_t arg), + word_t arg) { - SBI_HSM_CALL(SBI_HSM_HART_START, hart_id, start, arg); + sbi_hsm_ret_t ret = { 0 }; + SBI_HSM_CALL(SBI_HSM_HART_START, hart_id, start, arg, ret); + return ret; } diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index 872dac7a..ff851f84 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -171,6 +171,7 @@ int hsm_exists = 0; /* assembly startup code will initialise this */ #if CONFIG_MAX_NUM_NODES > 1 extern void secondary_harts(word_t hart_id, word_t core_id); +extern void hsm_entry_on_secondary_hart(void); int secondary_go = 0; int next_logical_core_id = 1; /* incremented by assembly code */ @@ -211,6 +212,58 @@ static int is_core_ready(int core_id) return (0 != __atomic_load_n(&core_ready[core_id], __ATOMIC_RELAXED)); } +static void start_secondary_harts(word_t primary_hart_id) +{ + /* Take the multicore lock first, then start all secondary cores. This + * ensures the boot process on the primary core can continue without running + * into concurrency issues, until things can really run in parallel. The + * main use case for this currently is printing nicely serialized boot + * messages, + */ + acquire_multicore_lock(); + set_secondary_cores_go(); + /* Start all cores */ + if (!hsm_exists) { + /* Without the HSM extension, we can't start the cores explicitly. But + * they might be running already, so we do nothing here and just hope + * things work out. If the secondary cores don't start we are stuck. + */ + printf("no HSM extension, let's hope secondary cores have been started\n"); + return; + } + + /* If we are running on a platform with SBI HSM extension support, no other + * hart is running. The system starts harts in a random hart, but the + * assembly startup code has done the migration to the designated primary + * hart already and stopped the others. The global variable logical_core_id + * must still be untouched here, otherwise something is badly wrong. + */ + if (1 != next_logical_core_id) { + printf("ERROR: logical core IDs have been assigned already\n"); + abort(); + UNREACHABLE(); + } + /* Start all harts */ + for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) { + word_t remote_hart_id = i + 1; /* hart IDs start at 1 */ + if (remote_hart_id == CONFIG_FIRST_HART_ID) { + assert(remote_hart_id == primary_hart_id); + continue; /* this is the current hart */ + } + /* Start secondary hart, there is nothing to pass as custom + * parameter thus it's 0. + */ + sbi_hsm_ret_t ret = sbi_hart_start(remote_hart_id, + hsm_entry_on_secondary_hart, + 0); + if (SBI_SUCCESS != ret.code) { + printf("ERROR: could not start hart %"PRIu_word", failure" + " (%d, %d)\n", remote_hart_id, ret.code, ret.data); + abort(); + UNREACHABLE(); + } + } +} #endif /* CONFIG_MAX_NUM_NODES > 1 */ @@ -332,22 +385,7 @@ void main(word_t hart_id, void *bootloader_dtb) } #if CONFIG_MAX_NUM_NODES > 1 - /* Take the multicore lock first, then start all secondary cores. This - * ensures the boot process on the primary core can continue without running - * into concurrency issues, until things can really run in parallel. The - * main use case for this currently is printing nicely serialized boot - * messages, - */ - acquire_multicore_lock(); - set_secondary_cores_go(); - word_t i = 0; - while (i < CONFIG_MAX_NUM_NODES && hsm_exists) { - i++; - if (i != hart_id) { - sbi_hart_start(i, secondary_harts, i); - } - } -} + start_secondary_harts(hart_id); #endif /* CONFIG_MAX_NUM_NODES > 1 */ boot_hart(hart_id, 0); diff --git a/elfloader-tool/src/arch-riscv/crt0.S b/elfloader-tool/src/arch-riscv/crt0.S index 105b88be..3169f2d4 100644 --- a/elfloader-tool/src/arch-riscv/crt0.S +++ b/elfloader-tool/src/arch-riscv/crt0.S @@ -101,12 +101,8 @@ hsm_switch_hart: mv a2, s2 /* dtb address to be passed in a1 when new hart starts is 3rd parameter */ la a1, _start1 /* where to start the hart */ ecall /* call SBI to start hart FIRST_HART_ID */ - - /* Since we are not the designated primary hart, continue the boot process as - * secondary hart - */ - mv a0, s0 /* restore a0 to hold hart ID passed by OpenSBI */ - j secondary_harts + /* Stop current hart, the boot code may bring it up again when needed. */ + j hsm_suspend_hart /*----------------------------------------------------------------------------*/ _start1: @@ -117,7 +113,7 @@ _start1: * All that is left to be done here is setting up the registers gp and sp to * have a proper C environment. The hart we are running on now could be a * different HART to the one that we have been on in _start. The original hart - * we came from will get a different stack in secondary_harts. + * we came from will get a different stack in hsm_entry_on_secondary_hart. */ .option push .option norelax @@ -130,8 +126,8 @@ _start1: jr s0 /*----------------------------------------------------------------------------*/ -.global secondary_harts -secondary_harts: +.global hsm_entry_on_secondary_hart +hsm_entry_on_secondary_hart: .option push .option norelax From 79bc266e372e15212171f1d8fb51e57dfe58366b Mon Sep 17 00:00:00 2001 From: Axel Heider Date: Mon, 15 Jan 2024 15:08:37 +0100 Subject: [PATCH 9/9] elfloader/risc-v: rework multi core handling - build code around SBI HSM extension - Ensure DTB is always passed to primary core boot - Drop variable hsm_exists, pass information as parameter - Print more log messages Signed-off-by: Axel Heider --- elfloader-tool/src/arch-riscv/boot.c | 37 +++--- elfloader-tool/src/arch-riscv/crt0.S | 187 +++++++++++++++++---------- 2 files changed, 143 insertions(+), 81 deletions(-) diff --git a/elfloader-tool/src/arch-riscv/boot.c b/elfloader-tool/src/arch-riscv/boot.c index ff851f84..58017b51 100644 --- a/elfloader-tool/src/arch-riscv/boot.c +++ b/elfloader-tool/src/arch-riscv/boot.c @@ -166,12 +166,11 @@ static int map_kernel_window(struct image_info *kernel_info) return 0; } -int hsm_exists = 0; /* assembly startup code will initialise this */ - #if CONFIG_MAX_NUM_NODES > 1 -extern void secondary_harts(word_t hart_id, word_t core_id); -extern void hsm_entry_on_secondary_hart(void); +/* entry if secondary harts are started via SBI HSM extension */ +extern void hsm_start_secondary_core(word_t hart_id, word_t core_id); +extern void hsm_entry_on_secondary_hart(word_t hard_id); int secondary_go = 0; int next_logical_core_id = 1; /* incremented by assembly code */ @@ -212,7 +211,7 @@ static int is_core_ready(int core_id) return (0 != __atomic_load_n(&core_ready[core_id], __ATOMIC_RELAXED)); } -static void start_secondary_harts(word_t primary_hart_id) +static void start_secondary_harts(word_t primary_hart_id, word_t hsm_exists) { /* Take the multicore lock first, then start all secondary cores. This * ensures the boot process on the primary core can continue without running @@ -222,6 +221,7 @@ static void start_secondary_harts(word_t primary_hart_id) */ acquire_multicore_lock(); set_secondary_cores_go(); + /* Start all cores */ if (!hsm_exists) { /* Without the HSM extension, we can't start the cores explicitly. But @@ -250,12 +250,10 @@ static void start_secondary_harts(word_t primary_hart_id) assert(remote_hart_id == primary_hart_id); continue; /* this is the current hart */ } - /* Start secondary hart, there is nothing to pass as custom - * parameter thus it's 0. - */ + /* Start secondary hart, pass logical hart ID. */ sbi_hsm_ret_t ret = sbi_hart_start(remote_hart_id, hsm_entry_on_secondary_hart, - 0); + next_logical_core_id++); if (SBI_SUCCESS != ret.code) { printf("ERROR: could not start hart %"PRIu_word", failure" " (%d, %d)\n", remote_hart_id, ret.code, ret.data); @@ -368,13 +366,22 @@ NORETURN void secondary_hart_main(word_t hart_id, word_t core_id) } #endif /* CONFIG_MAX_NUM_NODES > 1 */ -void main(word_t hart_id, void *bootloader_dtb) +void main(word_t hart_id, void *bootloader_dtb, UNUSED word_t hsm_exists) { /* Printing uses SBI, so there is no need to initialize any UART. */ - printf("ELF-loader started on (HART %"PRIu_word") (NODES %d)\n", - hart_id, (unsigned int)CONFIG_MAX_NUM_NODES); - - printf(" paddr=[%p..%p]\n", _text, _end - 1); + printf("ELF-loader started on hart %"PRIu_word"\n", hart_id); + printf(" MAX_NUM_NODES: %u, SBI HSM extension: %s\n", + (unsigned int)CONFIG_MAX_NUM_NODES, + hsm_exists ? "available" : "missing"); + printf(" phys area of binary: [%p..%p]\n", _text, _end - 1); + printf(" DTB from bootloader: %p\n", bootloader_dtb); + + if (hart_id != CONFIG_FIRST_HART_ID) { + printf("ERROR: ELF-loader not is running on FIRST_HART_ID (%d)\n", + (unsigned int)CONFIG_FIRST_HART_ID); + abort(); + UNREACHABLE(); + } int ret = run_elfloader(bootloader_dtb); if (0 != ret) { @@ -385,7 +392,7 @@ void main(word_t hart_id, void *bootloader_dtb) } #if CONFIG_MAX_NUM_NODES > 1 - start_secondary_harts(hart_id); + start_secondary_harts(hart_id, hsm_exists); #endif /* CONFIG_MAX_NUM_NODES > 1 */ boot_hart(hart_id, 0); diff --git a/elfloader-tool/src/arch-riscv/crt0.S b/elfloader-tool/src/arch-riscv/crt0.S index 3169f2d4..94eb78c3 100644 --- a/elfloader-tool/src/arch-riscv/crt0.S +++ b/elfloader-tool/src/arch-riscv/crt0.S @@ -10,7 +10,6 @@ .extern main .extern __global_pointer$ .extern elfloader_stack -.extern hsm_exists #if CONFIG_MAX_NUM_NODES > 1 .extern secondary_hart_main .extern next_logical_core_id @@ -49,23 +48,20 @@ .global _start _start: + /* save the parameters passed */ + mv s0, a0 /* preserve a0 (hart id) in s0 */ + mv s1, a1 /* preserve a1 (dtb) in s1 */ + + /* prepare a minimal C environment with registers gp and sp set up. */ .option push .option norelax 1:auipc gp, %pcrel_hi(__global_pointer$) addi gp, gp, %pcrel_lo(1b) .option pop - - /* save the parameters passed */ - mv s0, a0 /* preserve a0 (hart id) in s0 */ - mv s2, a1 /* preserve a1 (dtb) in s2 */ - - /* Attach the stack to sp before calling any C functions */ la sp, (elfloader_stack + BIT(CONFIG_KERNEL_STACK_BITS)) -#ifdef CONFIG_IMAGE_BINARY -/* Clear the BSS before we get to do anything more specific */ + /* Clear the BSS before we get to do anything more specific */ jal clear_bss -#endif /* Check if the Heart State Management (HSM) extension exists, so it can be * used to switch harts if we are not running on hart CONFIG_FIRST_HART_ID. @@ -77,57 +73,129 @@ _start: li a6, SBI_EXT_BASE_PROBE_EXT li a0, SBI_HSM_BASE ecall /* call SBI to probe for HSM extension */ - mv a2, a0 /* move SBI call generic return code to a2 as we need a0 */ - mv a3, a1 /* move SBI call error return code to a3 as we need a1 */ - mv a0, s0 /* restore a0 to hold hart ID passed by the boot loader */ - mv a1, s2 /* restore a1 to hold dtb address passed by the boot loader */ - bnez a2, _start1 /* goto _start1 if SBI did not return SBI_SUCCESS (0) */ - beqz a3, _start1 /* goto _start1 if HSM extension is missing */ - - /* Update global bool variable to tell boot code the HSM extension exists. */ - la t1, hsm_exists - li t2, 1 - amoadd.w t1, t2, (t1) - - /* Check if we are on CONFIG_FIRST_HART_ID */ - li s1, CONFIG_FIRST_HART_ID - beq a0, s1, _start1 /* goto _start1 if we are on CONFIG_FIRST_HART_ID */ - - /* Use HSM extension to start hart CONFIG_FIRST_HART_ID. */ -hsm_switch_hart: + seqz t0, a0 /* t0 = (a0 == 0) to check SBI returned SBI_SUCCESS (0) */ + snez t1, a1 /* t1 = (a1 != 0) to HSM extension exist */ + and a2, t0, t1 /* a2 = 1 if HSM extension is available, otherwise 0 */ + li t0, CONFIG_FIRST_HART_ID + bne s0, t0, start_on_secondary + mv a0, s0 /* restore a0 to hold hart ID */ + mv a1, s1 /* restore a1 to hold DTB passed on entry */ + /* a2 still hold the "HSM extension exists" flag, sp is already set up. Jump + * to C code via function address in a register, as this extends the jump + * range. Register t1 instead of t0 (x5) is used, because this is a designated + * additional link register making jr a call and not a jump then. + */ + la t1, main + jr t1 + +/*----------------------------------------------------------------------------*/ +start_on_secondary: + /* We end up here if the startup code has detected that SBI has started us on + * a hart that is not the designated primary hart. Try to switch to the + * primary hart and continue the boot process there. This must be supported + * even if CONFIG_MAX_NUM_NODES is set to 1. The register setup is: + * s0: hard ID + * s1: DTB passed from SBI + * a2: HSM extension exists flag + */ + beqz a2, no_hsm_start_secondary + /* Try to bring up the primary hart via the HSM extension */ li a7, SBI_HSM_BASE li a6, SBI_HSM_BASE_HART_START li a0, CONFIG_FIRST_HART_ID /* hart id to start */ - mv a2, s2 /* dtb address to be passed in a1 when new hart starts is 3rd parameter */ - la a1, _start1 /* where to start the hart */ + la a1, hsm_entry_on_primary_hart /* where to start the hart */ + mv a2, s1 /* custom parameter passed in a1 is the DTB */ ecall /* call SBI to start hart FIRST_HART_ID */ /* Stop current hart, the boot code may bring it up again when needed. */ - j hsm_suspend_hart +hsm_suspend_hart: + li a7, SBI_HSM_BASE + li a6, SBI_HSM_BASE_HART_STOP + ecall /* call SBI to suspend current HART */ + /* this is not supposed to return. Seem the hart could not be stopped, there + * is nothing we can do in this case as SBI seem broken. + */ +1: + wfi + j 1b /*----------------------------------------------------------------------------*/ -_start1: -/* This is basically an asm wrapper to jump to the C code at main(). The - * registers are already set up with the perameters for the C code: - * a0 holds current hard ID passed by bootloader - * a1 holds dtb address passed by bootloader - * All that is left to be done here is setting up the registers gp and sp to - * have a proper C environment. The hart we are running on now could be a - * different HART to the one that we have been on in _start. The original hart - * we came from will get a different stack in hsm_entry_on_secondary_hart. - */ +no_hsm_start_secondary: + /* We end up here if we were not started on the designated primary core and SBI + * does no implement the HSM extension, so we can't switch to the designated + * primary hart. Looks like we are running on a legacy platform where all harts + start in parallel. The register setup is: + * s0: hard ID + * s1: DTB passed from SBI + * a2: HSM extension exists flag, set to 0 + */ +#if CONFIG_MAX_NUM_NODES > 1 + /* Simulate an SBI HSM extension entry, where a0 holds the hart ID and a1 a + * custom value, which is the logical core ID in our usage. Determine it from + * an atomic increment operation on the global variable next_logical_core_id, + * what we use as our ID is the value it had before incrementing it. + */ + mv a0, s0 /* restore a0 with hart ID */ + la t0, next_logical_core_id + li t1, 1 + amoadd.w a1, t1, (t0) /* a1 is set to old value of next_logical_core_id */ + /* The logic core must be > 0, because we are not the primary core. Do another + * atomic increment if it is 0. + */ + bgtz a1, 1f + amoadd.w a1, t1, (t0) /* a1 is set to old value of next_logical_core_id */ +1: + /* The logical core ID is valid only if less than CONFIG_MAX_NUM_NODES. */ + li t0, CONFIG_MAX_NUM_NODES + blt a1, t0, hsm_entry_on_secondary_hart + +#endif /* CONFIG_MAX_NUM_NODES > 1*/ + + /* If we arrive here, this hart cannot be used because the number of supported + * secondary hart has been exceeded. Maybe multi core support is not even + * enabled at all. There is no SBI HSM extension here to turn off this hart, so + * all we can do is spinning over a WFI. However, this is not guaranteed to + * work forever, because the memory where the ELF loader keeps the loop can be + * reused and overwritten by the kernel. This will lead to undefined behavior, + * as we don't know what the new contents will be. If we are lucky, the loop + * keeps running from a hart specific instruction cache, so the new memory + * contents are ignored because no synchronization is triggered. + */ +1: + wfi + j 1b +/*----------------------------------------------------------------------------*/ +hsm_entry_on_primary_hart: + /* SBI has started us on a designated secondary hart, so we used the SBI HSM + * extension to switch to the designated primary hart. The secondary hart was + * shut down, so we can bring is up via the HSM extension when needed. + * The register setup is: + * a0: hard ID + * a1: custom parameter: DTB from bootloader + */ + .option push .option norelax 1:auipc gp, %pcrel_hi(__global_pointer$) addi gp, gp, %pcrel_lo(1b) .option pop la sp, (elfloader_stack + BIT(CONFIG_KERNEL_STACK_BITS)) - /* Jump via a register, as this can cover a bigger range. */ - la s0, main - jr s0 + li a2, 1 /* remember that the HSM extension is available */ + la t1, main + jr t1 /*----------------------------------------------------------------------------*/ +#if CONFIG_MAX_NUM_NODES > 1 + .global hsm_entry_on_secondary_hart hsm_entry_on_secondary_hart: + /* We enter here when the ELF-Loader starts a secondary hart via the SBI HSM + * extension. + * The register setup is: + * a0: hard ID + * a1: custom parameter: logical core ID + * All we have to do here is set up a the register gp and sp before jumping to + * C code. + */ .option push .option norelax @@ -135,30 +203,17 @@ hsm_entry_on_secondary_hart: addi gp, gp, %pcrel_lo(1b) .option pop -#if CONFIG_MAX_NUM_NODES > 1 - la a1, next_logical_core_id - li t2, 1 - /* atomically increment next_logical_core_id by one, afterwards a1 holds value - * before the update - which is our logical core ID. - */ - amoadd.w a1, t2, (a1) - li t2, CONFIG_MAX_NUM_NODES - bge a1, t2, hsm_suspend_hart - /* setup the hart specific stack pointer */ + /* setup stack based on the logical ID */ la sp, elfloader_stack addi t0, a1, 1 /* increment by one because we need to set sp to the end */ slli t0, t0, CONFIG_KERNEL_STACK_BITS /* t0 = t0 * BIT(CONFIG_KERNEL_STACK_BITS) */ add sp, sp, t0 - la s0, secondary_hart_main - jr s0 -#endif -/* If we get here then the HSM extension exists and the current - * HART is not going to be used and needs to be suspended. */ -hsm_suspend_hart: - li a7, SBI_HSM_BASE - li a6, SBI_HSM_BASE_HART_STOP - ecall /* call SBI to suspend current HART */ -spin_hart: - wfi - j spin_hart + * Jump to C code using a register, as this extends the jump range. Register + * t1 instead of t0 (x5) is used, because this is a designated additional link + * register making jr a call and not a jump then. + */ + la t1, secondary_hart_main + jr t1 + +#endif /* CONFIG_MAX_NUM_NODES > 1 */