diff --git a/kernel/Makefile b/kernel/Makefile index 621ff3af1..62faf7484 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -58,6 +58,13 @@ ifdef MULTICORE CFLAGS += -DMULTICORE endif +# Force all assembly to go through ASM_BKL_LOCK/ASM_BKL_UNLOCK in asm_std.h. +CFLAGS += -Dk0lock=DO_NOT_USE_K0LOCK -Dk0unlock=DO_NOT_USE_K0UNLOCK + +ifdef SOFT_BKL +CFLAGS += -DSOFT_BKL +endif + CFLAGS += -DH2K_KERNEL_PGSIZE=$(H2K_KERNEL_PGSIZE) -DH2K_KERNEL_ADDRBITS=$(H2K_KERNEL_ADDRBITS) -DH2K_GIT_COMMIT=$(H2K_GIT_COMMIT) CFLAGS += $(KERNEL_CFLAGS) $(H2K_EXTRA_CFLAGS) diff --git a/kernel/build/offsets/offsets.ref.c b/kernel/build/offsets/offsets.ref.c index 4c4864c6a..d3839cbd3 100644 --- a/kernel/build/offsets/offsets.ref.c +++ b/kernel/build/offsets/offsets.ref.c @@ -147,6 +147,9 @@ int main(int argc, char **argv) PRINT_KG_OFFSET(dma_version); PRINT_KG_OFFSET(core_id); PRINT_KG_OFFSET(core_count); +#ifdef SOFT_BKL + PRINT_KG_OFFSET(bkl); +#endif PRINT_KG_SUBSTRUCT_OFFSET(time,next_ticks); PRINT_KG_SUBSTRUCT_OFFSET(time,last_ticks); diff --git a/kernel/checkers/checker_kernel_locked/checker_kernel_locked.check.c b/kernel/checkers/checker_kernel_locked/checker_kernel_locked.check.c index 7f2e993c1..6f1a81ed1 100644 --- a/kernel/checkers/checker_kernel_locked/checker_kernel_locked.check.c +++ b/kernel/checkers/checker_kernel_locked/checker_kernel_locked.check.c @@ -10,27 +10,17 @@ s32_t checker_kernel_locked() { -#if ARCHV >= 3 - u32_t syscfg = H2K_get_syscfg(); - if ((syscfg & (1<<12)) == 0) { + if (!IS_BKL_LOCKED()) { FAIL("checker_kernel_locked: Kernel not locked."); } -#else -#error support less than v3 -#endif return 1; } s32_t checker_kernel_unlocked() { -#if ARCHV >= 3 - u32_t syscfg = H2K_get_syscfg(); - if ((syscfg & (1<<12)) != 0) { + if (IS_BKL_LOCKED()) { FAIL("checker_kernel_unlocked: Kernel is Locked."); } -#else -#error support less than v3 -#endif return 1; } diff --git a/kernel/data/globals/globals.h b/kernel/data/globals/globals.h index c034daf3b..9972d3e76 100644 --- a/kernel/data/globals/globals.h +++ b/kernel/data/globals/globals.h @@ -133,6 +133,9 @@ typedef struct { u32_t last_tlb_index_dma; u32_t tlb_size_dma; u32_t dma_tlb_start; +#endif +#ifdef SOFT_BKL + H2K_spinlock_t bkl; #endif H2K_spinlock_t tmpmap_lock; H2K_spinlock_t asid_spinlock; diff --git a/kernel/event/error/error.ref.S b/kernel/event/error/error.ref.S index 3b8e527c9..1b392303e 100644 --- a/kernel/event/error/error.ref.S +++ b/kernel/event/error/error.ref.S @@ -91,7 +91,8 @@ FUNC_START H2K_fatal_crash .text.vm.error .p2align 2 { memw(H2K_GP+#KG_is_nmi_soft) = #1 } - k0lock + // FIXME SOFT_BKL: crash path must NOT block on BKL if lock is already held + ASM_BKL_LOCK(r6, r7, r8, p0) r2 = htid r3 = #-1 r3 = togglebit(r3,r2) @@ -127,7 +128,7 @@ FUNC_START H2K_fatal_crash .text.vm.error .p2align 2 syscfg = r1 // We should now be in "Single Thread Mode": only one thread active, and that's the one executing here // Did we possibly nmi a thread holding a lock? We can fix it! - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) tlbunlock // Bit 1: Save off TLB and flush caches // EJP: for now, let's do this all the time. diff --git a/kernel/event/interrupt/interrupt.ref.S b/kernel/event/interrupt/interrupt.ref.S index f9eec196e..3d3ba06f0 100644 --- a/kernel/event/interrupt/interrupt.ref.S +++ b/kernel/event/interrupt/interrupt.ref.S @@ -181,7 +181,7 @@ FUNC_START H2K_interrupted_waitmode .text.core.interrupt .p2align 2 p0 = cmp.eq(INTINFO_FN,#0) // really, we shouldn't get NULL interrupt handlers, but if we do... if (!p0) callr INTINFO_FN r1:0 = #0 - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) jump H2K_switch FUNC_END H2K_interrupted_waitmode diff --git a/kernel/event/interrupt/interrupt.v3opt.S b/kernel/event/interrupt/interrupt.v3opt.S index 8e003f157..597369264 100644 --- a/kernel/event/interrupt/interrupt.v3opt.S +++ b/kernel/event/interrupt/interrupt.v3opt.S @@ -77,7 +77,7 @@ FUNC_START H2K_interrupted_waitmode .text.core.interrupt .falign { callr r4 } - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { r1:0 = combine(#0,#0) jump H2K_switch diff --git a/kernel/event/interrupt/interrupt.v4opt.S b/kernel/event/interrupt/interrupt.v4opt.S index f685f320d..5799fe7b4 100644 --- a/kernel/event/interrupt/interrupt.v4opt.S +++ b/kernel/event/interrupt/interrupt.v4opt.S @@ -178,9 +178,7 @@ FUNC_START H2K_interrupted_waitmode_or_preempt .text.core.interrupt .p2align 5 jumpr r31 } 1: // We were idle, go back to sleep - { - k0lock - } + ASM_BKL_LOCK(r6, r7, r8, p0) { r1:0 = #0 jump H2K_switch diff --git a/kernel/event/interrupt/test/test.c b/kernel/event/interrupt/test/test.c index c29b4378d..371624db2 100644 --- a/kernel/event/interrupt/test/test.c +++ b/kernel/event/interrupt/test/test.c @@ -11,6 +11,7 @@ #include #include #include +#include #include /* @@ -97,7 +98,7 @@ void H2K_switch(H2K_thread_context *from, H2K_thread_context *to) { if (from != NULL) FAIL("Unexpected FROM"); if (to != NULL) FAIL("Unexpected TO"); - asm volatile (" k0unlock "); + BKL_UNLOCK(); longjmp(env,1); } diff --git a/kernel/event/popup/popup.v4opt.S b/kernel/event/popup/popup.v4opt.S index 2f87d6c8c..d43b6e43c 100644 --- a/kernel/event/popup/popup.v4opt.S +++ b/kernel/event/popup/popup.v4opt.S @@ -25,7 +25,7 @@ FUNC_START H2K_popup_int .text.int.popup .p2align 5 #define NEGONE r13 #define NEWPRIO r14 #define PRIO r15 - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { WAIT_PRIOMASKS = memd(H2K_GP+#KG_lowprio_masks) // share w/ priomask? HANDLERBASE = add(H2K_GP,#KG_inthandlers) @@ -98,7 +98,7 @@ FUNC_START H2K_popup_int .text.int.popup .p2align 5 if (!p1) memd(ME) = PREV_HEAD } .Lnohandler: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) jumpr r31 #undef INTNO #undef ME @@ -147,7 +147,7 @@ FUNC_START H2K_popup_wait .text.int.popup .falign p1 = cmp.eq(INTNO, #L2_CORE_INTERRUPT) if (p1.new) jump:nt 9f } - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) #define TMP INTHANDLERS_ADDR { TMP = memub(ME+#CONTEXT_vmstatus) @@ -208,7 +208,7 @@ FUNC_START H2K_popup_wait .text.int.popup .falign jump H2K_dosched } 8: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) 9: { r0 = #-1 @@ -219,7 +219,7 @@ FUNC_START H2K_popup_wait .text.int.popup .falign r0 = ME call H2K_vm_do_work_withlock } - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { r0 = #-1 dealloc_return diff --git a/kernel/futex/futex_classic/futex_classic.v4opt.S b/kernel/futex/futex_classic/futex_classic.v4opt.S index 07fe2feca..247c8970e 100644 --- a/kernel/futex/futex_classic/futex_classic.v4opt.S +++ b/kernel/futex/futex_classic/futex_classic.v4opt.S @@ -24,7 +24,7 @@ FUNC_START H2K_futex_wait .text.core.futex .p2align 5 #define LOCKVAL r17 #define TMP r6 #define MASK r7 - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { TMP = memub(ME+#CONTEXT_vmstatus) r1 = #SAFEMEM_R @@ -99,7 +99,7 @@ FUNC_START H2K_futex_wait .text.core.futex .p2align 5 jump H2K_dosched } .Lwait_fail: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { r0 = #-1 dealloc_return @@ -109,7 +109,7 @@ FUNC_START H2K_futex_wait .text.core.futex .p2align 5 r0 = ME call H2K_vm_do_work_withlock } - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { r0 = #-1 dealloc_return @@ -175,7 +175,7 @@ FUNC_START H2K_futex_resume .text.core.futex .falign RING = tableidxw(HASH,#FUTEX_HASHBITS,#32-FUTEX_HASHBITS) PA_LO_SAV = PA_LO } - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { TMP = memw(RING) } @@ -208,7 +208,7 @@ FUNC_START H2K_futex_resume .text.core.futex .falign jump H2K_check_sanity_unlock } .Lresume_unlock_null: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) .Lresume_null: { r0 = #0 diff --git a/kernel/futex/futex_pi/futex_pi.v4opt.S b/kernel/futex/futex_pi/futex_pi.v4opt.S index aa4293f11..7fff1b094 100644 --- a/kernel/futex/futex_pi/futex_pi.v4opt.S +++ b/kernel/futex/futex_pi/futex_pi.v4opt.S @@ -44,7 +44,7 @@ FUNC_START H2K_futex_lock_pi .text.core.futex .p2align 5 ME_LOCK = combine(r1,r0) DEST = #H2K_VMSTATUS_VMWORK } - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { TMP = memub(ME+#CONTEXT_vmstatus) r1 = #SAFEMEM_RW @@ -316,13 +316,13 @@ FUNC_START H2K_futex_lock_pi .text.core.futex .p2align 5 deallocframe } .Ldone: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { r0 = #0 dealloc_return } .Llpi_unlock_retfail: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { r0 = #-1 dealloc_return @@ -332,7 +332,7 @@ FUNC_START H2K_futex_lock_pi .text.core.futex .p2align 5 call H2K_vm_do_work_withlock r0 = ME } - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { r0 = #-1 dealloc_return @@ -370,7 +370,7 @@ FUNC_START H2K_futex_unlock_pi .text.core.futex .p2align 5 allocframe(#8) ME_LOCK = combine(r1,r0) } - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { r1 = #SAFEMEM_RW r3:2 = combine(ME,r29) diff --git a/kernel/init/boot/boot.ref.S b/kernel/init/boot/boot.ref.S index 2ed83d785..337964a7f 100644 --- a/kernel/init/boot/boot.ref.S +++ b/kernel/init/boot/boot.ref.S @@ -535,10 +535,13 @@ boot_tcm_offset: .word 0x0 FUNC_START h2_init .text.init.boot .p2align 2 + SETCONST(H2K_GP, H2K_kg) jumpr r31 FUNC_END h2_init FUNC_START H2K_handle_reset .text.init.boot .p2align 2 + call 1f // initialize r31 for future mutex lock +1: SETCONST(TMP,_SDA_BASE_) gp = TMP @@ -549,6 +552,6 @@ FUNC_START H2K_handle_reset .text.init.boot .p2align 2 STACK_SIZE = #KERNEL_STACK_SIZE SP += mpyi(STACK_SIZE, TMP) r1:0 = #0 - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) jump H2K_switch FUNC_END H2K_handle_reset diff --git a/kernel/sched/check_sanity/check_sanity.v3opt.S b/kernel/sched/check_sanity/check_sanity.v3opt.S index cdff7763b..feb0c18d5 100644 --- a/kernel/sched/check_sanity/check_sanity.v3opt.S +++ b/kernel/sched/check_sanity/check_sanity.v3opt.S @@ -175,9 +175,7 @@ FUNC_START H2K_check_sanity_unlock .text.core.sanity .p2align 5 } .falign 2: - { - k0unlock - } + ASM_BKL_UNLOCK(r2, r3, r4, p0) { jumpr R31; } @@ -196,9 +194,7 @@ FUNC_START H2K_check_sanity_unlock .text.core.sanity .p2align 5 p0 = and(p0,!p1) if (!p0.new) jump:t 1b; } - { - k0unlock - } + ASM_BKL_UNLOCK(r2, r3, r4, p0) { jumpr R31; } diff --git a/kernel/sched/check_sanity/check_sanity.v4opt.S b/kernel/sched/check_sanity/check_sanity.v4opt.S index e121a64a4..ce36d60da 100644 --- a/kernel/sched/check_sanity/check_sanity.v4opt.S +++ b/kernel/sched/check_sanity/check_sanity.v4opt.S @@ -122,9 +122,7 @@ FUNC_START H2K_check_sanity_unlock .text.core.sanity .p2align 5 } .falign 2: - { - k0unlock - } + ASM_BKL_UNLOCK(r2, r3, r4, p0) { jumpr R31 } @@ -141,6 +139,6 @@ FUNC_START H2K_check_sanity .text.core.sanity .p2align 3 allocframe(#0) call H2K_check_sanity_unlock } - k0lock + ASM_BKL_LOCK(r2, r3, r4, p0) dealloc_return FUNC_END H2K_check_sanity diff --git a/kernel/sched/check_sanity/check_sanity.v68opt.S b/kernel/sched/check_sanity/check_sanity.v68opt.S index 5e586d211..341ad6f70 100644 --- a/kernel/sched/check_sanity/check_sanity.v68opt.S +++ b/kernel/sched/check_sanity/check_sanity.v68opt.S @@ -129,9 +129,7 @@ FUNC_START H2K_check_sanity_unlock .text.core.sanity .p2align 5 } .falign 2: - { - k0unlock - } + ASM_BKL_UNLOCK(r2, r3, r4, p0) { jumpr R31 } @@ -148,6 +146,6 @@ FUNC_START H2K_check_sanity .text.core.sanity .p2align 3 allocframe(#0) call H2K_check_sanity_unlock } - k0lock + ASM_BKL_LOCK(r2, r3, r4, p0) dealloc_return FUNC_END H2K_check_sanity diff --git a/kernel/sched/check_sanity/check_sanity.v81opt.S b/kernel/sched/check_sanity/check_sanity.v81opt.S index 36c12276d..8c53cb854 100644 --- a/kernel/sched/check_sanity/check_sanity.v81opt.S +++ b/kernel/sched/check_sanity/check_sanity.v81opt.S @@ -145,9 +145,7 @@ FUNC_START H2K_check_sanity_unlock .text.core.sanity .p2align 5 } .falign 2: - { - k0unlock - } + ASM_BKL_UNLOCK(r2, r3, r4, p0) { jumpr R31 } @@ -164,6 +162,6 @@ FUNC_START H2K_check_sanity .text.core.sanity .p2align 3 allocframe(#0) call H2K_check_sanity_unlock } - k0lock + ASM_BKL_LOCK(r2, r3, r4, p0) dealloc_return FUNC_END H2K_check_sanity diff --git a/kernel/sched/resched/resched.v3opt.S b/kernel/sched/resched/resched.v3opt.S index e83b26471..edcfd2b3f 100644 --- a/kernel/sched/resched/resched.v3opt.S +++ b/kernel/sched/resched/resched.v3opt.S @@ -28,9 +28,7 @@ FUNC_START H2K_resched .text.core.resched .p2align 3 { ciad(r0) // clear IAD } - { - k0lock // BKL_LOCK(&H2K_bkl) - } + ASM_BKL_LOCK(r6, r7, r8, p0) // BKL_LOCK(&H2K_bkl) { p0 = cmp.eq(r1,#0) // me == NULL if (!p0.new) jump:t .H2K_resched_NOTWAIT // if (me != NULL) then JUMP diff --git a/kernel/sched/resched/resched.v4opt.S b/kernel/sched/resched/resched.v4opt.S index 362bf503c..e19658b3c 100644 --- a/kernel/sched/resched/resched.v4opt.S +++ b/kernel/sched/resched/resched.v4opt.S @@ -27,9 +27,7 @@ FUNC_START H2K_resched .text.core.resched .p2align 3 { ciad(r0) // clear IAD } - { - k0lock // BKL_LOCK(&H2K_bkl) - } + ASM_BKL_LOCK(r6, r7, r8, p0) // BKL_LOCK(&H2K_bkl) { if (!p0) jump .H2K_resched_NOTWAIT // if (me != NULL) then JUMP r3 = clrbit(r3,r2) // mask = clrbit(-1, hwtnum) diff --git a/kernel/sched/switch/switch.ref.S b/kernel/sched/switch/switch.ref.S index b1f06acf4..35cf140bc 100644 --- a/kernel/sched/switch/switch.ref.S +++ b/kernel/sched/switch/switch.ref.S @@ -318,7 +318,7 @@ FUNC_START H2K_switch .text.core.switch .p2align 5 r8 = setbit(r8,r5) memw(H2K_GP+#KG_wait_mask) = r9 memw(H2K_GP+#KG_priomask) = r8 - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) // set SGP to NULL sgp0 = r1 diff --git a/kernel/sched/switch/switch.v4opt.S b/kernel/sched/switch/switch.v4opt.S index 698998726..bfdc3e214 100644 --- a/kernel/sched/switch/switch.v4opt.S +++ b/kernel/sched/switch/switch.v4opt.S @@ -427,8 +427,8 @@ FUNC_START H2K_switch .text.core.switch .p2align 5 memw(H2K_GP+#KG_priomask) |= r0 r0 = ##0x00040000 // insert(r5,#3,#16) } - k0unlock - IMASK = r3 + ASM_BKL_UNLOCK(r4, r5, r6, p0) + IMASK = r3 { ssr = r0 } diff --git a/kernel/time/timer/timer.v5opt.S b/kernel/time/timer/timer.v5opt.S index 510eff837..568de6c93 100644 --- a/kernel/time/timer/timer.v5opt.S +++ b/kernel/time/timer/timer.v5opt.S @@ -196,9 +196,7 @@ FUNC_START H2K_timer_int .text.time.timer r1 = add(H2K_GP,#KG_time_timeouts) DEVPTR = add(DEVPTR,#HW_COUNT_LO) } - { - k0lock - } + ASM_BKL_LOCK(r6, r7, r8, p0) { TIMER_INTNUM = memw(H2K_GP+#KG_timer_intnum) r2 = memw(r1) @@ -248,9 +246,7 @@ FUNC_START H2K_timer_int .text.time.timer #undef NEXTTICKS_LO #undef ROOT #undef LEFTPTR - { - k0unlock - } + ASM_BKL_UNLOCK(r6, r7, r8, p0) #define VICBASE r7 #define L2VICWORD r6 @@ -401,7 +397,7 @@ FUNC_START H2K_timer_set_timeout_tick .text.time.timer ME = r4 ZERO = #0 } - k0lock + ASM_BKL_LOCK(r12, r13, r14, p0) { p0 = cmp.gtu(TIMEOUT_TICK,LAST_TICKS) if (!p0.new) TIMEOUT_TICK = ZERO @@ -455,7 +451,7 @@ FUNC_START H2K_timer_set_timeout_tick .text.time.timer } .falign .Lskip_hw_set: - k0unlock + ASM_BKL_UNLOCK(r12, r13, r14, p0) { r1:0 = mpyu(TIMEOUT_TICK_LO,NSEC_PER_TICK) ME_DEVPTR = memd(r29+#0) diff --git a/kernel/time/timer/timer.v60opt.S b/kernel/time/timer/timer.v60opt.S index 6ae3d203d..e27c60367 100644 --- a/kernel/time/timer/timer.v60opt.S +++ b/kernel/time/timer/timer.v60opt.S @@ -216,9 +216,7 @@ FUNC_START H2K_timer_int .text.time.timer r1 = add(H2K_GP,#KG_time_timeouts) DEVPTR = add(DEVPTR,#HW_COUNT_LO) } - { - k0lock - } + ASM_BKL_LOCK(r6, r7, r8, p0) { TIMER_INTNUM = memw(H2K_GP+#KG_timer_intnum) r2 = memw(r1) @@ -268,9 +266,7 @@ FUNC_START H2K_timer_int .text.time.timer #undef NEXTTICKS_LO #undef ROOT #undef LEFTPTR - { - k0unlock - } + ASM_BKL_UNLOCK(r6, r7, r8, p0) #define VICBASE r7 #define L2VICWORD r6 @@ -439,7 +435,7 @@ FUNC_START H2K_timer_set_timeout_tick .text.time.timer ME = r4 ZERO = #0 } - k0lock + ASM_BKL_LOCK(r12, r13, r14, p0) { p0 = cmp.gtu(TIMEOUT_TICK,LAST_TICKS) if (!p0.new) TIMEOUT_TICK = ZERO @@ -493,7 +489,7 @@ FUNC_START H2K_timer_set_timeout_tick .text.time.timer } .falign .Lskip_hw_set: - k0unlock + ASM_BKL_UNLOCK(r12, r13, r14, p0) { r1:0 = mpyu(TIMEOUT_TICK_LO,NSEC_PER_TICK) ME_DEVPTR = memd(r29+#0) diff --git a/kernel/util/hw/hw.h b/kernel/util/hw/hw.h index bdcc6b5cb..483b4b2ba 100644 --- a/kernel/util/hw/hw.h +++ b/kernel/util/hw/hw.h @@ -415,8 +415,27 @@ static inline void H2K_gregs_restore(H2K_thread_context *me) { } #endif +#ifdef SOFT_BKL +#define BKL_LOCK(...) H2K_spinlock_lock(&H2K_gp->bkl) +#define BKL_UNLOCK(...) H2K_spinlock_unlock(&H2K_gp->bkl) +#else #define BKL_LOCK(...) H2K_mutex_lock_k0() #define BKL_UNLOCK(...) H2K_mutex_unlock_k0() +#endif /* SOFT_BKL */ + +/* For testing and assertions only — not a reliable check in production paths. + * Under SOFT_BKL reads the in-memory ticket spinlock word; locked when the + * upper half (next ticket) != lower half (now serving). Otherwise reads syscfg + * bit 12 (the k0lock hardware bit). */ +static inline int IS_BKL_LOCKED(void) +{ +#ifdef SOFT_BKL + u32_t v = H2K_gp->bkl; + return (u16_t)(v >> 16) != (u16_t)v; +#else + return (H2K_get_syscfg() & (1 << 12)) != 0; +#endif +} #endif #if (ARCHV >= 3) diff --git a/kernel/util/spinlock/spinlock.ref.S b/kernel/util/spinlock/spinlock.ref.S index f1957da62..fa32bee64 100644 --- a/kernel/util/spinlock/spinlock.ref.S +++ b/kernel/util/spinlock/spinlock.ref.S @@ -7,24 +7,23 @@ #include FUNC_START H2K_spinlock_lock .text.util.spinlock .p2align 5 - r1 = memw_locked(r0) - if (r1!=#0) jump:nt H2K_spinlock_lock - memw_locked(r0,p0) = r31 - if (!p0) jump H2K_spinlock_lock - { - dccleana(r0) - r0 = #0 - jumpr r31 - } + ASM_TICKET_LOCK(r0,r2,r3,p0) + jumpr r31 FUNC_END H2K_spinlock_lock FUNC_START H2K_spinlock_trylock .text.util.spinlock .p2align 5 r1 = memw_locked(r0) - if (r1!=#0) jump:nt .Lfail - memw_locked(r0,p0) = r31 + { + r2 = lsr(r1,#16) + r1 = add(r1,##0x10000) + } + { + p0 = cmph.eq(r1,r2) + if (!p0.new) jump:nt .Lfail + } + memw_locked(r0,p0) = r1 if (!p0) jump H2K_spinlock_trylock { - dccleana(r0) r0 = #0 jumpr r31 } @@ -36,12 +35,7 @@ FUNC_START H2K_spinlock_trylock .text.util.spinlock .p2align 5 FUNC_END H2K_spinlock_trylock FUNC_START H2K_spinlock_unlock .text.util.spinlock .p2align 5 - r1 = #0 - memw(r0) = r1 - { - dccleana(r0) - r0 = #0 - jumpr r31 - } + ASM_TICKET_UNLOCK(r0,r2,r3,p0) + jumpr r31 FUNC_END H2K_spinlock_unlock diff --git a/kernel/util/spinlock/test/test.c b/kernel/util/spinlock/test/test.c index 8d90a54cf..eae7625ad 100644 --- a/kernel/util/spinlock/test/test.c +++ b/kernel/util/spinlock/test/test.c @@ -19,22 +19,18 @@ H2K_spinlock_t lock; volatile int x; +/* True when the ticket lock is held: next ticket != now serving. */ +static inline int spinlock_is_locked(H2K_spinlock_t v) +{ + return (u16_t)(v >> 16) != (u16_t)v; +} + void thread_func(void *unused) { H2K_spinlock_lock(&lock); while (1) x = 1; } -/* - * For each test: - * * Build up the tree - * * Verify the tree is OK - * * 10 times: - * * Remove the head, verify tree - * * Add the former head back into the tree, verify tree - * * Bisect the tree - * * Destructive Iterate the tree, freeing nodes - */ int main() { int i; @@ -42,15 +38,15 @@ int main() H2K_spinlock_init(&lock); if (lock != 0) FAIL("init"); H2K_spinlock_lock(&lock); - if (lock == 0) FAIL("lock"); + if (!spinlock_is_locked(lock)) FAIL("lock"); if (H2K_spinlock_trylock(&lock) == 0) FAIL("trylock"); H2K_spinlock_unlock(&lock); - if (lock != 0) FAIL("unlock"); - if (H2K_spinlock_trylock(&lock) != 0) FAIL("trylock"); + if (spinlock_is_locked(lock)) FAIL("unlock"); + if (H2K_spinlock_trylock(&lock) != 0) FAIL("trylock2"); H2K_spinlock_unlock(&lock); x = 0; H2K_spinlock_lock(&lock); - if (lock == 0) FAIL("lock"); + if (!spinlock_is_locked(lock)) FAIL("lock2"); for (i = 0; i < 10000; i++) { asm volatile ( "nop" : : :"memory"); } diff --git a/kernel/util/std/asm_std.h b/kernel/util/std/asm_std.h index 9fc774385..f559bd6c8 100644 --- a/kernel/util/std/asm_std.h +++ b/kernel/util/std/asm_std.h @@ -46,5 +46,43 @@ #define sgp sgp0 #endif +/* BKL is an in-memory ticket spinlock at KG_bkl. Three scratch registers and a + * predicate register are required; all four are clobbered. */ + +#define ASM_TICKET_LOCK(addr, tmp1, tmp2, tmp_pred) \ +1: \ + tmp1 = memw_locked(addr) ; \ + { \ + tmp2 = lsr(tmp1,#16); \ + tmp1 = add(tmp1, # ## #0x10000); \ + } \ + memw_locked(addr,tmp_pred) = tmp1 ;\ + { \ + if (!tmp_pred) jump:nt 1b ;\ + tmp_pred = cmph.eq(tmp1,tmp2) ;\ + if (tmp_pred.new) jump:t 3f ;\ + } \ +2: \ + tmp1 = memuh(addr); \ + { tmp_pred = cmph.eq(tmp1,tmp2); if (tmp_pred.new) jump:t 3f } \ + /* pause */ \ + jump 2b; \ +3: + +#define ASM_TICKET_UNLOCK(addr,tmp1,tmp2,tmp_pred) \ +1: \ + { tmp2 = #1; tmp1 = memw_locked(addr); } \ + tmp1 = vaddh(tmp1,tmp2); /* add halfwords separately to avoid carry */ \ + memw_locked(addr,tmp_pred) = tmp1; \ + if (!tmp_pred) jump:nt 1b; \ + +#ifdef SOFT_BKL +#define ASM_BKL_LOCK(A,B,C,P) A = add(H2K_GP,#KG_bkl); ASM_TICKET_LOCK(A,B,C,P) +#define ASM_BKL_UNLOCK(A,B,C,P) A = add(H2K_GP,#KG_bkl); ASM_TICKET_UNLOCK(A,B,C,P) +#else +#define ASM_BKL_LOCK(tmp0, tmp1, tmp2, tmp_pred) K0LOCK +#define ASM_BKL_UNLOCK(tmp0, tmp1, tmp2, tmp_pred) K0UNLOCK +#endif /* SOFT_BKL */ + #endif diff --git a/kernel/vm/cpuint/test/test.c b/kernel/vm/cpuint/test/test.c index d673e6609..599166ac4 100644 --- a/kernel/vm/cpuint/test/test.c +++ b/kernel/vm/cpuint/test/test.c @@ -27,6 +27,7 @@ #include #include #include +#include void FAIL(const char *str) { @@ -35,6 +36,8 @@ void FAIL(const char *str) exit(1); } +H2K_kg_t H2K_kg; + H2K_vmblock_t TH_vmblock; H2K_thread_context a; @@ -294,6 +297,7 @@ void TH_test_status() int main() { int j; + __asm__ __volatile(GLOBAL_REG_STR " = %0 " : : "r"(&H2K_kg)); puts("0"); TH_init_vmblock(); diff --git a/kernel/vm/shint/test/test.c b/kernel/vm/shint/test/test.c index f15d3c3ce..41c45667c 100644 --- a/kernel/vm/shint/test/test.c +++ b/kernel/vm/shint/test/test.c @@ -41,6 +41,7 @@ #include #include #include +#include void FAIL(const char *str) { @@ -49,6 +50,8 @@ void FAIL(const char *str) exit(1); } +H2K_kg_t H2K_kg; + H2K_thread_context TH_threads[MAX_TEST_THREADS]; u32_t TH_pending_storage[MAX_TEST_INTERRUPTS/32]; u32_t TH_enable_storage[MAX_TEST_INTERRUPTS/32]; @@ -477,6 +480,7 @@ void TH_test_nexts() int main() { int i,j; + __asm__ __volatile(GLOBAL_REG_STR " = %0 " : : "r"(&H2K_kg)); TH_init_vmblock(); /* Check status */ diff --git a/kernel/vm/vmfuncs/test/test.c b/kernel/vm/vmfuncs/test/test.c index bd0bae517..d70a7d336 100644 --- a/kernel/vm/vmfuncs/test/test.c +++ b/kernel/vm/vmfuncs/test/test.c @@ -107,7 +107,7 @@ u32_t H2K_vm_do_work_withlock(H2K_thread_context *me) { if (!TH_expected_work) FAIL("Didn't expect work"); TH_expected_work = 0; - if ((H2K_get_syscfg() & (1<<12)) == 0) FAIL("no k0lock"); + if (!IS_BKL_LOCKED()) FAIL("no k0lock"); H2K_mutex_unlock_k0(); return TH_work_ret; } diff --git a/kernel/vm/vmfuncs/vmfuncs.v4opt.S b/kernel/vm/vmfuncs/vmfuncs.v4opt.S index daba2f352..f7942337d 100644 --- a/kernel/vm/vmfuncs/vmfuncs.v4opt.S +++ b/kernel/vm/vmfuncs/vmfuncs.v4opt.S @@ -161,7 +161,7 @@ FUNC_START H2K_vmtrap_wait .text.vm.funcs .falign #define NEWMASK r7:6 #define RUNLIST r8 #define RUNLIST_PRIOS r9 - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) { RET_MESAV = combine(r31,ME) call H2K_vm_do_work_withlock @@ -196,7 +196,7 @@ FUNC_START H2K_vmtrap_wait .text.vm.funcs .falign memh(RUNLIST_PRIOS+HTHREAD<<#1) = r10 } 9: - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) { memw(MESAV+#CONTEXT_r00) = r0 jumpr RET diff --git a/kernel/vm/vmint/vmint.v4opt.S b/kernel/vm/vmint/vmint.v4opt.S index 1d91cf009..2cb30de08 100644 --- a/kernel/vm/vmint/vmint.v4opt.S +++ b/kernel/vm/vmint/vmint.v4opt.S @@ -155,9 +155,9 @@ FUNC_END H2K_vm_int_deliver_locked FUNC_START H2K_vm_int_deliver .text.vm.int .p2align 2 allocframe(#0) - k0lock + ASM_BKL_LOCK(r6, r7, r8, p0) call H2K_vm_int_deliver_locked - k0unlock + ASM_BKL_UNLOCK(r6, r7, r8, p0) dealloc_return FUNC_END H2K_vm_int_deliver diff --git a/libs/h2/fatal/test/test.c b/libs/h2/fatal/test/test.c index fd27596d1..e2a4a4ea8 100644 --- a/libs/h2/fatal/test/test.c +++ b/libs/h2/fatal/test/test.c @@ -6,6 +6,7 @@ #include #include #include +#include void FAIL(const char *str) { @@ -16,7 +17,8 @@ void FAIL(const char *str) void PASS(const char *msg) { - asm volatile (" k0unlock; tlbunlock; \n"); + BKL_UNLOCK(); + H2K_mutex_unlock_tlb(); if (msg == NULL) FAIL("bad arg"); puts(msg); puts("OK?"); diff --git a/notes.md b/notes.md index 5a8e94ab9..3723b9103 100644 --- a/notes.md +++ b/notes.md @@ -748,6 +748,45 @@ C++ dtor: main binary ← fires at exit - Integrate dlopen support into the main test infrastructure - Consider promoting `sys_mmap` and fake `fstat` to a proper h2 library +## Soft BKL (SOFT_BKL) — BKL cutover to software ticket lock + +### What was done (2026-07-10) + +Redirect all BKL lock/unlock sites to an abstract macro layer, with the +in-memory ticket spinlock implementation gated on `SOFT_BKL`. + +**C-side changes:** +- `kernel/data/globals/globals.h`: added `H2K_spinlock_t bkl` to `H2K_kg_t` + under `#ifdef SOFT_BKL`. +- `kernel/util/hw/hw.h`: `BKL_LOCK`/`BKL_UNLOCK` macros redirect to + `H2K_spinlock_lock/unlock(&H2K_gp->bkl)` under `SOFT_BKL`; keep + `k0lock`/`k0unlock` otherwise. Macros are variadic so existing call sites + passing `&H2K_bkl` compile without change (arg is discarded). +- `kernel/checkers/checker_kernel_locked/checker_kernel_locked.check.c`: + `SOFT_BKL` branch checks `H2K_gp->bkl` word instead of the k0 syscfg bit. + +**Assembly-side changes:** +- `kernel/util/std/asm_std.h`: added `ASM_BKL_LOCK(tmp0,tmp1,tmp_pred)` and + `ASM_BKL_UNLOCK(tmp0,tmp1,tmp_pred)` macros. Under `SOFT_BKL` these expand + to an LL/SC spinlock on `KG_bkl`; otherwise they expand to `K0LOCK`/`K0UNLOCK` + (uppercase bypasses the poison `#define` but assembles identically). +- `kernel/Makefile`: always passes `-Dk0lock=DO_NOT_USE_K0LOCK + -Dk0unlock=DO_NOT_USE_K0UNLOCK` to enforce that all `.S` files go through the + macro. `SOFT_BKL=1` on the make command line enables the soft-lock build. +- All 16 unique opt `.S` files and all 4 `.ref.S` files converted from bare + `k0lock`/`k0unlock` to `ASM_BKL_LOCK`/`ASM_BKL_UNLOCK` with appropriate + scratch registers. Braces removed from solo `{ k0lock }` / `{ k0unlock }` + packets as required by macro expansion. + +**Crash path special case:** +- `event/error/error.ref.S` L94: marked with `FIXME SOFT_BKL` — the crash + path must NOT block on the BKL if the lock may already be held. Deferred. + +### Next task + +`offsets.ref.c` needs `PRINT_KG_OFFSET(bkl)` added under `#ifdef SOFT_BKL` +so `KG_bkl` is available to the assembler. + ## Architecture Versions (ARCHV) - v3/v4/v5: older versions, different interrupt mask conventions. - v60, v65, v68, v73, v81+: modern versions. Key differences: @@ -1633,3 +1672,52 @@ vmop.spec. Open for next session: #4 vmwork unconditional-clear test; #7 PI owner/waiter audit; #11 retry backstop; vmwork.ref.c unconditional clear (spec already done). + +## SOFT_BKL test run (2026-07-10) — resolved + +### Infrastructure fixes + +1. **`scripts/Makefile.inc.test`**: add `ifdef SOFT_BKL` block after + `endif #STANDALONE` so test sub-makes inherit the flag. Without this, test + stubs called hardware `k0unlock` while the library used the in-memory + spinlock -> lock word stuck -> spin forever. Fixed 20 hangs. + +2. **`kernel/vm/cpuint/test/test.c`** and **`kernel/vm/shint/test/test.c`**: add + `#include `, `H2K_kg_t H2K_kg;`, and r28 init at top of `main()`. + These STANDALONE tests call BKL-locked code via `cpuint_post`/`shint_post`; + under SOFT_BKL `BKL_LOCK()` dereferences `H2K_gp` (r28), which was not set + up. Pattern matches `vmfuncs/test/test.c`. Fixed 2 hangs. + +### PI futex test -- starvation under contention -> ticket lock + +The pi test demonstrated CPU starvation with the trivial LL/SC spinlock: under +high contention spinner threads starved pi_caller threads indefinitely (confirmed +via interactive PA peek -- `ready_valids` non-zero, BKL cycling, no forward +progress). Replaced with a fair ticket lock. + +**Ticket lock design (32-bit packed):** upper 16 bits = `next` ticket dispenser, +lower 16 bits = `now_serving`. Lock: LL/SC increments `next`, spins on `memuh` +until `now_serving` matches. Unlock: LL/SC increments `now_serving` via `vaddh` +(halfword add, no inter-half carry). `dccleana` removed from acquire path. +`IS_BKL_LOCKED()` updated to compare halfwords. + +CPP trick: `# ## #` in a `#define` emits a literal `##` into assembler output -- +used in `add(tmp1, ##0x10000)` for the Hexagon extended-immediate prefix. + +### stmode test -- two root causes in boot path + +`stmode/test` starts all hthreads simultaneously and hung before `main()`. + +1. `r31` is undefined at EVB entry (`H2K_handle_reset` has no prior `call`). + With the LL/SC lock each hthread wrote `r31` as a token; `r31 == 0` meant + the store wrote the "unlocked" value and all hthreads entered simultaneously. + **Fix:** `call 1f` / `1:` at top of `H2K_handle_reset` sets `r31` to a + known non-zero PC before the first `ASM_BKL_LOCK`. + +2. `H2K_gp` (r28) not initialized when `BKL_LOCK()` is first called from the + h2-compat path (`h2_init()` was `jumpr r31`). r28 pointed into instruction + memory -> read non-zero -> spinlock spun forever thinking it was held. + **Fix:** `SETCONST(H2K_GP, H2K_kg)` added to `h2_init()` so every + INC_KERNEL test gets r28 set before any `BKL_LOCK()`. + +**Result:** `make testall SOFT_BKL=1` -> **1224 passed, 0 failed**. diff --git a/scripts/Makefile.inc.test b/scripts/Makefile.inc.test index 94f96713a..95a2d1160 100644 --- a/scripts/Makefile.inc.test +++ b/scripts/Makefile.inc.test @@ -201,6 +201,11 @@ CFLAGS += -I ${INSTALLPATH}/include ASFLAGS += -mv${TOOLARCH} -DARCHV=${ARCHV} -Wall -I ${INSTALLPATH}/include endif #STANDALONE +ifdef SOFT_BKL +CFLAGS += -DSOFT_BKL +ASFLAGS += -DSOFT_BKL +endif + LDFLAGS+= -mv${TOOLARCH} -Qunused-arguments # Turn on call() support for assertion checking #CFLAGS += -DDEBUG