softbkl: replace hardware k0lock with software ticket lock (SOFT_BKL)

Introduce SOFT_BKL compile-time flag that redirects all BKL lock/unlock
sites from the hardware k0lock instruction to an in-memory ticket spinlock.

- BKL_LOCK/BKL_UNLOCK macros in hw.h dispatch to H2K_spinlock_{lock,unlock}
  when SOFT_BKL is set, k0lock/k0unlock otherwise
- ASM_BKL_LOCK/ASM_BKL_UNLOCK macros in asm_std.h wrap the LL/SC ticket lock
  for .S files; K0LOCK/K0UNLOCK (bypass poison) otherwise
- Makefile poisons bare k0lock/k0unlock in .S files to enforce macro use
- H2K_spinlock_t bkl field added to H2K_kg_t under SOFT_BKL
- All 20 .S files converted from bare k0lock/k0unlock to ASM_BKL_LOCK/UNLOCK
- ticket lock: 32-bit packed (upper 16 = next, lower 16 = now_serving),
  strictly fair FIFO, removes dccleana from acquire path
- h2_init() sets H2K_gp (r28) so INC_KERNEL tests work without per-test change
- boot.ref.S adds call 1f to establish r31 before first ASM_BKL_LOCK

make testall SOFT_BKL=1: 1224 passed, 0 failed

Signed-off-by: Erich Plondke <erich@qti.qualcomm.com>

Author: eplondke

kernel/Makefile

@@ -58,6 +58,13 @@ ifdef MULTICORE
 CFLAGS += -DMULTICORE
 endif
 
+# Force all assembly to go through ASM_BKL_LOCK/ASM_BKL_UNLOCK in asm_std.h.
+CFLAGS += -Dk0lock=DO_NOT_USE_K0LOCK -Dk0unlock=DO_NOT_USE_K0UNLOCK
+
+ifdef SOFT_BKL
+CFLAGS += -DSOFT_BKL
+endif
+
 CFLAGS += -DH2K_KERNEL_PGSIZE=$(H2K_KERNEL_PGSIZE) -DH2K_KERNEL_ADDRBITS=$(H2K_KERNEL_ADDRBITS) -DH2K_GIT_COMMIT=$(H2K_GIT_COMMIT)
 
 CFLAGS += $(KERNEL_CFLAGS) $(H2K_EXTRA_CFLAGS)

kernel/build/offsets/offsets.ref.c

@@ -147,6 +147,9 @@ int main(int argc, char **argv)
 	PRINT_KG_OFFSET(dma_version);
 	PRINT_KG_OFFSET(core_id);
 	PRINT_KG_OFFSET(core_count);
+#ifdef SOFT_BKL
+	PRINT_KG_OFFSET(bkl);
+#endif
 
 	PRINT_KG_SUBSTRUCT_OFFSET(time,next_ticks);
 	PRINT_KG_SUBSTRUCT_OFFSET(time,last_ticks);

kernel/checkers/checker_kernel_locked/checker_kernel_locked.check.c

@@ -10,27 +10,17 @@
 
 s32_t checker_kernel_locked()
 {
-#if ARCHV >= 3
-	u32_t syscfg = H2K_get_syscfg();
-	if ((syscfg & (1<<12)) == 0) {
+	if (!IS_BKL_LOCKED()) {
 		FAIL("checker_kernel_locked: Kernel not locked.");
 	}
-#else
-#error support less than v3
-#endif
 	return 1;
 }
 
 s32_t checker_kernel_unlocked()
 {
-#if ARCHV >= 3
-	u32_t syscfg = H2K_get_syscfg();
-	if ((syscfg & (1<<12)) != 0) {
+	if (IS_BKL_LOCKED()) {
 		FAIL("checker_kernel_unlocked: Kernel is Locked.");
 	}
-#else
-#error support less than v3
-#endif
 	return 1;
 }
 

kernel/data/globals/globals.h

@@ -133,6 +133,9 @@ typedef struct {
 	u32_t last_tlb_index_dma;
 	u32_t tlb_size_dma;
 	u32_t dma_tlb_start;
+#endif
+#ifdef SOFT_BKL
+	H2K_spinlock_t bkl;
 #endif
 	H2K_spinlock_t tmpmap_lock;
 	H2K_spinlock_t asid_spinlock;

kernel/event/error/error.ref.S

@@ -91,7 +91,8 @@ FUNC_START H2K_fatal_crash .text.vm.error .p2align 2
 	{
 		memw(H2K_GP+#KG_is_nmi_soft) = #1
 	}
-	k0lock
+	// FIXME SOFT_BKL: crash path must NOT block on BKL if lock is already held
+	ASM_BKL_LOCK(r6, r7, r8, p0)
 	r2 = htid
 	r3 = #-1
 	r3 = togglebit(r3,r2)
@@ -127,7 +128,7 @@ FUNC_START H2K_fatal_crash .text.vm.error .p2align 2
 	syscfg = r1
 	// We should now be in "Single Thread Mode": only one thread active, and that's the one executing here
 	// Did we possibly nmi a thread holding a lock? We can fix it!
-	k0unlock
+	ASM_BKL_UNLOCK(r6, r7, r8, p0)
 	tlbunlock
 	// Bit 1: Save off TLB and flush caches
 	// EJP: for now, let's do this all the time.

kernel/event/interrupt/interrupt.ref.S

@@ -181,7 +181,7 @@ FUNC_START H2K_interrupted_waitmode .text.core.interrupt .p2align 2
 	p0 = cmp.eq(INTINFO_FN,#0)	// really, we shouldn't get NULL interrupt handlers, but if we do...
 	if (!p0) callr INTINFO_FN
 	r1:0 = #0
-	k0lock
+	ASM_BKL_LOCK(r6, r7, r8, p0)
 	jump H2K_switch
 
 FUNC_END H2K_interrupted_waitmode

kernel/event/interrupt/interrupt.v3opt.S

@@ -77,7 +77,7 @@ FUNC_START H2K_interrupted_waitmode .text.core.interrupt .falign
 	{
 		callr r4
 	}
-	k0lock
+	ASM_BKL_LOCK(r6, r7, r8, p0)
 	{
 		r1:0 = combine(#0,#0)
 		jump H2K_switch

kernel/event/interrupt/interrupt.v4opt.S

@@ -178,9 +178,7 @@ FUNC_START H2K_interrupted_waitmode_or_preempt .text.core.interrupt .p2align 5
 		jumpr r31
 	}
 1:						// We were idle, go back to sleep
-	{
-		k0lock
-	}
+	ASM_BKL_LOCK(r6, r7, r8, p0)
 	{
 		r1:0 = #0
 		jump H2K_switch

kernel/event/interrupt/test/test.c

@@ -11,6 +11,7 @@
 #include <setjmp.h>
 #include <max.h>
 #include <globals.h>
+#include <hw.h>
 #include <trace.h>
 
 /*
@@ -97,7 +98,7 @@ void H2K_switch(H2K_thread_context *from, H2K_thread_context *to)
 {
 	if (from != NULL) FAIL("Unexpected FROM");
 	if (to != NULL) FAIL("Unexpected TO");
-	asm volatile (" k0unlock ");
+	BKL_UNLOCK();
 	longjmp(env,1);
 }
 

kernel/event/popup/popup.v4opt.S

@@ -25,7 +25,7 @@ FUNC_START H2K_popup_int .text.int.popup .p2align 5
 #define NEGONE r13
 #define NEWPRIO r14
 #define PRIO r15
-	k0lock
+	ASM_BKL_LOCK(r6, r7, r8, p0)
 	{
 		WAIT_PRIOMASKS = memd(H2K_GP+#KG_lowprio_masks)		// share w/ priomask?
 		HANDLERBASE = add(H2K_GP,#KG_inthandlers)
@@ -98,7 +98,7 @@ FUNC_START H2K_popup_int .text.int.popup .p2align 5
 		if (!p1) memd(ME) = PREV_HEAD
 	}
 .Lnohandler:
-	k0unlock
+	ASM_BKL_UNLOCK(r6, r7, r8, p0)
 	jumpr r31
 #undef INTNO
 #undef ME
@@ -147,7 +147,7 @@ FUNC_START H2K_popup_wait .text.int.popup .falign
 		p1 = cmp.eq(INTNO, #L2_CORE_INTERRUPT)
 		if (p1.new) jump:nt 9f
 	}
-	k0lock
+	ASM_BKL_LOCK(r6, r7, r8, p0)
 #define TMP INTHANDLERS_ADDR
 	{
 		TMP = memub(ME+#CONTEXT_vmstatus)
@@ -208,7 +208,7 @@ FUNC_START H2K_popup_wait .text.int.popup .falign
 		jump H2K_dosched
 	}
 8:
-	k0unlock
+	ASM_BKL_UNLOCK(r6, r7, r8, p0)
 9:
 	{
 		r0 = #-1
@@ -219,7 +219,7 @@ FUNC_START H2K_popup_wait .text.int.popup .falign
 		r0 = ME
 		call H2K_vm_do_work_withlock
 	}
-	k0unlock
+	ASM_BKL_UNLOCK(r6, r7, r8, p0)
 	{
 		r0 = #-1
 		dealloc_return