Skip to content

[WIP] Explore BESTWAIT hardware steering for scheduler optimization #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
zbelinsk wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

[WIP] Explore BESTWAIT hardware steering for scheduler optimization #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4239f0c
[WIP] Explore BESTWAIT hardware steering for scheduler optimization
zbelinsk Jun 21, 2026
6262329
sched: switch to BESTWAIT hardware steering, retire software lowprio
zbelinsk Jul 13, 2026
5a88e2c
Update dosched.ref.c
zbelinsk Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion kernel/checkers/checker_runlist/checker_runlist.check.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ s32_t checker_runlist()
{
u32_t i;
for (i = 0; i < H2K_gp->hthreads; i++) {
if (0 <= H2K_gp->runlist_prios[i] && H2K_gp->runlist_prios[i] <= MAX_PRIO) {
if (0 <= H2K_gp->runlist_prios[i] && H2K_gp->runlist_prios[i] <= MAX_READY_PRIO) {
if (H2K_gp->runlist_prios[i] != H2K_gp->runlist[i]->prio) FAIL("runlist_prios does not match priority of scheduled thread");
} else {
if (H2K_gp->runlist[i]) FAIL("Priority out of range but readylist non-null");
Expand Down
2 changes: 2 additions & 0 deletions kernel/data/globals/globals.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ typedef struct {
#endif
H2K_thread_context *runlist[MAX_HTHREADS];
s16_t runlist_prios[(MAX_HTHREADS+7)/8*8] __attribute__((aligned(8)));
H2K_thread_context *wip_dummy_runlist[MAX_HTHREADS];
s16_t wip_dummy_runlist_prios[(MAX_HTHREADS+7)/8*8] __attribute__((aligned(8)));
H2K_vmblock_t *vmblocks[H2K_ID_MAX_VMS];
u32_t phys_offset;
u32_t build_id;
Expand Down
2 changes: 1 addition & 1 deletion kernel/data/readylist/readylist.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ static inline u32_t H2K_ready_any_valid()
/* Check whether a thread at a given priority is ready */
static inline u32_t H2K_ready_prio_valid(u32_t prio)
{
if (prio > MAX_PRIO) return 0;

@eplondke Erich Plondke (eplondke) Jul 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

recommend keeping MAX_PRIO and setting it to 254 or smaller.

if (prio > WAITING_PRIO) return 0;
return (H2K_gp->ready_valids[prio >> 6] & (1ULL << (prio & 0x3f))) != 0;
}

Expand Down
39 changes: 6 additions & 33 deletions kernel/data/runlist/runlist.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,55 +11,28 @@
#include <hexagon_protos.h>
#include <globals.h>
#include <log.h>
#include <hw.h>

static inline void H2K_runlist_push(H2K_thread_context *newthread)
{
u32_t hthread = newthread->hthread;
u32_t prio = newthread->prio;
newthread->status = H2K_STATUS_RUNNING;
H2K_gp->runlist[hthread] = newthread;
H2K_gp->runlist_prios[hthread] = (s16_t)prio;
}

static inline u32_t H2K_runlist_worst_prio()
{
s32_t worst_prio = -1;
s32_t hthread = -1;
s32_t i;
for (i = 0; i < H2K_gp->hthreads; i++) {
if (H2K_gp->runlist_prios[i] IS_WORSE_THAN worst_prio) {
worst_prio = H2K_gp->runlist_prios[i];
hthread = i;
}
}
return hthread == -1 ? MAX_PRIOS : (u32_t)worst_prio;
}

static inline u32_t H2K_runlist_worst_prio_hthread()
{
s32_t worst_prio = -1;
s32_t hthread = -1;
s32_t i;
for (i = 0; i < H2K_gp->hthreads; i++) {
if (H2K_gp->runlist_prios[i] IS_WORSE_THAN worst_prio) {
worst_prio = H2K_gp->runlist_prios[i];
hthread = i;
}
}
return (u32_t)hthread;
H2K_gp->wip_dummy_runlist[hthread] = newthread;
H2K_gp->wip_dummy_runlist_prios[hthread] = (s16_t)prio;
}

static inline void H2K_runlist_remove(H2K_thread_context *thread)
{
u32_t hthread = thread->hthread;
H2K_gp->runlist[hthread] = NULL;
H2K_gp->runlist_prios[hthread] = -1;
H2K_gp->wip_dummy_runlist[hthread] = NULL;
H2K_gp->wip_dummy_runlist_prios[hthread] = -1;
}

static inline void H2K_runlist_set_thread_prio(H2K_thread_context *thread, u32_t new_prio)
{
thread->prio = (u8_t)new_prio;
H2K_gp->runlist_prios[thread->hthread] = (s16_t)new_prio;
H2K_gp->wip_dummy_runlist_prios[thread->hthread] = (s16_t)new_prio;
}

void H2K_runlist_init(void) IN_SECTION(".text.init.runlist");
Expand Down
4 changes: 0 additions & 4 deletions kernel/event/intpool/intpool.ref.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,10 @@ void H2K_intpool_int(u32_t intnum, H2K_thread_context *me, u32_t hwtnum, H2K_vmb
H2K_ring_remove(&vmblock->intpool,woken);
woken->r00 = intnum;
if (me != NULL) {
H2K_runlist_remove(me);
H2K_ready_append(me);
} else {
H2K_gp->wait_mask = (u32_t)Q6_R_clrbit_RR(H2K_gp->wait_mask,hwtnum);
}
H2K_gp->priomask = (u32_t)Q6_R_clrbit_RR(H2K_gp->priomask,hwtnum);
highprio_imask(hwtnum);
H2K_runlist_push(woken);
H2K_switch(me,woken);
}
Expand Down Expand Up @@ -88,7 +85,6 @@ int H2K_intpool_wait(u32_t int_ack_num, H2K_thread_context *me)
}
}
/* FIXME: check pending intpool interrupts */
H2K_runlist_remove(me);

@eplondke Erich Plondke (eplondke) Jul 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Getting rid of the runlist entirely is a little scary... very cool if we could do that though!

H2K_ring_append(&vmblock->intpool,me);
me->status = H2K_STATUS_INTBLOCKED; /* OR INTPOOL_BLOCKED? */
me->r00 = (u32_t)-1;
Expand Down
21 changes: 2 additions & 19 deletions kernel/event/intpool/test/test.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,25 +129,17 @@ int TH_call_intpool_wait(int i, H2K_thread_context *current)
*/
void TH_check_waiting(int i, H2K_thread_context *thread)
{
//if (H2K_gp->inthandlers[i].handler != H2K_intpool_int) FAIL("Wrong handler");
//if (H2K_gp->inthandlers[i].param != thread) FAIL("Wrong thread");
if (thread->status != H2K_STATUS_INTBLOCKED) FAIL("Wrong status");
if (H2K_gp->runlist[thread->hthread] == thread) FAIL("Thread still scheduled");
}

/*
* Check to make sure that thread is still running and not waiting on interrupt
*/
void TH_check_running(int i, H2K_thread_context *thread)
{
#if 0
printf("Checking i=%d thread=%p inthandlers[i].handler=%p inthandlers[i].param=%p\n",
i,thread,H2K_gp->inthandlers[i].handler,H2K_gp->inthandlers[i].param);
#endif
if (H2K_gp->inthandlers[i].handler != NULL) FAIL("Handler set");
if (H2K_gp->inthandlers[i].param == thread) FAIL("Handler thread set");
if (thread->status != H2K_STATUS_RUNNING) FAIL("Wrong status");
if (H2K_gp->runlist[thread->hthread] != thread) FAIL("Thread not scheduled");
}

void TH_check_old(H2K_thread_context *thread)
Expand Down Expand Up @@ -184,26 +176,19 @@ void TH_intpool_int(int i, H2K_thread_context *interrupted, int hthread, H2K_vmb
void TH_set_idle(int hthread)
{
H2K_gp->wait_mask = 1<<hthread;
H2K_gp->priomask = 1<<hthread;
H2K_runlist_init();
lowprio_imask(hthread);
}

void TH_set_running(int hthread, H2K_thread_context *thread)
{
H2K_gp->wait_mask &= ~(1<<hthread);
H2K_gp->priomask &= ~(1<<hthread);
H2K_runlist_push(thread);
lowprio_imask(hthread);
}

void TH_check_priowait_running(int hthread, H2K_thread_context *thread)
{
if ((1<<(hthread)) & H2K_gp->wait_mask) FAIL("wait_mask still set");
if ((1<<(hthread)) & H2K_gp->priomask) FAIL("priomask still set");
if ((get_imask(thread->hthread)) == 0) {
if ((get_imask(thread->hthread)) != 0) {
printf("ht=%x tht=%x imask=%x\n",hthread, thread->hthread, get_imask(thread->hthread));
FAIL("IMASK clear for hthread");
FAIL("IMASK set for hthread");
}
}

Expand Down Expand Up @@ -255,8 +240,6 @@ int main()
H2K_gp->l2_ack_base = fakeint+(0x200/sizeof(u32_t));
#endif
H2K_readylist_init();
H2K_runlist_init();
H2K_lowprio_init();
a.prio = b.prio = c.prio = MAX_PRIOS - 30;
a.vmblock = b.vmblock = c.vmblock = vmblock;

Expand Down
8 changes: 0 additions & 8 deletions kernel/event/popup/popup.ref.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,10 @@ void H2K_popup_int(u32_t intnum, H2K_thread_context *me, u32_t hwtnum, H2K_threa
return;
}
if (me != NULL) {
H2K_runlist_remove(me);
H2K_ready_append(me);
} else {
H2K_gp->wait_mask = (u32_t)Q6_R_clrbit_RR(H2K_gp->wait_mask,hwtnum);
}
/* Assume woken is better than interrupted thread.
* If not, check_sanity will detect.
* Let check_sanity find the new lowprio thread also...
*/
H2K_gp->priomask = (u32_t)Q6_R_clrbit_RR(H2K_gp->priomask,hwtnum);
highprio_imask(hwtnum);
H2K_runlist_push(woken);
H2K_switch(me,woken);
}
Expand All @@ -62,7 +55,6 @@ int H2K_popup_wait(u32_t intnum, H2K_thread_context *me)
}
H2K_gp->inthandlers[intnum].param = me;
H2K_gp->inthandlers[intnum].handler = H2K_popup_int;
H2K_runlist_remove(me);
me->status = H2K_STATUS_INTBLOCKED;
me->r0100 = intnum;
H2K_intcontrol_enable(intnum);
Expand Down
13 changes: 1 addition & 12 deletions kernel/event/popup/test/test.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,6 @@ void TH_check_waiting(int i, H2K_thread_context *thread)
if (H2K_gp->inthandlers[i].handler != H2K_popup_int) FAIL("Wrong handler");
if (H2K_gp->inthandlers[i].param != thread) FAIL("Wrong thread");
if (thread->status != H2K_STATUS_INTBLOCKED) FAIL("Wrong status");
if (H2K_gp->runlist[thread->hthread] == thread) FAIL("Thread still scheduled");
}

/*
Expand All @@ -145,7 +144,6 @@ void TH_check_running(int i, H2K_thread_context *thread)
if (H2K_gp->inthandlers[i].handler != NULL) FAIL("Handler set");
if (H2K_gp->inthandlers[i].param == thread) FAIL("Handler thread set");
if (thread->status != H2K_STATUS_RUNNING) FAIL("Wrong status");
if (H2K_gp->runlist[thread->hthread] != thread) FAIL("Thread not scheduled");
}

void TH_check_old(H2K_thread_context *thread)
Expand Down Expand Up @@ -190,9 +188,6 @@ void TH_popup_int(int i, H2K_thread_context *interrupted, int hthread, H2K_threa
void TH_set_idle(int hthread)
{
H2K_gp->wait_mask = 1<<hthread;
H2K_gp->priomask = 1<<hthread;
H2K_runlist_init();
lowprio_imask(hthread);
}

/*
Expand All @@ -201,9 +196,6 @@ void TH_set_idle(int hthread)
void TH_set_running(int hthread, H2K_thread_context *thread)
{
H2K_gp->wait_mask &= ~(1<<hthread);
H2K_gp->priomask &= ~(1<<hthread);
H2K_runlist_push(thread);
lowprio_imask(hthread);
}

/*
Expand All @@ -212,8 +204,7 @@ void TH_set_running(int hthread, H2K_thread_context *thread)
void TH_check_priowait_running(int hthread, H2K_thread_context *thread)
{
if ((1<<(hthread)) & H2K_gp->wait_mask) FAIL("wait_mask still set");
if ((1<<(hthread)) & H2K_gp->priomask) FAIL("priomask still set");
if ((get_imask(thread->hthread)) == 0) FAIL("IMASK clear for hthread");
if ((get_imask(thread->hthread)) != 0) FAIL("IMASK set for hthread");
}

/*
Expand Down Expand Up @@ -262,8 +253,6 @@ int main()
H2K_gp->l2_ack_base = fakeint+(0x200/sizeof(u32_t));
#endif
H2K_readylist_init();
H2K_runlist_init();
H2K_lowprio_init();
a.prio = b.prio = c.prio = MAX_PRIOS - 30;

/* First, test wait */
Expand Down
2 changes: 1 addition & 1 deletion kernel/futex/futex/test/tests/multi_wake/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ void vmmain(void *unused)

/* Figure out the max priority that can be woken up */
k = nr_to_wake;
for (j=0; j<MAX_PRIO; j++) {
for (j=0; j<MAX_READY_PRIO; j++) {
for (i=0; i<max_consumers; i++) {
if ((thread_info[i] >> 16) == j) {
k--;
Expand Down
1 change: 0 additions & 1 deletion kernel/futex/futex_classic/futex_classic.ref.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ s32_t H2K_futex_wait(u32_t *lock, u32_t val, H2K_thread_context *me)
return -1;
}
me->futex_ptr = pa;
H2K_runlist_remove(me);
me->r0100 = 0;
me->status = H2K_STATUS_BLOCKED;
H2K_futex_hash_add_ring(&H2K_gp->futexhash[FUTEX_HASHVAL(pa)],me);
Expand Down
13 changes: 7 additions & 6 deletions kernel/futex/futex_pi/futex_pi.ref.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ static inline void H2K_futex_pi_raise(u32_t prio, H2K_id_t destid)
H2K_ready_insert(dest);
} else if (dest->status == H2K_STATUS_RUNNING) {
H2K_runlist_set_thread_prio(dest, prio);
if (H2K_gp->priomask & (1<<dest->hthread)) {
H2K_raise_lowprio();
}
/* Sync hardware STID.PRIO so BESTWAIT sees the boosted priority
* immediately -- avoids spurious preemption of the holder. */
set_thread_stid_prio(dest->hthread, prio);

@eplondke Erich Plondke (eplondke) Jul 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should become the "setprio" instruction after the inline after the change, I think?

/* Need to update lowprio */
} else if (dest->status == H2K_STATUS_INTBLOCKED) {
/* Waiting on interrupt, but we want it to have high priority when it starts up */
Expand Down Expand Up @@ -92,7 +92,6 @@ s32_t H2K_futex_lock_pi(u32_t *lock, H2K_thread_context *me)
}
H2K_futex_pi_raise(me->prio,x.dest);
me->futex_ptr = pa;
H2K_runlist_remove(me);
me->r0100 = 0;
me->status = H2K_STATUS_BLOCKED;
H2K_futex_hash_add_ring(&H2K_gp->futexhash[FUTEX_HASHVAL(pa)],me);
Expand Down Expand Up @@ -130,8 +129,10 @@ s32_t H2K_futex_unlock_pi(u32_t *lock, H2K_thread_context *me)
H2K_atomic_swap(lock,H2K_id_from_context(ret).raw+1);
}
H2K_safemem_unlock();
/* Restore my priority */
/* Restore priority in software and hardware atomically -- BESTWAIT
* comparator reads STID.PRIO; without the hardware sync the stale
* boosted value hides me from the comparator causing a priority inversion. */
H2K_runlist_set_thread_prio(me, me->base_prio);
set_thread_stid_prio(me->hthread, me->base_prio);

@eplondke Erich Plondke (eplondke) Jul 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could also be the "setprio" instruction or re-writing our whole STID register

return (s32_t)H2K_check_sanity_unlock(0);
}

3 changes: 2 additions & 1 deletion kernel/init/boot/boot.ref.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,10 +348,11 @@ boot_continuation:
SETCONST(H2K_GP,H2K_kg)

/* Set up rising edge triggered */

@eplondke Erich Plondke (eplondke) Jul 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably want to fix the comments.

TMP = #-1
TMP = #0
/* Set imask to -1, since we are not lowest priority at boot */
/* (other idle threads) */
imask = TMP
TMP = #-1
#if ARCHV < 65
iel = TMP
iahl = TMP
Expand Down
6 changes: 3 additions & 3 deletions kernel/init/setup/setup.ref.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,11 @@ IN_SECTION(".text.init.setup") static H2K_vmblock_t *H2K_init_setup(u32_t multic
H2K_l2cache_init();
H2K_tcm_copy(last_tlb_index);
H2K_trace_init();
H2K_runlist_init();
H2K_readylist_init();
H2K_lowprio_init();
H2K_gp->wait_mask = 0;
H2K_futex_init();
H2K_intconfig_init(ssbase);
H2K_set_schedcfg(SCHEDCFG_EN | SCHEDCFG_INTNO(RESCHED_INT));
H2K_thread_init();
H2K_asid_table_init();

Expand Down Expand Up @@ -226,7 +226,7 @@ IN_SECTION(".text.init.boot") void H2K_thread_boot(u32_t multicore_shift, u32_t
#ifdef CLUSTER_SCHED
H2K_cluster_config();
#endif
H2K_runlist_push(boot);
H2K_runlist_push(boot); //removing this will make kernel/time/timer/test_h2 to fail
H2K_mutex_unlock_tlb();
H2K_switch(NULL,boot);
}
20 changes: 3 additions & 17 deletions kernel/init/setup/test/test.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,7 @@ u32_t TH_init_seen;
u32_t TH_switch_seen;

enum {
runlist_init = 0,
readylist_init,
lowprio_init,
readylist_init = 0,
futex_init,
intconfig_init,
kg_init,
Expand Down Expand Up @@ -89,9 +87,9 @@ u32_t H2K_trap_config(u32_t configtype, void *ptr, u32_t val2, u32_t val3, u32_t

#define HELPER_FUNC(X) void H2K_##X() { TH_init_seen |= 1<< X; }

HELPER_FUNC(runlist_init)

HELPER_FUNC(readylist_init)
HELPER_FUNC(lowprio_init)

HELPER_FUNC(futex_init)
//HELPER_FUNC(intconfig_init)
void H2K_intconfig_init(u32_t ssbase) { TH_init_seen |= 1<< intconfig_init; }
Expand Down Expand Up @@ -144,16 +142,12 @@ void H2K_thread_boot();
int main()
{
u32_t i;
u32_t found_thread;
__asm__ __volatile(GLOBAL_REG_STR " = %0 " : : "r"(&H2K_kg));

H2K_gp->logbuf_enable = 0;

H2K_gp->hthreads = get_hthreads();

for (i = 0; i < H2K_gp->hthreads; i++) {
H2K_gp->runlist[i] = 0;
}
TH_init_seen = 0;
TH_switch_seen = 0;
if (setjmp(env) == 0) {
Expand All @@ -166,14 +160,6 @@ int main()
}
if (boot->continuation != (H2K_interrupt_restore)) FAIL("Incorrect continuation");
if (boot->trapmask != 0xffffffffU) FAIL("boot thread trapmask");
found_thread = 0;
for (i = 0; i < H2K_gp->hthreads; i++) {
if (H2K_gp->runlist[i] == boot) {
if (H2K_gp->runlist_prios[i] != 0) FAIL("Didn't push into runlist (0)");
found_thread = 1;
}
}
if (!found_thread) FAIL("Didn't push into runlist (1)");
puts("TEST PASSED\n");
exit(0);
}
Expand Down
Loading
Loading