Skip to content

Commit e6979d0

Browse files
committed
tools/sched_ext: scx - Fix cmask_subset(), cmask_equal() and cmask_weight()
cmask_equal(), cmask_weight() and cmask_subset() bounded their word walks with CMASK_NR_WORDS(nr_cids), which pads by one word and can't tell the last word in use without @base. The walks could thus cover a slack word past the active range, which cmask_reframe() leaves non-zero: a stale bit there gave cmask_equal() a spurious mismatch, cmask_weight() an inflated count, and cmask_subset() a spurious violation. cmask_subset() could also read @b->bits[] one word past its allocation (within the arena's fault-recovered range, so harmless), and deviated from the kernel scx_cmask_subset() by failing any @A range that doesn't nest inside @b's even when the overhanging bits are all clear. Bound the cmask_equal() and cmask_weight() walks by the words the range actually spans, with early returns for empty ranges. Rewrite cmask_subset() to match the kernel semantics: scan @A's overhangs for set bits with cmask_next_set() and walk the words of the range intersection. cmask_subset() moves below cmask_next_set(), which it now uses. Padding bits don't need masking as every cmask helper keeps them clear. Fixes: a58e6b7 ("sched_ext: Add cmask, a base-windowed bitmap over cid space") Signed-off-by: Tejun Heo <tj@kernel.org> Reviewed-by: Andrea Righi <arighi@nvidia.com>
1 parent 49b3378 commit e6979d0

1 file changed

Lines changed: 55 additions & 33 deletions

File tree

tools/sched_ext/include/scx/cid.bpf.h

Lines changed: 55 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,9 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
391391

392392
if (a->base != b->base || a->nr_cids != b->nr_cids)
393393
return false;
394-
nr_words = CMASK_NR_WORDS(a->nr_cids);
394+
if (a->nr_cids == 0)
395+
return true;
396+
nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1;
395397

396398
bpf_for(i, 0, CMASK_MAX_WORDS) {
397399
if (i >= nr_words)
@@ -402,36 +404,6 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
402404
return true;
403405
}
404406

405-
/*
406-
* True iff every bit set in @a is also set in @b over the intersection of
407-
* their ranges. Bits of @a outside @b's range fail the test.
408-
*/
409-
static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
410-
const struct scx_cmask __arena *b)
411-
{
412-
u32 a_end = a->base + a->nr_cids;
413-
u32 b_end = b->base + b->nr_cids;
414-
u32 a_wbase = a->base / 64;
415-
u32 b_wbase = b->base / 64;
416-
u32 nr_words, i;
417-
418-
/* any bit of @a outside @b's range is a subset violation */
419-
if (a->base < b->base || a_end > b_end)
420-
return false;
421-
422-
nr_words = CMASK_NR_WORDS(a->nr_cids);
423-
bpf_for(i, 0, CMASK_MAX_WORDS) {
424-
u32 wi_b;
425-
426-
if (i >= nr_words)
427-
break;
428-
wi_b = a_wbase + i - b_wbase;
429-
if (a->bits[i] & ~b->bits[wi_b])
430-
return false;
431-
}
432-
return true;
433-
}
434-
435407
/**
436408
* cmask_next_set - find the first set bit at or after @cid
437409
* @m: cmask to search
@@ -488,16 +460,66 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m)
488460
(cid) < (m)->base + (m)->nr_cids; \
489461
(cid) = cmask_next_set((m), (cid) + 1))
490462

463+
/*
464+
* True iff every bit set in @a is also set in @b. Matches the kernel-side
465+
* scx_cmask_subset(): ranges don't need to nest, and set bits of @a outside
466+
* @b's range fail the test.
467+
*/
468+
static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
469+
const struct scx_cmask __arena *b)
470+
{
471+
u32 a_end = a->base + a->nr_cids;
472+
u32 b_end = b->base + b->nr_cids;
473+
u32 a_wbase = a->base / 64;
474+
u32 b_wbase = b->base / 64;
475+
u32 lo = a->base > b->base ? a->base : b->base;
476+
u32 hi = a_end < b_end ? a_end : b_end;
477+
u32 lo_word, hi_word, i;
478+
479+
/* set bits of @a outside @b's range can't be in @b */
480+
if (a->base < b->base &&
481+
cmask_next_set(a, a->base) < (b->base < a_end ? b->base : a_end))
482+
return false;
483+
if (a_end > b_end &&
484+
cmask_next_set(a, a->base > b_end ? a->base : b_end) < a_end)
485+
return false;
486+
487+
if (lo >= hi)
488+
return true;
489+
490+
/*
491+
* Walk the words the range intersection spans. Plain word tests
492+
* suffice: the scans above guarantee @a has no set bit outside @b's
493+
* range and padding bits are kept clear by all cmask helpers.
494+
*/
495+
lo_word = lo / 64;
496+
hi_word = (hi - 1) / 64;
497+
498+
bpf_for(i, 0, CMASK_MAX_WORDS) {
499+
u32 w = lo_word + i;
500+
501+
if (w > hi_word)
502+
break;
503+
if (a->bits[w - a_wbase] & ~b->bits[w - b_wbase])
504+
return false;
505+
}
506+
return true;
507+
}
508+
491509
/*
492510
* Population count over [base, base + nr_cids). Padding bits in the head/tail
493511
* words are guaranteed zero by the mutating helpers, so a flat popcount over
494-
* all words is correct.
512+
* the words the range spans is correct.
495513
*/
496514
static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
497515
{
498-
u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i;
516+
u32 nr_words, i;
499517
u32 count = 0;
500518

519+
if (!m->nr_cids)
520+
return 0;
521+
nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
522+
501523
bpf_for(i, 0, CMASK_MAX_WORDS) {
502524
if (i >= nr_words)
503525
break;

0 commit comments

Comments
 (0)