Skip to content

Commit c941d73

Browse files
htejunhtejun
committed
sched_ext: Close root-enable vs sched_ext_dead() race with SCX_TASK_INIT_BEGIN
scx_root_enable_workfn() drops the iter rq lock for ops.init_task() and a TASK_DEAD @p can fall through sched_ext_dead() in that window. The race hits when sched_ext_dead() observes SCX_TASK_INIT (the intermediate state before @p->scx.sched is published) and dereferences NULL via SCX_HAS_OP(NULL, exit_task), or observes SCX_TASK_NONE during the unlocked init window and skips cleanup so exit_task() never runs. Add SCX_TASK_INIT_BEGIN. The enable path writes NONE -> INIT_BEGIN under the iter rq lock, then takes the rq lock again after init to walk INIT_BEGIN -> INIT -> READY. sched_ext_dead() that wins the rq-lock race observes INIT_BEGIN and sets DEAD without calling into ops; the post-init recheck unwinds via scx_sub_init_cancel_task(). scx_fork() runs single-threaded against sched_ext_dead() (the task is not on scx_tasks until scx_post_fork() adds it) so its INIT_BEGIN -> INIT walk needs no rq-lock pairing; it rolls back to NONE on ops.init_task() failure. The validation matrix grows the INIT_BEGIN row and the INIT_BEGIN -> DEAD edge; INIT now requires INIT_BEGIN as the predecessor. scx_sub_disable()'s migration writes INIT_BEGIN as a synthetic predecessor to satisfy the tightened verification. The sub-sched paths still race with sched_ext_dead() during the unlocked init window. This will be fixed by the next patch. Reported-by: zhidao su <suzhidao@xiaomi.com> Link: https://lore.kernel.org/all/20260429133155.3825247-1-suzhidao@xiaomi.com/ Signed-off-by: Tejun Heo <tj@kernel.org> Reviewed-by: Andrea Righi <arighi@nvidia.com>
1 parent cceb8fa commit c941d73

2 files changed

Lines changed: 55 additions & 11 deletions

File tree

‎include/linux/sched/ext.h‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ enum scx_ent_flags {
106106
* Bits 8 to 10 are used to carry task state:
107107
*
108108
* NONE ops.init_task() not called yet
109+
* INIT_BEGIN ops.init_task() in flight; see sched_ext_dead()
109110
* INIT ops.init_task() succeeded, but task can be cancelled
110111
* READY fully initialized, but not in sched_ext
111112
* ENABLED fully initialized and in sched_ext
@@ -116,10 +117,11 @@ enum scx_ent_flags {
116117
SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT,
117118

118119
SCX_TASK_NONE = 0 << SCX_TASK_STATE_SHIFT,
119-
SCX_TASK_INIT = 1 << SCX_TASK_STATE_SHIFT,
120-
SCX_TASK_READY = 2 << SCX_TASK_STATE_SHIFT,
121-
SCX_TASK_ENABLED = 3 << SCX_TASK_STATE_SHIFT,
122-
SCX_TASK_DEAD = 4 << SCX_TASK_STATE_SHIFT,
120+
SCX_TASK_INIT_BEGIN = 1 << SCX_TASK_STATE_SHIFT,
121+
SCX_TASK_INIT = 2 << SCX_TASK_STATE_SHIFT,
122+
SCX_TASK_READY = 3 << SCX_TASK_STATE_SHIFT,
123+
SCX_TASK_ENABLED = 4 << SCX_TASK_STATE_SHIFT,
124+
SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT,
123125

124126
/*
125127
* Bits 12 and 13 are used to carry reenqueue reason. In addition to

‎kernel/sched/ext.c‎

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -725,8 +725,11 @@ static void scx_set_task_state(struct task_struct *p, u32 state)
725725
case SCX_TASK_NONE:
726726
warn = prev_state == SCX_TASK_DEAD;
727727
break;
728-
case SCX_TASK_INIT:
728+
case SCX_TASK_INIT_BEGIN:
729729
warn = prev_state != SCX_TASK_NONE;
730+
break;
731+
case SCX_TASK_INIT:
732+
warn = prev_state != SCX_TASK_INIT_BEGIN;
730733
p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
731734
break;
732735
case SCX_TASK_READY:
@@ -737,7 +740,8 @@ static void scx_set_task_state(struct task_struct *p, u32 state)
737740
warn = prev_state != SCX_TASK_READY;
738741
break;
739742
case SCX_TASK_DEAD:
740-
warn = prev_state != SCX_TASK_NONE;
743+
warn = !(prev_state == SCX_TASK_NONE ||
744+
prev_state == SCX_TASK_INIT_BEGIN);
741745
break;
742746
default:
743747
WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
@@ -3753,9 +3757,12 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs)
37533757
#else
37543758
struct scx_sched *sch = scx_root;
37553759
#endif
3760+
scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
37563761
ret = __scx_init_task(sch, p, true);
3757-
if (unlikely(ret))
3762+
if (unlikely(ret)) {
3763+
scx_set_task_state(p, SCX_TASK_NONE);
37583764
return ret;
3765+
}
37593766
scx_set_task_state(p, SCX_TASK_INIT);
37603767
scx_set_task_sched(p, sch);
37613768
}
@@ -3856,13 +3863,18 @@ void sched_ext_dead(struct task_struct *p)
38563863
* scx_task_iter_next_locked(). NONE tasks need no marking: cgroup
38573864
* iteration is only used from sub-sched paths, which require root
38583865
* enabled. Root enable transitions every live task to at least READY.
3866+
*
3867+
* %INIT_BEGIN means ops.init_task() is running for @p. Don't call
3868+
* into ops; transition to %DEAD so the post-init recheck unwinds
3869+
* via scx_sub_init_cancel_task().
38593870
*/
38603871
if (scx_get_task_state(p) != SCX_TASK_NONE) {
38613872
struct rq_flags rf;
38623873
struct rq *rq;
38633874

38643875
rq = task_rq_lock(p, &rf);
3865-
scx_disable_and_exit_task(scx_task_sched(p), p);
3876+
if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN)
3877+
scx_disable_and_exit_task(scx_task_sched(p), p);
38663878
scx_set_task_state(p, SCX_TASK_DEAD);
38673879
task_rq_unlock(rq, p, &rf);
38683880
}
@@ -5773,6 +5785,7 @@ static void scx_sub_disable(struct scx_sched *sch)
57735785
* $p having already been initialized, and then enable.
57745786
*/
57755787
scx_disable_and_exit_task(sch, p);
5788+
scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
57765789
scx_set_task_state(p, SCX_TASK_INIT);
57775790
scx_set_task_sched(p, parent);
57785791
scx_set_task_state(p, SCX_TASK_READY);
@@ -6878,6 +6891,9 @@ static void scx_root_enable_workfn(struct kthread_work *work)
68786891

68796892
scx_task_iter_start(&sti, NULL);
68806893
while ((p = scx_task_iter_next_locked(&sti))) {
6894+
struct rq_flags rf;
6895+
struct rq *rq;
6896+
68816897
/*
68826898
* @p may already be dead, have lost all its usages counts and
68836899
* be waiting for RCU grace period before being freed. @p can't
@@ -6886,21 +6902,47 @@ static void scx_root_enable_workfn(struct kthread_work *work)
68866902
if (!tryget_task_struct(p))
68876903
continue;
68886904

6905+
/*
6906+
* Set %INIT_BEGIN under the iter's rq lock so that a concurrent
6907+
* sched_ext_dead() does not call ops.exit_task() on @p while
6908+
* ops.init_task() is running. If sched_ext_dead() runs before
6909+
* this store, it has already removed @p from scx_tasks and the
6910+
* iter won't visit @p; if it runs after, it observes
6911+
* %INIT_BEGIN and transitions to %DEAD without calling ops,
6912+
* leaving the post-init recheck below to unwind.
6913+
*/
6914+
scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
68896915
scx_task_iter_unlock(&sti);
68906916

68916917
ret = __scx_init_task(sch, p, false);
6918+
6919+
rq = task_rq_lock(p, &rf);
6920+
68926921
if (unlikely(ret)) {
6922+
if (scx_get_task_state(p) != SCX_TASK_DEAD)
6923+
scx_set_task_state(p, SCX_TASK_NONE);
6924+
task_rq_unlock(rq, p, &rf);
68936925
put_task_struct(p);
68946926
scx_task_iter_stop(&sti);
68956927
scx_error(sch, "ops.init_task() failed (%d) for %s[%d]",
68966928
ret, p->comm, p->pid);
68976929
goto err_disable_unlock_all;
68986930
}
68996931

6900-
scx_set_task_state(p, SCX_TASK_INIT);
6901-
scx_set_task_sched(p, sch);
6902-
scx_set_task_state(p, SCX_TASK_READY);
6932+
if (scx_get_task_state(p) == SCX_TASK_DEAD) {
6933+
/*
6934+
* sched_ext_dead() observed %INIT_BEGIN and set %DEAD.
6935+
* ops.exit_task() is owed to the sched __scx_init_task()
6936+
* ran against; call it now.
6937+
*/
6938+
scx_sub_init_cancel_task(sch, p);
6939+
} else {
6940+
scx_set_task_state(p, SCX_TASK_INIT);
6941+
scx_set_task_sched(p, sch);
6942+
scx_set_task_state(p, SCX_TASK_READY);
6943+
}
69036944

6945+
task_rq_unlock(rq, p, &rf);
69046946
put_task_struct(p);
69056947
}
69066948
scx_task_iter_stop(&sti);

0 commit comments

Comments
 (0)