From 7a8464555d2e5f038758bb19e72ab4710b79e9cd Mon Sep 17 00:00:00 2001 From: zhidao su Date: Wed, 4 Mar 2026 13:37:30 +0800 Subject: [PATCH 1/9] sched_ext: Use WRITE_ONCE() for the write side of dsq->seq update bpf_iter_scx_dsq_new() reads dsq->seq via READ_ONCE() without holding any lock, making dsq->seq a lock-free concurrently accessed variable. However, dispatch_enqueue(), the sole writer of dsq->seq, uses a plain increment without the matching WRITE_ONCE() on the write side: dsq->seq++; ^^^^^^^^^^^ plain write -- KCSAN data race The KCSAN documentation requires that if one accessor uses READ_ONCE() or WRITE_ONCE() on a variable to annotate lock-free access, all other accesses must also use the appropriate accessor. A plain write leaves the pair incomplete and will trigger KCSAN warnings. Fix by using WRITE_ONCE() for the write side of the update: WRITE_ONCE(dsq->seq, dsq->seq + 1); This is consistent with bpf_iter_scx_dsq_new() and makes the concurrent access annotation complete and KCSAN-clean. Signed-off-by: zhidao su Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 1594987d637b..c56de568ed94 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -1103,7 +1103,7 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, } /* seq records the order tasks are queued, used by BPF DSQ iterator */ - dsq->seq++; + WRITE_ONCE(dsq->seq, dsq->seq + 1); p->scx.dsq_seq = dsq->seq; dsq_mod_nr(dsq, 1); From 6944e6d8a6d4c1e654de1da112da8fef1b30e623 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Thu, 5 Mar 2026 03:57:57 +0800 Subject: [PATCH 2/9] sched_ext/selftests: Fix format specifier and buffer length in file_write_long() Use %ld (not %lu) for signed long, and pass the actual string length returned by sprintf() to write_text() instead of sizeof(buf). Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/sched_ext/util.c b/tools/testing/selftests/sched_ext/util.c index e47769c91918..2111329ed289 100644 --- a/tools/testing/selftests/sched_ext/util.c +++ b/tools/testing/selftests/sched_ext/util.c @@ -60,11 +60,11 @@ int file_write_long(const char *path, long val) char buf[64]; int ret; - ret = sprintf(buf, "%lu", val); + ret = sprintf(buf, "%ld", val); if (ret < 0) return ret; - if (write_text(path, buf, sizeof(buf)) <= 0) + if (write_text(path, buf, ret) <= 0) return -1; return 0; From 0927780c90ce551869fb692279d66387a4b66af5 Mon Sep 17 00:00:00 2001 From: zhidao su Date: Thu, 5 Mar 2026 14:18:56 +0800 Subject: [PATCH 3/9] sched_ext: Use READ_ONCE() for lock-free reads of module param variables bypass_lb_cpu() reads scx_bypass_lb_intv_us and scx_slice_bypass_us without holding any lock, in timer callback context where module parameter writes via sysfs can happen concurrently: min_delta_us = scx_bypass_lb_intv_us / SCX_BYPASS_LB_MIN_DELTA_DIV; ^^^^^^^^^^^^^^^^^^^^ plain read -- KCSAN data race if (delta < DIV_ROUND_UP(min_delta_us, scx_slice_bypass_us)) ^^^^^^^^^^^^^^^^^ plain read -- KCSAN data race scx_bypass_lb_intv_us already uses READ_ONCE() in scx_bypass_lb_timerfn() and scx_bypass() for its other lock-free read sites, leaving bypass_lb_cpu() inconsistent. scx_slice_bypass_us has the same lock-free access pattern in the same function. Fix both plain reads by using READ_ONCE() to complete the concurrent access annotation and make the code KCSAN-clean. Signed-off-by: zhidao su Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index c56de568ed94..10866bfb88bf 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -3908,8 +3908,8 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq, * consider offloading iff the total queued duration is over the * threshold. */ - min_delta_us = scx_bypass_lb_intv_us / SCX_BYPASS_LB_MIN_DELTA_DIV; - if (delta < DIV_ROUND_UP(min_delta_us, scx_slice_bypass_us)) + min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; + if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) return 0; raw_spin_rq_lock_irq(rq); From 70f54f61a3d52af13b72248a63e98eddf4c990ac Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Thu, 5 Mar 2026 07:29:00 +0100 Subject: [PATCH 4/9] sched_ext: Document task ownership state machine The task ownership state machine in sched_ext is quite hard to follow from the code alone. The interaction of ownership states, memory ordering rules and cross-CPU "lock dancing" makes the overall model subtle. Extend the documentation next to scx_ops_state to provide a more structured and self-contained description of the state transitions and their synchronization rules. The new reference should make the code easier to reason about and maintain and can help future contributors understand the overall task-ownership workflow. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext_internal.h | 114 +++++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 16 deletions(-) diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 11ebb744d893..00b450597f3e 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -1035,26 +1035,108 @@ static const char *scx_enable_state_str[] = { }; /* - * sched_ext_entity->ops_state + * Task Ownership State Machine (sched_ext_entity->ops_state) * - * Used to track the task ownership between the SCX core and the BPF scheduler. - * State transitions look as follows: + * The sched_ext core uses this state machine to track task ownership + * between the SCX core and the BPF scheduler. This allows the BPF + * scheduler to dispatch tasks without strict ordering requirements, while + * the SCX core safely rejects invalid dispatches. * - * NONE -> QUEUEING -> QUEUED -> DISPATCHING - * ^ | | - * | v v - * \-------------------------------/ + * State Transitions * - * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call - * sites for explanations on the conditions being waited upon and why they are - * safe. Transitions out of them into NONE or QUEUED must store_release and the - * waiters should load_acquire. + * .------------> NONE (owned by SCX core) + * | | ^ + * | enqueue | | direct dispatch + * | v | + * | QUEUEING -------' + * | | + * | enqueue | + * | completes | + * | v + * | QUEUED (owned by BPF scheduler) + * | | + * | dispatch | + * | | + * | v + * | DISPATCHING + * | | + * | dispatch | + * | completes | + * `---------------' * - * Tracking scx_ops_state enables sched_ext core to reliably determine whether - * any given task can be dispatched by the BPF scheduler at all times and thus - * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler - * to try to dispatch any task anytime regardless of its state as the SCX core - * can safely reject invalid dispatches. + * State Descriptions + * + * - %SCX_OPSS_NONE: + * Task is owned by the SCX core. It's either on a run queue, running, + * or being manipulated by the core scheduler. The BPF scheduler has no + * claim on this task. + * + * - %SCX_OPSS_QUEUEING: + * Transitional state while transferring a task from the SCX core to + * the BPF scheduler. The task's rq lock is held during this state. + * Since QUEUEING is both entered and exited under the rq lock, dequeue + * can never observe this state (it would be a BUG). When finishing a + * dispatch, if the task is still in %SCX_OPSS_QUEUEING the completion + * path busy-waits for it to leave this state (via wait_ops_state()) + * before retrying. + * + * - %SCX_OPSS_QUEUED: + * Task is owned by the BPF scheduler. It's on a DSQ (dispatch queue) + * and the BPF scheduler is responsible for dispatching it. A QSEQ + * (queue sequence number) is embedded in this state to detect + * dispatch/dequeue races: if a task is dequeued and re-enqueued, the + * QSEQ changes and any in-flight dispatch operations targeting the old + * QSEQ are safely ignored. + * + * - %SCX_OPSS_DISPATCHING: + * Transitional state while transferring a task from the BPF scheduler + * back to the SCX core. This state indicates the BPF scheduler has + * selected the task for execution. When dequeue needs to take the task + * off a DSQ and it is still in %SCX_OPSS_DISPATCHING, the dequeue path + * busy-waits for it to leave this state (via wait_ops_state()) before + * proceeding. Exits to %SCX_OPSS_NONE when dispatch completes. + * + * Memory Ordering + * + * Transitions out of %SCX_OPSS_QUEUEING and %SCX_OPSS_DISPATCHING into + * %SCX_OPSS_NONE or %SCX_OPSS_QUEUED must use atomic_long_set_release() + * and waiters must use atomic_long_read_acquire(). This ensures proper + * synchronization between concurrent operations. + * + * Cross-CPU Task Migration + * + * When moving a task in the %SCX_OPSS_DISPATCHING state, we can't simply + * grab the target CPU's rq lock because a concurrent dequeue might be + * waiting on %SCX_OPSS_DISPATCHING while holding the source rq lock + * (deadlock). + * + * The sched_ext core uses a "lock dancing" protocol coordinated by + * p->scx.holding_cpu. When moving a task to a different rq: + * + * 1. Verify task can be moved (CPU affinity, migration_disabled, etc.) + * 2. Set p->scx.holding_cpu to the current CPU + * 3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING + * is set, so clearing DISPATCHING first prevents the circular wait + * (safe to lock the rq we need) + * 4. Unlock the current CPU's rq + * 5. Lock src_rq (where the task currently lives) + * 6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the + * race (dequeue clears holding_cpu to -1 when it takes the task), in + * this case migration is aborted + * 7. If src_rq == dst_rq: clear holding_cpu and enqueue directly + * into dst_rq's local DSQ (no lock swap needed) + * 8. Otherwise: call move_remote_task_to_local_dsq(), which releases + * src_rq, locks dst_rq, and performs the deactivate/activate + * migration cycle (dst_rq is held on return) + * 9. Unlock dst_rq and re-lock the current CPU's rq to restore + * the lock state expected by the caller + * + * If any verification fails, abort the migration. + * + * This state tracking allows the BPF scheduler to try to dispatch any task + * at any time regardless of its state. The SCX core can safely + * reject/ignore invalid dispatches, simplifying the BPF scheduler + * implementation. */ enum scx_ops_state { SCX_OPSS_NONE, /* owned by the SCX core */ From 8ce8d0524c136f60a1bddb0951db2999342d7217 Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Fri, 6 Mar 2026 10:30:51 +0000 Subject: [PATCH 5/9] sched_ext: Documentation: Mention scheduling class precedence Mention the scheduling class precedence of fair and sched_ext to clear up how sched_ext partial mode works. Signed-off-by: Christian Loehle Acked-by: Andrea Righi Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 9e2882d937b4..100d45963781 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -58,7 +58,8 @@ in ``ops->flags``, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE``, and However, when the BPF scheduler is loaded and ``SCX_OPS_SWITCH_PARTIAL`` is set in ``ops->flags``, only tasks with the ``SCHED_EXT`` policy are scheduled by sched_ext, while tasks with ``SCHED_NORMAL``, ``SCHED_BATCH`` and -``SCHED_IDLE`` policies are scheduled by the fair-class scheduler. +``SCHED_IDLE`` policies are scheduled by the fair-class scheduler which has +higher sched_class precedence than ``SCHED_EXT``. Terminating the sched_ext scheduler program, triggering `SysRq-S`, or detection of any internal error including stalled runnable tasks aborts the From 1dde502587657045b267f179d7a1ecc7b8a1a265 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 6 Mar 2026 04:50:55 +0000 Subject: [PATCH 6/9] sched_ext: Use READ_ONCE() for scx_slice_bypass_us in scx_bypass() Commit 0927780c90ce ("sched_ext: Use READ_ONCE() for lock-free reads of module param variables") annotated the plain reads of scx_slice_bypass_us and scx_bypass_lb_intv_us in bypass_lb_cpu(), but missed a third site in scx_bypass(): WRITE_ONCE(scx_slice_dfl, scx_slice_bypass_us * NSEC_PER_USEC); scx_slice_bypass_us is a module parameter writable via sysfs in process context through set_slice_us() -> param_set_uint_minmax(), which performs a plain store without holding bypass_lock. scx_bypass() reads the variable under bypass_lock, but since the writer does not take that lock, the two accesses are concurrent. WRITE_ONCE() only applies volatile semantics to the store of scx_slice_dfl -- the val expression containing scx_slice_bypass_us is evaluated as a plain read, providing no protection against concurrent writes. Wrap the read with READ_ONCE() to complete the annotation started by commit 0927780c90ce and make the access KCSAN-clean, consistent with the existing READ_ONCE(scx_slice_bypass_us) in bypass_lb_cpu(). Signed-off-by: David Carlier Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 10866bfb88bf..f323df7be180 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -4137,7 +4137,7 @@ static void scx_bypass(bool bypass) WARN_ON_ONCE(scx_bypass_depth <= 0); if (scx_bypass_depth != 1) goto unlock; - WRITE_ONCE(scx_slice_dfl, scx_slice_bypass_us * NSEC_PER_USEC); + WRITE_ONCE(scx_slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC); bypass_timestamp = ktime_get_ns(); if (sch) scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1); From 2a0596d516870951ce0e8edf510e48c87cb80761 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Sat, 7 Mar 2026 02:21:01 +0800 Subject: [PATCH 7/9] sched_ext: Documentation: Update sched-ext.rst - Remove CONFIG_PAHOLE_HAS_BTF_TAG from required config list - Document ext_idle.c as the built-in idle CPU selection policy - Add descriptions for example schedulers in tools/sched_ext/ Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 100d45963781..d74c2c2b9ef3 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -43,7 +43,6 @@ options should be enabled to use sched_ext: CONFIG_DEBUG_INFO_BTF=y CONFIG_BPF_JIT_ALWAYS_ON=y CONFIG_BPF_JIT_DEFAULT_ON=y - CONFIG_PAHOLE_HAS_BTF_TAG=y sched_ext is used only when the BPF scheduler is loaded and running. @@ -346,6 +345,8 @@ Where to Look The functions prefixed with ``scx_bpf_`` can be called from the BPF scheduler. +* ``kernel/sched/ext_idle.c`` contains the built-in idle CPU selection policy. + * ``tools/sched_ext/`` hosts example BPF scheduler implementations. * ``scx_simple[.bpf].c``: Minimal global FIFO scheduler example using a @@ -354,13 +355,35 @@ Where to Look * ``scx_qmap[.bpf].c``: A multi-level FIFO scheduler supporting five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``. + * ``scx_central[.bpf].c``: A central FIFO scheduler where all scheduling + decisions are made on one CPU, demonstrating ``LOCAL_ON`` dispatching, + tickless operation, and kthread preemption. + + * ``scx_cpu0[.bpf].c``: A scheduler that queues all tasks to a shared DSQ + and only dispatches them on CPU0 in FIFO order. Useful for testing bypass + behavior. + + * ``scx_flatcg[.bpf].c``: A flattened cgroup hierarchy scheduler + implementing hierarchical weight-based cgroup CPU control by compounding + each cgroup's share at every level into a single flat scheduling layer. + + * ``scx_pair[.bpf].c``: A core-scheduling example that always makes + sibling CPU pairs execute tasks from the same CPU cgroup. + + * ``scx_sdt[.bpf].c``: A variation of ``scx_simple`` demonstrating BPF + arena memory management for per-task data. + + * ``scx_userland[.bpf].c``: A minimal scheduler demonstrating user space + scheduling. Tasks with CPU affinity are direct-dispatched in FIFO order; + all others are scheduled in user space by a simple vruntime scheduler. + ABI Instability =============== The APIs provided by sched_ext to BPF schedulers programs have no stability guarantees. This includes the ops table callbacks and constants defined in ``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in -``kernel/sched/ext.c``. +``kernel/sched/ext.c`` and ``kernel/sched/ext_idle.c``. While we will attempt to provide a relatively stable API surface when possible, they are subject to change without warning between kernel From 57ccf5ccdc56954f2a91a7f66684fd31c566bde5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 7 Mar 2026 04:53:32 -1000 Subject: [PATCH 8/9] sched_ext: Fix enqueue_task_scx() truncation of upper enqueue flags enqueue_task_scx() takes int enq_flags from the sched_class interface. SCX enqueue flags starting at bit 32 (SCX_ENQ_PREEMPT and above) are silently truncated when passed through activate_task(). extra_enq_flags was added as a workaround - storing high bits in rq->scx.extra_enq_flags and OR-ing them back in enqueue_task_scx(). However, the OR target is still the int parameter, so the high bits are lost anyway. The current impact is limited as the only affected flag is SCX_ENQ_PREEMPT which is informational to the BPF scheduler - its loss means the scheduler doesn't know about preemption but doesn't cause incorrect behavior. Fix by renaming the int parameter to core_enq_flags and introducing a u64 enq_flags local that merges both sources. All downstream functions already take u64 enq_flags. Fixes: f0e1a0643a59 ("sched_ext: Implement BPF extensible scheduler class") Cc: stable@vger.kernel.org # v6.12+ Acked-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index f323df7be180..174e3650d7fe 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -1470,16 +1470,15 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } -static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) +static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags) { struct scx_sched *sch = scx_root; int sticky_cpu = p->scx.sticky_cpu; + u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; if (enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; - enq_flags |= rq->scx.extra_enq_flags; - if (sticky_cpu >= 0) p->scx.sticky_cpu = -1; From 2fcfe5951eb2e8440fc5e1dd6ea977336ff83a1d Mon Sep 17 00:00:00 2001 From: zhidao su Date: Mon, 9 Mar 2026 10:46:12 +0800 Subject: [PATCH 9/9] sched_ext: Use WRITE_ONCE() for the write side of scx_enable helper pointer scx_enable() uses double-checked locking to lazily initialize a static kthread_worker pointer. The fast path reads helper locklessly: if (!READ_ONCE(helper)) { // lockless read -- no helper_mutex The write side initializes helper under helper_mutex, but previously used a plain assignment: helper = kthread_run_worker(0, "scx_enable_helper"); ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ plain write -- KCSAN data race with READ_ONCE() above Since READ_ONCE() on the fast path and the plain write on the initialization path access the same variable without a common lock, they constitute a data race. KCSAN requires that all sides of a lock-free access use READ_ONCE()/WRITE_ONCE() consistently. Use a temporary variable to stage the result of kthread_run_worker(), and only WRITE_ONCE() into helper after confirming the pointer is valid. This avoids a window where a concurrent caller on the fast path could observe an ERR pointer via READ_ONCE(helper) before the error check completes. Fixes: b06ccbabe250 ("sched_ext: Fix starvation of scx_enable() under fair-class saturation") Signed-off-by: zhidao su Acked-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 174e3650d7fe..26a6ac2f8826 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -5258,13 +5258,14 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) if (!READ_ONCE(helper)) { mutex_lock(&helper_mutex); if (!helper) { - helper = kthread_run_worker(0, "scx_enable_helper"); - if (IS_ERR_OR_NULL(helper)) { - helper = NULL; + struct kthread_worker *w = + kthread_run_worker(0, "scx_enable_helper"); + if (IS_ERR_OR_NULL(w)) { mutex_unlock(&helper_mutex); return -ENOMEM; } - sched_set_fifo(helper->task); + sched_set_fifo(w->task); + WRITE_ONCE(helper, w); } mutex_unlock(&helper_mutex); }