From 9a8bc9bb4c3fb3218b4f151f98a722fbeb5b5c34 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Mon, 7 Sep 2026 14:38:55 +0200 Subject: [PATCH 1/4] sched/eevdf: Fix augmented max_slice Similarly to se->min_slice, init se->max_slice with se->slice before enqueueing the entity so the augmented callback computes it correctly at parent level. Fixes: 6e3c0a4e1ad1 ("sched/fair: Fix lag clamp") Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Link: https://patch.msgid.link/20260907123855.1297976-1-vincent.guittot@linaro.org --- kernel/sched/fair.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ade1eceb39b8..5b944f9a8a00 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1073,6 +1073,8 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; + se->max_slice = se->slice; + rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less, &min_vruntime_cb); } From 51b0e68cfa0ac69e3c3ea9d6753af7e15dfaab22 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Wed, 9 Sep 2026 17:05:22 +0200 Subject: [PATCH 2/4] sched/eevdf: Fix rb augmented with multi fields The eevdf rb tree maintains 3 augmented fields but only one is currently copied when balancing the tree. Add a more generic define that can be used when there are several augmented fields. In this case, we provide a function that takes care of copying all fields. Fixes: aef6987d8954 ("sched/eevdf: Propagate min_slice up the cgroup hierarchy") Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: K Prateek Nayak Tested-by: K Prateek Nayak Link: https://patch.msgid.link/20260909150522.858312-1-vincent.guittot@linaro.org --- include/linux/rbtree_augmented.h | 35 +++++++++++++++++++++++++------- kernel/sched/fair.c | 12 +++++++++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 6dbc5a1bf6a8..d2fa1c41bfd2 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -87,18 +87,18 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, } /* - * Template for declaring augmented rbtree callbacks (generic case) + * Template for declaring augmented rbtree callbacks (generic multi fields) * * RBSTATIC: 'static' or empty * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + * RBCOPY: name of function that copies the RBAUGMENTED datas + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED datas */ -#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +#define RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBCOPY, RBCOMPUTE) \ static inline void \ RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ { \ @@ -114,14 +114,14 @@ RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ } \ static void \ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ RBCOMPUTE(old, false); \ } \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ @@ -130,6 +130,27 @@ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .rotate = RBNAME ## _rotate \ }; +/* + * Template for declaring augmented rbtree callbacks (generic single field) + * + * RBSTATIC: 'static' or empty + * RBNAME: name of the rb_augment_callbacks structure + * RBSTRUCT: struct type of the tree nodes + * RBFIELD: name of struct rb_node field within RBSTRUCT + * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + */ + +#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +static inline void \ +RBNAME ## _copy_single(RBSTRUCT *new, RBSTRUCT *old) \ +{ \ + new->RBAUGMENTED = old->RBAUGMENTED; \ +} \ +RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBNAME ## _copy_single, RBCOMPUTE) + /* * Template for declaring augmented rbtree callbacks, * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5b944f9a8a00..944833e8056f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1032,6 +1032,13 @@ static inline void __max_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old) +{ + new->min_vruntime = old->min_vruntime; + new->min_slice = old->min_slice; + new->max_slice = old->max_slice; +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -1059,8 +1066,9 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) se->max_slice == old_max_slice; } -RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, - run_node, min_vruntime, min_vruntime_update); + +RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity, + run_node, min_vruntime_copy, min_vruntime_update); /* * Enqueue an entity into the rb-tree: From c23810313bdf6b02f39a1f2a1464c4b18bd39e31 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Fri, 4 Sep 2026 11:47:07 +0800 Subject: [PATCH 3/4] sched: Account cgroup CPU time to the execution context Proxy execution separates the scheduling context from the execution context. Commit aa4f74dfd42b ("sched: Fix runtime accounting w/ split exec & sched contexts") made per-task and thread-group runtime accounting follow the task that actually executes, while cgroup CPU usage is charged to the donor. When the donor and execution task belong to different cgroups, this makes a task's execution time count against a different cgroup from the one the task belongs to. Cgroup CPU usage should follow the execution context, matching the per-task, thread-group, and cgroup user/system accounting. Keep scheduling state associated with the donor, but charge cgroup CPU usage to rq->curr. A reproducer with the donor and execution task in separate cgroups showed the execution task accumulating runtime while cgroup CPU usage was charged to the donor's cgroup. With this change, the execution task's cgroup accumulates the CPU usage instead. The same behavior was verified with an RT donor and with legacy cpuacct accounting. Fixes: aa4f74dfd42b ("sched: Fix runtime accounting w/ split exec & sched contexts") Suggested-by: Tejun Heo Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Acked-by: Tejun Heo Acked-by: John Stultz Link: https://patch.msgid.link/20260904034707.268416-1-sh_def@163.com --- kernel/sched/fair.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 944833e8056f..7455a83a6a99 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1414,7 +1414,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) se->exec_start = now; if (entity_is_task(se)) { - struct task_struct *donor = task_of(se); struct task_struct *running = rq->curr; /* * If se is a task, we account the time against the running @@ -1427,8 +1426,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) account_group_exec_runtime(running, delta_exec); account_mm_sched(rq, running, delta_exec); - /* cgroup time is always accounted against the donor */ - cgroup_account_cputime(donor, delta_exec); + cgroup_account_cputime(running, delta_exec); } else { /* If not task, account the time against donor se */ se->sum_exec_runtime += delta_exec; From f5741d2b34519d387edf6e9798fc7030c20a35f3 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 2 Sep 2026 23:02:09 +0800 Subject: [PATCH 4/4] sched/core: Call wq_worker_tick() for the execution context wq_worker_tick() accounts CPU time and detects CPU-intensive work for the kworker that is actually running. With proxy execution, rq->donor is the scheduling context while rq->curr is the execution context. Calling the hook with rq->donor can skip workqueue accounting when a kworker is executing on behalf of a donor task. It can also account a blocked kworker when the donor is a worker but rq->curr is the task actually executing. The former can delay WORKER_CPU_INTENSIVE handling and pool concurrency management, which can delay pending kernel work and userspace operations depending on it. Use rq->curr for the workqueue tick hook while retaining rq->donor for scheduler accounting. Fixes: af0c8b2bf67b ("sched: Split scheduler and execution contexts") Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Acked-by: Tejun Heo Link: https://patch.msgid.link/20260902150208.1209922-2-sh_def@163.com --- kernel/sched/core.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b998ef6b87af..7885ff76e69f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5776,8 +5776,8 @@ void sched_tick(void) { int cpu = smp_processor_id(); struct rq *rq = cpu_rq(cpu); - /* accounting goes to the donor task */ - struct task_struct *donor; + /* scheduler accounting goes to the donor task */ + struct task_struct *curr, *donor; struct rq_flags rf; unsigned long hw_pressure; u64 resched_latency; @@ -5788,6 +5788,7 @@ void sched_tick(void) sched_clock_tick(); rq_lock(rq, &rf); + curr = rq->curr; donor = rq->donor; psi_account_irqtime(rq, donor, NULL); @@ -5813,8 +5814,8 @@ void sched_tick(void) perf_event_task_tick(); - if (donor->flags & PF_WQ_WORKER) - wq_worker_tick(donor); + if (curr->flags & PF_WQ_WORKER) + wq_worker_tick(curr); if (!scx_switched_all()) { rq->idle_balance = idle_cpu(cpu);