MIRRORS/linux-imx
1
0
Fork 0
mirror of https://github.com/nxp-imx/linux-imx.git synced 2025-07-13 04:39:36 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

sched/fair: Add EAS checks before updating root_domain::overutilized

Browse Source 
[ Upstream commit be3a51e68f ]

root_domain::overutilized is only used for EAS(energy aware scheduler)
to decide whether to do load balance or not. It is not used if EAS
not possible.

Currently enqueue_task_fair and task_tick_fair accesses, sometime updates
this field. In update_sd_lb_stats it is updated often. This causes cache
contention due to true sharing and burns a lot of cycles. ::overload and
::overutilized are part of the same cacheline. Updating it often invalidates
the cacheline. That causes access to ::overload to slow down due to
false sharing. Hence add EAS check before accessing/updating this field.
EAS check is optimized at compile time or it is a static branch.
Hence it shouldn't cost much.

With the patch, both enqueue_task_fair and newidle_balance don't show
up as hot routines in perf profile.

  6.8-rc4:
  7.18%  swapper          [kernel.vmlinux]              [k] enqueue_task_fair
  6.78%  s                [kernel.vmlinux]              [k] newidle_balance

  +patch:
  0.14%  swapper          [kernel.vmlinux]              [k] enqueue_task_fair
  0.00%  swapper          [kernel.vmlinux]              [k] newidle_balance

While at it: trace_sched_overutilized_tp expect that second argument to
be bool. So do a int to bool conversion for that.

Fixes: 2802bf3cd9 ("sched/fair: Add over-utilization/tipping point indicator")
Signed-off-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Qais Yousef <qyousef@layalina.io>
Reviewed-by: Srikar Dronamraju <srikar@linux.ibm.com>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lore.kernel.org/r/20240307085725.444486-2-sshegde@linux.ibm.com
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
Shrikanth Hegde 2024-03-07 14:27:23 +05:30 committed by Greg Kroah-Hartman
parent c078f2b492
commit 2bd572d421
1 changed files with 34 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
kernel/sched/fair.c
View File

@ -6564,22 +6564,42 @@ static inline void hrtick_update(struct rq *rq)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static inline bool cpu_overutilized(int cpu) static inline bool cpu_overutilized(int cpu)
{ {
unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); unsigned long rq_util_min, rq_util_max;
unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
if (!sched_energy_enabled())
return false;
rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN);
rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);
/* Return true only if the utilization doesn't fit CPU's capacity */ /* Return true only if the utilization doesn't fit CPU's capacity */
return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu);
} }
static inline void update_overutilized_status(struct rq *rq) static inline void set_rd_overutilized_status(struct root_domain *rd,
unsigned int status)
{ {
if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu)) { if (!sched_energy_enabled())
WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED); return;
trace_sched_overutilized_tp(rq->rd, SG_OVERUTILIZED);
} WRITE_ONCE(rd->overutilized, status);
trace_sched_overutilized_tp(rd, !!status);
}
static inline void check_update_overutilized_status(struct rq *rq)
{
/*
* overutilized field is used for load balancing decisions only
* if energy aware scheduler is being used
*/
if (!sched_energy_enabled())
return;
if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(rq->cpu))
set_rd_overutilized_status(rq->rd, SG_OVERUTILIZED);
} }
#else #else
static inline void update_overutilized_status(struct rq *rq) { } static inline void check_update_overutilized_status(struct rq *rq) { }
#endif #endif
/* Runqueue only has SCHED_IDLE tasks enqueued */ /* Runqueue only has SCHED_IDLE tasks enqueued */
@ -6680,7 +6700,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
* and the following generally works well enough in practice. * and the following generally works well enough in practice.
*/ */
if (!task_new) if (!task_new)
update_overutilized_status(rq); check_update_overutilized_status(rq);
enqueue_throttle: enqueue_throttle:
assert_list_leaf_cfs_rq(rq); assert_list_leaf_cfs_rq(rq);
@ -10500,19 +10520,14 @@ next_group:
env->fbq_type = fbq_classify_group(&sds->busiest_stat); env->fbq_type = fbq_classify_group(&sds->busiest_stat);
if (!env->sd->parent) { if (!env->sd->parent) {
struct root_domain *rd = env->dst_rq->rd;
/* update overload indicator if we are at root domain */ /* update overload indicator if we are at root domain */
WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD); WRITE_ONCE(env->dst_rq->rd->overload, sg_status & SG_OVERLOAD);
/* Update over-utilization (tipping point, U >= 0) indicator */ /* Update over-utilization (tipping point, U >= 0) indicator */
WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED); set_rd_overutilized_status(env->dst_rq->rd,
trace_sched_overutilized_tp(rd, sg_status & SG_OVERUTILIZED); sg_status & SG_OVERUTILIZED);
} else if (sg_status & SG_OVERUTILIZED) { } else if (sg_status & SG_OVERUTILIZED) {
struct root_domain *rd = env->dst_rq->rd; set_rd_overutilized_status(env->dst_rq->rd, SG_OVERUTILIZED);
WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED);
trace_sched_overutilized_tp(rd, SG_OVERUTILIZED);
} }
update_idle_cpu_scan(env, sum_util); update_idle_cpu_scan(env, sum_util);
@ -12503,7 +12518,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
task_tick_numa(rq, curr); task_tick_numa(rq, curr);
update_misfit_status(curr, rq); update_misfit_status(curr, rq);
update_overutilized_status(task_rq(curr)); check_update_overutilized_status(task_rq(curr));
task_tick_core(rq, curr); task_tick_core(rq, curr);
} }