From 06c9ba9cb2d8253dfb7c3fa4d20a4d302c1b6456 Mon Sep 17 00:00:00 2001 From: Sultan Alsawaf Date: Fri, 1 May 2020 16:38:46 -0700 Subject: [PATCH] cpuidle: Optimize pm_qos notifier callback and IPI semantics The pm_qos callback currently suffers from a number of pitfalls: it sends IPIs to CPUs that may not be idle, waits for those IPIs to finish propagating while preemption is disabled (resulting in a long busy wait for the pm_qos_update_target() caller), and needlessly calls a no-op function when the IPIs are processed. Optimize the pm_qos notifier by only sending IPIs to CPUs that are idle, and by using arch_send_wakeup_ipi_mask() instead of smp_call_function_many(). Using IPI_WAKEUP instead of IPI_CALL_FUNC, which is what smp_call_function_many() uses behind the scenes, has the benefit of doing zero work upon receipt of the IPI; IPI_WAKEUP is designed purely for sending an IPI without a payload, whereas IPI_CALL_FUNC does unwanted extra work just to run the empty smp_callback() function. Determining which CPUs are idle is done efficiently with an atomic bitmask instead of using the wake_up_if_idle() API, which checks the CPU's runqueue in an RCU read-side critical section and under a spin lock. Not very efficient in comparison to a simple, atomic bitwise operation. A cpumask isn't needed for this because NR_CPUS is guaranteed to fit within a word. Change-Id: Ic4dd7e4781172bb8e3b6eb13417a814256d44cf0 Signed-off-by: Sultan Alsawaf --- drivers/cpuidle/cpuidle.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index f823dd3f9480..3f115217c3ef 100755 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -44,15 +44,18 @@ static atomic_t idled = ATOMIC_INIT(0); #error idled CPU mask not big enough for NR_CPUS #endif -void cpuidle_set_idle_cpu(unsigned int cpu) +static void cpuidle_set_idle_cpu(unsigned int cpu) { atomic_or(BIT(cpu), &idled); } -void cpuidle_clear_idle_cpu(unsigned int cpu) +static void cpuidle_clear_idle_cpu(unsigned int cpu) { atomic_andnot(BIT(cpu), &idled); } +#else +static inline void cpuidle_set_idle_cpu(unsigned int cpu) { } +static inline void cpuidle_clear_idle_cpu(unsigned int cpu) { } #endif int cpuidle_disabled(void) @@ -237,7 +240,9 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, time_start = ns_to_ktime(local_clock()); stop_critical_timings(); + cpuidle_set_idle_cpu(dev->cpu); entered_state = target_state->enter(dev, drv, index); + cpuidle_clear_idle_cpu(dev->cpu); start_critical_timings(); sched_clock_idle_wakeup_event();