cpuidle: Mark CPUs idle as late as possible to avoid unneeded IPIs

It isn't guaranteed a CPU will idle upon calling lpm_cpuidle_enter(),
since it could abort early at the need_resched() check. In this case,
it's possible for an IPI to be sent to this "idle" CPU needlessly, thus
wasting power. For the same reason, it's also wasteful to keep a CPU
marked idle even after it's woken up.

Reduce the window that CPUs are marked idle to as small as it can be in
order to improve power consumption.

Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
Signed-off-by: Ruchit <ruchitmarathe@gmail.com>

drivers/cpuidle/cpuidle.c

@@ -44,18 +44,15 @@ static atomic_t idled = ATOMIC_INIT(0);
 #error idled CPU mask not big enough for NR_CPUS
 #endif
 
-static void cpuidle_set_idle_cpu(unsigned int cpu)
+void cpuidle_set_idle_cpu(unsigned int cpu)
 {
 	atomic_or(BIT(cpu), &idled);
 }
 
-static void cpuidle_clear_idle_cpu(unsigned int cpu)
+void cpuidle_clear_idle_cpu(unsigned int cpu)
 {
 	atomic_andnot(BIT(cpu), &idled);
 }
-#else
-static inline void cpuidle_set_idle_cpu(unsigned int cpu) { }
-static inline void cpuidle_clear_idle_cpu(unsigned int cpu) { }
 #endif
 
 int cpuidle_disabled(void)
@@ -240,9 +237,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
 	time_start = ns_to_ktime(local_clock());
 
 	stop_critical_timings();
-	cpuidle_set_idle_cpu(dev->cpu);
 	entered_state = target_state->enter(dev, drv, index);
-	cpuidle_clear_idle_cpu(dev->cpu);
 	start_critical_timings();
 
 	sched_clock_idle_wakeup_event();

drivers/cpuidle/lpm-levels.c

@@ -1476,7 +1476,9 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev,
 	success = psci_enter_sleep(cpu, idx, true);
 	sec_debug_sched_msg("-Idle(%s)", cluster->cpu->levels[idx].name);
 #else
+	cpuidle_set_idle_cpu(dev->cpu);
 	success = psci_enter_sleep(cpu, idx, true);
+	cpuidle_clear_idle_cpu(dev->cpu);
 #endif
 
 exit:

include/linux/cpuidle.h

@@ -277,4 +277,12 @@ static inline int cpuidle_register_governor(struct cpuidle_governor *gov)
 	__ret ? -1 : idx;					\
 })
 
+#ifdef CONFIG_SMP
+void cpuidle_set_idle_cpu(unsigned int cpu);
+void cpuidle_clear_idle_cpu(unsigned int cpu);
+#else
+static inline void cpuidle_set_idle_cpu(unsigned int cpu) { }
+static inline void cpuidle_clear_idle_cpu(unsigned int cpu) { }
+#endif
+
 #endif /* _LINUX_CPUIDLE_H */