PM / sleep: Re-implement suspend-to-idle handling

In preparation for adding support for quiescing timers in the final
stage of suspend-to-idle transitions, rework the freeze_enter()
function making the system wait on a wakeup event, the freeze_wake()
function terminating the suspend-to-idle loop and the mechanism by
which deep idle states are entered during suspend-to-idle.

First of all, introduce a simple state machine for suspend-to-idle
and make the code in question use it.

Second, prevent freeze_enter() from losing wakeup events due to race
conditions and ensure that the number of online CPUs won't change
while it is being executed.  In addition to that, make it force
all of the CPUs re-enter the idle loop in case they are in idle
states already (so they can enter deeper idle states if possible).

Next, drop cpuidle_use_deepest_state() and replace use_deepest_state
checks in cpuidle_select() and cpuidle_reflect() with a single
suspend-to-idle state check in cpuidle_idle_call().

Finally, introduce cpuidle_enter_freeze() that will simply find the
deepest idle state available to the given CPU and enter it using
cpuidle_enter().

Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>

2 files changed, 52 insertions, 7 deletions

diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index c347e3ce3a55..b7d6b3a721b1 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -37,7 +37,9 @@ const char *pm_states[PM_SUSPEND_MAX];
 static const struct platform_suspend_ops *suspend_ops;
 static const struct platform_freeze_ops *freeze_ops;
 static DECLARE_WAIT_QUEUE_HEAD(suspend_freeze_wait_head);
-static bool suspend_freeze_wake;
+
+enum freeze_state __read_mostly suspend_freeze_state;
+static DEFINE_SPINLOCK(suspend_freeze_lock);
 
 void freeze_set_ops(const struct platform_freeze_ops *ops)
 {
@@ -48,22 +50,49 @@ void freeze_set_ops(const struct platform_freeze_ops *ops)
 
 static void freeze_begin(void)
 {
-        suspend_freeze_wake = false;
+        suspend_freeze_state = FREEZE_STATE_NONE;
 }
 
 static void freeze_enter(void)
 {
-        cpuidle_use_deepest_state(true);
+        spin_lock_irq(&suspend_freeze_lock);
+        if (pm_wakeup_pending())
+                goto out;
+
+        suspend_freeze_state = FREEZE_STATE_ENTER;
+        spin_unlock_irq(&suspend_freeze_lock);
+
+        get_online_cpus();
         cpuidle_resume();
-        wait_event(suspend_freeze_wait_head, suspend_freeze_wake);
+
+        /* Push all the CPUs into the idle loop. */
+        wake_up_all_idle_cpus();
+        pr_debug("PM: suspend-to-idle\n");
+        /* Make the current CPU wait so it can enter the idle loop too. */
+        wait_event(suspend_freeze_wait_head,
+                   suspend_freeze_state == FREEZE_STATE_WAKE);
+        pr_debug("PM: resume from suspend-to-idle\n");
+
         cpuidle_pause();
-        cpuidle_use_deepest_state(false);
+        put_online_cpus();
+
+        spin_lock_irq(&suspend_freeze_lock);
+
+ out:
+        suspend_freeze_state = FREEZE_STATE_NONE;
+        spin_unlock_irq(&suspend_freeze_lock);
 }
 
 void freeze_wake(void)
 {
-        suspend_freeze_wake = true;
-        wake_up(&suspend_freeze_wait_head);
+        unsigned long flags;
+
+        spin_lock_irqsave(&suspend_freeze_lock, flags);
+        if (suspend_freeze_state > FREEZE_STATE_NONE) {
+                suspend_freeze_state = FREEZE_STATE_WAKE;
+                wake_up(&suspend_freeze_wait_head);
+        }
+        spin_unlock_irqrestore(&suspend_freeze_lock, flags);
 }
 EXPORT_SYMBOL_GPL(freeze_wake);
 
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index aaf1c1d5cf5d..94b2d7b88a27 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -7,6 +7,7 @@
 #include <linux/tick.h>
 #include <linux/mm.h>
 #include <linux/stackprotector.h>
+#include <linux/suspend.h>
 
 #include <asm/tlb.h>
 
@@ -105,6 +106,21 @@ static void cpuidle_idle_call(void)
         rcu_idle_enter();
 
         /*
+         * Suspend-to-idle ("freeze") is a system state in which all user space
+         * has been frozen, all I/O devices have been suspended and the only
+         * activity happens here and in iterrupts (if any).  In that case bypass
+         * the cpuidle governor and go stratight for the deepest idle state
+         * available.  Possibly also suspend the local tick and the entire
+         * timekeeping to prevent timer interrupts from kicking us out of idle
+         * until a proper wakeup interrupt happens.
+         */
+        if (idle_should_freeze()) {
+                cpuidle_enter_freeze();
+                local_irq_enable();
+                goto exit_idle;
+        }
+
+        /*
          * Ask the cpuidle framework to choose a convenient idle state.
          * Fall back to the default arch idle method on errors.
          */