1 files changed, 101 insertions, 61 deletions

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 513e4dfeeae7..6a4bae0a649d 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -205,76 +205,65 @@ exit_idle:
  *
  * Called with polling cleared.
  */
-static void cpu_idle_loop(void)
+static void do_idle(void)
 {
-        int cpu = smp_processor_id();
-
-        while (1) {
-                /*
-                 * If the arch has a polling bit, we maintain an invariant:
-                 *
-                 * Our polling bit is clear if we're not scheduled (i.e. if
-                 * rq->curr != rq->idle).  This means that, if rq->idle has
-                 * the polling bit set, then setting need_resched is
-                 * guaranteed to cause the cpu to reschedule.
-                 */
-
-                __current_set_polling();
-                quiet_vmstat();
-                tick_nohz_idle_enter();
+        /*
+         * If the arch has a polling bit, we maintain an invariant:
+         *
+         * Our polling bit is clear if we're not scheduled (i.e. if rq->curr !=
+         * rq->idle). This means that, if rq->idle has the polling bit set,
+         * then setting need_resched is guaranteed to cause the CPU to
+         * reschedule.
+         */
 
-                while (!need_resched()) {
-                        check_pgt_cache();
-                        rmb();
+        __current_set_polling();
+        tick_nohz_idle_enter();
 
-                        if (cpu_is_offline(cpu)) {
-                                cpuhp_report_idle_dead();
-                                arch_cpu_idle_dead();
-                        }
+        while (!need_resched()) {
+                check_pgt_cache();
+                rmb();
 
-                        local_irq_disable();
-                        arch_cpu_idle_enter();
-
-                        /*
-                         * In poll mode we reenable interrupts and spin.
-                         *
-                         * Also if we detected in the wakeup from idle
-                         * path that the tick broadcast device expired
-                         * for us, we don't want to go deep idle as we
-                         * know that the IPI is going to arrive right
-                         * away
-                         */
-                        if (cpu_idle_force_poll || tick_check_broadcast_expired())
-                                cpu_idle_poll();
-                        else
-                                cpuidle_idle_call();
-
-                        arch_cpu_idle_exit();
+                if (cpu_is_offline(smp_processor_id())) {
+                        cpuhp_report_idle_dead();
+                        arch_cpu_idle_dead();
                 }
 
-                /*
-                 * Since we fell out of the loop above, we know
-                 * TIF_NEED_RESCHED must be set, propagate it into
-                 * PREEMPT_NEED_RESCHED.
-                 *
-                 * This is required because for polling idle loops we will
-                 * not have had an IPI to fold the state for us.
-                 */
-                preempt_set_need_resched();
-                tick_nohz_idle_exit();
-                __current_clr_polling();
+                local_irq_disable();
+                arch_cpu_idle_enter();
 
                 /*
-                 * We promise to call sched_ttwu_pending and reschedule
-                 * if need_resched is set while polling is set.  That
-                 * means that clearing polling needs to be visible
-                 * before doing these things.
+                 * In poll mode we reenable interrupts and spin. Also if we
+                 * detected in the wakeup from idle path that the tick
+                 * broadcast device expired for us, we don't want to go deep
+                 * idle as we know that the IPI is going to arrive right away.
                  */
-                smp_mb__after_atomic();
-
-                sched_ttwu_pending();
-                schedule_preempt_disabled();
+                if (cpu_idle_force_poll || tick_check_broadcast_expired())
+                        cpu_idle_poll();
+                else
+                        cpuidle_idle_call();
+                arch_cpu_idle_exit();
         }
+
+        /*
+         * Since we fell out of the loop above, we know TIF_NEED_RESCHED must
+         * be set, propagate it into PREEMPT_NEED_RESCHED.
+         *
+         * This is required because for polling idle loops we will not have had
+         * an IPI to fold the state for us.
+         */
+        preempt_set_need_resched();
+        tick_nohz_idle_exit();
+        __current_clr_polling();
+
+        /*
+         * We promise to call sched_ttwu_pending() and reschedule if
+         * need_resched() is set while polling is set. That means that clearing
+         * polling needs to be visible before doing these things.
+         */
+        smp_mb__after_atomic();
+
+        sched_ttwu_pending();
+        schedule_preempt_disabled();
 }
 
 bool cpu_in_idle(unsigned long pc)
@@ -283,6 +272,56 @@ bool cpu_in_idle(unsigned long pc)
                 pc < (unsigned long)__cpuidle_text_end;
 }
 
+struct idle_timer {
+        struct hrtimer timer;
+        int done;
+};
+
+static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
+{
+        struct idle_timer *it = container_of(timer, struct idle_timer, timer);
+
+        WRITE_ONCE(it->done, 1);
+        set_tsk_need_resched(current);
+
+        return HRTIMER_NORESTART;
+}
+
+void play_idle(unsigned long duration_ms)
+{
+        struct idle_timer it;
+
+        /*
+         * Only FIFO tasks can disable the tick since they don't need the forced
+         * preemption.
+         */
+        WARN_ON_ONCE(current->policy != SCHED_FIFO);
+        WARN_ON_ONCE(current->nr_cpus_allowed != 1);
+        WARN_ON_ONCE(!(current->flags & PF_KTHREAD));
+        WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY));
+        WARN_ON_ONCE(!duration_ms);
+
+        rcu_sleep_check();
+        preempt_disable();
+        current->flags |= PF_IDLE;
+        cpuidle_use_deepest_state(true);
+
+        it.done = 0;
+        hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+        it.timer.function = idle_inject_timer_fn;
+        hrtimer_start(&it.timer, ms_to_ktime(duration_ms), HRTIMER_MODE_REL_PINNED);
+
+        while (!READ_ONCE(it.done))
+                do_idle();
+
+        cpuidle_use_deepest_state(false);
+        current->flags &= ~PF_IDLE;
+
+        preempt_fold_need_resched();
+        preempt_enable();
+}
+EXPORT_SYMBOL_GPL(play_idle);
+
 void cpu_startup_entry(enum cpuhp_state state)
 {
         /*
@@ -302,5 +341,6 @@ void cpu_startup_entry(enum cpuhp_state state)
 #endif
         arch_cpu_idle_prepare();
         cpuhp_online_idle(state);
-        cpu_idle_loop();
+        while (1)
+                do_idle();
 }