Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (64 commits)
  cpu: Export cpu_up()
  rcu: Apply ACCESS_ONCE() to rcu_boost() return value
  Revert "rcu: Permit rt_mutex_unlock() with irqs disabled"
  docs: Additional LWN links to RCU API
  rcu: Augment rcu_batch_end tracing for idle and callback state
  rcu: Add rcutorture tests for srcu_read_lock_raw()
  rcu: Make rcutorture test for hotpluggability before offlining CPUs
  driver-core/cpu: Expose hotpluggability to the rest of the kernel
  rcu: Remove redundant rcu_cpu_stall_suppress declaration
  rcu: Adaptive dyntick-idle preparation
  rcu: Keep invoking callbacks if CPU otherwise idle
  rcu: Irq nesting is always 0 on rcu_enter_idle_common
  rcu: Don't check irq nesting from rcu idle entry/exit
  rcu: Permit dyntick-idle with callbacks pending
  rcu: Document same-context read-side constraints
  rcu: Identify dyntick-idle CPUs on first force_quiescent_state() pass
  rcu: Remove dynticks false positives and RCU failures
  rcu: Reduce latency of rcu_prepare_for_idle()
  rcu: Eliminate RCU_FAST_NO_HZ grace-period hang
  rcu: Avoid needlessly IPIing CPUs at GP end
  ...

24 files changed, 97 insertions, 46 deletions

diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 3d0c6fb74ae4..e8e8fe505df1 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -183,7 +183,8 @@ void cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 leds_event(led_idle_start);
                 while (!need_resched()) {
 #ifdef CONFIG_HOTPLUG_CPU
@@ -213,7 +214,8 @@ void cpu_idle(void)
                         }
                 }
                 leds_event(led_idle_end);
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c
index ef5a2a08fcca..ea3395750324 100644
--- a/arch/avr32/kernel/process.c
+++ b/arch/avr32/kernel/process.c
@@ -34,10 +34,12 @@ void cpu_idle(void)
 {
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched())
                         cpu_idle_sleep();
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c
index 6a80a9e9fc4a..8dd0416673cb 100644
--- a/arch/blackfin/kernel/process.c
+++ b/arch/blackfin/kernel/process.c
@@ -88,10 +88,12 @@ void cpu_idle(void)
 #endif
                 if (!idle)
                         idle = default_idle;
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched())
                         idle();
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c
index 95cc295976a7..7dcb5bfffb75 100644
--- a/arch/microblaze/kernel/process.c
+++ b/arch/microblaze/kernel/process.c
@@ -103,10 +103,12 @@ void cpu_idle(void)
                 if (!idle)
                         idle = default_idle;
 
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched())
                         idle();
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
 
                 preempt_enable_no_resched();
                 schedule();
diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index c47f96e453c0..7955409051c4 100644
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -56,7 +56,8 @@ void __noreturn cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched() && cpu_online(cpu)) {
 #ifdef CONFIG_MIPS_MT_SMTC
                         extern void smtc_idle_loop_hook(void);
@@ -77,7 +78,8 @@ void __noreturn cpu_idle(void)
                      system_state == SYSTEM_BOOTING))
                         play_dead();
 #endif
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/openrisc/kernel/idle.c b/arch/openrisc/kernel/idle.c
index d5bc5f813e89..e5fc78877830 100644
--- a/arch/openrisc/kernel/idle.c
+++ b/arch/openrisc/kernel/idle.c
@@ -51,7 +51,8 @@ void cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
 
                 while (!need_resched()) {
                         check_pgt_cache();
@@ -69,7 +70,8 @@ void cpu_idle(void)
                         set_thread_flag(TIF_POLLING_NRFLAG);
                 }
 
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index 39a2baa6ad58..9c3cd490b1bd 100644
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -46,6 +46,12 @@ static int __init powersave_off(char *arg)
 }
 __setup("powersave=off", powersave_off);
 
+#if defined(CONFIG_PPC_PSERIES) && defined(CONFIG_TRACEPOINTS)
+static const bool idle_uses_rcu = 1;
+#else
+static const bool idle_uses_rcu;
+#endif
+
 /*
  * The body of the idle task.
  */
@@ -56,7 +62,10 @@ void cpu_idle(void)
 
         set_thread_flag(TIF_POLLING_NRFLAG);
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                if (!idle_uses_rcu)
+                        rcu_idle_enter();
+
                 while (!need_resched() && !cpu_should_die()) {
                         ppc64_runlatch_off();
 
@@ -93,7 +102,9 @@ void cpu_idle(void)
 
                 HMT_medium();
                 ppc64_runlatch_on();
-                tick_nohz_restart_sched_tick();
+                if (!idle_uses_rcu)
+                        rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 if (cpu_should_die())
                         cpu_die();
diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c
index ea0acbd8966d..8fc62586a973 100644
--- a/arch/powerpc/platforms/iseries/setup.c
+++ b/arch/powerpc/platforms/iseries/setup.c
@@ -563,7 +563,8 @@ static void yield_shared_processor(void)
 static void iseries_shared_idle(void)
 {
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched() && !hvlpevent_is_pending()) {
                         local_irq_disable();
                         ppc64_runlatch_off();
@@ -577,7 +578,8 @@ static void iseries_shared_idle(void)
                 }
 
                 ppc64_runlatch_on();
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
 
                 if (hvlpevent_is_pending())
                         process_iSeries_events();
@@ -593,7 +595,8 @@ static void iseries_dedicated_idle(void)
         set_thread_flag(TIF_POLLING_NRFLAG);
 
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 if (!need_resched()) {
                         while (!need_resched()) {
                                 ppc64_runlatch_off();
@@ -610,7 +613,8 @@ static void iseries_dedicated_idle(void)
                 }
 
                 ppc64_runlatch_on();
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index 27a49508b410..52d429be6c76 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -555,6 +555,8 @@ void __trace_hcall_entry(unsigned long opcode, unsigned long *args)
 
         (*depth)++;
         trace_hcall_entry(opcode, args);
+        if (opcode == H_CEDE)
+                rcu_idle_enter();
         (*depth)--;
 
 out:
@@ -575,6 +577,8 @@ void __trace_hcall_exit(long opcode, unsigned long retval,
                 goto out;
 
         (*depth)++;
+        if (opcode == H_CEDE)
+                rcu_idle_exit();
         trace_hcall_exit(opcode, retval, retbuf);
         (*depth)--;
 
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 9451b210a1b4..3201ae447990 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -91,10 +91,12 @@ static void default_idle(void)
 void cpu_idle(void)
 {
         for (;;) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched())
                         default_idle();
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c
index db4ecd731a00..406508d4ce74 100644
--- a/arch/sh/kernel/idle.c
+++ b/arch/sh/kernel/idle.c
@@ -89,7 +89,8 @@ void cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
 
                 while (!need_resched()) {
                         check_pgt_cache();
@@ -111,7 +112,8 @@ void cpu_idle(void)
                         start_critical_timings();
                 }
 
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index 3739a06a76cb..39d8b05201a2 100644
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -95,12 +95,14 @@ void cpu_idle(void)
         set_thread_flag(TIF_POLLING_NRFLAG);
 
         while(1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
 
                 while (!need_resched() && !cpu_is_offline(cpu))
                         sparc64_yield(cpu);
 
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
 
                 preempt_enable_no_resched();
 
diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c
index fe1e3fc31bc5..ffb883ddd0f0 100644
--- a/arch/sparc/kernel/setup_32.c
+++ b/arch/sparc/kernel/setup_32.c
@@ -84,7 +84,7 @@ static void prom_sync_me(void)
 
         prom_printf("PROM SYNC COMMAND...\n");
         show_free_areas(0);
-        if(current->pid != 0) {
+        if (!is_idle_task(current)) {
                 local_irq_enable();
                 sys_sync();
                 local_irq_disable();
diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c
index 9c45d8bbdf57..4c1ac6e5347a 100644
--- a/arch/tile/kernel/process.c
+++ b/arch/tile/kernel/process.c
@@ -85,7 +85,8 @@ void cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched()) {
                         if (cpu_is_offline(cpu))
                                 BUG();  /* no HOTPLUG_CPU */
@@ -105,7 +106,8 @@ void cpu_idle(void)
                                 local_irq_enable();
                         current_thread_info()->status |= TS_POLLING;
                 }
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c
index 25b7b90fd620..c1eaaa1fcc20 100644
--- a/arch/tile/mm/fault.c
+++ b/arch/tile/mm/fault.c
@@ -54,7 +54,7 @@ static noinline void force_sig_info_fault(const char *type, int si_signo,
         if (unlikely(tsk->pid < 2)) {
                 panic("Signal %d (code %d) at %#lx sent to %s!",
                       si_signo, si_code & 0xffff, address,
-                      tsk->pid ? "init" : "the idle task");
+                      is_idle_task(tsk) ? "the idle task" : "init");
         }
 
         info.si_signo = si_signo;
@@ -515,7 +515,7 @@ no_context:
 
         if (unlikely(tsk->pid < 2)) {
                 panic("Kernel page fault running %s!",
-                      tsk->pid ? "init" : "the idle task");
+                      is_idle_task(tsk) ? "the idle task" : "init");
         }
 
         /*
diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c
index c5338351aecd..69f24905abdc 100644
--- a/arch/um/kernel/process.c
+++ b/arch/um/kernel/process.c
@@ -246,10 +246,12 @@ void default_idle(void)
                 if (need_resched())
                         schedule();
 
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 nsecs = disable_timer();
                 idle_sleep(nsecs);
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
         }
 }
 
diff --git a/arch/unicore32/kernel/process.c b/arch/unicore32/kernel/process.c
index ba401df971ed..52edc2b62873 100644
--- a/arch/unicore32/kernel/process.c
+++ b/arch/unicore32/kernel/process.c
@@ -55,7 +55,8 @@ void cpu_idle(void)
 {
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched()) {
                         local_irq_disable();
                         stop_critical_timings();
@@ -63,7 +64,8 @@ void cpu_idle(void)
                         local_irq_enable();
                         start_critical_timings();
                 }
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index f98d84caf94c..2cd2d93643dc 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -876,8 +876,8 @@ void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
          * Besides, if we don't timer interrupts ignore the global
          * interrupt lock, which is the WrongThing (tm) to do.
          */
-        exit_idle();
         irq_enter();
+        exit_idle();
         local_apic_timer_interrupt();
         irq_exit();
 
@@ -1809,8 +1809,8 @@ void smp_spurious_interrupt(struct pt_regs *regs)
 {
         u32 v;
 
-        exit_idle();
         irq_enter();
+        exit_idle();
         /*
          * Check if this really is a spurious interrupt and ACK it
          * if it is a vectored one.  Just in case...
@@ -1846,8 +1846,8 @@ void smp_error_interrupt(struct pt_regs *regs)
                 "Illegal register address",     /* APIC Error Bit 7 */
         };
 
-        exit_idle();
         irq_enter();
+        exit_idle();
         /* First tickle the hardware, only then report what went on. -- REW */
         v0 = apic_read(APIC_ESR);
         apic_write(APIC_ESR, 0);
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 6d939d7847e2..898055585516 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -2421,8 +2421,8 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void)
         unsigned vector, me;
 
         ack_APIC_irq();
-        exit_idle();
         irq_enter();
+        exit_idle();
 
         me = smp_processor_id();
         for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 787e06c84ea6..ce215616d5b9 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -397,8 +397,8 @@ static void (*smp_thermal_vector)(void) = unexpected_thermal_interrupt;
 
 asmlinkage void smp_thermal_interrupt(struct pt_regs *regs)
 {
-        exit_idle();
         irq_enter();
+        exit_idle();
         inc_irq_stat(irq_thermal_count);
         smp_thermal_vector();
         irq_exit();
diff --git a/arch/x86/kernel/cpu/mcheck/threshold.c b/arch/x86/kernel/cpu/mcheck/threshold.c
index d746df2909c9..aa578cadb940 100644
--- a/arch/x86/kernel/cpu/mcheck/threshold.c
+++ b/arch/x86/kernel/cpu/mcheck/threshold.c
@@ -19,8 +19,8 @@ void (*mce_threshold_vector)(void) = default_threshold_interrupt;
 
 asmlinkage void smp_threshold_interrupt(void)
 {
-        exit_idle();
         irq_enter();
+        exit_idle();
         inc_irq_stat(irq_threshold_count);
         mce_threshold_vector();
         irq_exit();
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 429e0c92924e..5d31e5bdbf85 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -181,8 +181,8 @@ unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
         unsigned vector = ~regs->orig_ax;
         unsigned irq;
 
-        exit_idle();
         irq_enter();
+        exit_idle();
 
         irq = __this_cpu_read(vector_irq[vector]);
 
@@ -209,10 +209,10 @@ void smp_x86_platform_ipi(struct pt_regs *regs)
 
         ack_APIC_irq();
 
-        exit_idle();
-
         irq_enter();
 
+        exit_idle();
+
         inc_irq_stat(x86_platform_ipis);
 
         if (x86_platform_ipi_callback)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 795b79f984c2..485204f58cda 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -99,7 +99,8 @@ void cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
+                rcu_idle_enter();
                 while (!need_resched()) {
 
                         check_pgt_cache();
@@ -116,7 +117,8 @@ void cpu_idle(void)
                                 pm_idle();
                         start_critical_timings();
                 }
-                tick_nohz_restart_sched_tick();
+                rcu_idle_exit();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 3bd7e6eebf31..64e926c89a6f 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -122,7 +122,7 @@ void cpu_idle(void)
 
         /* endless idle loop with no priority at all */
         while (1) {
-                tick_nohz_stop_sched_tick(1);
+                tick_nohz_idle_enter();
                 while (!need_resched()) {
 
                         rmb();
@@ -139,8 +139,14 @@ void cpu_idle(void)
                         enter_idle();
                         /* Don't trace irqs off for idle */
                         stop_critical_timings();
+
+                        /* enter_idle() needs rcu for notifiers */
+                        rcu_idle_enter();
+
                         if (cpuidle_idle_call())
                                 pm_idle();
+
+                        rcu_idle_exit();
                         start_critical_timings();
 
                         /* In many cases the interrupt that ended idle
@@ -149,7 +155,7 @@ void cpu_idle(void)
                         __exit_idle();
                 }
 
-                tick_nohz_restart_sched_tick();
+                tick_nohz_idle_exit();
                 preempt_enable_no_resched();
                 schedule();
                 preempt_disable();