aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/audit.c2
-rw-r--r--kernel/audit_tree.c1
-rw-r--r--kernel/cpu.c19
-rw-r--r--kernel/events/core.c4
-rw-r--r--kernel/events/uprobes.c1
-rw-r--r--kernel/fork.c5
-rw-r--r--kernel/panic.c1
-rw-r--r--kernel/power/suspend.c4
-rw-r--r--kernel/rcu/Makefile2
-rw-r--r--kernel/rcu/rcu.h2
-rw-r--r--kernel/rcu/rcutorture.c1
-rw-r--r--kernel/rcu/tiny.c6
-rw-r--r--kernel/rcu/tree.c97
-rw-r--r--kernel/rcu/tree.h22
-rw-r--r--kernel/rcu/tree_plugin.h111
-rw-r--r--kernel/rcu/update.c89
-rw-r--r--kernel/sched/core.c73
-rw-r--r--kernel/sched/deadline.c2
-rw-r--r--kernel/sched/fair.c14
-rw-r--r--kernel/sched/idle_task.c5
-rw-r--r--kernel/sched/rt.c2
-rw-r--r--kernel/sched/sched.h2
-rw-r--r--kernel/sched/stop_task.c5
-rw-r--r--kernel/signal.c42
-rw-r--r--kernel/softirq.c2
-rw-r--r--kernel/time/posix-cpu-timers.c2
-rw-r--r--kernel/time/tick-sched.c2
-rw-r--r--kernel/time/timer.c3
-rw-r--r--kernel/trace/ring_buffer.c81
-rw-r--r--kernel/trace/trace.c33
30 files changed, 402 insertions, 233 deletions
diff --git a/kernel/audit.c b/kernel/audit.c
index 80983df92cd4..cebb11db4d34 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -739,7 +739,7 @@ static void audit_log_feature_change(int which, u32 old_feature, u32 new_feature
739 739
740 ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_FEATURE_CHANGE); 740 ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_FEATURE_CHANGE);
741 audit_log_task_info(ab, current); 741 audit_log_task_info(ab, current);
742 audit_log_format(ab, "feature=%s old=%u new=%u old_lock=%u new_lock=%u res=%d", 742 audit_log_format(ab, " feature=%s old=%u new=%u old_lock=%u new_lock=%u res=%d",
743 audit_feature_names[which], !!old_feature, !!new_feature, 743 audit_feature_names[which], !!old_feature, !!new_feature,
744 !!old_lock, !!new_lock, res); 744 !!old_lock, !!new_lock, res);
745 audit_log_end(ab); 745 audit_log_end(ab);
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index e242e3a9864a..80f29e015570 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -154,6 +154,7 @@ static struct audit_chunk *alloc_chunk(int count)
154 chunk->owners[i].index = i; 154 chunk->owners[i].index = i;
155 } 155 }
156 fsnotify_init_mark(&chunk->mark, audit_tree_destroy_watch); 156 fsnotify_init_mark(&chunk->mark, audit_tree_destroy_watch);
157 chunk->mark.mask = FS_IN_IGNORED;
157 return chunk; 158 return chunk;
158} 159}
159 160
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 90a3d017b90c..5d220234b3ca 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -86,6 +86,16 @@ static struct {
86#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) 86#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
87#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) 87#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
88 88
89static void apply_puts_pending(int max)
90{
91 int delta;
92
93 if (atomic_read(&cpu_hotplug.puts_pending) >= max) {
94 delta = atomic_xchg(&cpu_hotplug.puts_pending, 0);
95 cpu_hotplug.refcount -= delta;
96 }
97}
98
89void get_online_cpus(void) 99void get_online_cpus(void)
90{ 100{
91 might_sleep(); 101 might_sleep();
@@ -93,6 +103,7 @@ void get_online_cpus(void)
93 return; 103 return;
94 cpuhp_lock_acquire_read(); 104 cpuhp_lock_acquire_read();
95 mutex_lock(&cpu_hotplug.lock); 105 mutex_lock(&cpu_hotplug.lock);
106 apply_puts_pending(65536);
96 cpu_hotplug.refcount++; 107 cpu_hotplug.refcount++;
97 mutex_unlock(&cpu_hotplug.lock); 108 mutex_unlock(&cpu_hotplug.lock);
98} 109}
@@ -105,6 +116,7 @@ bool try_get_online_cpus(void)
105 if (!mutex_trylock(&cpu_hotplug.lock)) 116 if (!mutex_trylock(&cpu_hotplug.lock))
106 return false; 117 return false;
107 cpuhp_lock_acquire_tryread(); 118 cpuhp_lock_acquire_tryread();
119 apply_puts_pending(65536);
108 cpu_hotplug.refcount++; 120 cpu_hotplug.refcount++;
109 mutex_unlock(&cpu_hotplug.lock); 121 mutex_unlock(&cpu_hotplug.lock);
110 return true; 122 return true;
@@ -161,12 +173,7 @@ void cpu_hotplug_begin(void)
161 cpuhp_lock_acquire(); 173 cpuhp_lock_acquire();
162 for (;;) { 174 for (;;) {
163 mutex_lock(&cpu_hotplug.lock); 175 mutex_lock(&cpu_hotplug.lock);
164 if (atomic_read(&cpu_hotplug.puts_pending)) { 176 apply_puts_pending(1);
165 int delta;
166
167 delta = atomic_xchg(&cpu_hotplug.puts_pending, 0);
168 cpu_hotplug.refcount -= delta;
169 }
170 if (likely(!cpu_hotplug.refcount)) 177 if (likely(!cpu_hotplug.refcount))
171 break; 178 break;
172 __set_current_state(TASK_UNINTERRUPTIBLE); 179 __set_current_state(TASK_UNINTERRUPTIBLE);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 3e19d3ebc29c..af0a5ba4e21d 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7477,11 +7477,11 @@ SYSCALL_DEFINE5(perf_event_open,
7477 7477
7478 if (move_group) { 7478 if (move_group) {
7479 synchronize_rcu(); 7479 synchronize_rcu();
7480 perf_install_in_context(ctx, group_leader, event->cpu); 7480 perf_install_in_context(ctx, group_leader, group_leader->cpu);
7481 get_ctx(ctx); 7481 get_ctx(ctx);
7482 list_for_each_entry(sibling, &group_leader->sibling_list, 7482 list_for_each_entry(sibling, &group_leader->sibling_list,
7483 group_entry) { 7483 group_entry) {
7484 perf_install_in_context(ctx, sibling, event->cpu); 7484 perf_install_in_context(ctx, sibling, sibling->cpu);
7485 get_ctx(ctx); 7485 get_ctx(ctx);
7486 } 7486 }
7487 } 7487 }
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 1d0af8a2c646..ed8f2cde34c5 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -1640,7 +1640,6 @@ bool uprobe_deny_signal(void)
1640 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) { 1640 if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
1641 utask->state = UTASK_SSTEP_TRAPPED; 1641 utask->state = UTASK_SSTEP_TRAPPED;
1642 set_tsk_thread_flag(t, TIF_UPROBE); 1642 set_tsk_thread_flag(t, TIF_UPROBE);
1643 set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
1644 } 1643 }
1645 } 1644 }
1646 1645
diff --git a/kernel/fork.c b/kernel/fork.c
index 9b7d746d6d62..9ca84189cfc2 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1022,11 +1022,14 @@ void __cleanup_sighand(struct sighand_struct *sighand)
1022{ 1022{
1023 if (atomic_dec_and_test(&sighand->count)) { 1023 if (atomic_dec_and_test(&sighand->count)) {
1024 signalfd_cleanup(sighand); 1024 signalfd_cleanup(sighand);
1025 /*
1026 * sighand_cachep is SLAB_DESTROY_BY_RCU so we can free it
1027 * without an RCU grace period, see __lock_task_sighand().
1028 */
1025 kmem_cache_free(sighand_cachep, sighand); 1029 kmem_cache_free(sighand_cachep, sighand);
1026 } 1030 }
1027} 1031}
1028 1032
1029
1030/* 1033/*
1031 * Initialize POSIX timer handling for a thread group. 1034 * Initialize POSIX timer handling for a thread group.
1032 */ 1035 */
diff --git a/kernel/panic.c b/kernel/panic.c
index d09dc5c32c67..cf80672b7924 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -244,6 +244,7 @@ static const struct tnt tnts[] = {
244 * 'I' - Working around severe firmware bug. 244 * 'I' - Working around severe firmware bug.
245 * 'O' - Out-of-tree module has been loaded. 245 * 'O' - Out-of-tree module has been loaded.
246 * 'E' - Unsigned module has been loaded. 246 * 'E' - Unsigned module has been loaded.
247 * 'L' - A soft lockup has previously occurred.
247 * 248 *
248 * The string is overwritten by the next call to print_tainted(). 249 * The string is overwritten by the next call to print_tainted().
249 */ 250 */
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index 4ca9a33ff620..c347e3ce3a55 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -146,7 +146,7 @@ static int platform_suspend_prepare(suspend_state_t state)
146 146
147static int platform_suspend_prepare_late(suspend_state_t state) 147static int platform_suspend_prepare_late(suspend_state_t state)
148{ 148{
149 return state == PM_SUSPEND_FREEZE && freeze_ops->prepare ? 149 return state == PM_SUSPEND_FREEZE && freeze_ops && freeze_ops->prepare ?
150 freeze_ops->prepare() : 0; 150 freeze_ops->prepare() : 0;
151} 151}
152 152
@@ -164,7 +164,7 @@ static void platform_resume_noirq(suspend_state_t state)
164 164
165static void platform_resume_early(suspend_state_t state) 165static void platform_resume_early(suspend_state_t state)
166{ 166{
167 if (state == PM_SUSPEND_FREEZE && freeze_ops->restore) 167 if (state == PM_SUSPEND_FREEZE && freeze_ops && freeze_ops->restore)
168 freeze_ops->restore(); 168 freeze_ops->restore();
169} 169}
170 170
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
index 807ccfbf69b3..e6fae503d1bc 100644
--- a/kernel/rcu/Makefile
+++ b/kernel/rcu/Makefile
@@ -1,6 +1,6 @@
1obj-y += update.o srcu.o 1obj-y += update.o srcu.o
2obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o 2obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
3obj-$(CONFIG_TREE_RCU) += tree.o 3obj-$(CONFIG_TREE_RCU) += tree.o
4obj-$(CONFIG_TREE_PREEMPT_RCU) += tree.o 4obj-$(CONFIG_PREEMPT_RCU) += tree.o
5obj-$(CONFIG_TREE_RCU_TRACE) += tree_trace.o 5obj-$(CONFIG_TREE_RCU_TRACE) += tree_trace.o
6obj-$(CONFIG_TINY_RCU) += tiny.o 6obj-$(CONFIG_TINY_RCU) += tiny.o
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index ff1a6de62f17..07bb02eda844 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -135,4 +135,6 @@ int rcu_jiffies_till_stall_check(void);
135 */ 135 */
136#define TPS(x) tracepoint_string(x) 136#define TPS(x) tracepoint_string(x)
137 137
138void rcu_early_boot_tests(void);
139
138#endif /* __LINUX_RCU_H */ 140#endif /* __LINUX_RCU_H */
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 240fa9094f83..4d559baf06e0 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -812,6 +812,7 @@ rcu_torture_cbflood(void *arg)
812 cur_ops->cb_barrier(); 812 cur_ops->cb_barrier();
813 stutter_wait("rcu_torture_cbflood"); 813 stutter_wait("rcu_torture_cbflood");
814 } while (!torture_must_stop()); 814 } while (!torture_must_stop());
815 vfree(rhp);
815 torture_kthread_stopping("rcu_torture_cbflood"); 816 torture_kthread_stopping("rcu_torture_cbflood");
816 return 0; 817 return 0;
817} 818}
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index c0623fc47125..0db5649f8817 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -247,7 +247,7 @@ void rcu_bh_qs(void)
247 * be called from hardirq context. It is normally called from the 247 * be called from hardirq context. It is normally called from the
248 * scheduling-clock interrupt. 248 * scheduling-clock interrupt.
249 */ 249 */
250void rcu_check_callbacks(int cpu, int user) 250void rcu_check_callbacks(int user)
251{ 251{
252 RCU_TRACE(check_cpu_stalls()); 252 RCU_TRACE(check_cpu_stalls());
253 if (user || rcu_is_cpu_rrupt_from_idle()) 253 if (user || rcu_is_cpu_rrupt_from_idle())
@@ -380,7 +380,9 @@ void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
380} 380}
381EXPORT_SYMBOL_GPL(call_rcu_bh); 381EXPORT_SYMBOL_GPL(call_rcu_bh);
382 382
383void rcu_init(void) 383void __init rcu_init(void)
384{ 384{
385 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); 385 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
386
387 rcu_early_boot_tests();
386} 388}
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 9815447d22e0..7680fc275036 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -105,7 +105,7 @@ struct rcu_state sname##_state = { \
105 .name = RCU_STATE_NAME(sname), \ 105 .name = RCU_STATE_NAME(sname), \
106 .abbr = sabbr, \ 106 .abbr = sabbr, \
107}; \ 107}; \
108DEFINE_PER_CPU(struct rcu_data, sname##_data) 108DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data)
109 109
110RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); 110RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
111RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh); 111RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
@@ -152,19 +152,6 @@ EXPORT_SYMBOL_GPL(rcu_scheduler_active);
152 */ 152 */
153static int rcu_scheduler_fully_active __read_mostly; 153static int rcu_scheduler_fully_active __read_mostly;
154 154
155#ifdef CONFIG_RCU_BOOST
156
157/*
158 * Control variables for per-CPU and per-rcu_node kthreads. These
159 * handle all flavors of RCU.
160 */
161static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
162DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
163DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
164DEFINE_PER_CPU(char, rcu_cpu_has_work);
165
166#endif /* #ifdef CONFIG_RCU_BOOST */
167
168static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); 155static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
169static void invoke_rcu_core(void); 156static void invoke_rcu_core(void);
170static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); 157static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
@@ -286,11 +273,11 @@ static void rcu_momentary_dyntick_idle(void)
286 * and requires special handling for preemptible RCU. 273 * and requires special handling for preemptible RCU.
287 * The caller must have disabled preemption. 274 * The caller must have disabled preemption.
288 */ 275 */
289void rcu_note_context_switch(int cpu) 276void rcu_note_context_switch(void)
290{ 277{
291 trace_rcu_utilization(TPS("Start context switch")); 278 trace_rcu_utilization(TPS("Start context switch"));
292 rcu_sched_qs(); 279 rcu_sched_qs();
293 rcu_preempt_note_context_switch(cpu); 280 rcu_preempt_note_context_switch();
294 if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) 281 if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
295 rcu_momentary_dyntick_idle(); 282 rcu_momentary_dyntick_idle();
296 trace_rcu_utilization(TPS("End context switch")); 283 trace_rcu_utilization(TPS("End context switch"));
@@ -325,7 +312,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
325 unsigned long *maxj), 312 unsigned long *maxj),
326 bool *isidle, unsigned long *maxj); 313 bool *isidle, unsigned long *maxj);
327static void force_quiescent_state(struct rcu_state *rsp); 314static void force_quiescent_state(struct rcu_state *rsp);
328static int rcu_pending(int cpu); 315static int rcu_pending(void);
329 316
330/* 317/*
331 * Return the number of RCU-sched batches processed thus far for debug & stats. 318 * Return the number of RCU-sched batches processed thus far for debug & stats.
@@ -510,11 +497,11 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
510 * we really have entered idle, and must do the appropriate accounting. 497 * we really have entered idle, and must do the appropriate accounting.
511 * The caller must have disabled interrupts. 498 * The caller must have disabled interrupts.
512 */ 499 */
513static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval, 500static void rcu_eqs_enter_common(long long oldval, bool user)
514 bool user)
515{ 501{
516 struct rcu_state *rsp; 502 struct rcu_state *rsp;
517 struct rcu_data *rdp; 503 struct rcu_data *rdp;
504 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
518 505
519 trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting); 506 trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
520 if (!user && !is_idle_task(current)) { 507 if (!user && !is_idle_task(current)) {
@@ -531,7 +518,7 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
531 rdp = this_cpu_ptr(rsp->rda); 518 rdp = this_cpu_ptr(rsp->rda);
532 do_nocb_deferred_wakeup(rdp); 519 do_nocb_deferred_wakeup(rdp);
533 } 520 }
534 rcu_prepare_for_idle(smp_processor_id()); 521 rcu_prepare_for_idle();
535 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ 522 /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
536 smp_mb__before_atomic(); /* See above. */ 523 smp_mb__before_atomic(); /* See above. */
537 atomic_inc(&rdtp->dynticks); 524 atomic_inc(&rdtp->dynticks);
@@ -565,7 +552,7 @@ static void rcu_eqs_enter(bool user)
565 WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0); 552 WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
566 if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) { 553 if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {
567 rdtp->dynticks_nesting = 0; 554 rdtp->dynticks_nesting = 0;
568 rcu_eqs_enter_common(rdtp, oldval, user); 555 rcu_eqs_enter_common(oldval, user);
569 } else { 556 } else {
570 rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE; 557 rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;
571 } 558 }
@@ -589,7 +576,7 @@ void rcu_idle_enter(void)
589 576
590 local_irq_save(flags); 577 local_irq_save(flags);
591 rcu_eqs_enter(false); 578 rcu_eqs_enter(false);
592 rcu_sysidle_enter(this_cpu_ptr(&rcu_dynticks), 0); 579 rcu_sysidle_enter(0);
593 local_irq_restore(flags); 580 local_irq_restore(flags);
594} 581}
595EXPORT_SYMBOL_GPL(rcu_idle_enter); 582EXPORT_SYMBOL_GPL(rcu_idle_enter);
@@ -639,8 +626,8 @@ void rcu_irq_exit(void)
639 if (rdtp->dynticks_nesting) 626 if (rdtp->dynticks_nesting)
640 trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting); 627 trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
641 else 628 else
642 rcu_eqs_enter_common(rdtp, oldval, true); 629 rcu_eqs_enter_common(oldval, true);
643 rcu_sysidle_enter(rdtp, 1); 630 rcu_sysidle_enter(1);
644 local_irq_restore(flags); 631 local_irq_restore(flags);
645} 632}
646 633
@@ -651,16 +638,17 @@ void rcu_irq_exit(void)
651 * we really have exited idle, and must do the appropriate accounting. 638 * we really have exited idle, and must do the appropriate accounting.
652 * The caller must have disabled interrupts. 639 * The caller must have disabled interrupts.
653 */ 640 */
654static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval, 641static void rcu_eqs_exit_common(long long oldval, int user)
655 int user)
656{ 642{
643 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
644
657 rcu_dynticks_task_exit(); 645 rcu_dynticks_task_exit();
658 smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */ 646 smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */
659 atomic_inc(&rdtp->dynticks); 647 atomic_inc(&rdtp->dynticks);
660 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ 648 /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
661 smp_mb__after_atomic(); /* See above. */ 649 smp_mb__after_atomic(); /* See above. */
662 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); 650 WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
663 rcu_cleanup_after_idle(smp_processor_id()); 651 rcu_cleanup_after_idle();
664 trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting); 652 trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
665 if (!user && !is_idle_task(current)) { 653 if (!user && !is_idle_task(current)) {
666 struct task_struct *idle __maybe_unused = 654 struct task_struct *idle __maybe_unused =
@@ -691,7 +679,7 @@ static void rcu_eqs_exit(bool user)
691 rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE; 679 rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
692 } else { 680 } else {
693 rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; 681 rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
694 rcu_eqs_exit_common(rdtp, oldval, user); 682 rcu_eqs_exit_common(oldval, user);
695 } 683 }
696} 684}
697 685
@@ -712,7 +700,7 @@ void rcu_idle_exit(void)
712 700
713 local_irq_save(flags); 701 local_irq_save(flags);
714 rcu_eqs_exit(false); 702 rcu_eqs_exit(false);
715 rcu_sysidle_exit(this_cpu_ptr(&rcu_dynticks), 0); 703 rcu_sysidle_exit(0);
716 local_irq_restore(flags); 704 local_irq_restore(flags);
717} 705}
718EXPORT_SYMBOL_GPL(rcu_idle_exit); 706EXPORT_SYMBOL_GPL(rcu_idle_exit);
@@ -763,8 +751,8 @@ void rcu_irq_enter(void)
763 if (oldval) 751 if (oldval)
764 trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting); 752 trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
765 else 753 else
766 rcu_eqs_exit_common(rdtp, oldval, true); 754 rcu_eqs_exit_common(oldval, true);
767 rcu_sysidle_exit(rdtp, 1); 755 rcu_sysidle_exit(1);
768 local_irq_restore(flags); 756 local_irq_restore(flags);
769} 757}
770 758
@@ -2387,7 +2375,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
2387 * invoked from the scheduling-clock interrupt. If rcu_pending returns 2375 * invoked from the scheduling-clock interrupt. If rcu_pending returns
2388 * false, there is no point in invoking rcu_check_callbacks(). 2376 * false, there is no point in invoking rcu_check_callbacks().
2389 */ 2377 */
2390void rcu_check_callbacks(int cpu, int user) 2378void rcu_check_callbacks(int user)
2391{ 2379{
2392 trace_rcu_utilization(TPS("Start scheduler-tick")); 2380 trace_rcu_utilization(TPS("Start scheduler-tick"));
2393 increment_cpu_stall_ticks(); 2381 increment_cpu_stall_ticks();
@@ -2419,8 +2407,8 @@ void rcu_check_callbacks(int cpu, int user)
2419 2407
2420 rcu_bh_qs(); 2408 rcu_bh_qs();
2421 } 2409 }
2422 rcu_preempt_check_callbacks(cpu); 2410 rcu_preempt_check_callbacks();
2423 if (rcu_pending(cpu)) 2411 if (rcu_pending())
2424 invoke_rcu_core(); 2412 invoke_rcu_core();
2425 if (user) 2413 if (user)
2426 rcu_note_voluntary_context_switch(current); 2414 rcu_note_voluntary_context_switch(current);
@@ -2963,6 +2951,9 @@ static int synchronize_sched_expedited_cpu_stop(void *data)
2963 */ 2951 */
2964void synchronize_sched_expedited(void) 2952void synchronize_sched_expedited(void)
2965{ 2953{
2954 cpumask_var_t cm;
2955 bool cma = false;
2956 int cpu;
2966 long firstsnap, s, snap; 2957 long firstsnap, s, snap;
2967 int trycount = 0; 2958 int trycount = 0;
2968 struct rcu_state *rsp = &rcu_sched_state; 2959 struct rcu_state *rsp = &rcu_sched_state;
@@ -2997,11 +2988,26 @@ void synchronize_sched_expedited(void)
2997 } 2988 }
2998 WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id())); 2989 WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
2999 2990
2991 /* Offline CPUs, idle CPUs, and any CPU we run on are quiescent. */
2992 cma = zalloc_cpumask_var(&cm, GFP_KERNEL);
2993 if (cma) {
2994 cpumask_copy(cm, cpu_online_mask);
2995 cpumask_clear_cpu(raw_smp_processor_id(), cm);
2996 for_each_cpu(cpu, cm) {
2997 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
2998
2999 if (!(atomic_add_return(0, &rdtp->dynticks) & 0x1))
3000 cpumask_clear_cpu(cpu, cm);
3001 }
3002 if (cpumask_weight(cm) == 0)
3003 goto all_cpus_idle;
3004 }
3005
3000 /* 3006 /*
3001 * Each pass through the following loop attempts to force a 3007 * Each pass through the following loop attempts to force a
3002 * context switch on each CPU. 3008 * context switch on each CPU.
3003 */ 3009 */
3004 while (try_stop_cpus(cpu_online_mask, 3010 while (try_stop_cpus(cma ? cm : cpu_online_mask,
3005 synchronize_sched_expedited_cpu_stop, 3011 synchronize_sched_expedited_cpu_stop,
3006 NULL) == -EAGAIN) { 3012 NULL) == -EAGAIN) {
3007 put_online_cpus(); 3013 put_online_cpus();
@@ -3013,6 +3019,7 @@ void synchronize_sched_expedited(void)
3013 /* ensure test happens before caller kfree */ 3019 /* ensure test happens before caller kfree */
3014 smp_mb__before_atomic(); /* ^^^ */ 3020 smp_mb__before_atomic(); /* ^^^ */
3015 atomic_long_inc(&rsp->expedited_workdone1); 3021 atomic_long_inc(&rsp->expedited_workdone1);
3022 free_cpumask_var(cm);
3016 return; 3023 return;
3017 } 3024 }
3018 3025
@@ -3022,6 +3029,7 @@ void synchronize_sched_expedited(void)
3022 } else { 3029 } else {
3023 wait_rcu_gp(call_rcu_sched); 3030 wait_rcu_gp(call_rcu_sched);
3024 atomic_long_inc(&rsp->expedited_normal); 3031 atomic_long_inc(&rsp->expedited_normal);
3032 free_cpumask_var(cm);
3025 return; 3033 return;
3026 } 3034 }
3027 3035
@@ -3031,6 +3039,7 @@ void synchronize_sched_expedited(void)
3031 /* ensure test happens before caller kfree */ 3039 /* ensure test happens before caller kfree */
3032 smp_mb__before_atomic(); /* ^^^ */ 3040 smp_mb__before_atomic(); /* ^^^ */
3033 atomic_long_inc(&rsp->expedited_workdone2); 3041 atomic_long_inc(&rsp->expedited_workdone2);
3042 free_cpumask_var(cm);
3034 return; 3043 return;
3035 } 3044 }
3036 3045
@@ -3045,6 +3054,7 @@ void synchronize_sched_expedited(void)
3045 /* CPU hotplug operation in flight, use normal GP. */ 3054 /* CPU hotplug operation in flight, use normal GP. */
3046 wait_rcu_gp(call_rcu_sched); 3055 wait_rcu_gp(call_rcu_sched);
3047 atomic_long_inc(&rsp->expedited_normal); 3056 atomic_long_inc(&rsp->expedited_normal);
3057 free_cpumask_var(cm);
3048 return; 3058 return;
3049 } 3059 }
3050 snap = atomic_long_read(&rsp->expedited_start); 3060 snap = atomic_long_read(&rsp->expedited_start);
@@ -3052,6 +3062,9 @@ void synchronize_sched_expedited(void)
3052 } 3062 }
3053 atomic_long_inc(&rsp->expedited_stoppedcpus); 3063 atomic_long_inc(&rsp->expedited_stoppedcpus);
3054 3064
3065all_cpus_idle:
3066 free_cpumask_var(cm);
3067
3055 /* 3068 /*
3056 * Everyone up to our most recent fetch is covered by our grace 3069 * Everyone up to our most recent fetch is covered by our grace
3057 * period. Update the counter, but only if our work is still 3070 * period. Update the counter, but only if our work is still
@@ -3143,12 +3156,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
3143 * by the current CPU, returning 1 if so. This function is part of the 3156 * by the current CPU, returning 1 if so. This function is part of the
3144 * RCU implementation; it is -not- an exported member of the RCU API. 3157 * RCU implementation; it is -not- an exported member of the RCU API.
3145 */ 3158 */
3146static int rcu_pending(int cpu) 3159static int rcu_pending(void)
3147{ 3160{
3148 struct rcu_state *rsp; 3161 struct rcu_state *rsp;
3149 3162
3150 for_each_rcu_flavor(rsp) 3163 for_each_rcu_flavor(rsp)
3151 if (__rcu_pending(rsp, per_cpu_ptr(rsp->rda, cpu))) 3164 if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda)))
3152 return 1; 3165 return 1;
3153 return 0; 3166 return 0;
3154} 3167}
@@ -3158,7 +3171,7 @@ static int rcu_pending(int cpu)
3158 * non-NULL, store an indication of whether all callbacks are lazy. 3171 * non-NULL, store an indication of whether all callbacks are lazy.
3159 * (If there are no callbacks, all of them are deemed to be lazy.) 3172 * (If there are no callbacks, all of them are deemed to be lazy.)
3160 */ 3173 */
3161static int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy) 3174static int __maybe_unused rcu_cpu_has_callbacks(bool *all_lazy)
3162{ 3175{
3163 bool al = true; 3176 bool al = true;
3164 bool hc = false; 3177 bool hc = false;
@@ -3166,7 +3179,7 @@ static int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
3166 struct rcu_state *rsp; 3179 struct rcu_state *rsp;
3167 3180
3168 for_each_rcu_flavor(rsp) { 3181 for_each_rcu_flavor(rsp) {
3169 rdp = per_cpu_ptr(rsp->rda, cpu); 3182 rdp = this_cpu_ptr(rsp->rda);
3170 if (!rdp->nxtlist) 3183 if (!rdp->nxtlist)
3171 continue; 3184 continue;
3172 hc = true; 3185 hc = true;
@@ -3485,8 +3498,10 @@ static int rcu_cpu_notify(struct notifier_block *self,
3485 case CPU_DEAD_FROZEN: 3498 case CPU_DEAD_FROZEN:
3486 case CPU_UP_CANCELED: 3499 case CPU_UP_CANCELED:
3487 case CPU_UP_CANCELED_FROZEN: 3500 case CPU_UP_CANCELED_FROZEN:
3488 for_each_rcu_flavor(rsp) 3501 for_each_rcu_flavor(rsp) {
3489 rcu_cleanup_dead_cpu(cpu, rsp); 3502 rcu_cleanup_dead_cpu(cpu, rsp);
3503 do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu));
3504 }
3490 break; 3505 break;
3491 default: 3506 default:
3492 break; 3507 break;
@@ -3766,6 +3781,8 @@ void __init rcu_init(void)
3766 pm_notifier(rcu_pm_notify, 0); 3781 pm_notifier(rcu_pm_notify, 0);
3767 for_each_online_cpu(cpu) 3782 for_each_online_cpu(cpu)
3768 rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu); 3783 rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
3784
3785 rcu_early_boot_tests();
3769} 3786}
3770 3787
3771#include "tree_plugin.h" 3788#include "tree_plugin.h"
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index bbdc45d8d74f..8e7b1843896e 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -139,7 +139,7 @@ struct rcu_node {
139 unsigned long expmask; /* Groups that have ->blkd_tasks */ 139 unsigned long expmask; /* Groups that have ->blkd_tasks */
140 /* elements that need to drain to allow the */ 140 /* elements that need to drain to allow the */
141 /* current expedited grace period to */ 141 /* current expedited grace period to */
142 /* complete (only for TREE_PREEMPT_RCU). */ 142 /* complete (only for PREEMPT_RCU). */
143 unsigned long qsmaskinit; 143 unsigned long qsmaskinit;
144 /* Per-GP initial value for qsmask & expmask. */ 144 /* Per-GP initial value for qsmask & expmask. */
145 unsigned long grpmask; /* Mask to apply to parent qsmask. */ 145 unsigned long grpmask; /* Mask to apply to parent qsmask. */
@@ -530,10 +530,10 @@ DECLARE_PER_CPU(struct rcu_data, rcu_sched_data);
530extern struct rcu_state rcu_bh_state; 530extern struct rcu_state rcu_bh_state;
531DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); 531DECLARE_PER_CPU(struct rcu_data, rcu_bh_data);
532 532
533#ifdef CONFIG_TREE_PREEMPT_RCU 533#ifdef CONFIG_PREEMPT_RCU
534extern struct rcu_state rcu_preempt_state; 534extern struct rcu_state rcu_preempt_state;
535DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); 535DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data);
536#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ 536#endif /* #ifdef CONFIG_PREEMPT_RCU */
537 537
538#ifdef CONFIG_RCU_BOOST 538#ifdef CONFIG_RCU_BOOST
539DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); 539DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
@@ -547,7 +547,7 @@ DECLARE_PER_CPU(char, rcu_cpu_has_work);
547/* Forward declarations for rcutree_plugin.h */ 547/* Forward declarations for rcutree_plugin.h */
548static void rcu_bootup_announce(void); 548static void rcu_bootup_announce(void);
549long rcu_batches_completed(void); 549long rcu_batches_completed(void);
550static void rcu_preempt_note_context_switch(int cpu); 550static void rcu_preempt_note_context_switch(void);
551static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); 551static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
552#ifdef CONFIG_HOTPLUG_CPU 552#ifdef CONFIG_HOTPLUG_CPU
553static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, 553static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp,
@@ -561,12 +561,12 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
561 struct rcu_node *rnp, 561 struct rcu_node *rnp,
562 struct rcu_data *rdp); 562 struct rcu_data *rdp);
563#endif /* #ifdef CONFIG_HOTPLUG_CPU */ 563#endif /* #ifdef CONFIG_HOTPLUG_CPU */
564static void rcu_preempt_check_callbacks(int cpu); 564static void rcu_preempt_check_callbacks(void);
565void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 565void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
566#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) 566#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PREEMPT_RCU)
567static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, 567static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
568 bool wake); 568 bool wake);
569#endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) */ 569#endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PREEMPT_RCU) */
570static void __init __rcu_init_preempt(void); 570static void __init __rcu_init_preempt(void);
571static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); 571static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
572static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); 572static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
@@ -579,8 +579,8 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
579#endif /* #ifdef CONFIG_RCU_BOOST */ 579#endif /* #ifdef CONFIG_RCU_BOOST */
580static void __init rcu_spawn_boost_kthreads(void); 580static void __init rcu_spawn_boost_kthreads(void);
581static void rcu_prepare_kthreads(int cpu); 581static void rcu_prepare_kthreads(int cpu);
582static void rcu_cleanup_after_idle(int cpu); 582static void rcu_cleanup_after_idle(void);
583static void rcu_prepare_for_idle(int cpu); 583static void rcu_prepare_for_idle(void);
584static void rcu_idle_count_callbacks_posted(void); 584static void rcu_idle_count_callbacks_posted(void);
585static void print_cpu_stall_info_begin(void); 585static void print_cpu_stall_info_begin(void);
586static void print_cpu_stall_info(struct rcu_state *rsp, int cpu); 586static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
@@ -606,8 +606,8 @@ static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp);
606#endif /* #ifdef CONFIG_RCU_NOCB_CPU */ 606#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
607static void __maybe_unused rcu_kick_nohz_cpu(int cpu); 607static void __maybe_unused rcu_kick_nohz_cpu(int cpu);
608static bool init_nocb_callback_list(struct rcu_data *rdp); 608static bool init_nocb_callback_list(struct rcu_data *rdp);
609static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq); 609static void rcu_sysidle_enter(int irq);
610static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq); 610static void rcu_sysidle_exit(int irq);
611static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle, 611static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
612 unsigned long *maxj); 612 unsigned long *maxj);
613static bool is_sysidle_rcu_state(struct rcu_state *rsp); 613static bool is_sysidle_rcu_state(struct rcu_state *rsp);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index c1d7f27bd38f..3ec85cb5d544 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -30,14 +30,24 @@
30#include <linux/smpboot.h> 30#include <linux/smpboot.h>
31#include "../time/tick-internal.h" 31#include "../time/tick-internal.h"
32 32
33#define RCU_KTHREAD_PRIO 1
34
35#ifdef CONFIG_RCU_BOOST 33#ifdef CONFIG_RCU_BOOST
34
36#include "../locking/rtmutex_common.h" 35#include "../locking/rtmutex_common.h"
37#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO 36
38#else 37/* rcuc/rcub kthread realtime priority */
39#define RCU_BOOST_PRIO RCU_KTHREAD_PRIO 38static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
40#endif 39module_param(kthread_prio, int, 0644);
40
41/*
42 * Control variables for per-CPU and per-rcu_node kthreads. These
43 * handle all flavors of RCU.
44 */
45static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
46DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
47DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
48DEFINE_PER_CPU(char, rcu_cpu_has_work);
49
50#endif /* #ifdef CONFIG_RCU_BOOST */
41 51
42#ifdef CONFIG_RCU_NOCB_CPU 52#ifdef CONFIG_RCU_NOCB_CPU
43static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ 53static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
@@ -72,9 +82,6 @@ static void __init rcu_bootup_announce_oddness(void)
72#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE 82#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
73 pr_info("\tRCU torture testing starts during boot.\n"); 83 pr_info("\tRCU torture testing starts during boot.\n");
74#endif 84#endif
75#if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
76 pr_info("\tDump stacks of tasks blocking RCU-preempt GP.\n");
77#endif
78#if defined(CONFIG_RCU_CPU_STALL_INFO) 85#if defined(CONFIG_RCU_CPU_STALL_INFO)
79 pr_info("\tAdditional per-CPU info printed with stalls.\n"); 86 pr_info("\tAdditional per-CPU info printed with stalls.\n");
80#endif 87#endif
@@ -85,9 +92,12 @@ static void __init rcu_bootup_announce_oddness(void)
85 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf); 92 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
86 if (nr_cpu_ids != NR_CPUS) 93 if (nr_cpu_ids != NR_CPUS)
87 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids); 94 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
95#ifdef CONFIG_RCU_BOOST
96 pr_info("\tRCU kthread priority: %d.\n", kthread_prio);
97#endif
88} 98}
89 99
90#ifdef CONFIG_TREE_PREEMPT_RCU 100#ifdef CONFIG_PREEMPT_RCU
91 101
92RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu); 102RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
93static struct rcu_state *rcu_state_p = &rcu_preempt_state; 103static struct rcu_state *rcu_state_p = &rcu_preempt_state;
@@ -156,7 +166,7 @@ static void rcu_preempt_qs(void)
156 * 166 *
157 * Caller must disable preemption. 167 * Caller must disable preemption.
158 */ 168 */
159static void rcu_preempt_note_context_switch(int cpu) 169static void rcu_preempt_note_context_switch(void)
160{ 170{
161 struct task_struct *t = current; 171 struct task_struct *t = current;
162 unsigned long flags; 172 unsigned long flags;
@@ -167,7 +177,7 @@ static void rcu_preempt_note_context_switch(int cpu)
167 !t->rcu_read_unlock_special.b.blocked) { 177 !t->rcu_read_unlock_special.b.blocked) {
168 178
169 /* Possibly blocking in an RCU read-side critical section. */ 179 /* Possibly blocking in an RCU read-side critical section. */
170 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu); 180 rdp = this_cpu_ptr(rcu_preempt_state.rda);
171 rnp = rdp->mynode; 181 rnp = rdp->mynode;
172 raw_spin_lock_irqsave(&rnp->lock, flags); 182 raw_spin_lock_irqsave(&rnp->lock, flags);
173 smp_mb__after_unlock_lock(); 183 smp_mb__after_unlock_lock();
@@ -415,8 +425,6 @@ void rcu_read_unlock_special(struct task_struct *t)
415 } 425 }
416} 426}
417 427
418#ifdef CONFIG_RCU_CPU_STALL_VERBOSE
419
420/* 428/*
421 * Dump detailed information for all tasks blocking the current RCU 429 * Dump detailed information for all tasks blocking the current RCU
422 * grace period on the specified rcu_node structure. 430 * grace period on the specified rcu_node structure.
@@ -451,14 +459,6 @@ static void rcu_print_detail_task_stall(struct rcu_state *rsp)
451 rcu_print_detail_task_stall_rnp(rnp); 459 rcu_print_detail_task_stall_rnp(rnp);
452} 460}
453 461
454#else /* #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
455
456static void rcu_print_detail_task_stall(struct rcu_state *rsp)
457{
458}
459
460#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
461
462#ifdef CONFIG_RCU_CPU_STALL_INFO 462#ifdef CONFIG_RCU_CPU_STALL_INFO
463 463
464static void rcu_print_task_stall_begin(struct rcu_node *rnp) 464static void rcu_print_task_stall_begin(struct rcu_node *rnp)
@@ -621,7 +621,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
621 * 621 *
622 * Caller must disable hard irqs. 622 * Caller must disable hard irqs.
623 */ 623 */
624static void rcu_preempt_check_callbacks(int cpu) 624static void rcu_preempt_check_callbacks(void)
625{ 625{
626 struct task_struct *t = current; 626 struct task_struct *t = current;
627 627
@@ -630,8 +630,8 @@ static void rcu_preempt_check_callbacks(int cpu)
630 return; 630 return;
631 } 631 }
632 if (t->rcu_read_lock_nesting > 0 && 632 if (t->rcu_read_lock_nesting > 0 &&
633 per_cpu(rcu_preempt_data, cpu).qs_pending && 633 __this_cpu_read(rcu_preempt_data.qs_pending) &&
634 !per_cpu(rcu_preempt_data, cpu).passed_quiesce) 634 !__this_cpu_read(rcu_preempt_data.passed_quiesce))
635 t->rcu_read_unlock_special.b.need_qs = true; 635 t->rcu_read_unlock_special.b.need_qs = true;
636} 636}
637 637
@@ -919,7 +919,7 @@ void exit_rcu(void)
919 __rcu_read_unlock(); 919 __rcu_read_unlock();
920} 920}
921 921
922#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */ 922#else /* #ifdef CONFIG_PREEMPT_RCU */
923 923
924static struct rcu_state *rcu_state_p = &rcu_sched_state; 924static struct rcu_state *rcu_state_p = &rcu_sched_state;
925 925
@@ -945,7 +945,7 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed);
945 * Because preemptible RCU does not exist, we never have to check for 945 * Because preemptible RCU does not exist, we never have to check for
946 * CPUs being in quiescent states. 946 * CPUs being in quiescent states.
947 */ 947 */
948static void rcu_preempt_note_context_switch(int cpu) 948static void rcu_preempt_note_context_switch(void)
949{ 949{
950} 950}
951 951
@@ -1017,7 +1017,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
1017 * Because preemptible RCU does not exist, it never has any callbacks 1017 * Because preemptible RCU does not exist, it never has any callbacks
1018 * to check. 1018 * to check.
1019 */ 1019 */
1020static void rcu_preempt_check_callbacks(int cpu) 1020static void rcu_preempt_check_callbacks(void)
1021{ 1021{
1022} 1022}
1023 1023
@@ -1070,7 +1070,7 @@ void exit_rcu(void)
1070{ 1070{
1071} 1071}
1072 1072
1073#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ 1073#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
1074 1074
1075#ifdef CONFIG_RCU_BOOST 1075#ifdef CONFIG_RCU_BOOST
1076 1076
@@ -1326,7 +1326,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1326 smp_mb__after_unlock_lock(); 1326 smp_mb__after_unlock_lock();
1327 rnp->boost_kthread_task = t; 1327 rnp->boost_kthread_task = t;
1328 raw_spin_unlock_irqrestore(&rnp->lock, flags); 1328 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1329 sp.sched_priority = RCU_BOOST_PRIO; 1329 sp.sched_priority = kthread_prio;
1330 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 1330 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1331 wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ 1331 wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1332 return 0; 1332 return 0;
@@ -1343,7 +1343,7 @@ static void rcu_cpu_kthread_setup(unsigned int cpu)
1343{ 1343{
1344 struct sched_param sp; 1344 struct sched_param sp;
1345 1345
1346 sp.sched_priority = RCU_KTHREAD_PRIO; 1346 sp.sched_priority = kthread_prio;
1347 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); 1347 sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1348} 1348}
1349 1349
@@ -1512,10 +1512,10 @@ static void rcu_prepare_kthreads(int cpu)
1512 * any flavor of RCU. 1512 * any flavor of RCU.
1513 */ 1513 */
1514#ifndef CONFIG_RCU_NOCB_CPU_ALL 1514#ifndef CONFIG_RCU_NOCB_CPU_ALL
1515int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) 1515int rcu_needs_cpu(unsigned long *delta_jiffies)
1516{ 1516{
1517 *delta_jiffies = ULONG_MAX; 1517 *delta_jiffies = ULONG_MAX;
1518 return rcu_cpu_has_callbacks(cpu, NULL); 1518 return rcu_cpu_has_callbacks(NULL);
1519} 1519}
1520#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ 1520#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1521 1521
@@ -1523,7 +1523,7 @@ int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
1523 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up 1523 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1524 * after it. 1524 * after it.
1525 */ 1525 */
1526static void rcu_cleanup_after_idle(int cpu) 1526static void rcu_cleanup_after_idle(void)
1527{ 1527{
1528} 1528}
1529 1529
@@ -1531,7 +1531,7 @@ static void rcu_cleanup_after_idle(int cpu)
1531 * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n, 1531 * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1532 * is nothing. 1532 * is nothing.
1533 */ 1533 */
1534static void rcu_prepare_for_idle(int cpu) 1534static void rcu_prepare_for_idle(void)
1535{ 1535{
1536} 1536}
1537 1537
@@ -1624,15 +1624,15 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
1624 * The caller must have disabled interrupts. 1624 * The caller must have disabled interrupts.
1625 */ 1625 */
1626#ifndef CONFIG_RCU_NOCB_CPU_ALL 1626#ifndef CONFIG_RCU_NOCB_CPU_ALL
1627int rcu_needs_cpu(int cpu, unsigned long *dj) 1627int rcu_needs_cpu(unsigned long *dj)
1628{ 1628{
1629 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); 1629 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1630 1630
1631 /* Snapshot to detect later posting of non-lazy callback. */ 1631 /* Snapshot to detect later posting of non-lazy callback. */
1632 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted; 1632 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
1633 1633
1634 /* If no callbacks, RCU doesn't need the CPU. */ 1634 /* If no callbacks, RCU doesn't need the CPU. */
1635 if (!rcu_cpu_has_callbacks(cpu, &rdtp->all_lazy)) { 1635 if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) {
1636 *dj = ULONG_MAX; 1636 *dj = ULONG_MAX;
1637 return 0; 1637 return 0;
1638 } 1638 }
@@ -1666,12 +1666,12 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)
1666 * 1666 *
1667 * The caller must have disabled interrupts. 1667 * The caller must have disabled interrupts.
1668 */ 1668 */
1669static void rcu_prepare_for_idle(int cpu) 1669static void rcu_prepare_for_idle(void)
1670{ 1670{
1671#ifndef CONFIG_RCU_NOCB_CPU_ALL 1671#ifndef CONFIG_RCU_NOCB_CPU_ALL
1672 bool needwake; 1672 bool needwake;
1673 struct rcu_data *rdp; 1673 struct rcu_data *rdp;
1674 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); 1674 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
1675 struct rcu_node *rnp; 1675 struct rcu_node *rnp;
1676 struct rcu_state *rsp; 1676 struct rcu_state *rsp;
1677 int tne; 1677 int tne;
@@ -1679,7 +1679,7 @@ static void rcu_prepare_for_idle(int cpu)
1679 /* Handle nohz enablement switches conservatively. */ 1679 /* Handle nohz enablement switches conservatively. */
1680 tne = ACCESS_ONCE(tick_nohz_active); 1680 tne = ACCESS_ONCE(tick_nohz_active);
1681 if (tne != rdtp->tick_nohz_enabled_snap) { 1681 if (tne != rdtp->tick_nohz_enabled_snap) {
1682 if (rcu_cpu_has_callbacks(cpu, NULL)) 1682 if (rcu_cpu_has_callbacks(NULL))
1683 invoke_rcu_core(); /* force nohz to see update. */ 1683 invoke_rcu_core(); /* force nohz to see update. */
1684 rdtp->tick_nohz_enabled_snap = tne; 1684 rdtp->tick_nohz_enabled_snap = tne;
1685 return; 1685 return;
@@ -1688,7 +1688,7 @@ static void rcu_prepare_for_idle(int cpu)
1688 return; 1688 return;
1689 1689
1690 /* If this is a no-CBs CPU, no callbacks, just return. */ 1690 /* If this is a no-CBs CPU, no callbacks, just return. */
1691 if (rcu_is_nocb_cpu(cpu)) 1691 if (rcu_is_nocb_cpu(smp_processor_id()))
1692 return; 1692 return;
1693 1693
1694 /* 1694 /*
@@ -1712,7 +1712,7 @@ static void rcu_prepare_for_idle(int cpu)
1712 return; 1712 return;
1713 rdtp->last_accelerate = jiffies; 1713 rdtp->last_accelerate = jiffies;
1714 for_each_rcu_flavor(rsp) { 1714 for_each_rcu_flavor(rsp) {
1715 rdp = per_cpu_ptr(rsp->rda, cpu); 1715 rdp = this_cpu_ptr(rsp->rda);
1716 if (!*rdp->nxttail[RCU_DONE_TAIL]) 1716 if (!*rdp->nxttail[RCU_DONE_TAIL])
1717 continue; 1717 continue;
1718 rnp = rdp->mynode; 1718 rnp = rdp->mynode;
@@ -1731,10 +1731,10 @@ static void rcu_prepare_for_idle(int cpu)
1731 * any grace periods that elapsed while the CPU was idle, and if any 1731 * any grace periods that elapsed while the CPU was idle, and if any
1732 * callbacks are now ready to invoke, initiate invocation. 1732 * callbacks are now ready to invoke, initiate invocation.
1733 */ 1733 */
1734static void rcu_cleanup_after_idle(int cpu) 1734static void rcu_cleanup_after_idle(void)
1735{ 1735{
1736#ifndef CONFIG_RCU_NOCB_CPU_ALL 1736#ifndef CONFIG_RCU_NOCB_CPU_ALL
1737 if (rcu_is_nocb_cpu(cpu)) 1737 if (rcu_is_nocb_cpu(smp_processor_id()))
1738 return; 1738 return;
1739 if (rcu_try_advance_all_cbs()) 1739 if (rcu_try_advance_all_cbs())
1740 invoke_rcu_core(); 1740 invoke_rcu_core();
@@ -2573,9 +2573,13 @@ static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
2573 rdp->nocb_leader = rdp_spawn; 2573 rdp->nocb_leader = rdp_spawn;
2574 if (rdp_last && rdp != rdp_spawn) 2574 if (rdp_last && rdp != rdp_spawn)
2575 rdp_last->nocb_next_follower = rdp; 2575 rdp_last->nocb_next_follower = rdp;
2576 rdp_last = rdp; 2576 if (rdp == rdp_spawn) {
2577 rdp = rdp->nocb_next_follower; 2577 rdp = rdp->nocb_next_follower;
2578 rdp_last->nocb_next_follower = NULL; 2578 } else {
2579 rdp_last = rdp;
2580 rdp = rdp->nocb_next_follower;
2581 rdp_last->nocb_next_follower = NULL;
2582 }
2579 } while (rdp); 2583 } while (rdp);
2580 rdp_spawn->nocb_next_follower = rdp_old_leader; 2584 rdp_spawn->nocb_next_follower = rdp_old_leader;
2581 } 2585 }
@@ -2761,9 +2765,10 @@ static int full_sysidle_state; /* Current system-idle state. */
2761 * to detect full-system idle states, not RCU quiescent states and grace 2765 * to detect full-system idle states, not RCU quiescent states and grace
2762 * periods. The caller must have disabled interrupts. 2766 * periods. The caller must have disabled interrupts.
2763 */ 2767 */
2764static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq) 2768static void rcu_sysidle_enter(int irq)
2765{ 2769{
2766 unsigned long j; 2770 unsigned long j;
2771 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2767 2772
2768 /* If there are no nohz_full= CPUs, no need to track this. */ 2773 /* If there are no nohz_full= CPUs, no need to track this. */
2769 if (!tick_nohz_full_enabled()) 2774 if (!tick_nohz_full_enabled())
@@ -2832,8 +2837,10 @@ void rcu_sysidle_force_exit(void)
2832 * usermode execution does -not- count as idle here! The caller must 2837 * usermode execution does -not- count as idle here! The caller must
2833 * have disabled interrupts. 2838 * have disabled interrupts.
2834 */ 2839 */
2835static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq) 2840static void rcu_sysidle_exit(int irq)
2836{ 2841{
2842 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
2843
2837 /* If there are no nohz_full= CPUs, no need to track this. */ 2844 /* If there are no nohz_full= CPUs, no need to track this. */
2838 if (!tick_nohz_full_enabled()) 2845 if (!tick_nohz_full_enabled())
2839 return; 2846 return;
@@ -3127,11 +3134,11 @@ static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp)
3127 3134
3128#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ 3135#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3129 3136
3130static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq) 3137static void rcu_sysidle_enter(int irq)
3131{ 3138{
3132} 3139}
3133 3140
3134static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq) 3141static void rcu_sysidle_exit(int irq)
3135{ 3142{
3136} 3143}
3137 3144
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index 3ef8ba58694e..e0d31a345ee6 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -306,7 +306,7 @@ struct debug_obj_descr rcuhead_debug_descr = {
306EXPORT_SYMBOL_GPL(rcuhead_debug_descr); 306EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
307#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ 307#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
308 308
309#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE) 309#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE)
310void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp, 310void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp,
311 unsigned long secs, 311 unsigned long secs,
312 unsigned long c_old, unsigned long c) 312 unsigned long c_old, unsigned long c)
@@ -531,7 +531,8 @@ static int __noreturn rcu_tasks_kthread(void *arg)
531 struct rcu_head *next; 531 struct rcu_head *next;
532 LIST_HEAD(rcu_tasks_holdouts); 532 LIST_HEAD(rcu_tasks_holdouts);
533 533
534 /* FIXME: Add housekeeping affinity. */ 534 /* Run on housekeeping CPUs by default. Sysadm can move if desired. */
535 housekeeping_affine(current);
535 536
536 /* 537 /*
537 * Each pass through the following loop makes one check for 538 * Each pass through the following loop makes one check for
@@ -690,3 +691,87 @@ static void rcu_spawn_tasks_kthread(void)
690} 691}
691 692
692#endif /* #ifdef CONFIG_TASKS_RCU */ 693#endif /* #ifdef CONFIG_TASKS_RCU */
694
695#ifdef CONFIG_PROVE_RCU
696
697/*
698 * Early boot self test parameters, one for each flavor
699 */
700static bool rcu_self_test;
701static bool rcu_self_test_bh;
702static bool rcu_self_test_sched;
703
704module_param(rcu_self_test, bool, 0444);
705module_param(rcu_self_test_bh, bool, 0444);
706module_param(rcu_self_test_sched, bool, 0444);
707
708static int rcu_self_test_counter;
709
710static void test_callback(struct rcu_head *r)
711{
712 rcu_self_test_counter++;
713 pr_info("RCU test callback executed %d\n", rcu_self_test_counter);
714}
715
716static void early_boot_test_call_rcu(void)
717{
718 static struct rcu_head head;
719
720 call_rcu(&head, test_callback);
721}
722
723static void early_boot_test_call_rcu_bh(void)
724{
725 static struct rcu_head head;
726
727 call_rcu_bh(&head, test_callback);
728}
729
730static void early_boot_test_call_rcu_sched(void)
731{
732 static struct rcu_head head;
733
734 call_rcu_sched(&head, test_callback);
735}
736
737void rcu_early_boot_tests(void)
738{
739 pr_info("Running RCU self tests\n");
740
741 if (rcu_self_test)
742 early_boot_test_call_rcu();
743 if (rcu_self_test_bh)
744 early_boot_test_call_rcu_bh();
745 if (rcu_self_test_sched)
746 early_boot_test_call_rcu_sched();
747}
748
749static int rcu_verify_early_boot_tests(void)
750{
751 int ret = 0;
752 int early_boot_test_counter = 0;
753
754 if (rcu_self_test) {
755 early_boot_test_counter++;
756 rcu_barrier();
757 }
758 if (rcu_self_test_bh) {
759 early_boot_test_counter++;
760 rcu_barrier_bh();
761 }
762 if (rcu_self_test_sched) {
763 early_boot_test_counter++;
764 rcu_barrier_sched();
765 }
766
767 if (rcu_self_test_counter != early_boot_test_counter) {
768 WARN_ON(1);
769 ret = -1;
770 }
771
772 return ret;
773}
774late_initcall(rcu_verify_early_boot_tests);
775#else
776void rcu_early_boot_tests(void) {}
777#endif /* CONFIG_PROVE_RCU */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 240157c13ddc..e67a6e88e125 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2475,44 +2475,6 @@ EXPORT_PER_CPU_SYMBOL(kstat);
2475EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 2475EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
2476 2476
2477/* 2477/*
2478 * Return any ns on the sched_clock that have not yet been accounted in
2479 * @p in case that task is currently running.
2480 *
2481 * Called with task_rq_lock() held on @rq.
2482 */
2483static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
2484{
2485 u64 ns = 0;
2486
2487 /*
2488 * Must be ->curr _and_ ->on_rq. If dequeued, we would
2489 * project cycles that may never be accounted to this
2490 * thread, breaking clock_gettime().
2491 */
2492 if (task_current(rq, p) && task_on_rq_queued(p)) {
2493 update_rq_clock(rq);
2494 ns = rq_clock_task(rq) - p->se.exec_start;
2495 if ((s64)ns < 0)
2496 ns = 0;
2497 }
2498
2499 return ns;
2500}
2501
2502unsigned long long task_delta_exec(struct task_struct *p)
2503{
2504 unsigned long flags;
2505 struct rq *rq;
2506 u64 ns = 0;
2507
2508 rq = task_rq_lock(p, &flags);
2509 ns = do_task_delta_exec(p, rq);
2510 task_rq_unlock(rq, p, &flags);
2511
2512 return ns;
2513}
2514
2515/*
2516 * Return accounted runtime for the task. 2478 * Return accounted runtime for the task.
2517 * In case the task is currently running, return the runtime plus current's 2479 * In case the task is currently running, return the runtime plus current's
2518 * pending runtime that have not been accounted yet. 2480 * pending runtime that have not been accounted yet.
@@ -2521,7 +2483,7 @@ unsigned long long task_sched_runtime(struct task_struct *p)
2521{ 2483{
2522 unsigned long flags; 2484 unsigned long flags;
2523 struct rq *rq; 2485 struct rq *rq;
2524 u64 ns = 0; 2486 u64 ns;
2525 2487
2526#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 2488#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
2527 /* 2489 /*
@@ -2540,7 +2502,16 @@ unsigned long long task_sched_runtime(struct task_struct *p)
2540#endif 2502#endif
2541 2503
2542 rq = task_rq_lock(p, &flags); 2504 rq = task_rq_lock(p, &flags);
2543 ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq); 2505 /*
2506 * Must be ->curr _and_ ->on_rq. If dequeued, we would
2507 * project cycles that may never be accounted to this
2508 * thread, breaking clock_gettime().
2509 */
2510 if (task_current(rq, p) && task_on_rq_queued(p)) {
2511 update_rq_clock(rq);
2512 p->sched_class->update_curr(rq);
2513 }
2514 ns = p->se.sum_exec_runtime;
2544 task_rq_unlock(rq, p, &flags); 2515 task_rq_unlock(rq, p, &flags);
2545 2516
2546 return ns; 2517 return ns;
@@ -2802,7 +2773,7 @@ need_resched:
2802 preempt_disable(); 2773 preempt_disable();
2803 cpu = smp_processor_id(); 2774 cpu = smp_processor_id();
2804 rq = cpu_rq(cpu); 2775 rq = cpu_rq(cpu);
2805 rcu_note_context_switch(cpu); 2776 rcu_note_context_switch();
2806 prev = rq->curr; 2777 prev = rq->curr;
2807 2778
2808 schedule_debug(prev); 2779 schedule_debug(prev);
@@ -2903,10 +2874,14 @@ asmlinkage __visible void __sched schedule_user(void)
2903 * or we have been woken up remotely but the IPI has not yet arrived, 2874 * or we have been woken up remotely but the IPI has not yet arrived,
2904 * we haven't yet exited the RCU idle mode. Do it here manually until 2875 * we haven't yet exited the RCU idle mode. Do it here manually until
2905 * we find a better solution. 2876 * we find a better solution.
2877 *
2878 * NB: There are buggy callers of this function. Ideally we
2879 * should warn if prev_state != IN_USER, but that will trigger
2880 * too frequently to make sense yet.
2906 */ 2881 */
2907 user_exit(); 2882 enum ctx_state prev_state = exception_enter();
2908 schedule(); 2883 schedule();
2909 user_enter(); 2884 exception_exit(prev_state);
2910} 2885}
2911#endif 2886#endif
2912 2887
@@ -6368,6 +6343,10 @@ static void sched_init_numa(void)
6368 if (!sched_debug()) 6343 if (!sched_debug())
6369 break; 6344 break;
6370 } 6345 }
6346
6347 if (!level)
6348 return;
6349
6371 /* 6350 /*
6372 * 'level' contains the number of unique distances, excluding the 6351 * 'level' contains the number of unique distances, excluding the
6373 * identity distance node_distance(i,i). 6352 * identity distance node_distance(i,i).
@@ -7444,8 +7423,12 @@ void sched_move_task(struct task_struct *tsk)
7444 if (unlikely(running)) 7423 if (unlikely(running))
7445 put_prev_task(rq, tsk); 7424 put_prev_task(rq, tsk);
7446 7425
7447 tg = container_of(task_css_check(tsk, cpu_cgrp_id, 7426 /*
7448 lockdep_is_held(&tsk->sighand->siglock)), 7427 * All callers are synchronized by task_rq_lock(); we do not use RCU
7428 * which is pointless here. Thus, we pass "true" to task_css_check()
7429 * to prevent lockdep warnings.
7430 */
7431 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
7449 struct task_group, css); 7432 struct task_group, css);
7450 tg = autogroup_task_group(tsk, tg); 7433 tg = autogroup_task_group(tsk, tg);
7451 tsk->sched_task_group = tg; 7434 tsk->sched_task_group = tg;
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 5285332392d5..28fa9d9e9201 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1701,4 +1701,6 @@ const struct sched_class dl_sched_class = {
1701 .prio_changed = prio_changed_dl, 1701 .prio_changed = prio_changed_dl,
1702 .switched_from = switched_from_dl, 1702 .switched_from = switched_from_dl,
1703 .switched_to = switched_to_dl, 1703 .switched_to = switched_to_dl,
1704
1705 .update_curr = update_curr_dl,
1704}; 1706};
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 34baa60f8a7b..ef2b104b254c 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -726,6 +726,11 @@ static void update_curr(struct cfs_rq *cfs_rq)
726 account_cfs_rq_runtime(cfs_rq, delta_exec); 726 account_cfs_rq_runtime(cfs_rq, delta_exec);
727} 727}
728 728
729static void update_curr_fair(struct rq *rq)
730{
731 update_curr(cfs_rq_of(&rq->curr->se));
732}
733
729static inline void 734static inline void
730update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 735update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
731{ 736{
@@ -1180,6 +1185,13 @@ static void task_numa_compare(struct task_numa_env *env,
1180 raw_spin_unlock_irq(&dst_rq->lock); 1185 raw_spin_unlock_irq(&dst_rq->lock);
1181 1186
1182 /* 1187 /*
1188 * Because we have preemption enabled we can get migrated around and
1189 * end try selecting ourselves (current == env->p) as a swap candidate.
1190 */
1191 if (cur == env->p)
1192 goto unlock;
1193
1194 /*
1183 * "imp" is the fault differential for the source task between the 1195 * "imp" is the fault differential for the source task between the
1184 * source and destination node. Calculate the total differential for 1196 * source and destination node. Calculate the total differential for
1185 * the source task and potential destination task. The more negative 1197 * the source task and potential destination task. The more negative
@@ -7949,6 +7961,8 @@ const struct sched_class fair_sched_class = {
7949 7961
7950 .get_rr_interval = get_rr_interval_fair, 7962 .get_rr_interval = get_rr_interval_fair,
7951 7963
7964 .update_curr = update_curr_fair,
7965
7952#ifdef CONFIG_FAIR_GROUP_SCHED 7966#ifdef CONFIG_FAIR_GROUP_SCHED
7953 .task_move_group = task_move_group_fair, 7967 .task_move_group = task_move_group_fair,
7954#endif 7968#endif
diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c
index 67ad4e7f506a..c65dac8c97cd 100644
--- a/kernel/sched/idle_task.c
+++ b/kernel/sched/idle_task.c
@@ -75,6 +75,10 @@ static unsigned int get_rr_interval_idle(struct rq *rq, struct task_struct *task
75 return 0; 75 return 0;
76} 76}
77 77
78static void update_curr_idle(struct rq *rq)
79{
80}
81
78/* 82/*
79 * Simple, special scheduling class for the per-CPU idle tasks: 83 * Simple, special scheduling class for the per-CPU idle tasks:
80 */ 84 */
@@ -101,4 +105,5 @@ const struct sched_class idle_sched_class = {
101 105
102 .prio_changed = prio_changed_idle, 106 .prio_changed = prio_changed_idle,
103 .switched_to = switched_to_idle, 107 .switched_to = switched_to_idle,
108 .update_curr = update_curr_idle,
104}; 109};
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index d024e6ce30ba..20bca398084a 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2128,6 +2128,8 @@ const struct sched_class rt_sched_class = {
2128 2128
2129 .prio_changed = prio_changed_rt, 2129 .prio_changed = prio_changed_rt,
2130 .switched_to = switched_to_rt, 2130 .switched_to = switched_to_rt,
2131
2132 .update_curr = update_curr_rt,
2131}; 2133};
2132 2134
2133#ifdef CONFIG_SCHED_DEBUG 2135#ifdef CONFIG_SCHED_DEBUG
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 24156c8434d1..2df8ef067cc5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1135,6 +1135,8 @@ struct sched_class {
1135 unsigned int (*get_rr_interval) (struct rq *rq, 1135 unsigned int (*get_rr_interval) (struct rq *rq,
1136 struct task_struct *task); 1136 struct task_struct *task);
1137 1137
1138 void (*update_curr) (struct rq *rq);
1139
1138#ifdef CONFIG_FAIR_GROUP_SCHED 1140#ifdef CONFIG_FAIR_GROUP_SCHED
1139 void (*task_move_group) (struct task_struct *p, int on_rq); 1141 void (*task_move_group) (struct task_struct *p, int on_rq);
1140#endif 1142#endif
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 67426e529f59..79ffec45a6ac 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -102,6 +102,10 @@ get_rr_interval_stop(struct rq *rq, struct task_struct *task)
102 return 0; 102 return 0;
103} 103}
104 104
105static void update_curr_stop(struct rq *rq)
106{
107}
108
105/* 109/*
106 * Simple, special scheduling class for the per-CPU stop tasks: 110 * Simple, special scheduling class for the per-CPU stop tasks:
107 */ 111 */
@@ -128,4 +132,5 @@ const struct sched_class stop_sched_class = {
128 132
129 .prio_changed = prio_changed_stop, 133 .prio_changed = prio_changed_stop,
130 .switched_to = switched_to_stop, 134 .switched_to = switched_to_stop,
135 .update_curr = update_curr_stop,
131}; 136};
diff --git a/kernel/signal.c b/kernel/signal.c
index 8f0876f9f6dd..19e35135fc60 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -1275,7 +1275,17 @@ struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
1275 local_irq_restore(*flags); 1275 local_irq_restore(*flags);
1276 break; 1276 break;
1277 } 1277 }
1278 1278 /*
1279 * This sighand can be already freed and even reused, but
1280 * we rely on SLAB_DESTROY_BY_RCU and sighand_ctor() which
1281 * initializes ->siglock: this slab can't go away, it has
1282 * the same object type, ->siglock can't be reinitialized.
1283 *
1284 * We need to ensure that tsk->sighand is still the same
1285 * after we take the lock, we can race with de_thread() or
1286 * __exit_signal(). In the latter case the next iteration
1287 * must see ->sighand == NULL.
1288 */
1279 spin_lock(&sighand->siglock); 1289 spin_lock(&sighand->siglock);
1280 if (likely(sighand == tsk->sighand)) { 1290 if (likely(sighand == tsk->sighand)) {
1281 rcu_read_unlock(); 1291 rcu_read_unlock();
@@ -1331,23 +1341,21 @@ int kill_pid_info(int sig, struct siginfo *info, struct pid *pid)
1331 int error = -ESRCH; 1341 int error = -ESRCH;
1332 struct task_struct *p; 1342 struct task_struct *p;
1333 1343
1334 rcu_read_lock(); 1344 for (;;) {
1335retry: 1345 rcu_read_lock();
1336 p = pid_task(pid, PIDTYPE_PID); 1346 p = pid_task(pid, PIDTYPE_PID);
1337 if (p) { 1347 if (p)
1338 error = group_send_sig_info(sig, info, p); 1348 error = group_send_sig_info(sig, info, p);
1339 if (unlikely(error == -ESRCH)) 1349 rcu_read_unlock();
1340 /* 1350 if (likely(!p || error != -ESRCH))
1341 * The task was unhashed in between, try again. 1351 return error;
1342 * If it is dead, pid_task() will return NULL,
1343 * if we race with de_thread() it will find the
1344 * new leader.
1345 */
1346 goto retry;
1347 }
1348 rcu_read_unlock();
1349 1352
1350 return error; 1353 /*
1354 * The task was unhashed in between, try again. If it
1355 * is dead, pid_task() will return NULL, if we race with
1356 * de_thread() it will find the new leader.
1357 */
1358 }
1351} 1359}
1352 1360
1353int kill_proc_info(int sig, struct siginfo *info, pid_t pid) 1361int kill_proc_info(int sig, struct siginfo *info, pid_t pid)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 0699add19164..501baa9ac1be 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -656,7 +656,7 @@ static void run_ksoftirqd(unsigned int cpu)
656 * in the task stack here. 656 * in the task stack here.
657 */ 657 */
658 __do_softirq(); 658 __do_softirq();
659 rcu_note_context_switch(cpu); 659 rcu_note_context_switch();
660 local_irq_enable(); 660 local_irq_enable();
661 cond_resched(); 661 cond_resched();
662 return; 662 return;
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 492b986195d5..a16b67859e2a 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -553,7 +553,7 @@ static int cpu_timer_sample_group(const clockid_t which_clock,
553 *sample = cputime_to_expires(cputime.utime); 553 *sample = cputime_to_expires(cputime.utime);
554 break; 554 break;
555 case CPUCLOCK_SCHED: 555 case CPUCLOCK_SCHED:
556 *sample = cputime.sum_exec_runtime + task_delta_exec(p); 556 *sample = cputime.sum_exec_runtime;
557 break; 557 break;
558 } 558 }
559 return 0; 559 return 0;
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 7b5741fc4110..1f4356037a7d 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -585,7 +585,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
585 last_jiffies = jiffies; 585 last_jiffies = jiffies;
586 } while (read_seqretry(&jiffies_lock, seq)); 586 } while (read_seqretry(&jiffies_lock, seq));
587 587
588 if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || 588 if (rcu_needs_cpu(&rcu_delta_jiffies) ||
589 arch_needs_cpu() || irq_work_needs_cpu()) { 589 arch_needs_cpu() || irq_work_needs_cpu()) {
590 next_jiffies = last_jiffies + 1; 590 next_jiffies = last_jiffies + 1;
591 delta_jiffies = 1; 591 delta_jiffies = 1;
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 3260ffdb368f..2d3f5c504939 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -1377,12 +1377,11 @@ unsigned long get_next_timer_interrupt(unsigned long now)
1377void update_process_times(int user_tick) 1377void update_process_times(int user_tick)
1378{ 1378{
1379 struct task_struct *p = current; 1379 struct task_struct *p = current;
1380 int cpu = smp_processor_id();
1381 1380
1382 /* Note: this timer irq context must be accounted for as well. */ 1381 /* Note: this timer irq context must be accounted for as well. */
1383 account_process_tick(p, user_tick); 1382 account_process_tick(p, user_tick);
1384 run_local_timers(); 1383 run_local_timers();
1385 rcu_check_callbacks(cpu, user_tick); 1384 rcu_check_callbacks(user_tick);
1386#ifdef CONFIG_IRQ_WORK 1385#ifdef CONFIG_IRQ_WORK
1387 if (in_irq()) 1386 if (in_irq())
1388 irq_work_tick(); 1387 irq_work_tick();
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 2d75c94ae87d..a56e07c8d15b 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -538,16 +538,18 @@ static void rb_wake_up_waiters(struct irq_work *work)
538 * ring_buffer_wait - wait for input to the ring buffer 538 * ring_buffer_wait - wait for input to the ring buffer
539 * @buffer: buffer to wait on 539 * @buffer: buffer to wait on
540 * @cpu: the cpu buffer to wait on 540 * @cpu: the cpu buffer to wait on
541 * @full: wait until a full page is available, if @cpu != RING_BUFFER_ALL_CPUS
541 * 542 *
542 * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon 543 * If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
543 * as data is added to any of the @buffer's cpu buffers. Otherwise 544 * as data is added to any of the @buffer's cpu buffers. Otherwise
544 * it will wait for data to be added to a specific cpu buffer. 545 * it will wait for data to be added to a specific cpu buffer.
545 */ 546 */
546int ring_buffer_wait(struct ring_buffer *buffer, int cpu) 547int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
547{ 548{
548 struct ring_buffer_per_cpu *cpu_buffer; 549 struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
549 DEFINE_WAIT(wait); 550 DEFINE_WAIT(wait);
550 struct rb_irq_work *work; 551 struct rb_irq_work *work;
552 int ret = 0;
551 553
552 /* 554 /*
553 * Depending on what the caller is waiting for, either any 555 * Depending on what the caller is waiting for, either any
@@ -564,36 +566,61 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu)
564 } 566 }
565 567
566 568
567 prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE); 569 while (true) {
570 prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
568 571
569 /* 572 /*
570 * The events can happen in critical sections where 573 * The events can happen in critical sections where
571 * checking a work queue can cause deadlocks. 574 * checking a work queue can cause deadlocks.
572 * After adding a task to the queue, this flag is set 575 * After adding a task to the queue, this flag is set
573 * only to notify events to try to wake up the queue 576 * only to notify events to try to wake up the queue
574 * using irq_work. 577 * using irq_work.
575 * 578 *
576 * We don't clear it even if the buffer is no longer 579 * We don't clear it even if the buffer is no longer
577 * empty. The flag only causes the next event to run 580 * empty. The flag only causes the next event to run
578 * irq_work to do the work queue wake up. The worse 581 * irq_work to do the work queue wake up. The worse
579 * that can happen if we race with !trace_empty() is that 582 * that can happen if we race with !trace_empty() is that
580 * an event will cause an irq_work to try to wake up 583 * an event will cause an irq_work to try to wake up
581 * an empty queue. 584 * an empty queue.
582 * 585 *
583 * There's no reason to protect this flag either, as 586 * There's no reason to protect this flag either, as
584 * the work queue and irq_work logic will do the necessary 587 * the work queue and irq_work logic will do the necessary
585 * synchronization for the wake ups. The only thing 588 * synchronization for the wake ups. The only thing
586 * that is necessary is that the wake up happens after 589 * that is necessary is that the wake up happens after
587 * a task has been queued. It's OK for spurious wake ups. 590 * a task has been queued. It's OK for spurious wake ups.
588 */ 591 */
589 work->waiters_pending = true; 592 work->waiters_pending = true;
593
594 if (signal_pending(current)) {
595 ret = -EINTR;
596 break;
597 }
598
599 if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer))
600 break;
601
602 if (cpu != RING_BUFFER_ALL_CPUS &&
603 !ring_buffer_empty_cpu(buffer, cpu)) {
604 unsigned long flags;
605 bool pagebusy;
606
607 if (!full)
608 break;
609
610 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
611 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
612 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
613
614 if (!pagebusy)
615 break;
616 }
590 617
591 if ((cpu == RING_BUFFER_ALL_CPUS && ring_buffer_empty(buffer)) ||
592 (cpu != RING_BUFFER_ALL_CPUS && ring_buffer_empty_cpu(buffer, cpu)))
593 schedule(); 618 schedule();
619 }
594 620
595 finish_wait(&work->waiters, &wait); 621 finish_wait(&work->waiters, &wait);
596 return 0; 622
623 return ret;
597} 624}
598 625
599/** 626/**
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 8a528392b1f4..92f4a6cee172 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1076,13 +1076,14 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1076} 1076}
1077#endif /* CONFIG_TRACER_MAX_TRACE */ 1077#endif /* CONFIG_TRACER_MAX_TRACE */
1078 1078
1079static int wait_on_pipe(struct trace_iterator *iter) 1079static int wait_on_pipe(struct trace_iterator *iter, bool full)
1080{ 1080{
1081 /* Iterators are static, they should be filled or empty */ 1081 /* Iterators are static, they should be filled or empty */
1082 if (trace_buffer_iter(iter, iter->cpu_file)) 1082 if (trace_buffer_iter(iter, iter->cpu_file))
1083 return 0; 1083 return 0;
1084 1084
1085 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file); 1085 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1086 full);
1086} 1087}
1087 1088
1088#ifdef CONFIG_FTRACE_STARTUP_TEST 1089#ifdef CONFIG_FTRACE_STARTUP_TEST
@@ -4434,15 +4435,12 @@ static int tracing_wait_pipe(struct file *filp)
4434 4435
4435 mutex_unlock(&iter->mutex); 4436 mutex_unlock(&iter->mutex);
4436 4437
4437 ret = wait_on_pipe(iter); 4438 ret = wait_on_pipe(iter, false);
4438 4439
4439 mutex_lock(&iter->mutex); 4440 mutex_lock(&iter->mutex);
4440 4441
4441 if (ret) 4442 if (ret)
4442 return ret; 4443 return ret;
4443
4444 if (signal_pending(current))
4445 return -EINTR;
4446 } 4444 }
4447 4445
4448 return 1; 4446 return 1;
@@ -5372,16 +5370,12 @@ tracing_buffers_read(struct file *filp, char __user *ubuf,
5372 goto out_unlock; 5370 goto out_unlock;
5373 } 5371 }
5374 mutex_unlock(&trace_types_lock); 5372 mutex_unlock(&trace_types_lock);
5375 ret = wait_on_pipe(iter); 5373 ret = wait_on_pipe(iter, false);
5376 mutex_lock(&trace_types_lock); 5374 mutex_lock(&trace_types_lock);
5377 if (ret) { 5375 if (ret) {
5378 size = ret; 5376 size = ret;
5379 goto out_unlock; 5377 goto out_unlock;
5380 } 5378 }
5381 if (signal_pending(current)) {
5382 size = -EINTR;
5383 goto out_unlock;
5384 }
5385 goto again; 5379 goto again;
5386 } 5380 }
5387 size = 0; 5381 size = 0;
@@ -5500,7 +5494,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5500 }; 5494 };
5501 struct buffer_ref *ref; 5495 struct buffer_ref *ref;
5502 int entries, size, i; 5496 int entries, size, i;
5503 ssize_t ret; 5497 ssize_t ret = 0;
5504 5498
5505 mutex_lock(&trace_types_lock); 5499 mutex_lock(&trace_types_lock);
5506 5500
@@ -5538,13 +5532,16 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5538 int r; 5532 int r;
5539 5533
5540 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 5534 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5541 if (!ref) 5535 if (!ref) {
5536 ret = -ENOMEM;
5542 break; 5537 break;
5538 }
5543 5539
5544 ref->ref = 1; 5540 ref->ref = 1;
5545 ref->buffer = iter->trace_buffer->buffer; 5541 ref->buffer = iter->trace_buffer->buffer;
5546 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 5542 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5547 if (!ref->page) { 5543 if (!ref->page) {
5544 ret = -ENOMEM;
5548 kfree(ref); 5545 kfree(ref);
5549 break; 5546 break;
5550 } 5547 }
@@ -5582,19 +5579,19 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5582 5579
5583 /* did we read anything? */ 5580 /* did we read anything? */
5584 if (!spd.nr_pages) { 5581 if (!spd.nr_pages) {
5582 if (ret)
5583 goto out;
5584
5585 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) { 5585 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5586 ret = -EAGAIN; 5586 ret = -EAGAIN;
5587 goto out; 5587 goto out;
5588 } 5588 }
5589 mutex_unlock(&trace_types_lock); 5589 mutex_unlock(&trace_types_lock);
5590 ret = wait_on_pipe(iter); 5590 ret = wait_on_pipe(iter, true);
5591 mutex_lock(&trace_types_lock); 5591 mutex_lock(&trace_types_lock);
5592 if (ret) 5592 if (ret)
5593 goto out; 5593 goto out;
5594 if (signal_pending(current)) { 5594
5595 ret = -EINTR;
5596 goto out;
5597 }
5598 goto again; 5595 goto again;
5599 } 5596 }
5600 5597
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-29 22:38:57 -0500